From f34ca35a423bf3819dcc4f20cfae47166f75adce Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Thu, 14 Dec 2023 10:25:32 -0700 Subject: [PATCH 1/7] cp to tests --- tests/test_schema.py | 190 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 tests/test_schema.py diff --git a/tests/test_schema.py b/tests/test_schema.py new file mode 100644 index 000000000..8ec24fc49 --- /dev/null +++ b/tests/test_schema.py @@ -0,0 +1,190 @@ +from nose.tools import assert_false, assert_true, raises +import datajoint as dj +from inspect import getmembers +from . import schema +from . import schema_empty +from . import PREFIX, CONN_INFO, CONN_INFO_ROOT +from .schema_simple import schema as schema_simple + + +def relation_selector(attr): + try: + return issubclass(attr, dj.Table) + except TypeError: + return False + + +def part_selector(attr): + try: + return issubclass(attr, dj.Part) + except TypeError: + return False + + +def test_schema_size_on_disk(): + number_of_bytes = schema.schema.size_on_disk + assert_true(isinstance(number_of_bytes, int)) + + +def test_schema_list(): + schemas = dj.list_schemas() + assert_true(schema.schema.database in schemas) + + +@raises(dj.errors.AccessError) +def test_drop_unauthorized(): + info_schema = dj.schema("information_schema") + info_schema.drop() + + +def test_namespace_population(): + for name, rel in getmembers(schema, relation_selector): + assert_true( + hasattr(schema_empty, name), + "{name} not found in schema_empty".format(name=name), + ) + assert_true( + rel.__base__ is getattr(schema_empty, name).__base__, + "Wrong tier for {name}".format(name=name), + ) + + for name_part in dir(rel): + if name_part[0].isupper() and part_selector(getattr(rel, name_part)): + assert_true( + getattr(rel, name_part).__base__ is dj.Part, + "Wrong tier for {name}".format(name=name_part), + ) + + +@raises(dj.DataJointError) +def test_undecorated_table(): + """ + Undecorated user table classes should raise an informative exception upon first use + """ + + class UndecoratedClass(dj.Manual): + definition = "" + + a = UndecoratedClass() + print(a.full_table_name) + + +@raises(dj.DataJointError) +def test_reject_decorated_part(): + """ + Decorating a dj.Part table should raise an informative exception. + """ + + @schema.schema + class A(dj.Manual): + definition = ... + + @schema.schema + class B(dj.Part): + definition = ... + + +@raises(dj.DataJointError) +def test_unauthorized_database(): + """ + an attempt to create a database to which user has no privileges should raise an informative exception. + """ + dj.Schema("unauthorized_schema", connection=dj.conn(reset=True, **CONN_INFO)) + + +def test_drop_database(): + schema = dj.Schema( + PREFIX + "_drop_test", connection=dj.conn(reset=True, **CONN_INFO) + ) + assert_true(schema.exists) + schema.drop() + assert_false(schema.exists) + schema.drop() # should do nothing + + +def test_overlapping_name(): + test_schema = dj.Schema( + PREFIX + "_overlapping_schema", connection=dj.conn(**CONN_INFO) + ) + + @test_schema + class Unit(dj.Manual): + definition = """ + id: int # simple id + """ + + # hack to update the locals dictionary + locals() + + @test_schema + class Cell(dj.Manual): + definition = """ + type: varchar(32) # type of cell + """ + + class Unit(dj.Part): + definition = """ + -> master + -> Unit + """ + + test_schema.drop() + + +def test_list_tables(): + # https://github.com/datajoint/datajoint-python/issues/838 + assert set( + [ + "reserved_word", + "#l", + "#a", + "__d", + "__b", + "__b__c", + "__e", + "__e__f", + "#outfit_launch", + "#outfit_launch__outfit_piece", + "#i_j", + "#j_i", + "#t_test_update", + "#data_a", + "#data_b", + "f", + "#argmax_test", + "#website", + "profile", + "profile__website", + ] + ) == set(schema_simple.list_tables()) + + +def test_schema_save(): + assert_true("class Experiment(dj.Imported)" in schema.schema.code) + assert_true("class Experiment(dj.Imported)" in schema_empty.schema.code) + + +def test_uppercase_schema(): + # https://github.com/datajoint/datajoint-python/issues/564 + dj.conn(**CONN_INFO_ROOT, reset=True) + schema1 = dj.Schema("Schema_A") + + @schema1 + class Subject(dj.Manual): + definition = """ + name: varchar(32) + """ + + Schema_A = dj.VirtualModule("Schema_A", "Schema_A") + + schema2 = dj.Schema("schema_b") + + @schema2 + class Recording(dj.Manual): + definition = """ + -> Schema_A.Subject + id: smallint + """ + + schema2.drop() + schema1.drop() From 75984419cad06298fee47a679b6d0e84b2c91ce6 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Thu, 14 Dec 2023 10:25:48 -0700 Subject: [PATCH 2/7] nose2pytest test_schema --- tests/test_schema.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_schema.py b/tests/test_schema.py index 8ec24fc49..0e88a134a 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -23,12 +23,12 @@ def part_selector(attr): def test_schema_size_on_disk(): number_of_bytes = schema.schema.size_on_disk - assert_true(isinstance(number_of_bytes, int)) + assert isinstance(number_of_bytes, int) def test_schema_list(): schemas = dj.list_schemas() - assert_true(schema.schema.database in schemas) + assert schema.schema.database in schemas @raises(dj.errors.AccessError) @@ -96,9 +96,9 @@ def test_drop_database(): schema = dj.Schema( PREFIX + "_drop_test", connection=dj.conn(reset=True, **CONN_INFO) ) - assert_true(schema.exists) + assert schema.exists schema.drop() - assert_false(schema.exists) + assert not schema.exists schema.drop() # should do nothing @@ -160,8 +160,8 @@ def test_list_tables(): def test_schema_save(): - assert_true("class Experiment(dj.Imported)" in schema.schema.code) - assert_true("class Experiment(dj.Imported)" in schema_empty.schema.code) + assert "class Experiment(dj.Imported)" in schema.schema.code + assert "class Experiment(dj.Imported)" in schema_empty.schema.code def test_uppercase_schema(): From 55bf4ea716a541d579443155f2fd0746ca2a7e8b Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Thu, 14 Dec 2023 14:00:50 -0700 Subject: [PATCH 3/7] Add db_creds_test fixture --- tests/conftest.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5a38eef90..a9474b502 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -56,6 +56,15 @@ def enable_filepath_feature(monkeypatch): monkeypatch.delenv(FILEPATH_FEATURE_SWITCH, raising=True) +@pytest.fixture(scope="session") +def db_creds_test() -> Dict: + return dict( + host=os.getenv("DJ_TEST_HOST", "fakeservices.datajoint.io"), + user=os.getenv("DJ_TEST_USER", "datajoint"), + password=os.getenv("DJ_TEST_PASSWORD", "datajoint"), + ) + + @pytest.fixture(scope="session") def db_creds_root() -> Dict: return dict( @@ -142,12 +151,9 @@ def connection_root(connection_root_bare): @pytest.fixture(scope="session") -def connection_test(connection_root): +def connection_test(connection_root, db_creds_test): """Test user database connection.""" database = f"{PREFIX}%%" - credentials = dict( - host=os.getenv("DJ_HOST"), user="datajoint", password="datajoint" - ) permission = "ALL PRIVILEGES" # Create MySQL users @@ -157,14 +163,14 @@ def connection_test(connection_root): # create user if necessary on mysql8 connection_root.query( f""" - CREATE USER IF NOT EXISTS '{credentials["user"]}'@'%%' - IDENTIFIED BY '{credentials["password"]}'; + CREATE USER IF NOT EXISTS '{db_creds_test["user"]}'@'%%' + IDENTIFIED BY '{db_creds_test["password"]}'; """ ) connection_root.query( f""" GRANT {permission} ON `{database}`.* - TO '{credentials["user"]}'@'%%'; + TO '{db_creds_test["user"]}'@'%%'; """ ) else: @@ -173,14 +179,14 @@ def connection_test(connection_root): connection_root.query( f""" GRANT {permission} ON `{database}`.* - TO '{credentials["user"]}'@'%%' - IDENTIFIED BY '{credentials["password"]}'; + TO '{db_creds_test["user"]}'@'%%' + IDENTIFIED BY '{db_creds_test["password"]}'; """ ) - connection = dj.Connection(**credentials) + connection = dj.Connection(**db_creds_test) yield connection - connection_root.query(f"""DROP USER `{credentials["user"]}`""") + connection_root.query(f"""DROP USER `{db_creds_test["user"]}`""") connection.close() From 5fed6a515176e2cd20043f292fcc6d06d4358992 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Thu, 14 Dec 2023 14:01:03 -0700 Subject: [PATCH 4/7] First pass at migrating test_schema --- tests/test_schema.py | 108 +++++++++++++++++++++++++------------------ 1 file changed, 63 insertions(+), 45 deletions(-) diff --git a/tests/test_schema.py b/tests/test_schema.py index 0e88a134a..31825bc5d 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -1,10 +1,13 @@ -from nose.tools import assert_false, assert_true, raises +import pytest import datajoint as dj from inspect import getmembers from . import schema -from . import schema_empty -from . import PREFIX, CONN_INFO, CONN_INFO_ROOT -from .schema_simple import schema as schema_simple +from . import PREFIX + + +class Ephys(dj.Imported): + definition = """ # This is already declared in ./schema.py + """ def relation_selector(attr): @@ -21,42 +24,47 @@ def part_selector(attr): return False -def test_schema_size_on_disk(): - number_of_bytes = schema.schema.size_on_disk +@pytest.fixture +def schema_empty(connection_test, schema_any): + context = { + **schema.LOCALS_ANY, + "Ephys": Ephys + } + schema_emp = dj.Schema(PREFIX + "_test1", context=context, connection=connection_test) + schema_emp(Ephys) + # load the rest of the classes + schema_emp.spawn_missing_classes() + breakpoint() + yield schema_emp + schema_emp.drop() + + +def test_schema_size_on_disk(schema_any): + number_of_bytes = schema_any.size_on_disk assert isinstance(number_of_bytes, int) -def test_schema_list(): +def test_schema_list(schema_any): schemas = dj.list_schemas() - assert schema.schema.database in schemas + assert schema_any.database in schemas -@raises(dj.errors.AccessError) def test_drop_unauthorized(): info_schema = dj.schema("information_schema") - info_schema.drop() + with pytest.raises(dj.errors.AccessError): + info_schema.drop() -def test_namespace_population(): +def test_namespace_population(schema_empty, schema_any): for name, rel in getmembers(schema, relation_selector): - assert_true( - hasattr(schema_empty, name), - "{name} not found in schema_empty".format(name=name), - ) - assert_true( - rel.__base__ is getattr(schema_empty, name).__base__, - "Wrong tier for {name}".format(name=name), - ) + assert hasattr(schema_empty, name), "{name} not found in schema_empty".format(name=name) + assert rel.__base__ is getattr(schema_empty, name).__base__, "Wrong tier for {name}".format(name=name) for name_part in dir(rel): if name_part[0].isupper() and part_selector(getattr(rel, name_part)): - assert_true( - getattr(rel, name_part).__base__ is dj.Part, - "Wrong tier for {name}".format(name=name_part), - ) + assert getattr(rel, name_part).__base__ is dj.Part, "Wrong tier for {name}".format(name=name_part) -@raises(dj.DataJointError) def test_undecorated_table(): """ Undecorated user table classes should raise an informative exception upon first use @@ -66,35 +74,38 @@ class UndecoratedClass(dj.Manual): definition = "" a = UndecoratedClass() - print(a.full_table_name) + with pytest.raises(dj.DataJointError): + print(a.full_table_name) -@raises(dj.DataJointError) -def test_reject_decorated_part(): +def test_reject_decorated_part(schema_any): """ Decorating a dj.Part table should raise an informative exception. """ - @schema.schema class A(dj.Manual): definition = ... - @schema.schema class B(dj.Part): definition = ... -@raises(dj.DataJointError) -def test_unauthorized_database(): + with pytest.raises(dj.DataJointError): + schema_any(A.B) + schema_any(A) + + +def test_unauthorized_database(db_creds_test): """ an attempt to create a database to which user has no privileges should raise an informative exception. """ - dj.Schema("unauthorized_schema", connection=dj.conn(reset=True, **CONN_INFO)) + with pytest.raises(dj.DataJointError): + dj.Schema("unauthorized_schema", connection=dj.conn(reset=True, **db_creds_test)) -def test_drop_database(): +def test_drop_database(db_creds_test): schema = dj.Schema( - PREFIX + "_drop_test", connection=dj.conn(reset=True, **CONN_INFO) + PREFIX + "_drop_test", connection=dj.conn(reset=True, **db_creds_test) ) assert schema.exists schema.drop() @@ -102,9 +113,9 @@ def test_drop_database(): schema.drop() # should do nothing -def test_overlapping_name(): +def test_overlapping_name(connection_test): test_schema = dj.Schema( - PREFIX + "_overlapping_schema", connection=dj.conn(**CONN_INFO) + PREFIX + "_overlapping_schema", connection=connection_test ) @test_schema @@ -131,8 +142,10 @@ class Unit(dj.Part): test_schema.drop() -def test_list_tables(): - # https://github.com/datajoint/datajoint-python/issues/838 +def test_list_tables(schema_simp): + """ + https://github.com/datajoint/datajoint-python/issues/838 + """ assert set( [ "reserved_word", @@ -156,17 +169,22 @@ def test_list_tables(): "profile", "profile__website", ] - ) == set(schema_simple.list_tables()) + ) == set(schema_simp.list_tables()) + +def test_schema_save_any(schema_any): + assert "class Experiment(dj.Imported)" in schema_any.code -def test_schema_save(): - assert "class Experiment(dj.Imported)" in schema.schema.code - assert "class Experiment(dj.Imported)" in schema_empty.schema.code +def test_schema_save_empty(schema_empty): + assert "class Experiment(dj.Imported)" in schema_empty.code -def test_uppercase_schema(): - # https://github.com/datajoint/datajoint-python/issues/564 - dj.conn(**CONN_INFO_ROOT, reset=True) + +def test_uppercase_schema(db_creds_root): + """ + https://github.com/datajoint/datajoint-python/issues/564 + """ + dj.conn(**db_creds_root, reset=True) schema1 = dj.Schema("Schema_A") @schema1 From 58c6103f52c21995692d83b1fe62b25f3b635a5a Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Thu, 14 Dec 2023 15:07:50 -0700 Subject: [PATCH 5/7] Mock schema_empty module --- tests/test_schema.py | 46 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/tests/test_schema.py b/tests/test_schema.py index 31825bc5d..88b7422cf 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -1,12 +1,15 @@ +import types import pytest +import inspect import datajoint as dj +from unittest.mock import patch from inspect import getmembers from . import schema from . import PREFIX class Ephys(dj.Imported): - definition = """ # This is already declared in ./schema.py + definition = """ # This is already declare in ./schema.py """ @@ -24,19 +27,42 @@ def part_selector(attr): return False +@pytest.fixture +def schema_empty_module(schema_any, schema_empty): + # Mimic tests_old/schema_empty + namespace_dict = { + '_': schema_any, + 'schema': schema_empty, + 'Ephys': Ephys, + } + module = types.ModuleType('schema_empty') + + # Add classes to the module's namespace + for k, v in namespace_dict.items(): + setattr(module, k, v) + + # Spawn missing classes in the caller's (self) namespace. + # Then add them to the mock module's namespace. + module.schema.context = None + module.schema.spawn_missing_classes(context=None) + for k, v in locals().items(): + if inspect.isclass(v): + setattr(module, k, v) + return module + + @pytest.fixture def schema_empty(connection_test, schema_any): context = { **schema.LOCALS_ANY, "Ephys": Ephys } - schema_emp = dj.Schema(PREFIX + "_test1", context=context, connection=connection_test) - schema_emp(Ephys) + schema_empty = dj.Schema(PREFIX + "_test1", context=context, connection=connection_test) + schema_empty(Ephys) # load the rest of the classes - schema_emp.spawn_missing_classes() - breakpoint() - yield schema_emp - schema_emp.drop() + schema_empty.spawn_missing_classes(context=context) + yield schema_empty + schema_empty.drop() def test_schema_size_on_disk(schema_any): @@ -55,10 +81,10 @@ def test_drop_unauthorized(): info_schema.drop() -def test_namespace_population(schema_empty, schema_any): +def test_namespace_population(schema_empty_module): for name, rel in getmembers(schema, relation_selector): - assert hasattr(schema_empty, name), "{name} not found in schema_empty".format(name=name) - assert rel.__base__ is getattr(schema_empty, name).__base__, "Wrong tier for {name}".format(name=name) + assert hasattr(schema_empty_module, name), "{name} not found in schema_empty".format(name=name) + assert rel.__base__ is getattr(schema_empty_module, name).__base__, "Wrong tier for {name}".format(name=name) for name_part in dir(rel): if name_part[0].isupper() and part_selector(getattr(rel, name_part)): From b9ccb4fc23a431c4861a605347f44dabb331333c Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Thu, 14 Dec 2023 15:18:18 -0700 Subject: [PATCH 6/7] Move call to spawn_missing_classes to test --- tests/test_schema.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/test_schema.py b/tests/test_schema.py index 88b7422cf..09ae46ca0 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -29,7 +29,11 @@ def part_selector(attr): @pytest.fixture def schema_empty_module(schema_any, schema_empty): - # Mimic tests_old/schema_empty + """ + Mock the module tests_old.schema_empty. + The test `test_namespace_population` will check that the module contains all the + classes in schema_any, after running `spawn_missing_classes`. + """ namespace_dict = { '_': schema_any, 'schema': schema_empty, @@ -41,13 +45,6 @@ def schema_empty_module(schema_any, schema_empty): for k, v in namespace_dict.items(): setattr(module, k, v) - # Spawn missing classes in the caller's (self) namespace. - # Then add them to the mock module's namespace. - module.schema.context = None - module.schema.spawn_missing_classes(context=None) - for k, v in locals().items(): - if inspect.isclass(v): - setattr(module, k, v) return module @@ -82,6 +79,19 @@ def test_drop_unauthorized(): def test_namespace_population(schema_empty_module): + """ + With the schema_empty_module fixture, this test + mimics the behavior of `spawn_missing_classes`, as if the schema + was declared in a separate module and `spawn_missing_classes` was called in that namespace. + """ + # Spawn missing classes in the caller's (self) namespace. + schema_empty_module.schema.context = None + schema_empty_module.schema.spawn_missing_classes(context=None) + # Then add them to the mock module's namespace. + for k, v in locals().items(): + if inspect.isclass(v): + setattr(schema_empty_module, k, v) + for name, rel in getmembers(schema, relation_selector): assert hasattr(schema_empty_module, name), "{name} not found in schema_empty".format(name=name) assert rel.__base__ is getattr(schema_empty_module, name).__base__, "Wrong tier for {name}".format(name=name) From 7bf18f0c5562e9185e3468119c80c1f6bee36bc6 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Thu, 14 Dec 2023 15:26:53 -0700 Subject: [PATCH 7/7] Format with black --- tests/test_schema.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/tests/test_schema.py b/tests/test_schema.py index 09ae46ca0..7b262204f 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -35,11 +35,11 @@ def schema_empty_module(schema_any, schema_empty): classes in schema_any, after running `spawn_missing_classes`. """ namespace_dict = { - '_': schema_any, - 'schema': schema_empty, - 'Ephys': Ephys, + "_": schema_any, + "schema": schema_empty, + "Ephys": Ephys, } - module = types.ModuleType('schema_empty') + module = types.ModuleType("schema_empty") # Add classes to the module's namespace for k, v in namespace_dict.items(): @@ -50,11 +50,10 @@ def schema_empty_module(schema_any, schema_empty): @pytest.fixture def schema_empty(connection_test, schema_any): - context = { - **schema.LOCALS_ANY, - "Ephys": Ephys - } - schema_empty = dj.Schema(PREFIX + "_test1", context=context, connection=connection_test) + context = {**schema.LOCALS_ANY, "Ephys": Ephys} + schema_empty = dj.Schema( + PREFIX + "_test1", context=context, connection=connection_test + ) schema_empty(Ephys) # load the rest of the classes schema_empty.spawn_missing_classes(context=context) @@ -93,12 +92,18 @@ def test_namespace_population(schema_empty_module): setattr(schema_empty_module, k, v) for name, rel in getmembers(schema, relation_selector): - assert hasattr(schema_empty_module, name), "{name} not found in schema_empty".format(name=name) - assert rel.__base__ is getattr(schema_empty_module, name).__base__, "Wrong tier for {name}".format(name=name) + assert hasattr( + schema_empty_module, name + ), "{name} not found in schema_empty".format(name=name) + assert ( + rel.__base__ is getattr(schema_empty_module, name).__base__ + ), "Wrong tier for {name}".format(name=name) for name_part in dir(rel): if name_part[0].isupper() and part_selector(getattr(rel, name_part)): - assert getattr(rel, name_part).__base__ is dj.Part, "Wrong tier for {name}".format(name=name_part) + assert ( + getattr(rel, name_part).__base__ is dj.Part + ), "Wrong tier for {name}".format(name=name_part) def test_undecorated_table(): @@ -125,7 +130,6 @@ class A(dj.Manual): class B(dj.Part): definition = ... - with pytest.raises(dj.DataJointError): schema_any(A.B) schema_any(A) @@ -136,7 +140,9 @@ def test_unauthorized_database(db_creds_test): an attempt to create a database to which user has no privileges should raise an informative exception. """ with pytest.raises(dj.DataJointError): - dj.Schema("unauthorized_schema", connection=dj.conn(reset=True, **db_creds_test)) + dj.Schema( + "unauthorized_schema", connection=dj.conn(reset=True, **db_creds_test) + ) def test_drop_database(db_creds_test): @@ -150,9 +156,7 @@ def test_drop_database(db_creds_test): def test_overlapping_name(connection_test): - test_schema = dj.Schema( - PREFIX + "_overlapping_schema", connection=connection_test - ) + test_schema = dj.Schema(PREFIX + "_overlapping_schema", connection=connection_test) @test_schema class Unit(dj.Manual):