From 071d9ebd6b93ab4f9d5cef08ed78503ddc4b79e7 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Mon, 4 Dec 2023 23:46:23 -0600 Subject: [PATCH 1/5] adapted_attributes tests passing --- tests/conftest.py | 126 ++++++++++++++++++++++++++----- tests/test_adapted_attributes.py | 3 +- 2 files changed, 109 insertions(+), 20 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index aed3ca468..47ea656be 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,11 @@ import datajoint as dj from packaging import version import os +from os import environ, remove import minio import urllib3 import certifi +from distutils.version import LooseVersion import shutil import pytest import networkx as nx @@ -19,11 +21,13 @@ schema, schema_simple, schema_advanced, schema_adapted ) + @pytest.fixture(scope="session") def monkeysession(): with pytest.MonkeyPatch.context() as mp: yield mp + @pytest.fixture(scope="module") def monkeymodule(): with pytest.MonkeyPatch.context() as mp: @@ -31,26 +35,90 @@ def monkeymodule(): @pytest.fixture(scope="session") -def connection_root(): - """Root user database connection.""" - dj.config["safemode"] = False +def connection_root_bare(): connection = dj.Connection( host=os.getenv("DJ_HOST"), user=os.getenv("DJ_USER"), password=os.getenv("DJ_PASS"), ) yield connection - dj.config["safemode"] = True - connection.close() + + +@pytest.fixture(scope="session") +def connection_root(connection_root_bare): + """Root user database connection.""" + dj.config["safemode"] = False + conn_root = connection_root_bare + # Create MySQL users + if LooseVersion(conn_root.query("select @@version;").fetchone()[0]) >= LooseVersion( + "8.0.0" + ): + # create user if necessary on mysql8 + conn_root.query( + """ + CREATE USER IF NOT EXISTS 'datajoint'@'%%' + IDENTIFIED BY 'datajoint'; + """ + ) + conn_root.query( + """ + CREATE USER IF NOT EXISTS 'djview'@'%%' + IDENTIFIED BY 'djview'; + """ + ) + conn_root.query( + """ + CREATE USER IF NOT EXISTS 'djssl'@'%%' + IDENTIFIED BY 'djssl' + REQUIRE SSL; + """ + ) + conn_root.query("GRANT ALL PRIVILEGES ON `djtest%%`.* TO 'datajoint'@'%%';") + conn_root.query("GRANT SELECT ON `djtest%%`.* TO 'djview'@'%%';") + conn_root.query("GRANT SELECT ON `djtest%%`.* TO 'djssl'@'%%';") + else: + # grant permissions. For MySQL 5.7 this also automatically creates user + # if not exists + conn_root.query( + """ + GRANT ALL PRIVILEGES ON `djtest%%`.* TO 'datajoint'@'%%' + IDENTIFIED BY 'datajoint'; + """ + ) + conn_root.query( + "GRANT SELECT ON `djtest%%`.* TO 'djview'@'%%' IDENTIFIED BY 'djview';" + ) + conn_root.query( + """ + GRANT SELECT ON `djtest%%`.* TO 'djssl'@'%%' + IDENTIFIED BY 'djssl' + REQUIRE SSL; + """ + ) + + yield conn_root + + # Teardown + conn_root.query("SET FOREIGN_KEY_CHECKS=0") + cur = conn_root.query('SHOW DATABASES LIKE "{}\_%%"'.format(PREFIX)) + for db in cur.fetchall(): + conn_root.query("DROP DATABASE `{}`".format(db[0])) + conn_root.query("SET FOREIGN_KEY_CHECKS=1") + if os.path.exists("dj_local_conf.json"): + remove("dj_local_conf.json") + + # Remove created users + conn_root.query("DROP USER IF EXISTS `datajoint`") + conn_root.query("DROP USER IF EXISTS `djview`") + conn_root.query("DROP USER IF EXISTS `djssl`") + conn_root.close() @pytest.fixture(scope="session") def connection_test(connection_root): """Test user database connection.""" database = f"{PREFIX}%%" - credentials = dict( - host=os.getenv("DJ_HOST"), user="datajoint", password="datajoint" - ) + credentials = dict(host=os.getenv("DJ_HOST"), user="datajoint", password="datajoint") permission = "ALL PRIVILEGES" # Create MySQL users @@ -178,10 +246,10 @@ def schema_adv(connection_test): schema.drop() -@pytest.fixture -def httpClient(): +@pytest.fixture(scope='session') +def http_client(): # Initialize httpClient with relevant timeout. - httpClient = urllib3.PoolManager( + client = urllib3.PoolManager( timeout=30, cert_reqs="CERT_REQUIRED", ca_certs=certifi.where(), @@ -189,16 +257,38 @@ def httpClient(): total=3, backoff_factor=0.2, status_forcelist=[500, 502, 503, 504] ), ) - yield httpClient + yield client -@pytest.fixture -def minioClient(): - # Initialize minioClient with an endpoint and access/secret keys. - minioClient = minio.Minio( + +@pytest.fixture(scope='session') +def minio_client_bare(http_client): + client = minio.Minio( S3_CONN_INFO["endpoint"], access_key=S3_CONN_INFO["access_key"], secret_key=S3_CONN_INFO["secret_key"], secure=True, - http_client=httpClient, + http_client=http_client, ) - yield minioClient + return client + + +@pytest.fixture(scope='session') +def minio_client(minio_client_bare): + """Initialize MinIO with an endpoint and access/secret keys.""" + # Bootstrap MinIO bucket + aws_region = "us-east-1" + try: + minio_client_bare.make_bucket(S3_CONN_INFO["bucket"], location=aws_region) + except minio.error.S3Error as e: + if e.code != "BucketAlreadyOwnedByYou": + raise e + + yield minio_client_bare + + # Teardown S3 + objs = list(minio_client_bare.list_objects(S3_CONN_INFO["bucket"], recursive=True)) + objs = [ + minio_client_bare.remove_object(S3_CONN_INFO["bucket"], o.object_name.encode("utf-8")) + for o in objs + ] + minio_client_bare.remove_bucket(S3_CONN_INFO["bucket"]) diff --git a/tests/test_adapted_attributes.py b/tests/test_adapted_attributes.py index 2ec0c239f..03e8cfc1c 100644 --- a/tests/test_adapted_attributes.py +++ b/tests/test_adapted_attributes.py @@ -95,8 +95,7 @@ def test_adapted_type(schema_ad): c.delete() -@pytest.mark.skip(reason='misconfigured s3 fixtures') -def test_adapted_filepath_type(schema_ad): +def test_adapted_filepath_type(schema_ad, minio_client): """https://github.com/datajoint/datajoint-python/issues/684""" c = Connectivity() c.delete() From c2605d7562e06e4649d4567127c5c7b06de4ecea Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Tue, 5 Dec 2023 09:29:04 -0600 Subject: [PATCH 2/5] Format with black --- tests/conftest.py | 31 +++++++++++++++++++----------- tests/schema_adapted.py | 1 + tests/schema_advanced.py | 2 ++ tests/test_adapted_attributes.py | 33 +++++++++++++++++++++----------- tests/test_connection.py | 4 +++- tests/test_json.py | 7 ++++--- tests/test_plugin.py | 5 +++-- 7 files changed, 55 insertions(+), 28 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 47ea656be..43a336254 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,12 +13,15 @@ from pathlib import Path import tempfile from datajoint import errors -from datajoint.errors import ( - ADAPTED_TYPE_SWITCH, FILEPATH_FEATURE_SWITCH -) +from datajoint.errors import ADAPTED_TYPE_SWITCH, FILEPATH_FEATURE_SWITCH from . import ( - PREFIX, CONN_INFO, S3_CONN_INFO, - schema, schema_simple, schema_advanced, schema_adapted + PREFIX, + CONN_INFO, + S3_CONN_INFO, + schema, + schema_simple, + schema_advanced, + schema_adapted, ) @@ -118,7 +121,9 @@ def connection_root(connection_root_bare): def connection_test(connection_root): """Test user database connection.""" database = f"{PREFIX}%%" - credentials = dict(host=os.getenv("DJ_HOST"), user="datajoint", password="datajoint") + credentials = dict( + host=os.getenv("DJ_HOST"), user="datajoint", password="datajoint" + ) permission = "ALL PRIVILEGES" # Create MySQL users @@ -231,7 +236,9 @@ def schema_simp(connection_test): @pytest.fixture def schema_adv(connection_test): schema = dj.Schema( - PREFIX + "_advanced", schema_advanced.LOCALS_ADVANCED, connection=connection_test + PREFIX + "_advanced", + schema_advanced.LOCALS_ADVANCED, + connection=connection_test, ) schema(schema_advanced.Person) schema(schema_advanced.Parent) @@ -246,7 +253,7 @@ def schema_adv(connection_test): schema.drop() -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def http_client(): # Initialize httpClient with relevant timeout. client = urllib3.PoolManager( @@ -260,7 +267,7 @@ def http_client(): yield client -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def minio_client_bare(http_client): client = minio.Minio( S3_CONN_INFO["endpoint"], @@ -272,7 +279,7 @@ def minio_client_bare(http_client): return client -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def minio_client(minio_client_bare): """Initialize MinIO with an endpoint and access/secret keys.""" # Bootstrap MinIO bucket @@ -288,7 +295,9 @@ def minio_client(minio_client_bare): # Teardown S3 objs = list(minio_client_bare.list_objects(S3_CONN_INFO["bucket"], recursive=True)) objs = [ - minio_client_bare.remove_object(S3_CONN_INFO["bucket"], o.object_name.encode("utf-8")) + minio_client_bare.remove_object( + S3_CONN_INFO["bucket"], o.object_name.encode("utf-8") + ) for o in objs ] minio_client_bare.remove_bucket(S3_CONN_INFO["bucket"]) diff --git a/tests/schema_adapted.py b/tests/schema_adapted.py index 68a7e965a..ab9a02e76 100644 --- a/tests/schema_adapted.py +++ b/tests/schema_adapted.py @@ -48,6 +48,7 @@ class Connectivity(dj.Manual): conn_graph = null : """ + class Layout(dj.Manual): definition = """ # stores graph layout diff --git a/tests/schema_advanced.py b/tests/schema_advanced.py index 649ff186a..6a35cb34a 100644 --- a/tests/schema_advanced.py +++ b/tests/schema_advanced.py @@ -1,6 +1,7 @@ import datajoint as dj import inspect + class Person(dj.Manual): definition = """ person_id : int @@ -134,5 +135,6 @@ class GlobalSynapse(dj.Manual): -> Cell.proj(post_slice="slice", post_cell="cell") """ + LOCALS_ADVANCED = {k: v for k, v in locals().items() if inspect.isclass(v)} __all__ = list(LOCALS_ADVANCED) diff --git a/tests/test_adapted_attributes.py b/tests/test_adapted_attributes.py index 03e8cfc1c..bd0ce7713 100644 --- a/tests/test_adapted_attributes.py +++ b/tests/test_adapted_attributes.py @@ -17,14 +17,14 @@ def adapted_graph_instance(): @pytest.fixture def enable_adapted_types(monkeypatch): - monkeypatch.setenv(ADAPTED_TYPE_SWITCH, 'TRUE') + monkeypatch.setenv(ADAPTED_TYPE_SWITCH, "TRUE") yield monkeypatch.delenv(ADAPTED_TYPE_SWITCH, raising=True) @pytest.fixture def enable_filepath_feature(monkeypatch): - monkeypatch.setenv(FILEPATH_FEATURE_SWITCH, 'TRUE') + monkeypatch.setenv(FILEPATH_FEATURE_SWITCH, "TRUE") yield monkeypatch.delenv(FILEPATH_FEATURE_SWITCH, raising=True) @@ -37,22 +37,30 @@ def schema_name_custom_datatype(): @pytest.fixture def schema_ad( - schema_name_custom_datatype, connection_test, adapted_graph_instance, - enable_adapted_types, enable_filepath_feature + schema_name_custom_datatype, + connection_test, + adapted_graph_instance, + enable_adapted_types, + enable_filepath_feature, ): stores_config = { "repo-s3": dict( - S3_CONN_INFO, protocol="s3", location="adapted/repo", stage=tempfile.mkdtemp() + S3_CONN_INFO, + protocol="s3", + location="adapted/repo", + stage=tempfile.mkdtemp(), ) } dj.config["stores"] = stores_config layout_to_filepath = schema_adapted.LayoutToFilepath() context = { **schema_adapted.LOCALS_ADAPTED, - 'graph': adapted_graph_instance, - 'layout_to_filepath': layout_to_filepath, + "graph": adapted_graph_instance, + "layout_to_filepath": layout_to_filepath, } - schema = dj.schema(schema_name_custom_datatype, context=context, connection=connection_test) + schema = dj.schema( + schema_name_custom_datatype, context=context, connection=connection_test + ) graph = adapted_graph_instance schema(schema_adapted.Connectivity) schema(schema_adapted.Layout) @@ -70,10 +78,14 @@ def local_schema(schema_ad, schema_name_custom_datatype): @pytest.fixture -def schema_virtual_module(schema_ad, schema_name_custom_datatype, adapted_graph_instance): +def schema_virtual_module( + schema_ad, schema_name_custom_datatype, adapted_graph_instance +): """Fixture for testing virtual modules""" schema_virtual_module = dj.VirtualModule( - "virtual_module", schema_name_custom_datatype, add_objects={"graph": adapted_graph_instance} + "virtual_module", + schema_name_custom_datatype, + add_objects={"graph": adapted_graph_instance}, ) return schema_virtual_module @@ -130,7 +142,6 @@ def test_adapted_spawned(local_schema, enable_adapted_types): c.delete() - def test_adapted_virtual(schema_virtual_module): c = schema_virtual_module.Connectivity() graphs = [ diff --git a/tests/test_connection.py b/tests/test_connection.py index a73677aec..8cdbbbff5 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -12,7 +12,9 @@ @pytest.fixture def schema(connection_test): - schema = dj.Schema(PREFIX + "_transactions", context=dict(), connection=connection_test) + schema = dj.Schema( + PREFIX + "_transactions", context=dict(), connection=connection_test + ) yield schema schema.drop() diff --git a/tests/test_json.py b/tests/test_json.py index 37a33c825..c1caaeedd 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -107,6 +107,7 @@ def test_insert_update(schema): q.delete_quick() assert not q + def test_describe(schema): rel = Team() context = inspect.currentframe().f_globals @@ -114,6 +115,7 @@ def test_describe(schema): s2 = declare(rel.full_table_name, rel.describe(), context) assert s1 == s2 + def test_restrict(schema): # dict assert (Team & {"car.name": "Chaching"}).fetch1("name") == "business" @@ -139,9 +141,7 @@ def test_restrict(schema): assert (Team & {"car": None}).fetch1("name") == "marketing" - assert (Team & {"car.tire_pressure": [34, 30, 27, 32]}).fetch1( - "name" - ) == "business" + assert (Team & {"car.tire_pressure": [34, 30, 27, 32]}).fetch1("name") == "business" assert ( Team & {"car.headlights[1]": {"side": "right", "hyper_white": True}} @@ -175,6 +175,7 @@ def test_restrict(schema): & """`car`->>'$.headlights[1]' = '{"side": "right", "hyper_white": true}'""" ).fetch1("name") == "business", "2nd `headlight` object did not match" + def test_proj(schema): # proj necessary since we need to rename indexed value into a proper attribute name assert Team.proj(car_length="car.length").fetch( diff --git a/tests/test_plugin.py b/tests/test_plugin.py index e41224116..ddb8b3bfc 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -23,7 +23,7 @@ def test_normal_djerror(): assert e.__cause__ is None -@pytest.mark.parametrize('category', ('connection', )) +@pytest.mark.parametrize("category", ("connection",)) def test_verified_djerror(category): try: curr_plugins = getattr(p, "{}_plugins".format(category)) @@ -41,7 +41,8 @@ def test_verified_djerror(category): def test_verified_djerror_type(): test_verified_djerror(category="type") -@pytest.mark.parametrize('category', ('connection', )) + +@pytest.mark.parametrize("category", ("connection",)) def test_unverified_djerror(category): try: curr_plugins = getattr(p, "{}_plugins".format(category)) From 186d7cdc6b3b40936542f21517556941b1ad393e Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Tue, 5 Dec 2023 11:42:55 -0600 Subject: [PATCH 3/5] Merge #1116 changes --- tests/test_adapted_attributes.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/tests/test_adapted_attributes.py b/tests/test_adapted_attributes.py index bd0ce7713..82fefe9f1 100644 --- a/tests/test_adapted_attributes.py +++ b/tests/test_adapted_attributes.py @@ -9,6 +9,8 @@ from .schema_adapted import Connectivity, Layout from . import PREFIX, S3_CONN_INFO +SCHEMA_NAME = PREFIX + "_test_custom_datatype" + @pytest.fixture def adapted_graph_instance(): @@ -29,15 +31,8 @@ def enable_filepath_feature(monkeypatch): monkeypatch.delenv(FILEPATH_FEATURE_SWITCH, raising=True) -@pytest.fixture -def schema_name_custom_datatype(): - schema_name = PREFIX + "_test_custom_datatype" - return schema_name - - @pytest.fixture def schema_ad( - schema_name_custom_datatype, connection_test, adapted_graph_instance, enable_adapted_types, @@ -58,9 +53,7 @@ def schema_ad( "graph": adapted_graph_instance, "layout_to_filepath": layout_to_filepath, } - schema = dj.schema( - schema_name_custom_datatype, context=context, connection=connection_test - ) + schema = dj.schema(SCHEMA_NAME, context=context, connection=connection_test) graph = adapted_graph_instance schema(schema_adapted.Connectivity) schema(schema_adapted.Layout) @@ -69,23 +62,19 @@ def schema_ad( @pytest.fixture -def local_schema(schema_ad, schema_name_custom_datatype): +def local_schema(schema_ad): """Fixture for testing spawned classes""" - local_schema = dj.Schema(schema_name_custom_datatype) + local_schema = dj.Schema(SCHEMA_NAME) local_schema.spawn_missing_classes() yield local_schema local_schema.drop() @pytest.fixture -def schema_virtual_module( - schema_ad, schema_name_custom_datatype, adapted_graph_instance -): +def schema_virtual_module(schema_ad, adapted_graph_instance): """Fixture for testing virtual modules""" schema_virtual_module = dj.VirtualModule( - "virtual_module", - schema_name_custom_datatype, - add_objects={"graph": adapted_graph_instance}, + "virtual_module", SCHEMA_NAME, add_objects={"graph": adapted_graph_instance} ) return schema_virtual_module From a76bbd5a9c50009caeedbd9a39cff3d126e3f1af Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Tue, 5 Dec 2023 11:54:22 -0600 Subject: [PATCH 4/5] Format with black --- tests/conftest.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 667caec6e..23d42574d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,14 +33,12 @@ ) - @pytest.fixture(scope="session") def monkeysession(): with pytest.MonkeyPatch.context() as mp: yield mp - @pytest.fixture(scope="module") def monkeymodule(): with pytest.MonkeyPatch.context() as mp: From 124b1fb907a5585b0026a7f684fd1f1481d43016 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Tue, 5 Dec 2023 12:23:43 -0600 Subject: [PATCH 5/5] Remove duplicates from merge conflict resolution --- tests/conftest.py | 7 ------- tests/test_adapted_attributes.py | 2 -- 2 files changed, 9 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 23d42574d..dc984616c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,13 +23,6 @@ schema_simple, schema_advanced, schema_adapted, - PREFIX, - CONN_INFO, - S3_CONN_INFO, - schema, - schema_simple, - schema_advanced, - schema_adapted, ) diff --git a/tests/test_adapted_attributes.py b/tests/test_adapted_attributes.py index 61166f68f..82fefe9f1 100644 --- a/tests/test_adapted_attributes.py +++ b/tests/test_adapted_attributes.py @@ -11,8 +11,6 @@ SCHEMA_NAME = PREFIX + "_test_custom_datatype" -SCHEMA_NAME = PREFIX + "_test_custom_datatype" - @pytest.fixture def adapted_graph_instance():