diff --git a/tests/conftest.py b/tests/conftest.py index dc984616c..5a4858636 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,6 +23,7 @@ schema_simple, schema_advanced, schema_adapted, + schema_external, ) @@ -38,6 +39,20 @@ def monkeymodule(): yield mp +@pytest.fixture +def enable_adapted_types(monkeypatch): + monkeypatch.setenv(ADAPTED_TYPE_SWITCH, "TRUE") + yield + monkeypatch.delenv(ADAPTED_TYPE_SWITCH, raising=True) + + +@pytest.fixture +def enable_filepath_feature(monkeypatch): + monkeypatch.setenv(FILEPATH_FEATURE_SWITCH, "TRUE") + yield + monkeypatch.delenv(FILEPATH_FEATURE_SWITCH, raising=True) + + @pytest.fixture(scope="session") def connection_root_bare(): connection = dj.Connection( @@ -161,6 +176,24 @@ def connection_test(connection_root): connection.close() +@pytest.fixture(scope="session") +def stores_config(): + stores_config = { + "raw": dict(protocol="file", location=tempfile.mkdtemp()), + "repo": dict( + stage=tempfile.mkdtemp(), protocol="file", location=tempfile.mkdtemp() + ), + "repo-s3": dict( + S3_CONN_INFO, protocol="s3", location="dj/repo", stage=tempfile.mkdtemp() + ), + "local": dict(protocol="file", location=tempfile.mkdtemp(), subfolding=(1, 1)), + "share": dict( + S3_CONN_INFO, protocol="s3", location="dj/store/repo", subfolding=(2, 4) + ), + } + return stores_config + + @pytest.fixture def schema_any(connection_test): schema_any = dj.Schema( @@ -254,6 +287,28 @@ def schema_adv(connection_test): schema.drop() +@pytest.fixture +def schema_ext(connection_test, stores_config, enable_filepath_feature): + schema = dj.Schema( + PREFIX + "_extern", + context=schema_external.LOCALS_EXTERNAL, + connection=connection_test, + ) + dj.config["stores"] = stores_config + dj.config["cache"] = tempfile.mkdtemp() + + schema(schema_external.Simple) + schema(schema_external.SimpleRemote) + schema(schema_external.Seed) + schema(schema_external.Dimension) + schema(schema_external.Image) + schema(schema_external.Attach) + schema(schema_external.Filepath) + schema(schema_external.FilepathS3) + yield schema + schema.drop() + + @pytest.fixture(scope="session") def http_client(): # Initialize httpClient with relevant timeout. @@ -270,6 +325,7 @@ def http_client(): @pytest.fixture(scope="session") def minio_client_bare(http_client): + """Initialize MinIO with an endpoint and access/secret keys.""" client = minio.Minio( S3_CONN_INFO["endpoint"], access_key=S3_CONN_INFO["access_key"], @@ -282,8 +338,8 @@ def minio_client_bare(http_client): @pytest.fixture(scope="session") def minio_client(minio_client_bare): - """Initialize MinIO with an endpoint and access/secret keys.""" - # Bootstrap MinIO bucket + """Initialize a MinIO client and create buckets for testing session.""" + # Setup MinIO bucket aws_region = "us-east-1" try: minio_client_bare.make_bucket(S3_CONN_INFO["bucket"], location=aws_region) diff --git a/tests/schema_external.py b/tests/schema_external.py new file mode 100644 index 000000000..294ecb070 --- /dev/null +++ b/tests/schema_external.py @@ -0,0 +1,89 @@ +""" +A schema for testing external attributes +""" + +import tempfile +import inspect +import datajoint as dj +from . import PREFIX, CONN_INFO, S3_CONN_INFO +import numpy as np + + +class Simple(dj.Manual): + definition = """ + simple : int + --- + item : blob@local + """ + + +class SimpleRemote(dj.Manual): + definition = """ + simple : int + --- + item : blob@share + """ + + +class Seed(dj.Lookup): + definition = """ + seed : int + """ + contents = zip(range(4)) + + +class Dimension(dj.Lookup): + definition = """ + dim : int + --- + dimensions : blob + """ + contents = ([0, [100, 50]], [1, [3, 4, 8, 6]]) + + +class Image(dj.Computed): + definition = """ + # table for storing + -> Seed + -> Dimension + ---- + img : blob@share # objects are stored as specified by dj.config['stores']['share'] + neg : blob@local # objects are stored as specified by dj.config['stores']['local'] + """ + + def make(self, key): + np.random.seed(key["seed"]) + img = np.random.rand(*(Dimension() & key).fetch1("dimensions")) + self.insert1(dict(key, img=img, neg=-img.astype(np.float32))) + + +class Attach(dj.Manual): + definition = """ + # table for storing attachments + attach : int + ---- + img : attach@share # attachments are stored as specified by: dj.config['stores']['raw'] + txt : attach # attachments are stored directly in the database + """ + + +class Filepath(dj.Manual): + definition = """ + # table for file management + fnum : int # test comment containing : + --- + img : filepath@repo # managed files + """ + + +class FilepathS3(dj.Manual): + definition = """ + # table for file management + fnum : int + --- + img : filepath@repo-s3 # managed files + """ + + +LOCALS_EXTERNAL = {k: v for k, v in locals().items() if inspect.isclass(v)} +__all__ = list(LOCALS_EXTERNAL) diff --git a/tests/test_adapted_attributes.py b/tests/test_adapted_attributes.py index 82fefe9f1..bbe8456f5 100644 --- a/tests/test_adapted_attributes.py +++ b/tests/test_adapted_attributes.py @@ -2,7 +2,6 @@ import pytest import tempfile import datajoint as dj -from datajoint.errors import ADAPTED_TYPE_SWITCH, FILEPATH_FEATURE_SWITCH import networkx as nx from itertools import zip_longest from . import schema_adapted @@ -17,20 +16,6 @@ def adapted_graph_instance(): yield schema_adapted.GraphAdapter() -@pytest.fixture -def enable_adapted_types(monkeypatch): - monkeypatch.setenv(ADAPTED_TYPE_SWITCH, "TRUE") - yield - monkeypatch.delenv(ADAPTED_TYPE_SWITCH, raising=True) - - -@pytest.fixture -def enable_filepath_feature(monkeypatch): - monkeypatch.setenv(FILEPATH_FEATURE_SWITCH, "TRUE") - yield - monkeypatch.delenv(FILEPATH_FEATURE_SWITCH, raising=True) - - @pytest.fixture def schema_ad( connection_test, diff --git a/tests/test_s3.py b/tests/test_s3.py new file mode 100644 index 000000000..090d6acf0 --- /dev/null +++ b/tests/test_s3.py @@ -0,0 +1,50 @@ +import pytest +import urllib3 +import certifi +from .schema_external import SimpleRemote +from datajoint.errors import DataJointError +from datajoint.hash import uuid_from_buffer +from datajoint.blob import pack +from . import S3_CONN_INFO +from minio import Minio + + +class TestS3: + def test_connection(self, http_client, minio_client): + assert minio_client.bucket_exists(S3_CONN_INFO["bucket"]) + + def test_connection_secure(self, minio_client): + assert minio_client.bucket_exists(S3_CONN_INFO["bucket"]) + + def test_remove_object_exception(self, schema_ext): + # https://github.com/datajoint/datajoint-python/issues/952 + + # Insert some test data and remove it so that the external table is populated + test = [1, [1, 2, 3]] + SimpleRemote.insert1(test) + SimpleRemote.delete() + + # Save the old external table minio client + old_client = schema_ext.external["share"].s3.client + + # Apply our new minio client which has a user that does not exist + schema_ext.external["share"].s3.client = Minio( + S3_CONN_INFO["endpoint"], + access_key="jeffjeff", + secret_key="jeffjeff", + secure=False, + ) + + # This method returns a list of errors + error_list = schema_ext.external["share"].delete( + delete_external_files=True, errors_as_string=False + ) + + # Teardown + schema_ext.external["share"].s3.client = old_client + schema_ext.external["share"].delete(delete_external_files=True) + + with pytest.raises(DataJointError): + # Raise the error we want if the error matches the expected uuid + if str(error_list[0][0]) == str(uuid_from_buffer(pack(test[1]))): + raise error_list[0][2]