diff --git a/datajoint/s3.py b/datajoint/s3.py index 72e0f4f06..0e3efb2dd 100644 --- a/datajoint/s3.py +++ b/datajoint/s3.py @@ -37,8 +37,11 @@ def get(self, name): logger.debug('get: {}:{}'.format(self.bucket, name)) try: return self.client.get_object(self.bucket, str(name)).data - except minio.error.NoSuchKey: - raise errors.MissingExternalFile('Missing s3 key %s' % name) from None + except minio.error.S3Error as e: + if e.code == 'NoSuchKey': + raise errors.MissingExternalFile('Missing s3 key %s' % name) from None + else: + raise e def fget(self, name, local_filepath): """get file from object name to local filepath""" @@ -59,16 +62,22 @@ def exists(self, name): logger.debug('exists: {}:{}'.format(self.bucket, name)) try: self.client.stat_object(self.bucket, str(name)) - except minio.error.NoSuchKey: - return False + except minio.error.S3Error as e: + if e.code == 'NoSuchKey': + return False + else: + raise e return True def get_size(self, name): logger.debug('get_size: {}:{}'.format(self.bucket, name)) try: return self.client.stat_object(self.bucket, str(name)).size - except minio.error.NoSuchKey: - raise errors.MissingExternalFile from None + except minio.error.S3Error as e: + if e.code == 'NoSuchKey': + raise errors.MissingExternalFile from None + else: + raise e def remove_object(self, name): logger.debug('remove_object: {}:{}'.format(self.bucket, name)) diff --git a/requirements.txt b/requirements.txt index 5c84e822c..fa3882448 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,5 +6,5 @@ pandas tqdm networkx pydot -minio +minio>=7.0.0 matplotlib diff --git a/tests/__init__.py b/tests/__init__.py index bb0b814c4..6d578f95d 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -136,8 +136,9 @@ def setup_package(): region = "us-east-1" try: minioClient.make_bucket(S3_MIGRATE_BUCKET, location=region) - except minio.error.BucketAlreadyOwnedByYou: - pass + except minio.error.S3Error as e: + if e.code != 'BucketAlreadyOwnedByYou': + raise e pathlist = Path(source).glob('**/*') for path in pathlist: @@ -149,8 +150,9 @@ def setup_package(): # Add S3 try: minioClient.make_bucket(S3_CONN_INFO['bucket'], location=region) - except minio.error.BucketAlreadyOwnedByYou: - pass + except minio.error.S3Error as e: + if e.code != 'BucketAlreadyOwnedByYou': + raise e # Add old File Content try: @@ -179,14 +181,14 @@ def teardown_package(): remove("dj_local_conf.json") # Remove old S3 - objs = list(minioClient.list_objects_v2( + objs = list(minioClient.list_objects( S3_MIGRATE_BUCKET, recursive=True)) objs = [minioClient.remove_object(S3_MIGRATE_BUCKET, o.object_name.encode('utf-8')) for o in objs] minioClient.remove_bucket(S3_MIGRATE_BUCKET) # Remove S3 - objs = list(minioClient.list_objects_v2(S3_CONN_INFO['bucket'], recursive=True)) + objs = list(minioClient.list_objects(S3_CONN_INFO['bucket'], recursive=True)) objs = [minioClient.remove_object(S3_CONN_INFO['bucket'], o.object_name.encode('utf-8')) for o in objs] minioClient.remove_bucket(S3_CONN_INFO['bucket'])