diff --git a/fsspec/implementations/tests/test_local.py b/fsspec/implementations/tests/test_local.py index aed971113..fefa3398d 100644 --- a/fsspec/implementations/tests/test_local.py +++ b/fsspec/implementations/tests/test_local.py @@ -6,6 +6,7 @@ import pickle import sys from contextlib import contextmanager +from distutils.version import LooseVersion import posixpath import tempfile @@ -325,8 +326,11 @@ def test_get_pyarrow_filesystem(): pa = pytest.importorskip("pyarrow") fs = LocalFileSystem() - assert isinstance(fs, pa.filesystem.FileSystem) - assert fs._get_pyarrow_filesystem() is fs + if LooseVersion(pa.__version__) < LooseVersion("2.0"): + assert isinstance(fs, pa.filesystem.FileSystem) + assert fs._get_pyarrow_filesystem() is fs + else: + assert not isinstance(fs, pa.filesystem.FileSystem) class UnknownFileSystem(object): pass diff --git a/fsspec/spec.py b/fsspec/spec.py index 300f88857..3ed137842 100644 --- a/fsspec/spec.py +++ b/fsspec/spec.py @@ -2,6 +2,7 @@ import logging import os import warnings +from distutils.version import LooseVersion from errno import ESPIPE from hashlib import sha256 from glob import has_magic @@ -70,12 +71,15 @@ def __call__(cls, *args, **kwargs): try: # optionally derive from pyarrow's FileSystem, if available - # TODO: it should be possible to disable this import pyarrow as pa - - up = pa.filesystem.DaskFileSystem except ImportError: up = object +else: + # only derive from the legacy pyarrow's FileSystem for older pyarrow versions + if LooseVersion(pa.__version__) < LooseVersion("2.0"): + up = pa.filesystem.DaskFileSystem + else: + up = object class AbstractFileSystem(up, metaclass=_Cached): @@ -1109,6 +1113,13 @@ def sign(self, path, expiration=100, **kwargs): """ raise NotImplementedError("Sign is not implemented for this filesystem") + def _isfilestore(self): + # Originally inherited from pyarrow DaskFileSystem. Keeping this + # here for backwards compatibility as long as pyarrow uses its + # legacy ffspec-compatible filesystems and thus accepts fsspec + # filesystems as well + return False + class AbstractBufferedFile(io.IOBase): """Convenient class to derive from to provide buffering