Merge pull request #411 from jorisvandenbossche/remove-deprecated-filesystem

Stop inheriting from pyarrow.filesystem for pyarrow>=2.0

Author: martindurant

fsspec/implementations/tests/test_local.py

@@ -6,6 +6,7 @@
 import pickle
 import sys
 from contextlib import contextmanager
+from distutils.version import LooseVersion
 import posixpath
 import tempfile
 
@@ -325,8 +326,11 @@ def test_get_pyarrow_filesystem():
     pa = pytest.importorskip("pyarrow")
 
     fs = LocalFileSystem()
-    assert isinstance(fs, pa.filesystem.FileSystem)
-    assert fs._get_pyarrow_filesystem() is fs
+    if LooseVersion(pa.__version__) < LooseVersion("2.0"):
+        assert isinstance(fs, pa.filesystem.FileSystem)
+        assert fs._get_pyarrow_filesystem() is fs
+    else:
+        assert not isinstance(fs, pa.filesystem.FileSystem)
 
     class UnknownFileSystem(object):
         pass

fsspec/spec.py

@@ -2,6 +2,7 @@
 import logging
 import os
 import warnings
+from distutils.version import LooseVersion
 from errno import ESPIPE
 from hashlib import sha256
 from glob import has_magic
@@ -70,12 +71,15 @@ def __call__(cls, *args, **kwargs):
 
 
 try:  # optionally derive from pyarrow's FileSystem, if available
-    # TODO: it should be possible to disable this
     import pyarrow as pa
-
-    up = pa.filesystem.DaskFileSystem
 except ImportError:
     up = object
+else:
+    # only derive from the legacy pyarrow's FileSystem for older pyarrow versions
+    if LooseVersion(pa.__version__) < LooseVersion("2.0"):
+        up = pa.filesystem.DaskFileSystem
+    else:
+        up = object
 
 
 class AbstractFileSystem(up, metaclass=_Cached):
@@ -1109,6 +1113,13 @@ def sign(self, path, expiration=100, **kwargs):
         """
         raise NotImplementedError("Sign is not implemented for this filesystem")
 
+    def _isfilestore(self):
+        # Originally inherited from pyarrow DaskFileSystem. Keeping this
+        # here for backwards compatibility as long as pyarrow uses its
+        # legacy ffspec-compatible filesystems and thus accepts fsspec
+        # filesystems as well
+        return False
+
 
 class AbstractBufferedFile(io.IOBase):
     """Convenient class to derive from to provide buffering