Skip to content

Commit f23346e

Browse files
authored
Merge pull request #411 from jorisvandenbossche/remove-deprecated-filesystem
Stop inheriting from pyarrow.filesystem for pyarrow>=2.0
2 parents 7fb2faa + 741565f commit f23346e

File tree

2 files changed

+20
-5
lines changed

2 files changed

+20
-5
lines changed

fsspec/implementations/tests/test_local.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import pickle
77
import sys
88
from contextlib import contextmanager
9+
from distutils.version import LooseVersion
910
import posixpath
1011
import tempfile
1112

@@ -325,8 +326,11 @@ def test_get_pyarrow_filesystem():
325326
pa = pytest.importorskip("pyarrow")
326327

327328
fs = LocalFileSystem()
328-
assert isinstance(fs, pa.filesystem.FileSystem)
329-
assert fs._get_pyarrow_filesystem() is fs
329+
if LooseVersion(pa.__version__) < LooseVersion("2.0"):
330+
assert isinstance(fs, pa.filesystem.FileSystem)
331+
assert fs._get_pyarrow_filesystem() is fs
332+
else:
333+
assert not isinstance(fs, pa.filesystem.FileSystem)
330334

331335
class UnknownFileSystem(object):
332336
pass

fsspec/spec.py

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import logging
33
import os
44
import warnings
5+
from distutils.version import LooseVersion
56
from errno import ESPIPE
67
from hashlib import sha256
78
from glob import has_magic
@@ -70,12 +71,15 @@ def __call__(cls, *args, **kwargs):
7071

7172

7273
try: # optionally derive from pyarrow's FileSystem, if available
73-
# TODO: it should be possible to disable this
7474
import pyarrow as pa
75-
76-
up = pa.filesystem.DaskFileSystem
7775
except ImportError:
7876
up = object
77+
else:
78+
# only derive from the legacy pyarrow's FileSystem for older pyarrow versions
79+
if LooseVersion(pa.__version__) < LooseVersion("2.0"):
80+
up = pa.filesystem.DaskFileSystem
81+
else:
82+
up = object
7983

8084

8185
class AbstractFileSystem(up, metaclass=_Cached):
@@ -1109,6 +1113,13 @@ def sign(self, path, expiration=100, **kwargs):
11091113
"""
11101114
raise NotImplementedError("Sign is not implemented for this filesystem")
11111115

1116+
def _isfilestore(self):
1117+
# Originally inherited from pyarrow DaskFileSystem. Keeping this
1118+
# here for backwards compatibility as long as pyarrow uses its
1119+
# legacy ffspec-compatible filesystems and thus accepts fsspec
1120+
# filesystems as well
1121+
return False
1122+
11121123

11131124
class AbstractBufferedFile(io.IOBase):
11141125
"""Convenient class to derive from to provide buffering

0 commit comments

Comments
 (0)