Skip to content

Commit 97516b1

Browse files
committed
RF: add a helper add_cache to avoid duplication (.name) and allow for "formalized" addition of caches
Aiming to possibly to expand with another cache without (initially) contributing to fsspec, thought to define helper.
1 parent 55c5d71 commit 97516b1

File tree

2 files changed

+47
-13
lines changed

2 files changed

+47
-13
lines changed

fsspec/caching.py

Lines changed: 39 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -766,15 +766,42 @@ def _read_cache(self, start, end, start_block_number, end_block_number):
766766
return b"".join(out)
767767

768768

769-
caches = {
770-
"none": BaseCache,
771-
None: BaseCache,
772-
"mmap": MMapCache,
773-
"bytes": BytesCache,
774-
"readahead": ReadAheadCache,
775-
"block": BlockCache,
776-
"first": FirstChunkCache,
777-
"all": AllBytes,
778-
"parts": KnownPartsOfAFile,
779-
"background": BackgroundBlockCache,
780-
}
769+
caches = {}
770+
771+
def add_cache(cls, overload=False):
772+
"""'Register' cache implementation.
773+
774+
Parameters
775+
----------
776+
overload: bool, optional
777+
If overload is set to True (default is False) - allow to overload existing
778+
entry.
779+
780+
Raises
781+
------
782+
ValueError
783+
"""
784+
name = cls.name
785+
if not overload and name in caches:
786+
raise ValueError(f"Cache with name {name!r} is already known: {caches[name]}")
787+
caches[name] = cls
788+
# custom handling for None
789+
if name == 'none':
790+
if not overload:
791+
if None in caches:
792+
raise ValueError(f"Cache for None is already known: {caches[None]}")
793+
caches[None] = cls
794+
795+
[
796+
add_cache(c) for c in (
797+
BaseCache,
798+
MMapCache,
799+
BytesCache,
800+
ReadAheadCache,
801+
BlockCache,
802+
FirstChunkCache,
803+
AllBytes,
804+
KnownPartsOfAFile,
805+
BackgroundBlockCache,
806+
)
807+
]

fsspec/tests/test_caches.py

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
import pytest
55

6-
from fsspec.caching import BlockCache, FirstChunkCache, caches
6+
from fsspec.caching import BlockCache, FirstChunkCache, caches, add_cache
77

88

99
def test_cache_getitem(Cache_imp):
@@ -138,3 +138,10 @@ def wrapped(*a, **kw):
138138
f.read(1)
139139
time.sleep(0.1) # second block is loading
140140
assert len(thread_ids) == 2
141+
142+
143+
def test_add_cache():
144+
# just test that we have them populated and fail to re-add again unless overload
145+
with pytest.raises(ValueError):
146+
add_cache(BlockCache)
147+
add_cache(BlockCache, overload=True)

0 commit comments

Comments
 (0)