RF: add a helper add_cache to avoid duplication (.name) and allow for "formalized" addition of caches

Aiming to possibly to expand with another cache without (initially)
contributing to fsspec, thought to define helper.

Author: yarikoptic

fsspec/caching.py

@@ -766,15 +766,45 @@ def _read_cache(self, start, end, start_block_number, end_block_number):
             return b"".join(out)
 
 
-caches = {
-    "none": BaseCache,
-    None: BaseCache,
-    "mmap": MMapCache,
-    "bytes": BytesCache,
-    "readahead": ReadAheadCache,
-    "block": BlockCache,
-    "first": FirstChunkCache,
-    "all": AllBytes,
-    "parts": KnownPartsOfAFile,
-    "background": BackgroundBlockCache,
-}
+caches = {}
+
+
+def add_cache(cls, overload=False):
+    """'Register' cache implementation.
+
+    Parameters
+    ----------
+    overload: bool, optional
+        If overload is set to True (default is False) - allow to overload existing
+        entry.
+
+    Raises
+    ------
+    ValueError
+    """
+    name = cls.name
+    if not overload and name in caches:
+        raise ValueError(f"Cache with name {name!r} is already known: {caches[name]}")
+    caches[name] = cls
+    # custom handling for None
+    if name == "none":
+        if not overload:
+            if None in caches:
+                raise ValueError(f"Cache for None is already known: {caches[None]}")
+        caches[None] = cls
+
+
+[
+    add_cache(c)
+    for c in (
+        BaseCache,
+        MMapCache,
+        BytesCache,
+        ReadAheadCache,
+        BlockCache,
+        FirstChunkCache,
+        AllBytes,
+        KnownPartsOfAFile,
+        BackgroundBlockCache,
+    )
+]

fsspec/tests/test_caches.py

@@ -3,7 +3,7 @@
 
 import pytest
 
-from fsspec.caching import BlockCache, FirstChunkCache, caches
+from fsspec.caching import BlockCache, FirstChunkCache, add_cache, caches
 
 
 def test_cache_getitem(Cache_imp):
@@ -138,3 +138,10 @@ def wrapped(*a, **kw):
     f.read(1)
     time.sleep(0.1)  # second block is loading
     assert len(thread_ids) == 2
+
+
+def test_add_cache():
+    # just test that we have them populated and fail to re-add again unless overload
+    with pytest.raises(ValueError):
+        add_cache(BlockCache)
+    add_cache(BlockCache, overload=True)