Merge pull request #649 from intake/fix/http-ls

Raise FileNotFoundError from HTTP.ls

Author: martindurant

fsspec/implementations/http.py

@@ -138,7 +138,7 @@ async def _ls_real(self, url, detail=True, **kwargs):
         logger.debug(url)
         session = await self.set_session()
         async with session.get(url, **self.kwargs) as r:
-            r.raise_for_status()
+            self._raise_not_found_for_status(r, url)
             text = await r.text()
         if self.simple_links:
             links = ex2.findall(text) + [u[2] for u in ex.findall(text)]
@@ -190,6 +190,14 @@ async def _ls(self, url, detail=True, **kwargs):
 
     ls = sync_wrapper(_ls)
 
+    def _raise_not_found_for_status(self, response, url):
+        """
+        Raises FileNotFoundError for 404s, otherwise uses raise_for_status.
+        """
+        if response.status == 404:
+            raise FileNotFoundError(url)
+        response.raise_for_status()
+
     async def _cat_file(self, url, start=None, end=None, **kwargs):
         kw = self.kwargs.copy()
         kw.update(kwargs)
@@ -225,9 +233,7 @@ async def _cat_file(self, url, start=None, end=None, **kwargs):
             kw["headers"] = headers
         session = await self.set_session()
         async with session.get(url, **kw) as r:
-            if r.status == 404:
-                raise FileNotFoundError(url)
-            r.raise_for_status()
+            self._raise_not_found_for_status(r, url)
             out = await r.read()
         return out
 
@@ -237,9 +243,7 @@ async def _get_file(self, rpath, lpath, chunk_size=5 * 2 ** 20, **kwargs):
         logger.debug(rpath)
         session = await self.set_session()
         async with session.get(rpath, **self.kwargs) as r:
-            if r.status == 404:
-                raise FileNotFoundError(rpath)
-            r.raise_for_status()
+            self._raise_not_found_for_status(r, rpath)
             with open(lpath, "wb") as fd:
                 chunk = True
                 while chunk:

fsspec/implementations/tests/test_http.py

@@ -201,6 +201,13 @@ def test_list_cache_reuse(server):
     assert len(h.dircache) == 0
 
 
+def test_ls_raises_filenotfound(server):
+    h = fsspec.filesystem("http")
+
+    with pytest.raises(FileNotFoundError):
+        h.ls(server + "/not-a-key")
+
+
 def test_list_cache_with_max_paths(server):
     h = fsspec.filesystem("http", use_listings_cache=True, max_paths=5)
     out = h.glob(server + "/index/*")