handle fsspec inconsistency in PyArrowHDFS

Author: f4hy

pytorch_lightning/loggers/tensorboard.py

@@ -232,7 +232,8 @@ def _get_next_version(self):
             return 0
 
         existing_versions = []
-        for d in self._fs.ls(root_dir):
+        for listing in self._fs.listdir(root_dir):
+            d = listing["name"]
             bn = os.path.basename(d)
             if self._fs.isdir(d) and bn.startswith("version_"):
                 dir_ver = bn.split("_")[1].replace('/', '')

pytorch_lightning/trainer/connectors/checkpoint_connector.py

@@ -194,7 +194,7 @@ def restore_hpc_weights_if_needed(self, model: LightningModule):
         folderpath = str(self.trainer.weights_save_path)
         fs = get_filesystem(folderpath)
         if fs.exists(folderpath):
-            files = [os.path.basename(f) for f in fs.ls(folderpath)]
+            files = [os.path.basename(f['name']) for f in fs.listdir(folderpath)]
             hpc_weight_paths = [x for x in files if 'hpc_ckpt' in x]
 
             # if hpc weights exist restore model
@@ -333,7 +333,7 @@ def hpc_load(self, folderpath, on_gpu):
 
     def max_ckpt_in_folder(self, path, name_key='ckpt_'):
         fs = get_filesystem(path)
-        files = [os.path.basename(f) for f in fs.ls(path)]
+        files = [os.path.basename(f["name"]) for f in fs.listdir(path)]
         files = [x for x in files if name_key in x]
         if len(files) == 0:
             return 0