refactor uses of UPath (#693)

Author: normanrz

webknossos/poetry.lock

@@ -31,7 +31,7 @@ zarr = "^2.11.0"
 numcodecs = "^0.9.1"
 zipp = "^3.5.0"
 fsspec = "^2022.2.0"
-universal-pathlib = "^0.0.14"
+universal-pathlib = "0.0.16"
 
 
 [tool.poetry.dev-dependencies]

webknossos/pyproject.toml

@@ -141,7 +141,9 @@ def __init__(self, path: Path):
 
     @classmethod
     def open(cls, path: Path) -> "WKWArray":
-        if (path / "header.wkw").is_file():
+        header_path = path / "header.wkw"
+
+        if header_path.exists() and header_path.is_file():
             return cls(path)
         raise ArrayException(
             f"Could not open WKW array at {path}. `header.wkw` not found."
@@ -288,7 +290,9 @@ def __init__(self, path: Path):
 
     @classmethod
     def open(cls, path: Path) -> "ZarrArray":
-        if (path / ".zarray").is_file():
+        zarray_path = path / ".zarray"
+
+        if zarray_path.exists() and zarray_path.is_file():
             return cls(path)
         raise ArrayException(
             f"Could not open Zarr array at {path}. `.zarray` not found."

webknossos/webknossos/dataset/_array.py

@@ -23,6 +23,7 @@
 import attr
 import numpy as np
 from boltons.typeutils import make_sentinel
+from upath import UPath
 
 from ..geometry.vec3_int import Vec3IntLike
 from ._array import ArrayException, ArrayInfo, BaseArray, DataFormat
@@ -36,8 +37,6 @@
     copytree,
     get_executor_for_args,
     is_fs_path,
-    is_symlink,
-    make_upath,
     rmtree,
 )
 from ._utils.infer_bounding_box_existing_files import infer_bounding_box_existing_files
@@ -124,7 +123,7 @@ def __init__(
         Currently exist_ok=True is the deprecated default and will change in future releases.
         Please use `Dataset.open` if you intend to open an existing dataset and don't want/need the creation behavior.
         """
-        dataset_path = make_upath(dataset_path)
+        dataset_path = UPath(dataset_path)
 
         dataset_existed_already = (
             dataset_path.exists()
@@ -183,7 +182,7 @@ def __init__(
         for layer_properties in self._properties.data_layers:
             num_channels = _extract_num_channels(
                 layer_properties.num_channels,
-                make_upath(dataset_path),
+                UPath(dataset_path),
                 layer_properties.name,
                 layer_properties.mags[0].mag
                 if len(layer_properties.mags) > 0
@@ -222,7 +221,7 @@ def open(cls, dataset_path: Union[str, PathLike]) -> "Dataset":
 
         The `dataset_path` refers to the top level directory of the dataset (excluding layer or magnification names).
         """
-        dataset_path = make_upath(dataset_path)
+        dataset_path = UPath(dataset_path)
         assert (
             dataset_path.exists()
         ), f"Cannot open Dataset: Couldn't find {dataset_path}"
@@ -648,7 +647,7 @@ def delete_layer(self, layer_name: str) -> None:
             layer for layer in self._properties.data_layers if layer.name != layer_name
         ]
         # delete files on disk
-        if is_symlink(layer_path):
+        if layer_path.is_symlink():
             layer_path.unlink()
         else:
             # rmtree does not recurse into linked dirs, but removes the link
@@ -674,7 +673,7 @@ def add_symlink_layer(
         if isinstance(foreign_layer, Layer):
             foreign_layer_path = foreign_layer.path
         else:
-            foreign_layer_path = make_upath(foreign_layer)
+            foreign_layer_path = UPath(foreign_layer)
 
         foreign_layer_name = foreign_layer_path.name
         layer_name = (
@@ -726,7 +725,7 @@ def add_copy_layer(
         if isinstance(foreign_layer, Layer):
             foreign_layer_path = foreign_layer.path
         else:
-            foreign_layer_path = make_upath(foreign_layer)
+            foreign_layer_path = UPath(foreign_layer)
 
         foreign_layer_name = foreign_layer_path.name
         layer_name = (
@@ -778,7 +777,7 @@ def copy_dataset(
             file_len=file_len,
         )
 
-        new_dataset_path = make_upath(new_dataset_path)
+        new_dataset_path = UPath(new_dataset_path)
 
         if data_format == DataFormat.WKW:
             assert is_fs_path(

webknossos/webknossos/dataset/dataset.py

@@ -9,6 +9,7 @@
 from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
 
 import numpy as np
+from upath import UPath
 
 from webknossos.geometry import BoundingBox, Mag, Vec3Int, Vec3IntLike
 
@@ -40,7 +41,6 @@
     copytree,
     get_executor_for_args,
     is_fs_path,
-    make_upath,
     named_partial,
     rmtree,
     warn_deprecated,
@@ -495,7 +495,7 @@ def _add_foreign_mag(
             # local import to prevent circular dependency
             from .dataset import Dataset
 
-            foreign_mag_view_path = make_upath(foreign_mag_view_or_path)
+            foreign_mag_view_path = UPath(foreign_mag_view_or_path)
             foreign_mag_view = (
                 Dataset.open(foreign_mag_view_path.parent.parent)
                 .get_layer(foreign_mag_view_path.parent.name)

webknossos/webknossos/dataset/layer.py

@@ -6,15 +6,10 @@
 from uuid import uuid4
 
 import numpy as np
+from upath import UPath
 
 from ..geometry import BoundingBox, Mag, Vec3Int, Vec3IntLike
-from ..utils import (
-    get_executor_for_args,
-    is_fs_path,
-    make_upath,
-    rmtree,
-    wait_and_ensure_success,
-)
+from ..utils import get_executor_for_args, is_fs_path, rmtree, wait_and_ensure_success
 from ._array import ArrayInfo, BaseArray
 from .properties import MagViewProperties
 
@@ -284,7 +279,7 @@ def compress(
                     self.path
                 ), "Cannot compress a remote mag without `target_path`."
         else:
-            target_path = make_upath(target_path)
+            target_path = UPath(target_path)
 
         uncompressed_full_path = self.path
         compressed_dataset_path = (

webknossos/webknossos/dataset/mag_view.py

@@ -11,7 +11,6 @@
 from datetime import datetime
 from inspect import getframeinfo, stack
 from multiprocessing import cpu_count
-from os import PathLike
 from os.path import relpath
 from pathlib import Path
 from shutil import copyfileobj
@@ -213,34 +212,21 @@ def warn_deprecated(deprecated_item: str, alternative_item: str) -> None:
     )
 
 
-def make_upath(maybe_path: Union[str, PathLike, Path]) -> UPath:
-    return maybe_path if isinstance(maybe_path, UPath) else UPath(maybe_path)
-
-
 def is_fs_path(path: Path) -> bool:
-    # Distinguish between `pathlib.*Path` and `UPath` through a `UPath`-specific attribute
-    return not hasattr(path, "_url")
-
-
-def is_symlink(path: Path) -> bool:
-    try:
-        return path.is_symlink()
-    except NotImplementedError:
-        # `Path` raises `NotImplmentedError` for some methods, including `is_symlink`
-        return False
+    return not isinstance(path, UPath)
 
 
 def rmtree(path: Path) -> None:
     def _walk(path: Path) -> Iterator[Path]:
         if path.exists():
-            if path.is_dir() and not is_symlink(path):
+            if path.is_dir() and not path.is_symlink():
                 for p in path.iterdir():
                     yield from _walk(p)
             yield path
 
     for sub_path in _walk(path):
         try:
-            if sub_path.is_file() or is_symlink(sub_path):
+            if sub_path.is_file() or sub_path.is_symlink():
                 sub_path.unlink()
             elif sub_path.is_dir():
                 sub_path.rmdir()

webknossos/webknossos/utils.py

@@ -29,7 +29,7 @@ tifffile = "^2020.11.26"
 webknossos = { path = "../webknossos/", develop = true }
 wkw = "1.1.11"
 zarr = "^2.11.0"
-universal-pathlib = "^0.0.14"
+universal-pathlib = "0.0.16"
 
 [tool.poetry.dev-dependencies]
 black = "^20.8b1"