From ec9817b9d1262cf0116bae54ee98310d6ecb0610 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Fri, 18 Dec 2020 09:03:43 -0500 Subject: [PATCH 1/4] Don't import pyarrow just to check version --- fsspec/spec.py | 21 ++++++++++++--------- fsspec/utils.py | 38 +++++++++++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/fsspec/spec.py b/fsspec/spec.py index a39671605..aad504c50 100644 --- a/fsspec/spec.py +++ b/fsspec/spec.py @@ -9,7 +9,13 @@ from .dircache import DirCache from .transaction import Transaction -from .utils import read_block, tokenize, stringify_path, other_paths +from .utils import ( + read_block, + tokenize, + stringify_path, + other_paths, + get_package_version_without_import, +) from .config import apply_config logger = logging.getLogger("fsspec") @@ -72,16 +78,13 @@ def __call__(cls, *args, **kwargs): return obj -try: # optionally derive from pyarrow's FileSystem, if available +pa_version = get_package_version_without_import("pyarrow") +if LooseVersion(pa_version) < LooseVersion("2.0"): import pyarrow as pa -except ImportError: - up = object + + up = pa.filesystem.DaskFileSystem else: - # only derive from the legacy pyarrow's FileSystem for older pyarrow versions - if LooseVersion(pa.__version__) < LooseVersion("2.0"): - up = pa.filesystem.DaskFileSystem - else: - up = object + up = object class AbstractFileSystem(up, metaclass=_Cached): diff --git a/fsspec/utils.py b/fsspec/utils.py index 78df77e70..e5824a319 100644 --- a/fsspec/utils.py +++ b/fsspec/utils.py @@ -3,6 +3,7 @@ import os import pathlib import re +import sys from urllib.parse import urlsplit @@ -383,10 +384,13 @@ def can_be_local(path): return False -def setup_logger(logname, level="DEBUG"): +def setup_logger(logname, level="DEBUG", clear=True): + """Add standard logging handler to logger of given name""" import logging logger = logging.getLogger(logname) + if clear: + logger.handlers.clear() handle = logging.StreamHandler() formatter = logging.Formatter( "%(asctime)s - %(name)s - %(levelname)s " "- %(message)s" @@ -395,3 +399,35 @@ def setup_logger(logname, level="DEBUG"): logger.addHandler(handle) logger.setLevel(level) return logger + + +def get_package_version_without_import(name): + """For given package name, try to find the version without importing it + + Import and package.__version__ is still the backup here, so an import + *might* happen. + + Returns either the version string, or None if the package + or the version was not readily found. + """ + if name in sys.modules: + mod = sys.modules[name] + if hasattr(mod, "__version__"): + return mod.__version__ + if sys.version_info >= (3, 8): + import importlib.metadata + + return importlib.metadata.distribution(name).version + try: + import importlib_metadata + + return importlib_metadata.distribution(name).version + except ImportError: + pass + try: + import importlib + + mod = importlib.import_module(name) + return mod.__version__ + except (ImportError, AttributeError): + return None From 1b9ef9274c878c3998fbe1b270a73b5de020b8d9 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Fri, 18 Dec 2020 09:16:09 -0500 Subject: [PATCH 2/4] surprise import error --- fsspec/utils.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/fsspec/utils.py b/fsspec/utils.py index e5824a319..e2efee5b1 100644 --- a/fsspec/utils.py +++ b/fsspec/utils.py @@ -415,15 +415,19 @@ def get_package_version_without_import(name): if hasattr(mod, "__version__"): return mod.__version__ if sys.version_info >= (3, 8): - import importlib.metadata + try: + import importlib.metadata - return importlib.metadata.distribution(name).version - try: - import importlib_metadata + return importlib.metadata.distribution(name).version + except ImportError: + pass + else: + try: + import importlib_metadata - return importlib_metadata.distribution(name).version - except ImportError: - pass + return importlib_metadata.distribution(name).version + except ImportError: + pass try: import importlib From 950731db4a33a0667be70c7abbec5088ed46b819 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Fri, 18 Dec 2020 09:57:29 -0500 Subject: [PATCH 3/4] fix when no version --- fsspec/spec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fsspec/spec.py b/fsspec/spec.py index aad504c50..a6ae3ea31 100644 --- a/fsspec/spec.py +++ b/fsspec/spec.py @@ -2,7 +2,6 @@ import logging import os import warnings -from distutils.version import LooseVersion from errno import ESPIPE from hashlib import sha256 from glob import has_magic @@ -12,6 +11,7 @@ from .utils import ( read_block, tokenize, + loose_version, stringify_path, other_paths, get_package_version_without_import, @@ -79,7 +79,7 @@ def __call__(cls, *args, **kwargs): pa_version = get_package_version_without_import("pyarrow") -if LooseVersion(pa_version) < LooseVersion("2.0"): +if pa_version and loose_version(pa_version) < loose_version("2.0"): import pyarrow as pa up = pa.filesystem.DaskFileSystem From e93bb32963999ffddc2d589b79d48a9ef8c9ae8d Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Fri, 18 Dec 2020 10:05:34 -0500 Subject: [PATCH 4/4] reversion --- fsspec/spec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fsspec/spec.py b/fsspec/spec.py index a6ae3ea31..b2fd144e4 100644 --- a/fsspec/spec.py +++ b/fsspec/spec.py @@ -4,6 +4,7 @@ import warnings from errno import ESPIPE from hashlib import sha256 +from distutils.version import LooseVersion from glob import has_magic from .dircache import DirCache @@ -11,7 +12,6 @@ from .utils import ( read_block, tokenize, - loose_version, stringify_path, other_paths, get_package_version_without_import, @@ -79,7 +79,7 @@ def __call__(cls, *args, **kwargs): pa_version = get_package_version_without_import("pyarrow") -if pa_version and loose_version(pa_version) < loose_version("2.0"): +if pa_version and LooseVersion(pa_version) < LooseVersion("2.0"): import pyarrow as pa up = pa.filesystem.DaskFileSystem