Refactor the plugin manager.

docs/source/changes.md

@@ -25,6 +25,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and
 - {pull}`536` allows partialed functions to be task functions.
 - {pull}`540` changes the CLI entry-point and allow `pytask.build(tasks=task_func)` as
   the signatures suggested.
+- {pull}`542` refactors the plugin manager.
 
 ## 0.4.4 - 2023-12-04
 

pyproject.toml

@@ -146,7 +146,7 @@ ignore = [
 convention = "numpy"
 
 [tool.pytest.ini_options]
-testpaths = ["tests"]
+testpaths = ["src", "tests"]
 markers = [
     "wip: Tests that are work-in-progress.",
     "unit: Flag for unit tests which target mainly a single function.",

src/_pytask/build.py

@@ -15,7 +15,6 @@
 from _pytask.capture_utils import CaptureMethod
 from _pytask.capture_utils import ShowCapture
 from _pytask.click import ColoredCommand
-from _pytask.config import hookimpl
 from _pytask.config_utils import find_project_root_and_config
 from _pytask.config_utils import read_config
 from _pytask.console import console
@@ -26,6 +25,8 @@
 from _pytask.outcomes import ExitCode
 from _pytask.path import HashPathCache
 from _pytask.pluginmanager import get_plugin_manager
+from _pytask.pluginmanager import hookimpl
+from _pytask.pluginmanager import storage
 from _pytask.session import Session
 from _pytask.shared import parse_paths
 from _pytask.shared import to_list
@@ -62,7 +63,7 @@ def pytask_unconfigure(session: Session) -> None:
     path.write_text(json.dumps(HashPathCache._cache))
 
 
-def build(  # noqa: C901, PLR0912, PLR0913
+def build(  # noqa: C901, PLR0912, PLR0913, PLR0915
     *,
     capture: Literal["fd", "no", "sys", "tee-sys"] | CaptureMethod = CaptureMethod.FD,
     check_casing_of_paths: bool = True,
@@ -177,8 +178,6 @@ def build(  # noqa: C901, PLR0912, PLR0913
 
     """
     try:
-        pm = get_plugin_manager()
-
         raw_config = {
             "capture": capture,
             "check_casing_of_paths": check_casing_of_paths,
@@ -212,6 +211,12 @@ def build(  # noqa: C901, PLR0912, PLR0913
             **kwargs,
         }
 
+        if "command" not in raw_config:
+            pm = get_plugin_manager()
+            storage.store(pm)
+        else:
+            pm = storage.get()
+
         # If someone called the programmatic interface, we need to do some parsing.
         if "command" not in raw_config:
             raw_config["command"] = "build"

src/_pytask/capture.py

@@ -44,7 +44,7 @@
 from _pytask.capture_utils import CaptureMethod
 from _pytask.capture_utils import ShowCapture
 from _pytask.click import EnumChoice
-from _pytask.config import hookimpl
+from _pytask.pluginmanager import hookimpl
 from _pytask.shared import convert_to_enum
 
 if TYPE_CHECKING:

src/_pytask/clean.py

@@ -14,7 +14,6 @@
 import click
 from _pytask.click import ColoredCommand
 from _pytask.click import EnumChoice
-from _pytask.config import hookimpl
 from _pytask.console import console
 from _pytask.exceptions import CollectionError
 from _pytask.git import get_all_files
@@ -26,7 +25,8 @@
 from _pytask.outcomes import ExitCode
 from _pytask.path import find_common_ancestor
 from _pytask.path import relative_to
-from _pytask.pluginmanager import get_plugin_manager
+from _pytask.pluginmanager import hookimpl
+from _pytask.pluginmanager import storage
 from _pytask.session import Session
 from _pytask.shared import to_list
 from _pytask.traceback import Traceback
@@ -97,12 +97,11 @@ def pytask_parse_config(config: dict[str, Any]) -> None:
 )
 def clean(**raw_config: Any) -> NoReturn:  # noqa: C901, PLR0912
     """Clean the provided paths by removing files unknown to pytask."""
+    pm = storage.get()
     raw_config["command"] = "clean"
 
     try:
         # Duplication of the same mechanism in :func:`pytask.build`.
-        pm = get_plugin_manager()
-
         config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config)
         session = Session.from_config(config)
 

src/_pytask/cli.py

@@ -2,34 +2,29 @@
 from __future__ import annotations
 
 from typing import Any
-from typing import TYPE_CHECKING
 
 import click
 from _pytask.click import ColoredGroup
-from _pytask.config import hookimpl
-from _pytask.pluginmanager import get_plugin_manager
+from _pytask.pluginmanager import storage
 from packaging.version import parse as parse_version
 
-if TYPE_CHECKING:
-    import pluggy
-
 
 _CONTEXT_SETTINGS: dict[str, Any] = {
     "help_option_names": ("-h", "--help"),
     "show_default": True,
 }
 
 
-if parse_version(click.__version__) < parse_version("8"):  # pragma: no cover
-    _VERSION_OPTION_KWARGS: dict[str, Any] = {}
-else:  # pragma: no cover
+if parse_version(click.__version__) >= parse_version("8"):  # pragma: no cover
     _VERSION_OPTION_KWARGS = {"package_name": "pytask"}
+else:  # pragma: no cover
+    _VERSION_OPTION_KWARGS = {}
 
 
 def _extend_command_line_interface(cli: click.Group) -> click.Group:
     """Add parameters from plugins to the commandline interface."""
-    pm = get_plugin_manager()
-    pm.hook.pytask_extend_command_line_interface(cli=cli)
+    pm = storage.create()
+    pm.hook.pytask_extend_command_line_interface.call_historic(kwargs={"cli": cli})
     _sort_options_for_each_command_alphabetically(cli)
     return cli
 
@@ -42,54 +37,6 @@ def _sort_options_for_each_command_alphabetically(cli: click.Group) -> None:
         )
 
 
-@hookimpl
-def pytask_add_hooks(pm: pluggy.PluginManager) -> None:
-    """Add hooks."""
-    from _pytask import build
-    from _pytask import capture
-    from _pytask import clean
-    from _pytask import collect
-    from _pytask import collect_command
-    from _pytask import config
-    from _pytask import database
-    from _pytask import debugging
-    from _pytask import execute
-    from _pytask import dag_command
-    from _pytask import live
-    from _pytask import logging
-    from _pytask import mark
-    from _pytask import nodes
-    from _pytask import parameters
-    from _pytask import persist
-    from _pytask import profile
-    from _pytask import dag
-    from _pytask import skipping
-    from _pytask import task
-    from _pytask import warnings
-
-    pm.register(build)
-    pm.register(capture)
-    pm.register(clean)
-    pm.register(collect)
-    pm.register(collect_command)
-    pm.register(config)
-    pm.register(database)
-    pm.register(debugging)
-    pm.register(execute)
-    pm.register(dag_command)
-    pm.register(live)
-    pm.register(logging)
-    pm.register(mark)
-    pm.register(nodes)
-    pm.register(parameters)
-    pm.register(persist)
-    pm.register(profile)
-    pm.register(dag)
-    pm.register(skipping)
-    pm.register(task)
-    pm.register(warnings)
-
-
 @click.group(
     cls=ColoredGroup,
     context_settings=_CONTEXT_SETTINGS,

src/_pytask/collect.py

@@ -15,7 +15,6 @@
 from _pytask.collect_utils import create_name_of_python_node
 from _pytask.collect_utils import parse_dependencies_from_task_function
 from _pytask.collect_utils import parse_products_from_task_function
-from _pytask.config import hookimpl
 from _pytask.config import IS_FILE_SYSTEM_CASE_SENSITIVE
 from _pytask.console import console
 from _pytask.console import create_summary_panel
@@ -36,6 +35,7 @@
 from _pytask.path import find_case_sensitive_path
 from _pytask.path import import_path
 from _pytask.path import shorten_path
+from _pytask.pluginmanager import hookimpl
 from _pytask.reports import CollectionReport
 from _pytask.shared import find_duplicates
 from _pytask.shared import to_list

src/_pytask/collect_command.py

@@ -8,7 +8,6 @@
 
 import click
 from _pytask.click import ColoredCommand
-from _pytask.config import hookimpl
 from _pytask.console import console
 from _pytask.console import create_url_style_for_path
 from _pytask.console import FILE_ICON
@@ -27,7 +26,8 @@
 from _pytask.outcomes import ExitCode
 from _pytask.path import find_common_ancestor
 from _pytask.path import relative_to
-from _pytask.pluginmanager import get_plugin_manager
+from _pytask.pluginmanager import hookimpl
+from _pytask.pluginmanager import storage
 from _pytask.session import Session
 from _pytask.tree_util import tree_leaves
 from rich.text import Text
@@ -54,10 +54,10 @@ def pytask_extend_command_line_interface(cli: click.Group) -> None:
 )
 def collect(**raw_config: Any | None) -> NoReturn:
     """Collect tasks and report information about them."""
+    pm = storage.get()
     raw_config["command"] = "collect"
 
     try:
-        pm = get_plugin_manager()
         config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config)
         session = Session.from_config(config)
 

src/_pytask/compat.py

@@ -1,10 +1,10 @@
 """Contains functions to assess compatibility and optional dependencies."""
 from __future__ import annotations
 
-import importlib
 import shutil
 import sys
 import warnings
+from importlib import import_module
 from typing import TYPE_CHECKING
 
 from packaging.version import parse as parse_version
@@ -89,7 +89,9 @@ def import_optional_dependency(
         f"Use pip or conda to install {install_name!r}."
     )
     try:
-        module = importlib.import_module(name)
+        # The from import is used to avoid monkeypatching errors in some tests. See
+        # https://stackoverflow.com/a/31746577 for more information.
+        module = import_module(name)
     except ImportError:
         if errors == "raise":
             raise ImportError(msg) from None

src/_pytask/config.py

@@ -4,14 +4,15 @@
 import tempfile
 from pathlib import Path
 from typing import Any
+from typing import TYPE_CHECKING
 
-import pluggy
+from _pytask.pluginmanager import hookimpl
 from _pytask.shared import parse_markers
 from _pytask.shared import parse_paths
 from _pytask.shared import to_list
 
-
-hookimpl = pluggy.HookimplMarker("pytask")
+if TYPE_CHECKING:
+    from pluggy import PluginManager
 
 
 _IGNORED_FOLDERS: list[str] = [".git/*", ".venv/*"]
@@ -59,9 +60,7 @@ def is_file_system_case_sensitive() -> bool:
 
 
 @hookimpl
-def pytask_configure(
-    pm: pluggy.PluginManager, raw_config: dict[str, Any]
-) -> dict[str, Any]:
+def pytask_configure(pm: PluginManager, raw_config: dict[str, Any]) -> dict[str, Any]:
     """Configure pytask."""
     # Add all values by default so that many plugins do not need to copy over values.
     config = {"pm": pm, "markers": {}, **raw_config}

src/_pytask/dag.py

@@ -6,7 +6,6 @@
 from typing import TYPE_CHECKING
 
 import networkx as nx
-from _pytask.config import hookimpl
 from _pytask.console import ARROW_DOWN_ICON
 from _pytask.console import console
 from _pytask.console import FILE_ICON
@@ -19,6 +18,7 @@
 from _pytask.node_protocols import PNode
 from _pytask.node_protocols import PTask
 from _pytask.nodes import PythonNode
+from _pytask.pluginmanager import hookimpl
 from _pytask.reports import DagReport
 from _pytask.shared import reduce_names_of_multiple_nodes
 from _pytask.tree_util import tree_map

src/_pytask/dag_command.py

@@ -12,7 +12,6 @@
 from _pytask.click import EnumChoice
 from _pytask.compat import check_for_optional_program
 from _pytask.compat import import_optional_dependency
-from _pytask.config import hookimpl
 from _pytask.config_utils import find_project_root_and_config
 from _pytask.config_utils import read_config
 from _pytask.console import console
@@ -21,6 +20,8 @@
 from _pytask.exceptions import ResolvingDependenciesError
 from _pytask.outcomes import ExitCode
 from _pytask.pluginmanager import get_plugin_manager
+from _pytask.pluginmanager import hookimpl
+from _pytask.pluginmanager import storage
 from _pytask.session import Session
 from _pytask.shared import parse_paths
 from _pytask.shared import reduce_names_of_multiple_nodes
@@ -80,7 +81,7 @@ def pytask_extend_command_line_interface(cli: click.Group) -> None:
 def dag(**raw_config: Any) -> int:
     """Create a visualization of the project's directed acyclic graph."""
     try:
-        pm = get_plugin_manager()
+        pm = storage.get()
         config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config)
         session = Session.from_config(config)
 
@@ -143,6 +144,7 @@ def build_dag(raw_config: dict[str, Any]) -> nx.DiGraph:
     """
     try:
         pm = get_plugin_manager()
+        storage.store(pm)
 
         # If someone called the programmatic interface, we need to do some parsing.
         if "command" not in raw_config: