From 51bb82b397198a469c24690a56e5bdff62b4e2ab Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Wed, 27 Dec 2023 17:04:39 +0100 Subject: [PATCH 01/11] Move hookimpl, moved add hooks to pluginmanager. --- src/_pytask/build.py | 2 +- src/_pytask/capture.py | 2 +- src/_pytask/clean.py | 2 +- src/_pytask/cli.py | 53 ---------------------------------- src/_pytask/collect.py | 2 +- src/_pytask/collect_command.py | 2 +- src/_pytask/compat.py | 4 +-- src/_pytask/config.py | 7 +++-- src/_pytask/dag.py | 2 +- src/_pytask/dag_command.py | 2 +- src/_pytask/database.py | 2 +- src/_pytask/debugging.py | 2 +- src/_pytask/execute.py | 2 +- src/_pytask/live.py | 2 +- src/_pytask/logging.py | 2 +- src/_pytask/mark/__init__.py | 2 +- src/_pytask/parameters.py | 2 +- src/_pytask/persist.py | 2 +- src/_pytask/pluginmanager.py | 52 +++++++++++++++++++++++++++++++-- src/_pytask/profile.py | 2 +- src/_pytask/skipping.py | 2 +- src/_pytask/task.py | 2 +- src/_pytask/warnings.py | 2 +- src/pytask/__init__.py | 2 +- tests/test_dag_command.py | 8 ++--- 25 files changed, 79 insertions(+), 85 deletions(-) diff --git a/src/_pytask/build.py b/src/_pytask/build.py index 85e1c883..9313a95a 100644 --- a/src/_pytask/build.py +++ b/src/_pytask/build.py @@ -15,7 +15,6 @@ from _pytask.capture_utils import CaptureMethod from _pytask.capture_utils import ShowCapture from _pytask.click import ColoredCommand -from _pytask.config import hookimpl from _pytask.config_utils import find_project_root_and_config from _pytask.config_utils import read_config from _pytask.console import console @@ -26,6 +25,7 @@ from _pytask.outcomes import ExitCode from _pytask.path import HashPathCache from _pytask.pluginmanager import get_plugin_manager +from _pytask.pluginmanager import hookimpl from _pytask.session import Session from _pytask.shared import parse_paths from _pytask.shared import to_list diff --git a/src/_pytask/capture.py b/src/_pytask/capture.py index fa46d1ab..7d448e70 100644 --- a/src/_pytask/capture.py +++ b/src/_pytask/capture.py @@ -44,7 +44,7 @@ from _pytask.capture_utils import CaptureMethod from _pytask.capture_utils import ShowCapture from _pytask.click import EnumChoice -from _pytask.config import hookimpl +from _pytask.pluginmanager import hookimpl from _pytask.shared import convert_to_enum if TYPE_CHECKING: diff --git a/src/_pytask/clean.py b/src/_pytask/clean.py index 15ea6add..cba8435c 100644 --- a/src/_pytask/clean.py +++ b/src/_pytask/clean.py @@ -14,7 +14,6 @@ import click from _pytask.click import ColoredCommand from _pytask.click import EnumChoice -from _pytask.config import hookimpl from _pytask.console import console from _pytask.exceptions import CollectionError from _pytask.git import get_all_files @@ -27,6 +26,7 @@ from _pytask.path import find_common_ancestor from _pytask.path import relative_to from _pytask.pluginmanager import get_plugin_manager +from _pytask.pluginmanager import hookimpl from _pytask.session import Session from _pytask.shared import to_list from _pytask.traceback import Traceback diff --git a/src/_pytask/cli.py b/src/_pytask/cli.py index f056b2e1..bf0111f1 100644 --- a/src/_pytask/cli.py +++ b/src/_pytask/cli.py @@ -2,17 +2,12 @@ from __future__ import annotations from typing import Any -from typing import TYPE_CHECKING import click from _pytask.click import ColoredGroup -from _pytask.config import hookimpl from _pytask.pluginmanager import get_plugin_manager from packaging.version import parse as parse_version -if TYPE_CHECKING: - import pluggy - _CONTEXT_SETTINGS: dict[str, Any] = { "help_option_names": ("-h", "--help"), @@ -42,54 +37,6 @@ def _sort_options_for_each_command_alphabetically(cli: click.Group) -> None: ) -@hookimpl -def pytask_add_hooks(pm: pluggy.PluginManager) -> None: - """Add hooks.""" - from _pytask import build - from _pytask import capture - from _pytask import clean - from _pytask import collect - from _pytask import collect_command - from _pytask import config - from _pytask import database - from _pytask import debugging - from _pytask import execute - from _pytask import dag_command - from _pytask import live - from _pytask import logging - from _pytask import mark - from _pytask import nodes - from _pytask import parameters - from _pytask import persist - from _pytask import profile - from _pytask import dag - from _pytask import skipping - from _pytask import task - from _pytask import warnings - - pm.register(build) - pm.register(capture) - pm.register(clean) - pm.register(collect) - pm.register(collect_command) - pm.register(config) - pm.register(database) - pm.register(debugging) - pm.register(execute) - pm.register(dag_command) - pm.register(live) - pm.register(logging) - pm.register(mark) - pm.register(nodes) - pm.register(parameters) - pm.register(persist) - pm.register(profile) - pm.register(dag) - pm.register(skipping) - pm.register(task) - pm.register(warnings) - - @click.group( cls=ColoredGroup, context_settings=_CONTEXT_SETTINGS, diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index c04335c3..3b262568 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -15,7 +15,6 @@ from _pytask.collect_utils import create_name_of_python_node from _pytask.collect_utils import parse_dependencies_from_task_function from _pytask.collect_utils import parse_products_from_task_function -from _pytask.config import hookimpl from _pytask.config import IS_FILE_SYSTEM_CASE_SENSITIVE from _pytask.console import console from _pytask.console import create_summary_panel @@ -36,6 +35,7 @@ from _pytask.path import find_case_sensitive_path from _pytask.path import import_path from _pytask.path import shorten_path +from _pytask.pluginmanager import hookimpl from _pytask.reports import CollectionReport from _pytask.shared import find_duplicates from _pytask.shared import to_list diff --git a/src/_pytask/collect_command.py b/src/_pytask/collect_command.py index 0ae8648d..9a71129e 100644 --- a/src/_pytask/collect_command.py +++ b/src/_pytask/collect_command.py @@ -8,7 +8,6 @@ import click from _pytask.click import ColoredCommand -from _pytask.config import hookimpl from _pytask.console import console from _pytask.console import create_url_style_for_path from _pytask.console import FILE_ICON @@ -28,6 +27,7 @@ from _pytask.path import find_common_ancestor from _pytask.path import relative_to from _pytask.pluginmanager import get_plugin_manager +from _pytask.pluginmanager import hookimpl from _pytask.session import Session from _pytask.tree_util import tree_leaves from rich.text import Text diff --git a/src/_pytask/compat.py b/src/_pytask/compat.py index c5213c3d..e4151e2b 100644 --- a/src/_pytask/compat.py +++ b/src/_pytask/compat.py @@ -1,10 +1,10 @@ """Contains functions to assess compatibility and optional dependencies.""" from __future__ import annotations -import importlib import shutil import sys import warnings +from importlib import import_module from typing import TYPE_CHECKING from packaging.version import parse as parse_version @@ -89,7 +89,7 @@ def import_optional_dependency( f"Use pip or conda to install {install_name!r}." ) try: - module = importlib.import_module(name) + module = import_module(name) except ImportError: if errors == "raise": raise ImportError(msg) from None diff --git a/src/_pytask/config.py b/src/_pytask/config.py index 7cb6d9e6..48151046 100644 --- a/src/_pytask/config.py +++ b/src/_pytask/config.py @@ -4,14 +4,15 @@ import tempfile from pathlib import Path from typing import Any +from typing import TYPE_CHECKING -import pluggy +from _pytask.pluginmanager import hookimpl from _pytask.shared import parse_markers from _pytask.shared import parse_paths from _pytask.shared import to_list - -hookimpl = pluggy.HookimplMarker("pytask") +if TYPE_CHECKING: + import pluggy _IGNORED_FOLDERS: list[str] = [".git/*", ".venv/*"] diff --git a/src/_pytask/dag.py b/src/_pytask/dag.py index 1278b8aa..89941b74 100644 --- a/src/_pytask/dag.py +++ b/src/_pytask/dag.py @@ -6,7 +6,6 @@ from typing import TYPE_CHECKING import networkx as nx -from _pytask.config import hookimpl from _pytask.console import ARROW_DOWN_ICON from _pytask.console import console from _pytask.console import FILE_ICON @@ -19,6 +18,7 @@ from _pytask.node_protocols import PNode from _pytask.node_protocols import PTask from _pytask.nodes import PythonNode +from _pytask.pluginmanager import hookimpl from _pytask.reports import DagReport from _pytask.shared import reduce_names_of_multiple_nodes from _pytask.tree_util import tree_map diff --git a/src/_pytask/dag_command.py b/src/_pytask/dag_command.py index 5eeef7da..b1a64e39 100644 --- a/src/_pytask/dag_command.py +++ b/src/_pytask/dag_command.py @@ -12,7 +12,6 @@ from _pytask.click import EnumChoice from _pytask.compat import check_for_optional_program from _pytask.compat import import_optional_dependency -from _pytask.config import hookimpl from _pytask.config_utils import find_project_root_and_config from _pytask.config_utils import read_config from _pytask.console import console @@ -21,6 +20,7 @@ from _pytask.exceptions import ResolvingDependenciesError from _pytask.outcomes import ExitCode from _pytask.pluginmanager import get_plugin_manager +from _pytask.pluginmanager import hookimpl from _pytask.session import Session from _pytask.shared import parse_paths from _pytask.shared import reduce_names_of_multiple_nodes diff --git a/src/_pytask/database.py b/src/_pytask/database.py index eb40fd2b..2a19a459 100644 --- a/src/_pytask/database.py +++ b/src/_pytask/database.py @@ -4,8 +4,8 @@ from pathlib import Path from typing import Any -from _pytask.config import hookimpl from _pytask.database_utils import create_database +from _pytask.pluginmanager import hookimpl from sqlalchemy.engine import make_url diff --git a/src/_pytask/debugging.py b/src/_pytask/debugging.py index 7da0d985..83d20f7b 100644 --- a/src/_pytask/debugging.py +++ b/src/_pytask/debugging.py @@ -10,10 +10,10 @@ from typing import TYPE_CHECKING import click -from _pytask.config import hookimpl from _pytask.console import console from _pytask.node_protocols import PTask from _pytask.outcomes import Exit +from _pytask.pluginmanager import hookimpl from _pytask.traceback import Traceback diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py index ccc87d66..e58f3173 100644 --- a/src/_pytask/execute.py +++ b/src/_pytask/execute.py @@ -7,7 +7,6 @@ from typing import Any from typing import TYPE_CHECKING -from _pytask.config import hookimpl from _pytask.config import IS_FILE_SYSTEM_CASE_SENSITIVE from _pytask.console import console from _pytask.console import create_summary_panel @@ -33,6 +32,7 @@ from _pytask.outcomes import SkippedUnchanged from _pytask.outcomes import TaskOutcome from _pytask.outcomes import WouldBeExecuted +from _pytask.pluginmanager import hookimpl from _pytask.reports import ExecutionReport from _pytask.traceback import remove_traceback_from_exc_info from _pytask.tree_util import tree_leaves diff --git a/src/_pytask/live.py b/src/_pytask/live.py index 32974426..77982d59 100644 --- a/src/_pytask/live.py +++ b/src/_pytask/live.py @@ -7,11 +7,11 @@ from typing import TYPE_CHECKING import click -from _pytask.config import hookimpl from _pytask.console import console from _pytask.console import format_task_name from _pytask.outcomes import CollectionOutcome from _pytask.outcomes import TaskOutcome +from _pytask.pluginmanager import hookimpl from attrs import define from attrs import field from rich.box import ROUNDED diff --git a/src/_pytask/logging.py b/src/_pytask/logging.py index 14332917..6dedd1c4 100644 --- a/src/_pytask/logging.py +++ b/src/_pytask/logging.py @@ -12,9 +12,9 @@ import _pytask import click import pluggy -from _pytask.config import hookimpl from _pytask.console import console from _pytask.console import IS_WINDOWS_TERMINAL +from _pytask.pluginmanager import hookimpl from _pytask.reports import ExecutionReport from _pytask.traceback import Traceback from rich.text import Text diff --git a/src/_pytask/mark/__init__.py b/src/_pytask/mark/__init__.py index 20266a13..c8fddf63 100644 --- a/src/_pytask/mark/__init__.py +++ b/src/_pytask/mark/__init__.py @@ -8,7 +8,6 @@ import click from _pytask.click import ColoredCommand -from _pytask.config import hookimpl from _pytask.console import console from _pytask.dag_utils import task_and_preceding_tasks from _pytask.exceptions import ConfigurationError @@ -20,6 +19,7 @@ from _pytask.mark.structures import MarkGenerator from _pytask.outcomes import ExitCode from _pytask.pluginmanager import get_plugin_manager +from _pytask.pluginmanager import hookimpl from _pytask.session import Session from _pytask.shared import parse_markers from attrs import define diff --git a/src/_pytask/parameters.py b/src/_pytask/parameters.py index ddf7f907..36e547fe 100644 --- a/src/_pytask/parameters.py +++ b/src/_pytask/parameters.py @@ -4,8 +4,8 @@ from pathlib import Path import click -from _pytask.config import hookimpl from _pytask.config_utils import set_defaults_from_config +from _pytask.pluginmanager import hookimpl from click import Context from sqlalchemy.engine import make_url from sqlalchemy.engine import URL diff --git a/src/_pytask/persist.py b/src/_pytask/persist.py index 49b61510..bb74d3bb 100644 --- a/src/_pytask/persist.py +++ b/src/_pytask/persist.py @@ -4,13 +4,13 @@ from typing import Any from typing import TYPE_CHECKING -from _pytask.config import hookimpl from _pytask.dag_utils import node_and_neighbors from _pytask.database_utils import has_node_changed from _pytask.database_utils import update_states_in_database from _pytask.mark_utils import has_mark from _pytask.outcomes import Persisted from _pytask.outcomes import TaskOutcome +from _pytask.pluginmanager import hookimpl if TYPE_CHECKING: diff --git a/src/_pytask/pluginmanager.py b/src/_pytask/pluginmanager.py index aa3acac7..6deac51d 100644 --- a/src/_pytask/pluginmanager.py +++ b/src/_pytask/pluginmanager.py @@ -1,8 +1,56 @@ """Contains the plugin manager.""" from __future__ import annotations +import importlib +from typing import Iterable + import pluggy from _pytask import hookspecs +from pluggy import HookimplMarker +from pluggy import PluginManager + + +hookimpl = HookimplMarker("pytask") + + +def register_hook_impls_from_modules( + plugin_manager: PluginManager, module_names: Iterable[str] +) -> None: + """Register hook implementations from modules.""" + for module_name in module_names: + module = importlib.import_module(module_name) + plugin_manager.register(module) + + +class BuiltinPlugins: + @staticmethod + @hookimpl + def pytask_add_hooks(pm: pluggy.PluginManager) -> None: + """Add hooks.""" + builtin_hook_impl_modules = ( + "_pytask.build", + "_pytask.capture", + "_pytask.clean", + "_pytask.collect", + "_pytask.collect_command", + "_pytask.config", + "_pytask.dag", + "_pytask.dag_command", + "_pytask.database", + "_pytask.debugging", + "_pytask.execute", + "_pytask.live", + "_pytask.logging", + "_pytask.mark", + "_pytask.nodes", + "_pytask.parameters", + "_pytask.persist", + "_pytask.profile", + "_pytask.skipping", + "_pytask.task", + "_pytask.warnings", + ) + register_hook_impls_from_modules(pm, builtin_hook_impl_modules) def get_plugin_manager() -> pluggy.PluginManager: @@ -11,9 +59,7 @@ def get_plugin_manager() -> pluggy.PluginManager: pm.add_hookspecs(hookspecs) pm.load_setuptools_entrypoints("pytask") - from _pytask import cli - - pm.register(cli) + pm.register(BuiltinPlugins) pm.hook.pytask_add_hooks(pm=pm) return pm diff --git a/src/_pytask/profile.py b/src/_pytask/profile.py index 4c873f5d..34b34978 100644 --- a/src/_pytask/profile.py +++ b/src/_pytask/profile.py @@ -14,7 +14,6 @@ import click from _pytask.click import ColoredCommand from _pytask.click import EnumChoice -from _pytask.config import hookimpl from _pytask.console import console from _pytask.console import format_task_name from _pytask.database_utils import BaseTable @@ -26,6 +25,7 @@ from _pytask.outcomes import ExitCode from _pytask.outcomes import TaskOutcome from _pytask.pluginmanager import get_plugin_manager +from _pytask.pluginmanager import hookimpl from _pytask.session import Session from _pytask.traceback import Traceback from rich.table import Table diff --git a/src/_pytask/skipping.py b/src/_pytask/skipping.py index baba42d6..7264095e 100644 --- a/src/_pytask/skipping.py +++ b/src/_pytask/skipping.py @@ -4,7 +4,6 @@ from typing import Any from typing import TYPE_CHECKING -from _pytask.config import hookimpl from _pytask.dag_utils import descending_tasks from _pytask.mark import Mark from _pytask.mark_utils import get_marks @@ -13,6 +12,7 @@ from _pytask.outcomes import SkippedAncestorFailed from _pytask.outcomes import SkippedUnchanged from _pytask.outcomes import TaskOutcome +from _pytask.pluginmanager import hookimpl if TYPE_CHECKING: diff --git a/src/_pytask/task.py b/src/_pytask/task.py index 2309695b..895b2d2d 100644 --- a/src/_pytask/task.py +++ b/src/_pytask/task.py @@ -5,8 +5,8 @@ from typing import Callable from typing import TYPE_CHECKING -from _pytask.config import hookimpl from _pytask.console import format_strings_as_flat_tree +from _pytask.pluginmanager import hookimpl from _pytask.shared import find_duplicates from _pytask.task_utils import COLLECTED_TASKS from _pytask.task_utils import parse_collected_tasks_with_task_marker diff --git a/src/_pytask/warnings.py b/src/_pytask/warnings.py index 9ecee42f..e643c70f 100644 --- a/src/_pytask/warnings.py +++ b/src/_pytask/warnings.py @@ -7,8 +7,8 @@ from typing import TYPE_CHECKING import click -from _pytask.config import hookimpl from _pytask.console import console +from _pytask.pluginmanager import hookimpl from _pytask.warnings_utils import catch_warnings_for_item from _pytask.warnings_utils import parse_filterwarnings from _pytask.warnings_utils import WarningReport diff --git a/src/pytask/__init__.py b/src/pytask/__init__.py index 1b2f25c5..9c3053b0 100644 --- a/src/pytask/__init__.py +++ b/src/pytask/__init__.py @@ -15,7 +15,6 @@ from _pytask.collect_utils import produces from _pytask.compat import check_for_optional_program from _pytask.compat import import_optional_dependency -from _pytask.config import hookimpl from _pytask.console import console from _pytask.dag_command import build_dag from _pytask.data_catalog import DataCatalog @@ -59,6 +58,7 @@ from _pytask.outcomes import SkippedAncestorFailed from _pytask.outcomes import SkippedUnchanged from _pytask.outcomes import TaskOutcome +from _pytask.pluginmanager import hookimpl from _pytask.profile import Runtime from _pytask.reports import CollectionReport from _pytask.reports import DagReport diff --git a/tests/test_dag_command.py b/tests/test_dag_command.py index 30ae7561..96143212 100644 --- a/tests/test_dag_command.py +++ b/tests/test_dag_command.py @@ -115,7 +115,7 @@ def task_example(): pass tmp_path.joinpath("input.txt").touch() monkeypatch.setattr( - "_pytask.compat.importlib.import_module", + "_pytask.compat.import_module", lambda x: _raise_exc(ImportError("pygraphviz not found")), # noqa: ARG005 ) @@ -150,7 +150,7 @@ def task_create_graph(): tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) monkeypatch.setattr( - "_pytask.compat.importlib.import_module", + "_pytask.compat.import_module", lambda x: _raise_exc(ImportError("pygraphviz not found")), # noqa: ARG005 ) @@ -169,7 +169,7 @@ def test_raise_error_with_graph_via_cli_missing_optional_program( monkeypatch, tmp_path, runner ): monkeypatch.setattr( - "_pytask.compat.importlib.import_module", + "_pytask.compat.import_module", lambda x: None, # noqa: ARG005 ) monkeypatch.setattr("_pytask.compat.shutil.which", lambda x: None) # noqa: ARG005 @@ -200,7 +200,7 @@ def test_raise_error_with_graph_via_task_missing_optional_program( monkeypatch, tmp_path, runner ): monkeypatch.setattr( - "_pytask.compat.importlib.import_module", + "_pytask.compat.import_module", lambda x: None, # noqa: ARG005 ) monkeypatch.setattr("_pytask.compat.shutil.which", lambda x: None) # noqa: ARG005 From 314ea28d2803e8b2b6e0580cc003e629b39ea1be Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Wed, 27 Dec 2023 17:09:28 +0100 Subject: [PATCH 02/11] Simplify regitering builtin hooks. --- src/_pytask/pluginmanager.py | 59 ++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/src/_pytask/pluginmanager.py b/src/_pytask/pluginmanager.py index 6deac51d..f560385b 100644 --- a/src/_pytask/pluginmanager.py +++ b/src/_pytask/pluginmanager.py @@ -2,6 +2,7 @@ from __future__ import annotations import importlib +import sys from typing import Iterable import pluggy @@ -22,35 +23,33 @@ def register_hook_impls_from_modules( plugin_manager.register(module) -class BuiltinPlugins: - @staticmethod - @hookimpl - def pytask_add_hooks(pm: pluggy.PluginManager) -> None: - """Add hooks.""" - builtin_hook_impl_modules = ( - "_pytask.build", - "_pytask.capture", - "_pytask.clean", - "_pytask.collect", - "_pytask.collect_command", - "_pytask.config", - "_pytask.dag", - "_pytask.dag_command", - "_pytask.database", - "_pytask.debugging", - "_pytask.execute", - "_pytask.live", - "_pytask.logging", - "_pytask.mark", - "_pytask.nodes", - "_pytask.parameters", - "_pytask.persist", - "_pytask.profile", - "_pytask.skipping", - "_pytask.task", - "_pytask.warnings", - ) - register_hook_impls_from_modules(pm, builtin_hook_impl_modules) +@hookimpl +def pytask_add_hooks(pm: pluggy.PluginManager) -> None: + """Add hooks.""" + builtin_hook_impl_modules = ( + "_pytask.build", + "_pytask.capture", + "_pytask.clean", + "_pytask.collect", + "_pytask.collect_command", + "_pytask.config", + "_pytask.dag", + "_pytask.dag_command", + "_pytask.database", + "_pytask.debugging", + "_pytask.execute", + "_pytask.live", + "_pytask.logging", + "_pytask.mark", + "_pytask.nodes", + "_pytask.parameters", + "_pytask.persist", + "_pytask.profile", + "_pytask.skipping", + "_pytask.task", + "_pytask.warnings", + ) + register_hook_impls_from_modules(pm, builtin_hook_impl_modules) def get_plugin_manager() -> pluggy.PluginManager: @@ -59,7 +58,7 @@ def get_plugin_manager() -> pluggy.PluginManager: pm.add_hookspecs(hookspecs) pm.load_setuptools_entrypoints("pytask") - pm.register(BuiltinPlugins) + pm.register(sys.modules[__name__]) pm.hook.pytask_add_hooks(pm=pm) return pm From 3dd4bcd658da779b79c7baefb9df4c3ff8abbd22 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Wed, 27 Dec 2023 17:14:53 +0100 Subject: [PATCH 03/11] Remove remainings from cyclic import error hack. --- src/pytask/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/pytask/__init__.py b/src/pytask/__init__.py index 9c3053b0..7244b9eb 100644 --- a/src/pytask/__init__.py +++ b/src/pytask/__init__.py @@ -6,6 +6,7 @@ from _pytask.build import build from _pytask.capture_utils import CaptureMethod from _pytask.capture_utils import ShowCapture +from _pytask.cli import cli from _pytask.click import ColoredCommand from _pytask.click import ColoredGroup from _pytask.click import EnumChoice @@ -74,10 +75,6 @@ from _pytask.warnings_utils import WarningReport -# This import must come last, otherwise a circular import occurs. -from _pytask.cli import cli # noreorder - - __all__ = [ "BaseTable", "CaptureMethod", From 195b70a43c3c6fd273d2e3b597ce3c9fce9c90a8 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Wed, 27 Dec 2023 17:24:02 +0100 Subject: [PATCH 04/11] Little if clause change. --- src/_pytask/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/_pytask/cli.py b/src/_pytask/cli.py index bf0111f1..aa5eda2d 100644 --- a/src/_pytask/cli.py +++ b/src/_pytask/cli.py @@ -15,10 +15,10 @@ } -if parse_version(click.__version__) < parse_version("8"): # pragma: no cover - _VERSION_OPTION_KWARGS: dict[str, Any] = {} -else: # pragma: no cover +if parse_version(click.__version__) >= parse_version("8"): # pragma: no cover _VERSION_OPTION_KWARGS = {"package_name": "pytask"} +else: # pragma: no cover + _VERSION_OPTION_KWARGS = {} def _extend_command_line_interface(cli: click.Group) -> click.Group: From 3364de591d94fe2e8df307a86f4075c55b1f736c Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Wed, 27 Dec 2023 17:30:28 +0100 Subject: [PATCH 05/11] Make two hooks historic. --- pyproject.toml | 2 +- src/_pytask/cli.py | 2 +- src/_pytask/hookspecs.py | 4 ++-- src/_pytask/pluginmanager.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index dce6e6ca..d9d00557 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -146,7 +146,7 @@ ignore = [ convention = "numpy" [tool.pytest.ini_options] -testpaths = ["tests"] +testpaths = ["src", "tests"] markers = [ "wip: Tests that are work-in-progress.", "unit: Flag for unit tests which target mainly a single function.", diff --git a/src/_pytask/cli.py b/src/_pytask/cli.py index aa5eda2d..d6df216e 100644 --- a/src/_pytask/cli.py +++ b/src/_pytask/cli.py @@ -24,7 +24,7 @@ def _extend_command_line_interface(cli: click.Group) -> click.Group: """Add parameters from plugins to the commandline interface.""" pm = get_plugin_manager() - pm.hook.pytask_extend_command_line_interface(cli=cli) + pm.hook.pytask_extend_command_line_interface.call_historic(kwargs={"cli": cli}) _sort_options_for_each_command_alphabetically(cli) return cli diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py index bea498e4..82183ae1 100644 --- a/src/_pytask/hookspecs.py +++ b/src/_pytask/hookspecs.py @@ -30,7 +30,7 @@ hookspec = pluggy.HookspecMarker("pytask") -@hookspec +@hookspec(historic=True) def pytask_add_hooks(pm: pluggy.PluginManager) -> None: """Add hook specifications and implementations to the plugin manager. @@ -46,7 +46,7 @@ def pytask_add_hooks(pm: pluggy.PluginManager) -> None: # Hooks for the command-line interface. -@hookspec +@hookspec(historic=True) def pytask_extend_command_line_interface(cli: click.Group) -> None: """Extend the command line interface. diff --git a/src/_pytask/pluginmanager.py b/src/_pytask/pluginmanager.py index f560385b..b717cae6 100644 --- a/src/_pytask/pluginmanager.py +++ b/src/_pytask/pluginmanager.py @@ -59,6 +59,6 @@ def get_plugin_manager() -> pluggy.PluginManager: pm.load_setuptools_entrypoints("pytask") pm.register(sys.modules[__name__]) - pm.hook.pytask_add_hooks(pm=pm) + pm.hook.pytask_add_hooks.call_historic(kwargs={"pm": pm}) return pm From 7ce28d84f8c7b34063d2feed67ca0446cf18d9e3 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 29 Dec 2023 11:02:29 +0100 Subject: [PATCH 06/11] Find better solution with storage. --- src/_pytask/build.py | 7 ++++++- src/_pytask/clean.py | 5 ++--- src/_pytask/cli.py | 4 ++-- src/_pytask/collect_command.py | 4 ++-- src/_pytask/dag_command.py | 3 ++- src/_pytask/mark/__init__.py | 4 ++-- src/_pytask/pluginmanager.py | 34 ++++++++++++++++++++++++++++++++++ src/_pytask/profile.py | 4 ++-- src/pytask/__init__.py | 4 ++++ tests/conftest.py | 2 ++ tests/test_clean.py | 2 ++ 11 files changed, 60 insertions(+), 13 deletions(-) diff --git a/src/_pytask/build.py b/src/_pytask/build.py index 9313a95a..204c4c24 100644 --- a/src/_pytask/build.py +++ b/src/_pytask/build.py @@ -26,6 +26,7 @@ from _pytask.path import HashPathCache from _pytask.pluginmanager import get_plugin_manager from _pytask.pluginmanager import hookimpl +from _pytask.pluginmanager import storage from _pytask.session import Session from _pytask.shared import parse_paths from _pytask.shared import to_list @@ -34,6 +35,7 @@ if TYPE_CHECKING: + from pluggy import PluginManager from _pytask.node_protocols import PTask from typing import NoReturn @@ -82,6 +84,7 @@ def build( # noqa: C901, PLR0912, PLR0913 paths: str | Path | Iterable[str | Path] = (), pdb: bool = False, pdb_cls: str = "", + pm: PluginManager | None = None, s: bool = False, show_capture: Literal["no", "stdout", "stderr", "all"] | ShowCapture = ShowCapture.ALL, @@ -177,7 +180,8 @@ def build( # noqa: C901, PLR0912, PLR0913 """ try: - pm = get_plugin_manager() + if not pm: + pm = get_plugin_manager() raw_config = { "capture": capture, @@ -330,6 +334,7 @@ def build_command(**raw_config: Any) -> NoReturn: current working directory, executes them and reports the results. """ + raw_config["pm"] = storage._plugin_manager raw_config["command"] = "build" session = build(**raw_config) sys.exit(session.exit_code) diff --git a/src/_pytask/clean.py b/src/_pytask/clean.py index cba8435c..61e9ee08 100644 --- a/src/_pytask/clean.py +++ b/src/_pytask/clean.py @@ -25,8 +25,8 @@ from _pytask.outcomes import ExitCode from _pytask.path import find_common_ancestor from _pytask.path import relative_to -from _pytask.pluginmanager import get_plugin_manager from _pytask.pluginmanager import hookimpl +from _pytask.pluginmanager import storage from _pytask.session import Session from _pytask.shared import to_list from _pytask.traceback import Traceback @@ -97,12 +97,11 @@ def pytask_parse_config(config: dict[str, Any]) -> None: ) def clean(**raw_config: Any) -> NoReturn: # noqa: C901, PLR0912 """Clean the provided paths by removing files unknown to pytask.""" + pm = storage.get() raw_config["command"] = "clean" try: # Duplication of the same mechanism in :func:`pytask.build`. - pm = get_plugin_manager() - config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config) session = Session.from_config(config) diff --git a/src/_pytask/cli.py b/src/_pytask/cli.py index d6df216e..f6d6b450 100644 --- a/src/_pytask/cli.py +++ b/src/_pytask/cli.py @@ -5,7 +5,7 @@ import click from _pytask.click import ColoredGroup -from _pytask.pluginmanager import get_plugin_manager +from _pytask.pluginmanager import storage from packaging.version import parse as parse_version @@ -23,7 +23,7 @@ def _extend_command_line_interface(cli: click.Group) -> click.Group: """Add parameters from plugins to the commandline interface.""" - pm = get_plugin_manager() + pm = storage.create() pm.hook.pytask_extend_command_line_interface.call_historic(kwargs={"cli": cli}) _sort_options_for_each_command_alphabetically(cli) return cli diff --git a/src/_pytask/collect_command.py b/src/_pytask/collect_command.py index 9a71129e..f9181643 100644 --- a/src/_pytask/collect_command.py +++ b/src/_pytask/collect_command.py @@ -26,8 +26,8 @@ from _pytask.outcomes import ExitCode from _pytask.path import find_common_ancestor from _pytask.path import relative_to -from _pytask.pluginmanager import get_plugin_manager from _pytask.pluginmanager import hookimpl +from _pytask.pluginmanager import storage from _pytask.session import Session from _pytask.tree_util import tree_leaves from rich.text import Text @@ -54,10 +54,10 @@ def pytask_extend_command_line_interface(cli: click.Group) -> None: ) def collect(**raw_config: Any | None) -> NoReturn: """Collect tasks and report information about them.""" + pm = storage.get() raw_config["command"] = "collect" try: - pm = get_plugin_manager() config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config) session = Session.from_config(config) diff --git a/src/_pytask/dag_command.py b/src/_pytask/dag_command.py index b1a64e39..16af6288 100644 --- a/src/_pytask/dag_command.py +++ b/src/_pytask/dag_command.py @@ -21,6 +21,7 @@ from _pytask.outcomes import ExitCode from _pytask.pluginmanager import get_plugin_manager from _pytask.pluginmanager import hookimpl +from _pytask.pluginmanager import storage from _pytask.session import Session from _pytask.shared import parse_paths from _pytask.shared import reduce_names_of_multiple_nodes @@ -80,7 +81,7 @@ def pytask_extend_command_line_interface(cli: click.Group) -> None: def dag(**raw_config: Any) -> int: """Create a visualization of the project's directed acyclic graph.""" try: - pm = get_plugin_manager() + pm = storage.get() config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config) session = Session.from_config(config) diff --git a/src/_pytask/mark/__init__.py b/src/_pytask/mark/__init__.py index c8fddf63..22055cea 100644 --- a/src/_pytask/mark/__init__.py +++ b/src/_pytask/mark/__init__.py @@ -18,8 +18,8 @@ from _pytask.mark.structures import MarkDecorator from _pytask.mark.structures import MarkGenerator from _pytask.outcomes import ExitCode -from _pytask.pluginmanager import get_plugin_manager from _pytask.pluginmanager import hookimpl +from _pytask.pluginmanager import storage from _pytask.session import Session from _pytask.shared import parse_markers from attrs import define @@ -49,9 +49,9 @@ def markers(**raw_config: Any) -> NoReturn: """Show all registered markers.""" raw_config["command"] = "markers" + pm = storage.get() try: - pm = get_plugin_manager() config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config) session = Session.from_config(config) diff --git a/src/_pytask/pluginmanager.py b/src/_pytask/pluginmanager.py index b717cae6..fbc32fd3 100644 --- a/src/_pytask/pluginmanager.py +++ b/src/_pytask/pluginmanager.py @@ -7,6 +7,7 @@ import pluggy from _pytask import hookspecs +from attrs import define from pluggy import HookimplMarker from pluggy import PluginManager @@ -62,3 +63,36 @@ def get_plugin_manager() -> pluggy.PluginManager: pm.hook.pytask_add_hooks.call_historic(kwargs={"pm": pm}) return pm + + +@define +class PluginManagerStorage: + """A class to store the plugin manager. + + This storage is needed to harmonize the two different ways to call pytask, via the + CLI or the API. + + When pytask is called from the CLI, the plugin manager is created in + :mod:`_pytask.cli` outside the click command to extend the command line interface. + Afterwards, it needs to be accessed in the different commands. + + When pytask is called from the API, the plugin manager needs to be created inside + the function, for example, :func:`~pytask.build` to ensure each call can start from + a blank slate and is able to register any plugins. + + """ + + _plugin_manager: PluginManager | None = None + + def get(self) -> PluginManager: + """Get the plugin manager.""" + assert self._plugin_manager + return self._plugin_manager + + def create(self) -> PluginManager: + """Create the plugin manager.""" + self._plugin_manager = get_plugin_manager() + return self._plugin_manager + + +storage = PluginManagerStorage() diff --git a/src/_pytask/profile.py b/src/_pytask/profile.py index 34b34978..3147042a 100644 --- a/src/_pytask/profile.py +++ b/src/_pytask/profile.py @@ -24,8 +24,8 @@ from _pytask.node_protocols import PTask from _pytask.outcomes import ExitCode from _pytask.outcomes import TaskOutcome -from _pytask.pluginmanager import get_plugin_manager from _pytask.pluginmanager import hookimpl +from _pytask.pluginmanager import storage from _pytask.session import Session from _pytask.traceback import Traceback from rich.table import Table @@ -111,10 +111,10 @@ def _create_or_update_runtime(task_signature: str, start: float, end: float) -> ) def profile(**raw_config: Any) -> NoReturn: """Show information about tasks like runtime and memory consumption of products.""" + pm = storage.get() raw_config["command"] = "profile" try: - pm = get_plugin_manager() config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config) session = Session.from_config(config) diff --git a/src/pytask/__init__.py b/src/pytask/__init__.py index 7244b9eb..612dc537 100644 --- a/src/pytask/__init__.py +++ b/src/pytask/__init__.py @@ -59,7 +59,9 @@ from _pytask.outcomes import SkippedAncestorFailed from _pytask.outcomes import SkippedUnchanged from _pytask.outcomes import TaskOutcome +from _pytask.pluginmanager import get_plugin_manager from _pytask.pluginmanager import hookimpl +from _pytask.pluginmanager import storage from _pytask.profile import Runtime from _pytask.reports import CollectionReport from _pytask.reports import DagReport @@ -133,6 +135,7 @@ "depends_on", "get_all_marks", "get_marks", + "get_plugin_manager", "has_mark", "hash_value", "hookimpl", @@ -146,6 +149,7 @@ "remove_internal_traceback_frames_from_exc_info", "remove_marks", "set_marks", + "storage", "task", "warning_record_to_str", ] diff --git a/tests/conftest.py b/tests/conftest.py index a17802a0..756200e7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,7 @@ from click.testing import CliRunner from packaging import version from pytask import console +from pytask import storage @pytest.fixture(autouse=True) @@ -97,6 +98,7 @@ def _restore_sys_path_and_module_after_test_execution(): class CustomCliRunner(CliRunner): def invoke(self, *args, **kwargs): """Restore sys.path and sys.modules after an invocation.""" + storage.create() with restore_sys_path_and_module_after_test_execution(): return super().invoke(*args, **kwargs) diff --git a/tests/test_clean.py b/tests/test_clean.py index 085ec1b9..91c59f0e 100644 --- a/tests/test_clean.py +++ b/tests/test_clean.py @@ -9,6 +9,7 @@ from _pytask.git import init_repo from pytask import cli from pytask import ExitCode +from pytask import storage _PROJECT_TASK = """ @@ -86,6 +87,7 @@ def test_clean_database_ignored(project, runner): os.chdir(project) result = runner.invoke(cli, ["build"]) assert result.exit_code == ExitCode.OK + storage.create() result = runner.invoke(cli, ["clean"]) assert result.exit_code == ExitCode.OK os.chdir(cwd) From 7f55533a859cbdced2f1214fc6a677cd3c54829e Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 29 Dec 2023 11:03:57 +0100 Subject: [PATCH 07/11] to changes. --- docs/source/changes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/changes.md b/docs/source/changes.md index 8e44dec1..31eef098 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -25,6 +25,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and - {pull}`536` allows partialed functions to be task functions. - {pull}`540` changes the CLI entry-point and allow `pytask.build(tasks=task_func)` as the signatures suggested. +- {pull}`542` refactors the plugin manager. ## 0.4.4 - 2023-12-04 From 3f34a091eb11ac95b517642a5f749604ee310c7d Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 29 Dec 2023 11:10:17 +0100 Subject: [PATCH 08/11] fix. --- src/_pytask/build.py | 2 +- src/_pytask/data_catalog.py | 6 ++---- src/_pytask/pluginmanager.py | 12 ++++++++++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/_pytask/build.py b/src/_pytask/build.py index 204c4c24..68fcbc09 100644 --- a/src/_pytask/build.py +++ b/src/_pytask/build.py @@ -334,7 +334,7 @@ def build_command(**raw_config: Any) -> NoReturn: current working directory, executes them and reports the results. """ - raw_config["pm"] = storage._plugin_manager + raw_config["pm"] = storage.get() raw_config["command"] = "build" session = build(**raw_config) sys.exit(session.exit_code) diff --git a/src/_pytask/data_catalog.py b/src/_pytask/data_catalog.py index a25f1ae7..ecc99668 100644 --- a/src/_pytask/data_catalog.py +++ b/src/_pytask/data_catalog.py @@ -16,7 +16,7 @@ from _pytask.node_protocols import PNode from _pytask.node_protocols import PPathNode from _pytask.nodes import PickleNode -from _pytask.pluginmanager import get_plugin_manager +from _pytask.pluginmanager import storage from _pytask.session import Session from attrs import define from attrs import field @@ -36,9 +36,7 @@ def _get_parent_path_of_data_catalog_module(stacklevel: int = 2) -> Path: def _create_default_session() -> Session: """Create a default session to use the hooks and collect nodes.""" - return Session( - config={"check_casing_of_paths": True}, hook=get_plugin_manager().hook - ) + return Session(config={"check_casing_of_paths": True}, hook=storage.get().hook) @define(kw_only=True) diff --git a/src/_pytask/pluginmanager.py b/src/_pytask/pluginmanager.py index fbc32fd3..02e09c52 100644 --- a/src/_pytask/pluginmanager.py +++ b/src/_pytask/pluginmanager.py @@ -12,6 +12,14 @@ from pluggy import PluginManager +__all__ = [ + "get_plugin_manager", + "hookimpl", + "register_hook_impls_from_modules", + "storage", +] + + hookimpl = HookimplMarker("pytask") @@ -66,7 +74,7 @@ def get_plugin_manager() -> pluggy.PluginManager: @define -class PluginManagerStorage: +class _PluginManagerStorage: """A class to store the plugin manager. This storage is needed to harmonize the two different ways to call pytask, via the @@ -95,4 +103,4 @@ def create(self) -> PluginManager: return self._plugin_manager -storage = PluginManagerStorage() +storage = _PluginManagerStorage() From fd42324a82468700f56d924e536ce924e70906b2 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 29 Dec 2023 11:26:50 +0100 Subject: [PATCH 09/11] Remove pm argument for build. --- src/_pytask/build.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/_pytask/build.py b/src/_pytask/build.py index 68fcbc09..5aff0343 100644 --- a/src/_pytask/build.py +++ b/src/_pytask/build.py @@ -35,7 +35,6 @@ if TYPE_CHECKING: - from pluggy import PluginManager from _pytask.node_protocols import PTask from typing import NoReturn @@ -84,7 +83,6 @@ def build( # noqa: C901, PLR0912, PLR0913 paths: str | Path | Iterable[str | Path] = (), pdb: bool = False, pdb_cls: str = "", - pm: PluginManager | None = None, s: bool = False, show_capture: Literal["no", "stdout", "stderr", "all"] | ShowCapture = ShowCapture.ALL, @@ -180,9 +178,6 @@ def build( # noqa: C901, PLR0912, PLR0913 """ try: - if not pm: - pm = get_plugin_manager() - raw_config = { "capture": capture, "check_casing_of_paths": check_casing_of_paths, @@ -216,6 +211,8 @@ def build( # noqa: C901, PLR0912, PLR0913 **kwargs, } + pm = get_plugin_manager() if "command" not in raw_config else storage.get() + # If someone called the programmatic interface, we need to do some parsing. if "command" not in raw_config: raw_config["command"] = "build" @@ -334,7 +331,6 @@ def build_command(**raw_config: Any) -> NoReturn: current working directory, executes them and reports the results. """ - raw_config["pm"] = storage.get() raw_config["command"] = "build" session = build(**raw_config) sys.exit(session.exit_code) From 1ca119b1bf557c5b45414dea86a29b72eb59252d Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 29 Dec 2023 12:01:54 +0100 Subject: [PATCH 10/11] Fix type imports. --- src/_pytask/compat.py | 2 ++ src/_pytask/config.py | 6 ++---- src/_pytask/debugging.py | 5 ++--- src/_pytask/hookspecs.py | 7 +++---- src/_pytask/pluginmanager.py | 7 +++---- 5 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/_pytask/compat.py b/src/_pytask/compat.py index e4151e2b..d392576e 100644 --- a/src/_pytask/compat.py +++ b/src/_pytask/compat.py @@ -89,6 +89,8 @@ def import_optional_dependency( f"Use pip or conda to install {install_name!r}." ) try: + # The from import is used to avoid monkeypatching errors in some tests. See + # https://stackoverflow.com/a/31746577 for more information. module = import_module(name) except ImportError: if errors == "raise": diff --git a/src/_pytask/config.py b/src/_pytask/config.py index 48151046..ef176100 100644 --- a/src/_pytask/config.py +++ b/src/_pytask/config.py @@ -12,7 +12,7 @@ from _pytask.shared import to_list if TYPE_CHECKING: - import pluggy + from pluggy import PluginManager _IGNORED_FOLDERS: list[str] = [".git/*", ".venv/*"] @@ -60,9 +60,7 @@ def is_file_system_case_sensitive() -> bool: @hookimpl -def pytask_configure( - pm: pluggy.PluginManager, raw_config: dict[str, Any] -) -> dict[str, Any]: +def pytask_configure(pm: PluginManager, raw_config: dict[str, Any]) -> dict[str, Any]: """Configure pytask.""" # Add all values by default so that many plugins do not need to copy over values. config = {"pm": pm, "markers": {}, **raw_config} diff --git a/src/_pytask/debugging.py b/src/_pytask/debugging.py index 83d20f7b..c95f79d3 100644 --- a/src/_pytask/debugging.py +++ b/src/_pytask/debugging.py @@ -16,9 +16,8 @@ from _pytask.pluginmanager import hookimpl from _pytask.traceback import Traceback - if TYPE_CHECKING: - import pluggy + from pluggy import PluginManager from _pytask.session import Session from types import TracebackType from types import FrameType @@ -111,7 +110,7 @@ def pytask_unconfigure() -> None: class PytaskPDB: """Pseudo PDB that defers to the real pdb.""" - _pluginmanager: pluggy.PluginManager | None = None + _pluginmanager: PluginManager | None = None _config: dict[str, Any] | None = None _saved: ClassVar[list[tuple[Any, ...]]] = [] _recursive_debug: int = 0 diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py index 82183ae1..a1d34a69 100644 --- a/src/_pytask/hookspecs.py +++ b/src/_pytask/hookspecs.py @@ -25,13 +25,14 @@ from _pytask.reports import CollectionReport from _pytask.reports import ExecutionReport from _pytask.reports import DagReport + from pluggy import PluginManager hookspec = pluggy.HookspecMarker("pytask") @hookspec(historic=True) -def pytask_add_hooks(pm: pluggy.PluginManager) -> None: +def pytask_add_hooks(pm: PluginManager) -> None: """Add hook specifications and implementations to the plugin manager. This hook is the first to be called to let plugins register their hook @@ -65,9 +66,7 @@ def pytask_extend_command_line_interface(cli: click.Group) -> None: @hookspec(firstresult=True) -def pytask_configure( - pm: pluggy.PluginManager, raw_config: dict[str, Any] -) -> dict[str, Any]: +def pytask_configure(pm: PluginManager, raw_config: dict[str, Any]) -> dict[str, Any]: """Configure pytask. The main hook implementation which controls the configuration and calls subordinated diff --git a/src/_pytask/pluginmanager.py b/src/_pytask/pluginmanager.py index 02e09c52..a72408c7 100644 --- a/src/_pytask/pluginmanager.py +++ b/src/_pytask/pluginmanager.py @@ -5,7 +5,6 @@ import sys from typing import Iterable -import pluggy from _pytask import hookspecs from attrs import define from pluggy import HookimplMarker @@ -33,7 +32,7 @@ def register_hook_impls_from_modules( @hookimpl -def pytask_add_hooks(pm: pluggy.PluginManager) -> None: +def pytask_add_hooks(pm: PluginManager) -> None: """Add hooks.""" builtin_hook_impl_modules = ( "_pytask.build", @@ -61,9 +60,9 @@ def pytask_add_hooks(pm: pluggy.PluginManager) -> None: register_hook_impls_from_modules(pm, builtin_hook_impl_modules) -def get_plugin_manager() -> pluggy.PluginManager: +def get_plugin_manager() -> PluginManager: """Get the plugin manager.""" - pm = pluggy.PluginManager("pytask") + pm = PluginManager("pytask") pm.add_hookspecs(hookspecs) pm.load_setuptools_entrypoints("pytask") From 0654ba76b370ba6fd184913a5ab7dcf22b80f6a4 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 29 Dec 2023 12:14:40 +0100 Subject: [PATCH 11/11] Use latest pluginmanager in data catalog. --- src/_pytask/build.py | 8 ++++++-- src/_pytask/dag_command.py | 1 + src/_pytask/data_catalog.py | 18 +++++++++--------- src/_pytask/pluginmanager.py | 12 ++++++++---- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/_pytask/build.py b/src/_pytask/build.py index 5aff0343..45444992 100644 --- a/src/_pytask/build.py +++ b/src/_pytask/build.py @@ -63,7 +63,7 @@ def pytask_unconfigure(session: Session) -> None: path.write_text(json.dumps(HashPathCache._cache)) -def build( # noqa: C901, PLR0912, PLR0913 +def build( # noqa: C901, PLR0912, PLR0913, PLR0915 *, capture: Literal["fd", "no", "sys", "tee-sys"] | CaptureMethod = CaptureMethod.FD, check_casing_of_paths: bool = True, @@ -211,7 +211,11 @@ def build( # noqa: C901, PLR0912, PLR0913 **kwargs, } - pm = get_plugin_manager() if "command" not in raw_config else storage.get() + if "command" not in raw_config: + pm = get_plugin_manager() + storage.store(pm) + else: + pm = storage.get() # If someone called the programmatic interface, we need to do some parsing. if "command" not in raw_config: diff --git a/src/_pytask/dag_command.py b/src/_pytask/dag_command.py index 16af6288..1dbcdb92 100644 --- a/src/_pytask/dag_command.py +++ b/src/_pytask/dag_command.py @@ -144,6 +144,7 @@ def build_dag(raw_config: dict[str, Any]) -> nx.DiGraph: """ try: pm = get_plugin_manager() + storage.store(pm) # If someone called the programmatic interface, we need to do some parsing. if "command" not in raw_config: diff --git a/src/_pytask/data_catalog.py b/src/_pytask/data_catalog.py index ecc99668..20b63b56 100644 --- a/src/_pytask/data_catalog.py +++ b/src/_pytask/data_catalog.py @@ -9,6 +9,7 @@ import inspect import pickle from pathlib import Path +from typing import Any from _pytask.config_utils import find_project_root_and_config from _pytask.exceptions import NodeNotCollectedError @@ -34,11 +35,6 @@ def _get_parent_path_of_data_catalog_module(stacklevel: int = 2) -> Path: return Path.cwd() -def _create_default_session() -> Session: - """Create a default session to use the hooks and collect nodes.""" - return Session(config={"check_casing_of_paths": True}, hook=storage.get().hook) - - @define(kw_only=True) class DataCatalog: """A data catalog. @@ -65,12 +61,14 @@ class DataCatalog: entries: dict[str, PNode] = field(factory=dict) name: str = "default" path: Path | None = None - _session: Session = field(factory=_create_default_session) + _session_config: dict[str, Any] = field( + factory=lambda *x: {"check_casing_of_paths": True} # noqa: ARG005 + ) _instance_path: Path = field(factory=_get_parent_path_of_data_catalog_module) def __attrs_post_init__(self) -> None: root_path, _ = find_project_root_and_config((self._instance_path,)) - self._session.config["paths"] = (root_path,) + self._session_config["paths"] = (root_path,) if not self.path: self.path = root_path / ".pytask" / "data_catalogs" / self.name @@ -113,8 +111,10 @@ def add(self, name: str, node: PNode | None = None) -> None: elif isinstance(node, PNode): self.entries[name] = node else: - collected_node = self._session.hook.pytask_collect_node( - session=self._session, + # Acquire the latest pluginmanager. + session = Session(config=self._session_config, hook=storage.get().hook) + collected_node = session.hook.pytask_collect_node( + session=session, path=self._instance_path, node_info=NodeInfo( arg_name=name, path=(), value=node, task_path=None, task_name="" diff --git a/src/_pytask/pluginmanager.py b/src/_pytask/pluginmanager.py index a72408c7..931e12a8 100644 --- a/src/_pytask/pluginmanager.py +++ b/src/_pytask/pluginmanager.py @@ -91,15 +91,19 @@ class _PluginManagerStorage: _plugin_manager: PluginManager | None = None + def create(self) -> PluginManager: + """Create the plugin manager.""" + self._plugin_manager = get_plugin_manager() + return self._plugin_manager + def get(self) -> PluginManager: """Get the plugin manager.""" assert self._plugin_manager return self._plugin_manager - def create(self) -> PluginManager: - """Create the plugin manager.""" - self._plugin_manager = get_plugin_manager() - return self._plugin_manager + def store(self, pm: PluginManager) -> None: + """Store the plugin manager.""" + self._plugin_manager = pm storage = _PluginManagerStorage()