diff --git a/docs/source/changes.md b/docs/source/changes.md index 7f826fec..60c38125 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -19,6 +19,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and - {pull}`557` fixes an issue with `@task(after=...)` in notebooks and terminals. - {pull}`566` makes universal-pathlib an official dependency. - {pull}`568` restricts `task_files` to a list of patterns and raises a better error. +- {pull}`569` removes the hooks related to the creation of the DAG. ## 0.4.5 - 2024-01-09 diff --git a/docs/source/reference_guides/hookspecs.md b/docs/source/reference_guides/hookspecs.md index 0586d192..64c01031 100644 --- a/docs/source/reference_guides/hookspecs.md +++ b/docs/source/reference_guides/hookspecs.md @@ -62,25 +62,6 @@ The following hooks traverse directories and collect tasks from files. .. autofunction:: pytask_collect_log ``` -## Resolving Dependencies - -The following hooks are designed to build a DAG from tasks and dependencies and check -which files have changed and need to be re-run. - -```{warning} -This step is still experimental and likely to change in the future. If you are planning -to write a plugin which extends pytask in this dimension, please, start a discussion -before writing a plugin. It may make your life easier if changes in pytask anticipate -your plugin. -``` - -```{eval-rst} -.. autofunction:: pytask_dag -.. autofunction:: pytask_dag_create_dag -.. autofunction:: pytask_dag_log - -``` - ## Execution The following hooks execute the tasks and log information on the result in the terminal. diff --git a/src/_pytask/build.py b/src/_pytask/build.py index 5c598705..e93c9ada 100644 --- a/src/_pytask/build.py +++ b/src/_pytask/build.py @@ -21,6 +21,7 @@ from _pytask.config_utils import find_project_root_and_config from _pytask.config_utils import read_config from _pytask.console import console +from _pytask.dag import create_dag from _pytask.exceptions import CollectionError from _pytask.exceptions import ConfigurationError from _pytask.exceptions import ExecutionError @@ -265,7 +266,7 @@ def build( # noqa: C901, PLR0912, PLR0913, PLR0915 try: session.hook.pytask_log_session_header(session=session) session.hook.pytask_collect(session=session) - session.hook.pytask_dag(session=session) + session.dag = create_dag(session=session) session.hook.pytask_execute(session=session) except CollectionError: diff --git a/src/_pytask/collect_command.py b/src/_pytask/collect_command.py index 79f8dfcb..2e953e5c 100644 --- a/src/_pytask/collect_command.py +++ b/src/_pytask/collect_command.py @@ -19,6 +19,7 @@ from _pytask.console import create_url_style_for_path from _pytask.console import format_node_name from _pytask.console import format_task_name +from _pytask.dag import create_dag from _pytask.exceptions import CollectionError from _pytask.exceptions import ConfigurationError from _pytask.exceptions import ResolvingDependenciesError @@ -70,7 +71,7 @@ def collect(**raw_config: Any | None) -> NoReturn: try: session.hook.pytask_log_session_header(session=session) session.hook.pytask_collect(session=session) - session.hook.pytask_dag(session=session) + session.dag = create_dag(session=session) tasks = _select_tasks_by_expressions_and_marker(session) task_with_path = [t for t in tasks if isinstance(t, PTaskWithPath)] diff --git a/src/_pytask/dag.py b/src/_pytask/dag.py index 9ebc7425..14d15401 100644 --- a/src/_pytask/dag.py +++ b/src/_pytask/dag.py @@ -19,10 +19,10 @@ from _pytask.console import render_to_string from _pytask.exceptions import ResolvingDependenciesError from _pytask.mark import select_by_after_keyword +from _pytask.mark import select_tasks_by_marks_and_expressions from _pytask.node_protocols import PNode from _pytask.node_protocols import PTask from _pytask.nodes import PythonNode -from _pytask.pluginmanager import hookimpl from _pytask.reports import DagReport from _pytask.shared import reduce_names_of_multiple_nodes from _pytask.tree_util import tree_map @@ -33,28 +33,28 @@ from _pytask.session import Session -@hookimpl -def pytask_dag(session: Session) -> bool | None: +__all__ = ["create_dag"] + + +def create_dag(session: Session) -> nx.DiGraph: """Create a directed acyclic graph (DAG) for the workflow.""" try: - session.dag = session.hook.pytask_dag_create_dag( - session=session, tasks=session.tasks - ) - session.hook.pytask_dag_modify_dag(session=session, dag=session.dag) + dag = _create_dag(tasks=session.tasks) + _check_if_dag_has_cycles(dag) + _check_if_tasks_have_the_same_products(dag, session.config["paths"]) + _modify_dag(session=session, dag=dag) + select_tasks_by_marks_and_expressions(session=session, dag=dag) except Exception: # noqa: BLE001 report = DagReport.from_exception(sys.exc_info()) - session.hook.pytask_dag_log(session=session, report=report) + _log_dag(report=report) session.dag_report = report raise ResolvingDependenciesError from None - - else: - return True + return dag -@hookimpl -def pytask_dag_create_dag(session: Session, tasks: list[PTask]) -> nx.DiGraph: +def _create_dag(tasks: list[PTask]) -> nx.DiGraph: """Create the DAG from tasks, dependencies and products.""" def _add_dependency(dag: nx.DiGraph, task: PTask, node: PNode) -> None: @@ -90,15 +90,10 @@ def _add_product(dag: nx.DiGraph, task: PTask, node: PNode) -> None: else None, task.depends_on, ) - - _check_if_dag_has_cycles(dag) - _check_if_tasks_have_the_same_products(dag, session.config["paths"]) - return dag -@hookimpl -def pytask_dag_modify_dag(session: Session, dag: nx.DiGraph) -> None: +def _modify_dag(session: Session, dag: nx.DiGraph) -> None: """Create dependencies between tasks when using ``@task(after=...)``.""" temporary_id_to_task = { task.attributes["collection_id"]: task @@ -194,8 +189,7 @@ def _check_if_tasks_have_the_same_products(dag: nx.DiGraph, paths: list[Path]) - raise ResolvingDependenciesError(msg) -@hookimpl -def pytask_dag_log(report: DagReport) -> None: +def _log_dag(report: DagReport) -> None: """Log errors which happened while resolving dependencies.""" console.print() console.rule( diff --git a/src/_pytask/dag_command.py b/src/_pytask/dag_command.py index ef84fa47..b8df2f26 100644 --- a/src/_pytask/dag_command.py +++ b/src/_pytask/dag_command.py @@ -19,6 +19,7 @@ from _pytask.config_utils import find_project_root_and_config from _pytask.config_utils import read_config from _pytask.console import console +from _pytask.dag import create_dag from _pytask.exceptions import CollectionError from _pytask.exceptions import ConfigurationError from _pytask.exceptions import ResolvingDependenciesError @@ -101,7 +102,7 @@ def dag(**raw_config: Any) -> int: "can install with conda.", ) session.hook.pytask_collect(session=session) - session.hook.pytask_dag(session=session) + session.dag = create_dag(session=session) dag = _refine_dag(session) _write_graph(dag, session.config["output_path"], session.config["layout"]) @@ -198,7 +199,7 @@ def build_dag(raw_config: dict[str, Any]) -> nx.DiGraph: "can install with conda.", ) session.hook.pytask_collect(session=session) - session.hook.pytask_dag(session=session) + session.dag = create_dag(session=session) session.hook.pytask_unconfigure(session=session) dag = _refine_dag(session) diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py index 9ca83367..0a66da3e 100644 --- a/src/_pytask/hookspecs.py +++ b/src/_pytask/hookspecs.py @@ -16,7 +16,6 @@ from pathlib import Path import click - import networkx as nx from pluggy import PluginManager from _pytask.models import NodeInfo @@ -25,7 +24,6 @@ from _pytask.outcomes import CollectionOutcome from _pytask.outcomes import TaskOutcome from _pytask.reports import CollectionReport - from _pytask.reports import DagReport from _pytask.reports import ExecutionReport from _pytask.session import Session @@ -212,44 +210,6 @@ def pytask_collect_log( """ -# Hooks for resolving dependencies. - - -@hookspec(firstresult=True) -def pytask_dag(session: Session) -> None: - """Create a DAG. - - The main hook implementation which controls the resolution of dependencies and calls - subordinated hooks. - - """ - - -@hookspec(firstresult=True) -def pytask_dag_create_dag(session: Session, tasks: list[PTask]) -> nx.DiGraph: - """Create the DAG. - - This hook creates the DAG from tasks, dependencies and products. The DAG can be used - by a scheduler to find an execution order. - - """ - - -@hookspec -def pytask_dag_modify_dag(session: Session, dag: nx.DiGraph) -> None: - """Modify the DAG. - - This hook allows to make some changes to the DAG before it is validated and tasks - are selected. - - """ - - -@hookspec -def pytask_dag_log(session: Session, report: DagReport) -> None: - """Log errors during resolving dependencies.""" - - # Hooks for running tasks. diff --git a/src/_pytask/mark/__init__.py b/src/_pytask/mark/__init__.py index 1c116c7a..84e5d387 100644 --- a/src/_pytask/mark/__init__.py +++ b/src/_pytask/mark/__init__.py @@ -45,6 +45,7 @@ "select_by_after_keyword", "select_by_keyword", "select_by_mark", + "select_tasks_by_marks_and_expressions", ] @@ -234,8 +235,7 @@ def _deselect_others_with_mark( task.markers.append(mark) -@hookimpl -def pytask_dag_modify_dag(session: Session, dag: nx.DiGraph) -> None: +def select_tasks_by_marks_and_expressions(session: Session, dag: nx.DiGraph) -> None: """Modify the tasks which are executed with expressions and markers.""" remaining = select_by_keyword(session, dag) if remaining is not None: diff --git a/src/_pytask/profile.py b/src/_pytask/profile.py index 50e25985..71ec5158 100644 --- a/src/_pytask/profile.py +++ b/src/_pytask/profile.py @@ -21,6 +21,7 @@ from _pytask.click import EnumChoice from _pytask.console import console from _pytask.console import format_task_name +from _pytask.dag import create_dag from _pytask.database_utils import BaseTable from _pytask.database_utils import DatabaseSession from _pytask.exceptions import CollectionError @@ -128,7 +129,7 @@ def profile(**raw_config: Any) -> NoReturn: try: session.hook.pytask_log_session_header(session=session) session.hook.pytask_collect(session=session) - session.hook.pytask_dag(session=session) + session.dag = create_dag(session=session) profile: dict[str, dict[str, Any]] = { task.name: {} for task in session.tasks diff --git a/tests/test_dag.py b/tests/test_dag.py index 05044aa5..0d4d711d 100644 --- a/tests/test_dag.py +++ b/tests/test_dag.py @@ -5,10 +5,9 @@ from pathlib import Path import pytest -from _pytask.dag import pytask_dag_create_dag +from _pytask.dag import _create_dag from pytask import ExitCode from pytask import PathNode -from pytask import Session from pytask import Task from pytask import build from pytask import cli @@ -16,7 +15,7 @@ @pytest.mark.unit() @pytest.mark.skipif(sys.platform == "win32", reason="Hashes match only on unix.") -def test_pytask_dag_create_dag(): +def test_create_dag(): root = Path("src") task = Task( base_name="task_dummy", @@ -27,8 +26,7 @@ def test_pytask_dag_create_dag(): 1: PathNode.from_path(root / "node_2"), }, ) - session = Session.from_config({"paths": (root,)}) - dag = pytask_dag_create_dag(session=session, tasks=[task]) + dag = _create_dag(tasks=[task]) for signature in ( "90bb899a1b60da28ff70352cfb9f34a8bed485597c7f40eed9bd4c6449147525",