Skip to content

Add more explanation when PNode.load() fails during execution. #455

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Oct 18, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/source/changes.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and
- {pull}`449` simplifies the code building the plugin manager.
- {pull}`451` improves `collect_command.py` and renames `graph.py` to `dag_command.py`.
- {pull}`454` removes more `.svg`s and replaces them with animations.
- {pull}`455` adds more explanation when {meth}`~pytask.PNode.load` fails during the
execution.

## 0.4.1 - 2023-10-11

Expand Down
2 changes: 1 addition & 1 deletion environment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ channels:
- nodefaults

dependencies:
- python >=3.8
- python >=3.8,<3.12
- pip
- setuptools_scm
- toml
Expand Down
4 changes: 4 additions & 0 deletions src/_pytask/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ class NodeNotCollectedError(PytaskError):
"""Exception for nodes which could not be collected."""


class NodeLoadError(PytaskError):
"""Exception for nodes whose value could not be loaded."""


class ConfigurationError(PytaskError):
"""Exception during the configuration."""

Expand Down
17 changes: 15 additions & 2 deletions src/_pytask/execute.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,11 @@
from _pytask.database_utils import update_states_in_database
from _pytask.enums import ShowCapture
from _pytask.exceptions import ExecutionError
from _pytask.exceptions import NodeLoadError
from _pytask.exceptions import NodeNotFoundError
from _pytask.mark import Mark
from _pytask.mark_utils import has_mark
from _pytask.node_protocols import PNode
from _pytask.node_protocols import PPathNode
from _pytask.node_protocols import PTask
from _pytask.outcomes import count_outcomes
Expand All @@ -31,6 +33,7 @@
from _pytask.report import ExecutionReport
from _pytask.shared import reduce_node_name
from _pytask.traceback import format_exception_without_traceback
from _pytask.traceback import remove_internal_traceback_frames_from_exception
from _pytask.traceback import remove_traceback_from_exc_info
from _pytask.traceback import render_exc_info
from _pytask.tree_util import tree_leaves
Expand Down Expand Up @@ -142,6 +145,16 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None:
raise WouldBeExecuted


def _safe_load(node: PNode, task: PTask) -> Any:
try:
return node.load()
except Exception as e: # noqa: BLE001
e = remove_internal_traceback_frames_from_exception(e)
task_name = getattr(task, "display_name", task.name)
msg = f"Exception while loading node {node.name!r} of task {task_name!r}"
raise NodeLoadError(msg) from e


@hookimpl(trylast=True)
def pytask_execute_task(session: Session, task: PTask) -> bool:
"""Execute task."""
Expand All @@ -152,11 +165,11 @@ def pytask_execute_task(session: Session, task: PTask) -> bool:

kwargs = {}
for name, value in task.depends_on.items():
kwargs[name] = tree_map(lambda x: x.load(), value)
kwargs[name] = tree_map(lambda x: _safe_load(x, task), value)

for name, value in task.produces.items():
if name in parameters:
kwargs[name] = tree_map(lambda x: x.load(), value)
kwargs[name] = tree_map(lambda x: _safe_load(x, task), value)

out = task.execute(**kwargs)

Expand Down
2 changes: 2 additions & 0 deletions tests/test_collect_command.py
Original file line number Diff line number Diff line change
Expand Up @@ -512,6 +512,7 @@ def task_example(
assert "Product" in captured


@pytest.mark.end_to_end()
def test_node_protocol_for_custom_nodes(runner, tmp_path):
source = """
from typing_extensions import Annotated
Expand Down Expand Up @@ -543,6 +544,7 @@ def task_example(
assert "<Dependency custom>" in result.output


@pytest.mark.end_to_end()
def test_node_protocol_for_custom_nodes_with_paths(runner, tmp_path):
source = """
from typing_extensions import Annotated
Expand Down
40 changes: 40 additions & 0 deletions tests/test_execute.py
Original file line number Diff line number Diff line change
Expand Up @@ -774,6 +774,7 @@ def test_pass_non_task_to_functional_api_that_are_ignored():
@pytest.mark.end_to_end()
def test_multiple_product_annotations(runner, tmp_path):
source = """
from __future__ import annotations
from pytask import Product
from typing_extensions import Annotated
from pathlib import Path
Expand All @@ -793,3 +794,42 @@ def task_second(
tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
result = runner.invoke(cli, [tmp_path.as_posix()])
assert result.exit_code == ExitCode.OK


@pytest.mark.end_to_end()
def test_errors_during_loading_nodes_have_info(runner, tmp_path):
source = """
from __future__ import annotations
from pathlib import Path
from typing import Any
import attrs
import pickle

@attrs.define
class PickleNode:
name: str
path: Path

def state(self) -> str | None:
if self.path.exists():
return str(self.path.stat().st_mtime)
return None

def load(self) -> Any:
return pickle.loads(self.path.read_bytes())

def save(self, value: Any) -> None:
self.path.write_bytes(pickle.dumps(value))

def task_example(
value=PickleNode(name="node", path=Path(__file__).parent / "file.txt")
): pass
"""
tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source))
tmp_path.joinpath("file.txt").touch()

result = runner.invoke(cli, [tmp_path.as_posix()])
assert result.exit_code == ExitCode.FAILED
assert "task_example.py::task_example" in result.output
assert "Exception while loading node" in result.output
assert "_pytask/execute.py" not in result.output
3 changes: 3 additions & 0 deletions tests/test_node_protocols.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@
import pickle
import textwrap

import pytest
from pytask import cli
from pytask import ExitCode


@pytest.mark.end_to_end()
def test_node_protocol_for_custom_nodes(runner, tmp_path):
source = """
from typing_extensions import Annotated
Expand Down Expand Up @@ -41,6 +43,7 @@ def task_example(
assert tmp_path.joinpath("out.txt").read_text() == "text"


@pytest.mark.end_to_end()
def test_node_protocol_for_custom_nodes_with_paths(runner, tmp_path):
source = """
from typing_extensions import Annotated
Expand Down
1 change: 1 addition & 0 deletions tests/test_path.py
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,7 @@ def test_insert_missing_modules(
assert not modules


@pytest.mark.unit()
def test_importlib_package(monkeypatch: pytest.MonkeyPatch, tmp_path: Path):
"""
Importing a package using --importmode=importlib should not import the
Expand Down
2 changes: 2 additions & 0 deletions tests/test_typing.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@

import functools

import pytest
from _pytask.typing import is_task_function


@pytest.mark.unit()
def test_is_task_function():
def func():
pass
Expand Down