Add more explanation when PNode.load() fails during execution. (#455)

Author: tobiasraabe

docs/source/changes.md

@@ -10,6 +10,8 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and
 - {pull}`449` simplifies the code building the plugin manager.
 - {pull}`451` improves `collect_command.py` and renames `graph.py` to `dag_command.py`.
 - {pull}`454` removes more `.svg`s and replaces them with animations.
+- {pull}`455` adds more explanation when {meth}`~pytask.PNode.load` fails during the
+  execution.
 
 ## 0.4.1 - 2023-10-11
 

environment.yml

@@ -5,7 +5,7 @@ channels:
   - nodefaults
 
 dependencies:
-  - python >=3.8
+  - python >=3.8,<3.12
   - pip
   - setuptools_scm
   - toml

src/_pytask/exceptions.py

@@ -14,6 +14,10 @@ class NodeNotCollectedError(PytaskError):
     """Exception for nodes which could not be collected."""
 
 
+class NodeLoadError(PytaskError):
+    """Exception for nodes whose value could not be loaded."""
+
+
 class ConfigurationError(PytaskError):
     """Exception during the configuration."""
 

src/_pytask/execute.py

@@ -19,9 +19,11 @@
 from _pytask.database_utils import update_states_in_database
 from _pytask.enums import ShowCapture
 from _pytask.exceptions import ExecutionError
+from _pytask.exceptions import NodeLoadError
 from _pytask.exceptions import NodeNotFoundError
 from _pytask.mark import Mark
 from _pytask.mark_utils import has_mark
+from _pytask.node_protocols import PNode
 from _pytask.node_protocols import PPathNode
 from _pytask.node_protocols import PTask
 from _pytask.outcomes import count_outcomes
@@ -31,6 +33,7 @@
 from _pytask.report import ExecutionReport
 from _pytask.shared import reduce_node_name
 from _pytask.traceback import format_exception_without_traceback
+from _pytask.traceback import remove_internal_traceback_frames_from_exception
 from _pytask.traceback import remove_traceback_from_exc_info
 from _pytask.traceback import render_exc_info
 from _pytask.tree_util import tree_leaves
@@ -142,6 +145,16 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None:
         raise WouldBeExecuted
 
 
+def _safe_load(node: PNode, task: PTask) -> Any:
+    try:
+        return node.load()
+    except Exception as e:  # noqa: BLE001
+        e = remove_internal_traceback_frames_from_exception(e)
+        task_name = getattr(task, "display_name", task.name)
+        msg = f"Exception while loading node {node.name!r} of task {task_name!r}"
+        raise NodeLoadError(msg) from e
+
+
 @hookimpl(trylast=True)
 def pytask_execute_task(session: Session, task: PTask) -> bool:
     """Execute task."""
@@ -152,11 +165,11 @@ def pytask_execute_task(session: Session, task: PTask) -> bool:
 
     kwargs = {}
     for name, value in task.depends_on.items():
-        kwargs[name] = tree_map(lambda x: x.load(), value)
+        kwargs[name] = tree_map(lambda x: _safe_load(x, task), value)
 
     for name, value in task.produces.items():
         if name in parameters:
-            kwargs[name] = tree_map(lambda x: x.load(), value)
+            kwargs[name] = tree_map(lambda x: _safe_load(x, task), value)
 
     out = task.execute(**kwargs)
 

tests/test_collect_command.py

@@ -512,6 +512,7 @@ def task_example(
     assert "Product" in captured
 
 
+@pytest.mark.end_to_end()
 def test_node_protocol_for_custom_nodes(runner, tmp_path):
     source = """
     from typing_extensions import Annotated
@@ -543,6 +544,7 @@ def task_example(
     assert "<Dependency custom>" in result.output
 
 
+@pytest.mark.end_to_end()
 def test_node_protocol_for_custom_nodes_with_paths(runner, tmp_path):
     source = """
     from typing_extensions import Annotated

tests/test_execute.py

@@ -774,6 +774,7 @@ def test_pass_non_task_to_functional_api_that_are_ignored():
 @pytest.mark.end_to_end()
 def test_multiple_product_annotations(runner, tmp_path):
     source = """
+    from __future__ import annotations
     from pytask import Product
     from typing_extensions import Annotated
     from pathlib import Path
@@ -793,3 +794,42 @@ def task_second(
     tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
     result = runner.invoke(cli, [tmp_path.as_posix()])
     assert result.exit_code == ExitCode.OK
+
+
+@pytest.mark.end_to_end()
+def test_errors_during_loading_nodes_have_info(runner, tmp_path):
+    source = """
+    from __future__ import annotations
+    from pathlib import Path
+    from typing import Any
+    import attrs
+    import pickle
+
+    @attrs.define
+    class PickleNode:
+        name: str
+        path: Path
+
+        def state(self) -> str | None:
+            if self.path.exists():
+                return str(self.path.stat().st_mtime)
+            return None
+
+        def load(self) -> Any:
+            return pickle.loads(self.path.read_bytes())
+
+        def save(self, value: Any) -> None:
+            self.path.write_bytes(pickle.dumps(value))
+
+    def task_example(
+        value=PickleNode(name="node", path=Path(__file__).parent / "file.txt")
+    ): pass
+    """
+    tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source))
+    tmp_path.joinpath("file.txt").touch()
+
+    result = runner.invoke(cli, [tmp_path.as_posix()])
+    assert result.exit_code == ExitCode.FAILED
+    assert "task_example.py::task_example" in result.output
+    assert "Exception while loading node" in result.output
+    assert "_pytask/execute.py" not in result.output

tests/test_node_protocols.py

@@ -3,10 +3,12 @@
 import pickle
 import textwrap
 
+import pytest
 from pytask import cli
 from pytask import ExitCode
 
 
+@pytest.mark.end_to_end()
 def test_node_protocol_for_custom_nodes(runner, tmp_path):
     source = """
     from typing_extensions import Annotated
@@ -41,6 +43,7 @@ def task_example(
     assert tmp_path.joinpath("out.txt").read_text() == "text"
 
 
+@pytest.mark.end_to_end()
 def test_node_protocol_for_custom_nodes_with_paths(runner, tmp_path):
     source = """
     from typing_extensions import Annotated

tests/test_path.py

@@ -318,6 +318,7 @@ def test_insert_missing_modules(
     assert not modules
 
 
+@pytest.mark.unit()
 def test_importlib_package(monkeypatch: pytest.MonkeyPatch, tmp_path: Path):
     """
     Importing a package using --importmode=importlib should not import the

tests/test_typing.py

@@ -2,9 +2,11 @@
 
 import functools
 
+import pytest
 from _pytask.typing import is_task_function
 
 
+@pytest.mark.unit()
 def test_is_task_function():
     def func():
         pass