diff --git a/docs/source/changes.md b/docs/source/changes.md index 65740f41..78f89b0a 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -14,6 +14,9 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and caused flaky tests. - {pull}`486` adds default names to {class}`~pytask.PPathNode`. - {pull}`488` raises an error when an invalid value is used in a return annotation. +- {pull}`489` simplifies parsing products and does not raise an error when a product + annotation is used with the argument name `produces`. And, allow `produces` to intake + any node. ## 0.4.2 - 2023-11-8 diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py index 6cb892dd..89d51376 100644 --- a/src/_pytask/collect_utils.py +++ b/src/_pytask/collect_utils.py @@ -264,7 +264,7 @@ def parse_dependencies_from_task_function( kwargs = {**signature_defaults, **task_kwargs} kwargs.pop("produces", None) - # Parse products from task decorated with @task and that uses produces. + # Parse dependencies from task when @task is used. if "depends_on" in kwargs: has_depends_on_argument = True dependencies["depends_on"] = tree_map( @@ -375,7 +375,7 @@ def _find_args_with_node_annotation(func: Callable[..., Any]) -> dict[str, PNode _ERROR_MULTIPLE_PRODUCT_DEFINITIONS = """The task uses multiple ways to define \ products. Products should be defined with either -- 'typing.Annotated[Path(...), Product]' (recommended) +- 'typing.Annotated[Path, Product] = Path(...)' (recommended) - '@pytask.mark.task(kwargs={'produces': Path(...)})' - as a default argument for 'produces': 'produces = Path(...)' - '@pytask.mark.produces(Path(...))' (deprecated) @@ -384,7 +384,7 @@ def _find_args_with_node_annotation(func: Callable[..., Any]) -> dict[str, PNode """ -def parse_products_from_task_function( +def parse_products_from_task_function( # noqa: C901 session: Session, task_path: Path | None, task_name: str, node_path: Path, obj: Any ) -> dict[str, Any]: """Parse products from task function. @@ -415,26 +415,14 @@ def parse_products_from_task_function( parameters_with_product_annot = _find_args_with_product_annotation(obj) parameters_with_node_annot = _find_args_with_node_annotation(obj) - # Parse products from task decorated with @task and that uses produces. + # Allow to collect products from 'produces'. if "produces" in kwargs: - has_produces_argument = True - collected_products = tree_map_with_path( - lambda p, x: _collect_product( - session, - node_path, - task_name, - NodeInfo( - arg_name="produces", - path=p, - value=x, - task_path=task_path, - task_name=task_name, - ), - is_string_allowed=True, - ), - kwargs["produces"], - ) - out = {"produces": collected_products} + if "produces" not in parameters_with_product_annot: + parameters_with_product_annot.append("produces") + # If there are more parameters with a product annotation, we want to raise an + # error later to warn about mixing different interfaces. + if set(parameters_with_product_annot) - {"produces"}: + has_produces_argument = True if parameters_with_product_annot: out = {} @@ -473,7 +461,7 @@ def parse_products_from_task_function( task_path=task_path, task_name=task_name, ), - is_string_allowed=False, + convert_string_to_path=parameter_name == "produces", # noqa: B023 ), value, ) @@ -493,7 +481,7 @@ def parse_products_from_task_function( task_path=task_path, task_name=task_name, ), - is_string_allowed=False, + convert_string_to_path=False, ), parameters_with_node_annot["return"], ) @@ -514,7 +502,7 @@ def parse_products_from_task_function( task_path=task_path, task_name=task_name, ), - is_string_allowed=False, + convert_string_to_path=False, ), task_produces, ) @@ -641,7 +629,7 @@ def _collect_product( path: Path, task_name: str, node_info: NodeInfo, - is_string_allowed: bool = False, + convert_string_to_path: bool = False, ) -> PNode: """Collect products for a task. @@ -655,17 +643,9 @@ def _collect_product( """ node = node_info.value - # For historical reasons, task.kwargs is like the deco and supports str and Path. - if not isinstance(node, (str, Path)) and is_string_allowed: - msg = ( - f"`@pytask.mark.task(kwargs={{'produces': ...}}` can only accept values of " - "type 'str' and 'pathlib.Path' or the same values nested in tuples, lists, " - f"and dictionaries. Here, {node} has type {type(node)}." - ) - raise ValueError(msg) # If we encounter a string and it is allowed, convert it to a path. - if isinstance(node, str) and is_string_allowed: + if isinstance(node, str) and convert_string_to_path: node = Path(node) node_info = node_info._replace(value=node) diff --git a/tests/test_execute.py b/tests/test_execute.py index 8dcf6dab..a78af8ae 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -43,8 +43,7 @@ def test_task_did_not_produce_node(tmp_path): import pytask @pytask.mark.produces("out.txt") - def task_example(): - pass + def task_example(): ... """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -61,8 +60,7 @@ def test_task_did_not_produce_multiple_nodes_and_all_are_shown(runner, tmp_path) import pytask @pytask.mark.produces(["1.txt", "2.txt"]) - def task_example(): - pass + def task_example(): ... """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -425,14 +423,16 @@ def test_task_is_not_reexecuted_when_modification_changed_file_not(runner, tmp_p @pytest.mark.end_to_end() -def test_task_with_product_annotation(tmp_path): - source = """ +@pytest.mark.parametrize("arg_name", ["path", "produces"]) +def test_task_with_product_annotation(tmp_path, arg_name): + """Using 'produces' with a product annotation should not cause an error.""" + source = f""" from pathlib import Path from typing_extensions import Annotated from pytask import Product - def task_example(path_to_file: Annotated[Path, Product] = Path("out.txt")) -> None: - path_to_file.touch() + def task_example({arg_name}: Annotated[Path, Product] = Path("out.txt")) -> None: + {arg_name}.touch() """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -441,7 +441,7 @@ def task_example(path_to_file: Annotated[Path, Product] = Path("out.txt")) -> No assert session.exit_code == ExitCode.OK assert len(session.tasks) == 1 task = session.tasks[0] - assert "path_to_file" in task.produces + assert arg_name in task.produces @pytest.mark.end_to_end() @@ -506,41 +506,18 @@ def task_example( @pytest.mark.end_to_end() -def test_error_with_multiple_dep_annotations(runner, tmp_path): - source = """ - from pathlib import Path +@pytest.mark.parametrize( + "second_node", ["PythonNode()", "PathNode(path=Path('a.txt'))"] +) +def test_error_with_multiple_dependency_annotations(runner, tmp_path, second_node): + source = f""" from typing_extensions import Annotated - from pytask import Product, PythonNode - from typing import Any - - def task_example( - dependency: Annotated[Any, PythonNode(), PythonNode()] = "hello", - path: Annotated[Path, Product] = Path("out.txt") - ) -> None: - path.write_text(dependency) - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - - result = runner.invoke(cli, [tmp_path.as_posix()]) - assert result.exit_code == ExitCode.COLLECTION_FAILED - assert "Parameter 'dependency'" in result.output - - -@pytest.mark.end_to_end() -def test_error_with_multiple_different_dep_annotations(runner, tmp_path): - source = """ + from pytask import PythonNode, PathNode from pathlib import Path - from typing_extensions import Annotated - from pytask import Product, PythonNode, PathNode - from typing import Any - - annotation = Annotated[Any, PythonNode(), PathNode(name="a", path=Path("a.txt"))] def task_example( - dependency: annotation = "hello", - path: Annotated[Path, Product] = Path("out.txt") - ) -> None: - path.write_text(dependency) + dependency: Annotated[str, PythonNode(), {second_node}] = "hello" + ) -> None: ... """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -553,9 +530,7 @@ def task_example( def test_return_with_path_annotation_as_return(runner, tmp_path): source = """ from pathlib import Path - from typing import Any from typing_extensions import Annotated - from pytask import PathNode def task_example() -> Annotated[str, Path("file.txt")]: return "Hello, World!" @@ -570,13 +545,10 @@ def task_example() -> Annotated[str, Path("file.txt")]: def test_return_with_pathnode_annotation_as_return(runner, tmp_path): source = """ from pathlib import Path - from typing import Any from typing_extensions import Annotated from pytask import PathNode - node = PathNode.from_path(Path(__file__).parent.joinpath("file.txt")) - - def task_example() -> Annotated[str, node]: + def task_example() -> Annotated[str, PathNode(path=Path("file.txt"))]: return "Hello, World!" """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -586,29 +558,19 @@ def task_example() -> Annotated[str, node]: @pytest.mark.end_to_end() -def test_return_with_tuple_pathnode_annotation_as_return(runner, tmp_path): - source = """ - from pathlib import Path - from typing import Any - from typing_extensions import Annotated - from pytask import PathNode - - node1 = PathNode.from_path(Path(__file__).parent.joinpath("file1.txt")) - node2 = PathNode.from_path(Path(__file__).parent.joinpath("file2.txt")) - - def task_example() -> Annotated[str, (node1, node2)]: - return "Hello,", "World!" - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - result = runner.invoke(cli, [tmp_path.as_posix()]) - assert result.exit_code == ExitCode.OK - assert tmp_path.joinpath("file1.txt").read_text() == "Hello," - assert tmp_path.joinpath("file2.txt").read_text() == "World!" - - -@pytest.mark.end_to_end() -def test_return_with_custom_node_and_return_annotation(runner, tmp_path): - source = """ +@pytest.mark.parametrize( + ("product_def", "return_def"), + [ + ("produces=PickleNode(path=Path('data.pkl')))", "produces.save(1)"), + ( + "node: Annotated[PickleNode, PickleNode(path=Path('data.pkl')), Product])", + "node.save(1)", + ), + (") -> Annotated[int, PickleNode(path=Path('data.pkl'))]", "return 1"), + ], +) +def test_custom_node_as_product(runner, tmp_path, product_def, return_def): + source = f""" from __future__ import annotations from pathlib import Path @@ -616,11 +578,12 @@ def test_return_with_custom_node_and_return_annotation(runner, tmp_path): from typing import Any from typing_extensions import Annotated import attrs + from pytask import Product @attrs.define class PickleNode: - name: str path: Path + name: str = "" signature: str = "id" def state(self) -> str | None: @@ -636,10 +599,8 @@ def load(self, is_product) -> Any: def save(self, value: Any) -> None: self.path.write_bytes(pickle.dumps(value)) - node = PickleNode("pickled_data", Path(__file__).parent.joinpath("data.pkl")) - - def task_example() -> Annotated[int, node]: - return 1 + def task_example({product_def}: + {return_def} """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) result = runner.invoke(cli, [tmp_path.as_posix()]) @@ -650,53 +611,23 @@ def task_example() -> Annotated[int, node]: @pytest.mark.end_to_end() -def test_return_with_custom_node_with_product_annotation(runner, tmp_path): +def test_return_with_tuple_pathnode_annotation_as_return(runner, tmp_path): source = """ - from __future__ import annotations - from pathlib import Path - import pickle - from typing import Any from typing_extensions import Annotated - import attrs - from pytask import Product - from _pytask._hashlib import hash_value - import hashlib - - @attrs.define - class PickleNode: - name: str - path: Path - - @property - def signature(self) -> str: - raw_key = "".join(str(hash_value(arg)) for arg in (self.name, self.path)) - return hashlib.sha256(raw_key.encode()).hexdigest() - - def state(self) -> str | None: - if self.path.exists(): - return str(self.path.stat().st_mtime) - return None - - def load(self, is_product) -> Any: - if is_product: - return self - return pickle.loads(self.path.read_bytes()) - - def save(self, value: Any) -> None: - self.path.write_bytes(pickle.dumps(value)) + from pytask import PathNode - node = PickleNode("pickled_data", Path(__file__).parent.joinpath("data.pkl")) + node1 = PathNode(path=Path("file1.txt")) + node2 = PathNode(path=Path("file2.txt")) - def task_example(node: Annotated[PickleNode, node, Product]) -> None: - node.save(1) + def task_example() -> Annotated[str, (node1, node2)]: + return "Hello,", "World!" """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) result = runner.invoke(cli, [tmp_path.as_posix()]) assert result.exit_code == ExitCode.OK - - data = pickle.loads(tmp_path.joinpath("data.pkl").read_bytes()) # noqa: S301 - assert data == 1 + assert tmp_path.joinpath("file1.txt").read_text() == "Hello," + assert tmp_path.joinpath("file2.txt").read_text() == "World!" @pytest.mark.end_to_end() @@ -707,8 +638,8 @@ def test_error_when_return_pytree_mismatch(runner, tmp_path): from typing_extensions import Annotated from pytask import PathNode - node1 = PathNode.from_path(Path(__file__).parent.joinpath("file1.txt")) - node2 = PathNode.from_path(Path(__file__).parent.joinpath("file2.txt")) + node1 = PathNode(path=Path("file1.txt")) + node2 = PathNode(path=Path("file2.txt")) def task_example() -> Annotated[str, (node1, node2)]: return "Hello," @@ -774,11 +705,7 @@ def test_more_nested_pytree_and_python_node_as_return(runner, tmp_path): from pytask import PythonNode from typing import Dict - nodes = [ - PythonNode(), - (PythonNode(), PythonNode()), - PythonNode() - ] + nodes = [PythonNode(), (PythonNode(), PythonNode()), PythonNode()] def task_example() -> Annotated[Dict[str, str], nodes]: return [{"first": "a", "second": "b"}, (1, 2), 1] @@ -806,7 +733,7 @@ def test_execute_tasks_and_pass_values_only_by_python_nodes(runner, tmp_path): def task_create_text() -> Annotated[int, node_text]: return "This is the text." - node_file = PathNode.from_path(Path(__file__).parent.joinpath("file.txt")) + node_file = PathNode(path=Path("file.txt")) def task_create_file(text: Annotated[int, node_text]) -> Annotated[str, node_file]: return text @@ -833,7 +760,7 @@ def test_execute_tasks_via_functional_api(tmp_path): def create_text() -> Annotated[int, node_text]: return "This is the text." - node_file = PathNode.from_path(Path(__file__).parent.joinpath("file.txt")) + node_file = PathNode(path=Path("file.txt")) def create_file(content: Annotated[str, node_text]) -> Annotated[str, node_file]: return content diff --git a/tests/test_task.py b/tests/test_task.py index 4d8680ad..012558a4 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -534,17 +534,16 @@ def task_example( @pytest.mark.end_to_end() -def test_return_with_task_decorator(runner, tmp_path): - source = """ +@pytest.mark.parametrize( + "node_def", ["PathNode(path=Path('file.txt'))", "Path('file.txt')"] +) +def test_return_with_task_decorator(runner, tmp_path, node_def): + source = f""" from pathlib import Path - from typing import Any from typing_extensions import Annotated - from pytask import PathNode - import pytask - - node = PathNode.from_path(Path(__file__).parent.joinpath("file.txt")) + from pytask import task, PathNode - @pytask.mark.task(produces=node) + @task(produces={node_def}) def task_example(): return "Hello, World!" """ @@ -555,18 +554,20 @@ def task_example(): @pytest.mark.end_to_end() -def test_return_with_tuple_and_task_decorator(runner, tmp_path): - source = """ +@pytest.mark.parametrize( + "node_def", + [ + "(PathNode(path=Path('file1.txt')), PathNode(path=Path('file2.txt')))", + "(Path('file1.txt'), Path('file2.txt'))", + ], +) +def test_return_with_tuple_and_task_decorator(runner, tmp_path, node_def): + source = f""" from pathlib import Path - from typing import Any from typing_extensions import Annotated - from pytask import PathNode - import pytask - - node1 = PathNode.from_path(Path(__file__).parent.joinpath("file1.txt")) - node2 = PathNode.from_path(Path(__file__).parent.joinpath("file2.txt")) + from pytask import task, PathNode - @pytask.mark.task(produces=(node1, node2)) + @task(produces={node_def}) def task_example(): return "Hello,", "World!" """ @@ -581,15 +582,12 @@ def test_error_when_function_is_defined_outside_loop_body(runner, tmp_path): source = """ from pathlib import Path from typing_extensions import Annotated - from pytask import task - from pytask import Product + from pytask import task, Product def func(path: Annotated[Path, Product]): path.touch() - _PATH = Path.cwd() - - for path in (_PATH.joinpath("a.txt"), _PATH.joinpath("b.txt")): + for path in (Path("a.txt"), Path("b.txt")): task(kwargs={"path": path})(func) """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -609,9 +607,7 @@ def test_error_when_function_is_defined_outside_loop_body_with_id(runner, tmp_pa def func(path: Annotated[Path, Product]): path.touch() - _PATH = Path.cwd() - - for path in (_PATH.joinpath("a.txt"), _PATH.joinpath("b.txt")): + for path in (Path("a.txt"), Path("b.txt")): task(kwargs={"path": path}, id=path.name)(func) """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))