From f1d6ea9b191c5ce6c1845e3dec274ea37c9f066f Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Thu, 20 Jan 2022 12:13:29 +0100 Subject: [PATCH 1/6] Add task marker. --- docs/source/changes.rst | 5 ++++- src/_pytask/cli.py | 2 ++ src/_pytask/task.py | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 src/_pytask/task.py diff --git a/docs/source/changes.rst b/docs/source/changes.rst index 979e2346..b7c12d7d 100644 --- a/docs/source/changes.rst +++ b/docs/source/changes.rst @@ -14,7 +14,10 @@ all releases are available on `PyPI `_ and - :gh:`193` adds more figures to the documentation. - :gh:`194` updates the ``README.rst``. - :gh:`196` references the two new cookiecutters for projects and plugins. -- :gh:`198` fixes the documentation of ``@pytask.mark.skipif``. (Closes :gh:`195`) +- :gh:`198` fixes the documentation of :func:`@pytask.mark.skipif + <_pytask.skipping.skipif>`. (Closes :gh:`195`) +- :gh:`199` implements the :func:`@pytask.mark.task <_pytask.task.task>` decorator to + mark functions as tasks regardless whether they are prefixed with ``task_`` or not. 0.1.5 - 2022-01-10 diff --git a/src/_pytask/cli.py b/src/_pytask/cli.py index d898b6d1..480ebdd4 100644 --- a/src/_pytask/cli.py +++ b/src/_pytask/cli.py @@ -66,6 +66,7 @@ def pytask_add_hooks(pm: pluggy.PluginManager) -> None: from _pytask import profile from _pytask import resolve_dependencies from _pytask import skipping + from _pytask import task pm.register(build) pm.register(capture) @@ -86,6 +87,7 @@ def pytask_add_hooks(pm: pluggy.PluginManager) -> None: pm.register(profile) pm.register(resolve_dependencies) pm.register(skipping) + pm.register(task) @click.group( diff --git a/src/_pytask/task.py b/src/_pytask/task.py new file mode 100644 index 00000000..e6230aad --- /dev/null +++ b/src/_pytask/task.py @@ -0,0 +1,36 @@ +from pathlib import Path +from typing import Any +from typing import Dict +from typing import Optional + +from _pytask.config import hookimpl +from _pytask.nodes import PythonFunctionTask +from _pytask.session import Session + + +def task(*, name: str) -> str: + return name + + +@hookimpl +def pytask_parse_config(config: Dict[str, Any]) -> None: + config["markers"]["task"] = "Mark a function as a task regardless of its name." + + +@hookimpl +def pytask_collect_task( + session: Session, path: Path, name: str, obj: Any +) -> Optional[PythonFunctionTask]: + """Collect a task which is a function. + + There is some discussion on how to detect functions in this `thread + `_. :class:`types.FunctionType` does not + detect built-ins which is not possible anyway. + + """ + if name.startswith("task_") and callable(obj): + return PythonFunctionTask.from_path_name_function_session( + path, name, obj, session + ) + else: + return None From 77129692b8f9c6fbc908fcaf7db137e408b09082 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 21 Jan 2022 17:20:15 +0100 Subject: [PATCH 2/6] Add functionality for normal and parametrized tasks. --- src/_pytask/mark/structures.py | 2 +- src/_pytask/mark_utils.py | 10 ++++++++ src/_pytask/parametrize.py | 22 ++++++++-------- src/_pytask/task.py | 11 ++++---- src/_pytask/task_utils.py | 27 ++++++++++++++++++++ tests/test_task.py | 46 ++++++++++++++++++++++++++++++++++ 6 files changed, 100 insertions(+), 18 deletions(-) create mode 100644 src/_pytask/task_utils.py create mode 100644 tests/test_task.py diff --git a/src/_pytask/mark/structures.py b/src/_pytask/mark/structures.py index 7ca30e67..c1f9f321 100644 --- a/src/_pytask/mark/structures.py +++ b/src/_pytask/mark/structures.py @@ -182,7 +182,7 @@ def store_mark(obj: Callable[..., Any], mark: Mark) -> None: assert isinstance(mark, Mark), mark # Always reassign name to avoid updating pytaskmark in a reference that was only # borrowed. - obj.pytaskmark = get_unpacked_marks(obj) + [mark] # type: ignore + obj.pytaskmark = get_unpacked_marks(obj) + [mark] # type: ignore[attr-defined] class MarkGenerator: diff --git a/src/_pytask/mark_utils.py b/src/_pytask/mark_utils.py index 28f330a1..6af4c428 100644 --- a/src/_pytask/mark_utils.py +++ b/src/_pytask/mark_utils.py @@ -5,6 +5,7 @@ """ from typing import Any from typing import List +from typing import Tuple from typing import TYPE_CHECKING @@ -30,3 +31,12 @@ def get_marks_from_obj(obj: Any, marker_name: str) -> "List[Mark]": def has_marker(obj: Any, marker_name: str) -> bool: """Determine whether a task function has a certain marker.""" return any(marker.name == marker_name for marker in getattr(obj, "pytaskmark", [])) + + +def remove_markers_from_func(obj: Any, marker_name: str) -> Tuple[Any, List["Mark"]]: + """Remove parametrize markers from the object.""" + markers = [i for i in obj.pytaskmark if i.name == marker_name] + others = [i for i in obj.pytaskmark if i.name != marker_name] + obj.pytaskmark = others + + return obj, markers diff --git a/src/_pytask/parametrize.py b/src/_pytask/parametrize.py index 31b8021e..0d29cf02 100644 --- a/src/_pytask/parametrize.py +++ b/src/_pytask/parametrize.py @@ -18,8 +18,11 @@ from _pytask.console import TASK_ICON from _pytask.mark import Mark from _pytask.mark import MARK_GEN as mark # noqa: N811 +from _pytask.mark_utils import has_marker +from _pytask.mark_utils import remove_markers_from_func from _pytask.nodes import find_duplicates from _pytask.session import Session +from _pytask.task_utils import parse_task_marker def parametrize( @@ -93,7 +96,7 @@ def pytask_parametrize_task( """ if callable(obj): - obj, markers = _remove_parametrize_markers_from_func(obj) + obj, markers = remove_markers_from_func(obj, "parametrize") if len(markers) > 1: raise NotImplementedError( @@ -101,6 +104,9 @@ def pytask_parametrize_task( "not possible to define products for tasks from a Cartesian product." ) + if has_marker(obj, "task"): + name = parse_task_marker(obj) + base_arg_names, arg_names, arg_values = _parse_parametrize_markers( markers, name ) @@ -119,8 +125,9 @@ def pytask_parametrize_task( # Copy function and attributes to allow in-place changes. func = _copy_func(obj) # type: ignore - func.pytaskmark = copy.deepcopy(obj.pytaskmark) # type: ignore - + func.pytaskmark = copy.deepcopy( # type: ignore[attr-defined] + obj.pytaskmark # type: ignore[attr-defined] + ) # Convert parametrized dependencies and products to decorator. session.hook.pytask_parametrize_kwarg_to_marker(obj=func, kwargs=kwargs) @@ -148,15 +155,6 @@ def pytask_parametrize_task( return names_and_functions -def _remove_parametrize_markers_from_func(obj: Any) -> Tuple[Any, List[Mark]]: - """Remove parametrize markers from the object.""" - parametrize_markers = [i for i in obj.pytaskmark if i.name == "parametrize"] - others = [i for i in obj.pytaskmark if i.name != "parametrize"] - obj.pytaskmark = others - - return obj, parametrize_markers - - def _parse_parametrize_marker( marker: Mark, name: str ) -> Tuple[Tuple[str, ...], List[Tuple[str, ...]], List[Tuple[Any, ...]]]: diff --git a/src/_pytask/task.py b/src/_pytask/task.py index e6230aad..41f2f9f6 100644 --- a/src/_pytask/task.py +++ b/src/_pytask/task.py @@ -4,12 +4,10 @@ from typing import Optional from _pytask.config import hookimpl +from _pytask.mark_utils import has_marker from _pytask.nodes import PythonFunctionTask from _pytask.session import Session - - -def task(*, name: str) -> str: - return name +from _pytask.task_utils import parse_task_marker @hookimpl @@ -28,7 +26,10 @@ def pytask_collect_task( detect built-ins which is not possible anyway. """ - if name.startswith("task_") and callable(obj): + if has_marker(obj, "task") and callable(obj): + parsed_name = parse_task_marker(obj) + if parsed_name is not None: + name = parsed_name return PythonFunctionTask.from_path_name_function_session( path, name, obj, session ) diff --git a/src/_pytask/task_utils.py b/src/_pytask/task_utils.py new file mode 100644 index 00000000..f95d8fff --- /dev/null +++ b/src/_pytask/task_utils.py @@ -0,0 +1,27 @@ +from typing import Any +from typing import Callable +from typing import Optional + +from _pytask.mark import Mark +from _pytask.mark_utils import remove_markers_from_func + + +def task(name: Optional[str] = None) -> str: + return name + + +def parse_task_marker(obj: Callable[..., Any]) -> str: + obj, task_markers = remove_markers_from_func(obj, "task") + + if len(task_markers) != 1: + raise ValueError( + "The @pytask.mark.task decorator cannot be applied more than once to a " + "single task." + ) + task_marker = task_markers[0] + + name = task(*task_marker.args, **task_marker.kwargs) + + obj.pytaskmark.append(Mark("task", (), {})) # type: ignore[attr-defined] + + return name diff --git a/tests/test_task.py b/tests/test_task.py new file mode 100644 index 00000000..909a9c67 --- /dev/null +++ b/tests/test_task.py @@ -0,0 +1,46 @@ +import textwrap + +from _pytask.nodes import create_task_name +from _pytask.outcomes import ExitCode +from pytask import main + + +def test_task_with_task_decorator(tmp_path): + source = """ + import pytask + + @pytask.mark.task("the_only_task") + @pytask.mark.produces("out.txt") + def task_example(produces): + produces.write_text("Hello. It's me.") + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + session = main({"paths": tmp_path}) + + assert session.exit_code == ExitCode.OK + assert session.tasks[0].name == create_task_name( + tmp_path.joinpath("task_module.py"), "the_only_task" + ) + + +def test_task_with_task_decorator_with_parametrize(tmp_path): + source = """ + import pytask + + @pytask.mark.task("the_parametrized_task") + @pytask.mark.parametrize("produces", ["out_1.txt", "out_2.txt"]) + def task_example(produces): + produces.write_text("Hello. It's me.") + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + session = main({"paths": tmp_path}) + + assert session.exit_code == ExitCode.OK + assert session.tasks[0].name == create_task_name( + tmp_path.joinpath("task_module.py"), "the_parametrized_task[produces0]" + ) + assert session.tasks[1].name == create_task_name( + tmp_path.joinpath("task_module.py"), "the_parametrized_task[produces1]" + ) From f3a290b8cdbab69b4b8670c341c7fa3acff80bc0 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 21 Jan 2022 17:28:02 +0100 Subject: [PATCH 3/6] Extend tests. --- tests/test_task.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/test_task.py b/tests/test_task.py index 909a9c67..b86d02f8 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -1,15 +1,17 @@ import textwrap +import pytest from _pytask.nodes import create_task_name from _pytask.outcomes import ExitCode from pytask import main -def test_task_with_task_decorator(tmp_path): - source = """ +@pytest.mark.parametrize("task_name", ["'the_only_task'", None]) +def test_task_with_task_decorator(tmp_path, task_name): + source = f""" import pytask - @pytask.mark.task("the_only_task") + @pytask.mark.task({task_name}) @pytask.mark.produces("out.txt") def task_example(produces): produces.write_text("Hello. It's me.") @@ -19,9 +21,14 @@ def task_example(produces): session = main({"paths": tmp_path}) assert session.exit_code == ExitCode.OK - assert session.tasks[0].name == create_task_name( - tmp_path.joinpath("task_module.py"), "the_only_task" - ) + if task_name: + assert session.tasks[0].name == create_task_name( + tmp_path.joinpath("task_module.py"), "the_only_task" + ) + else: + assert session.tasks[0].name == create_task_name( + tmp_path.joinpath("task_module.py"), "task_example" + ) def test_task_with_task_decorator_with_parametrize(tmp_path): @@ -39,8 +46,8 @@ def task_example(produces): assert session.exit_code == ExitCode.OK assert session.tasks[0].name == create_task_name( - tmp_path.joinpath("task_module.py"), "the_parametrized_task[produces0]" + tmp_path.joinpath("task_module.py"), "the_parametrized_task[out_1.txt]" ) assert session.tasks[1].name == create_task_name( - tmp_path.joinpath("task_module.py"), "the_parametrized_task[produces1]" + tmp_path.joinpath("task_module.py"), "the_parametrized_task[out_2.txt]" ) From 708112f7d7364d9b7b77d3469c1b6fc0b0d8a1ba Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 21 Jan 2022 17:53:23 +0100 Subject: [PATCH 4/6] some amelioration. --- src/_pytask/mark_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/_pytask/mark_utils.py b/src/_pytask/mark_utils.py index 6af4c428..a535ace8 100644 --- a/src/_pytask/mark_utils.py +++ b/src/_pytask/mark_utils.py @@ -35,8 +35,8 @@ def has_marker(obj: Any, marker_name: str) -> bool: def remove_markers_from_func(obj: Any, marker_name: str) -> Tuple[Any, List["Mark"]]: """Remove parametrize markers from the object.""" - markers = [i for i in obj.pytaskmark if i.name == marker_name] - others = [i for i in obj.pytaskmark if i.name != marker_name] + markers = [i for i in getattr(obj, "pytaskmark", []) if i.name == marker_name] + others = [i for i in getattr(obj, "pytaskmark", []) if i.name != marker_name] obj.pytaskmark = others return obj, markers From 7c4d4fa32c5ed7d44cd6cc7ae252fe2124d4fd95 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 23 Jan 2022 18:04:32 +0100 Subject: [PATCH 5/6] Caught error when parametrized task did not receive a name via the decorator. --- src/_pytask/parametrize.py | 4 +++- tests/test_task.py | 47 +++++++++++++++++++++++++------------- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/src/_pytask/parametrize.py b/src/_pytask/parametrize.py index 0d29cf02..f30eb53c 100644 --- a/src/_pytask/parametrize.py +++ b/src/_pytask/parametrize.py @@ -105,7 +105,9 @@ def pytask_parametrize_task( ) if has_marker(obj, "task"): - name = parse_task_marker(obj) + parsed_name = parse_task_marker(obj) + if parsed_name is not None: + name = parsed_name base_arg_names, arg_names, arg_values = _parse_parametrize_markers( markers, name diff --git a/tests/test_task.py b/tests/test_task.py index b86d02f8..511f8b20 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -6,14 +6,16 @@ from pytask import main -@pytest.mark.parametrize("task_name", ["'the_only_task'", None]) -def test_task_with_task_decorator(tmp_path, task_name): +@pytest.mark.parametrize("func_name", ["task_example", "func"]) +@pytest.mark.parametrize("task_name", ["the_only_task", None]) +def test_task_with_task_decorator(tmp_path, func_name, task_name): + task_decorator_input = f"{task_name!r}" if task_name else task_name source = f""" import pytask - @pytask.mark.task({task_name}) + @pytask.mark.task({task_decorator_input}) @pytask.mark.produces("out.txt") - def task_example(produces): + def {func_name}(produces): produces.write_text("Hello. It's me.") """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -21,23 +23,27 @@ def task_example(produces): session = main({"paths": tmp_path}) assert session.exit_code == ExitCode.OK + if task_name: assert session.tasks[0].name == create_task_name( - tmp_path.joinpath("task_module.py"), "the_only_task" + tmp_path.joinpath("task_module.py"), task_name ) else: assert session.tasks[0].name == create_task_name( - tmp_path.joinpath("task_module.py"), "task_example" + tmp_path.joinpath("task_module.py"), func_name ) -def test_task_with_task_decorator_with_parametrize(tmp_path): - source = """ +@pytest.mark.parametrize("func_name", ["task_example", "func"]) +@pytest.mark.parametrize("task_name", ["the_only_task", None]) +def test_task_with_task_decorator_with_parametrize(tmp_path, func_name, task_name): + task_decorator_input = f"{task_name!r}" if task_name else task_name + source = f""" import pytask - @pytask.mark.task("the_parametrized_task") + @pytask.mark.task({task_decorator_input}) @pytask.mark.parametrize("produces", ["out_1.txt", "out_2.txt"]) - def task_example(produces): + def {func_name}(produces): produces.write_text("Hello. It's me.") """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -45,9 +51,18 @@ def task_example(produces): session = main({"paths": tmp_path}) assert session.exit_code == ExitCode.OK - assert session.tasks[0].name == create_task_name( - tmp_path.joinpath("task_module.py"), "the_parametrized_task[out_1.txt]" - ) - assert session.tasks[1].name == create_task_name( - tmp_path.joinpath("task_module.py"), "the_parametrized_task[out_2.txt]" - ) + + if task_name: + assert session.tasks[0].name == create_task_name( + tmp_path.joinpath("task_module.py"), f"{task_name}[out_1.txt]" + ) + assert session.tasks[1].name == create_task_name( + tmp_path.joinpath("task_module.py"), f"{task_name}[out_2.txt]" + ) + else: + assert session.tasks[0].name == create_task_name( + tmp_path.joinpath("task_module.py"), f"{func_name}[out_1.txt]" + ) + assert session.tasks[1].name == create_task_name( + tmp_path.joinpath("task_module.py"), f"{func_name}[out_2.txt]" + ) From 9461ccb41cc22eefc7a9871e99026932e3a6aa7e Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 23 Jan 2022 20:00:12 +0100 Subject: [PATCH 6/6] Document marker. --- docs/source/tutorials/how_to_write_a_task.rst | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/docs/source/tutorials/how_to_write_a_task.rst b/docs/source/tutorials/how_to_write_a_task.rst index 938cd58c..a8e0eba0 100644 --- a/docs/source/tutorials/how_to_write_a_task.rst +++ b/docs/source/tutorials/how_to_write_a_task.rst @@ -4,8 +4,11 @@ How to write a task Starting from the project structure in the :doc:`previous tutorial `, this tutorial teaches you how to write your first task. -The task will be defined in ``src/my_project/task_data_preparation.py`` and it will -generate artificial data which will be stored in ``bld/data.pkl``. We will call the +By default, pytask will look for tasks in modules whose name is prefixed with ``task_``. +Tasks are functions in these modules whose name also starts with ``task_``. + +Our first task will be defined in ``src/my_project/task_data_preparation.py`` and it +will generate artificial data which will be stored in ``bld/data.pkl``. We will call the function in the module :func:`task_create_random_data`. .. code-block:: @@ -62,10 +65,35 @@ subsequent directories. .. image:: /_static/images/how-to-write-a-task.png -.. important:: - By default, pytask assumes that tasks are functions and both, the function name and - the module name, must be prefixed with ``task_``. +Customize task names +-------------------- + +Use the :func:`@pytask.mark.task <_pytask.task_utils.task>` decorator to mark a function +as a task regardless of its function name. You can optionally pass a new name for the +task. Otherwise, the function name is used. + +.. code-block:: python + + # The id will be '.../task_data_preparation.py::create_random_data' + + + @pytask.mark.task + def create_random_data(): + ... + + + # The id will be '.../task_data_preparation.py::create_data' + + + @pytask.mark.task(name="create_data") + def create_random_data(): + ... + + +Customize task module names +--------------------------- - Use the configuration value :confval:`task_files` if you prefer a different naming - scheme for the task modules. +Use the configuration value :confval:`task_files` if you prefer a different naming +scheme for the task modules. By default, it is set to ``task_*.py``. You can specify one +or multiple patterns to collect tasks from other files.