diff --git a/docs/source/explanations/index.md b/docs/source/explanations/index.md index 1ce7d262..c2cab053 100644 --- a/docs/source/explanations/index.md +++ b/docs/source/explanations/index.md @@ -8,7 +8,6 @@ systems in general as well as its design. maxdepth: 1 --- why_pytask -interfaces_for_dependencies_products comparison_to_other_tools pluggy ``` diff --git a/docs/source/explanations/interfaces_for_dependencies_products.md b/docs/source/explanations/interfaces_for_dependencies_products.md deleted file mode 100644 index cb6ab800..00000000 --- a/docs/source/explanations/interfaces_for_dependencies_products.md +++ /dev/null @@ -1,30 +0,0 @@ -# Interfaces for dependencies and products - -There are different interfaces for dependencies and products and it might be confusing -when to use what. The tables gives you an overview to decide which interface is most -suitable for you. - -## Legend - -- ✅ = True -- ❌ = False -- ➖ = Does not apply - -## Products - -| | `Annotated[..., PNode, Product]` | `@task(produces=...)` | `produces` | `defd task() -> Annotated[..., PNode]` | `@pytask.mark.produces(...)` | -| --------------------------------------------------------- | :------------------------------: | :-------------------: | :--------: | :------------------------------------: | :--------------------------: | -| Not deprecated | ✅ | ✅ | ✅ | ✅ | ❌ | -| No type annotations required | ❌ | ✅ | ✅ | ❌ | ✅ | -| Flexible choice of argument name | ✅ | ✅ | ❌ | ➖ | ❌ | -| Supports third-party functions as tasks | ❌ | ✅ | ❌ | ❌ | ❌ | -| Allows to pass custom node while preserving type of value | ✅ | ✅ | ✅ | ✅ | ✅ | - -## Dependencies - -| | `Annotated[..., PNode]` | `@task(kwargs=...)` | `@pytask.mark.depends_on(...)` | -| --------------------------------------- | :---------------------: | :-----------------: | :----------------------------: | -| Not deprecated | ✅ | ✅ | ❌ | -| No type annotations required | ❌ | ✅ | ✅ | -| Flexible choice of argument name | ✅ | ✅ | ❌ | -| Supports third-party functions as tasks | ❌ | ✅ | ❌ | diff --git a/docs/source/how_to_guides/functional_interface.ipynb b/docs/source/how_to_guides/functional_interface.ipynb index 31337eaa..d4b657ef 100644 --- a/docs/source/how_to_guides/functional_interface.ipynb +++ b/docs/source/how_to_guides/functional_interface.ipynb @@ -273,7 +273,7 @@ { "data": { "text/plain": [ - "Session(config={'pm': , 'markers': {'depends_on': 'Add dependencies to a task. See this tutorial for more information: [link https://bit.ly/3JlxylS]https://bit.ly/3JlxylS[/].', 'filterwarnings': 'Add a filter for a warning to a task.', 'persist': 'Prevent execution of a task if all products exist and even if something has changed (dependencies, source file, products). This decorator might be useful for expensive tasks where only the formatting of the file has changed. The state of the files which have changed will also be remembered and another run will skip the task with success.', 'produces': 'Add products to a task. See this tutorial for more information: [link https://bit.ly/3JlxylS]https://bit.ly/3JlxylS[/].', 'skip': 'Skip a task and all its dependent tasks.', 'skip_ancestor_failed': 'Internal decorator applied to tasks if any of its preceding tasks failed.', 'skip_unchanged': 'Internal decorator applied to tasks which have already been executed and have not been changed.', 'skipif': 'Skip a task and all its dependent tasks if a condition is met.', 'task': 'Mark a function as a task regardless of its name. Or mark tasks which are repeated in a loop. See this tutorial for more information: [link https://bit.ly/3DWrXS3]https://bit.ly/3DWrXS3[/].', 'try_first': 'Try to execute a task a early as possible.', 'try_last': 'Try to execute a task a late as possible.'}, 'config': None, 'database_url': sqlite:////home/tobia/git/pytask/.pytask/.pytask.sqlite3, 'editor_url_scheme': 'file', 'export': <_ExportFormats.NO: 'no'>, 'ignore': ['.codecov.yml', '.gitignore', '.pre-commit-config.yaml', '.readthedocs.yml', '.readthedocs.yaml', 'readthedocs.yml', 'readthedocs.yaml', 'environment.yml', 'pyproject.toml', 'setup.cfg', 'tox.ini', '.git/*', '.venv/*', '*.egg-info/*', '.ipynb_checkpoints/*', '.mypy_cache/*', '.nox/*', '.tox/*', '_build/*', '__pycache__/*', 'build/*', 'dist/*', 'pytest_cache/*'], 'paths': [], 'layout': 'dot', 'output_path': 'dag.pdf', 'rank_direction': <_RankDirection.TB: 'TB'>, 'expression': '', 'marker_expression': '', 'nodes': False, 'strict_markers': False, 'directories': False, 'exclude': [None, '.git/*'], 'mode': <_CleanMode.DRY_RUN: 'dry-run'>, 'quiet': False, 'capture': , 'debug_pytask': False, 'disable_warnings': False, 'dry_run': False, 'force': False, 'max_failures': inf, 'n_entries_in_table': 15, 'pdb': False, 'pdbcls': None, 's': False, 'show_capture': True, 'show_errors_immediately': False, 'show_locals': False, 'show_traceback': True, 'sort_table': True, 'trace': False, 'verbose': 1, 'stop_after_first_failure': False, 'check_casing_of_paths': True, 'pdb_cls': '', 'tasks': [, , at 0x7f3c1b407d80>], 'task_files': ['task_*.py'], 'command': 'build', 'root': PosixPath('/home/tobia/git/pytask'), 'filterwarnings': []}, hook=, collection_reports=[CollectionReport(outcome=, node=TaskWithoutPath(name='task_create_first_file', function=, depends_on={}, produces={'return': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/first.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/first.txt'))}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'duration': (1696055304.0767577, 1696055304.077608)}), exc_info=None), CollectionReport(outcome=, node=TaskWithoutPath(name='task_merge_files', function=, depends_on={'first': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/first.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/first.txt')), 'second': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/second.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/second.txt'))}, produces={'return': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/hello_world.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/hello_world.txt'))}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'duration': (1696055304.123595, 1696055304.1244528)}), exc_info=None), CollectionReport(outcome=, node=TaskWithoutPath(name='task_create_second_file', function= at 0x7f3c1b407d80>, depends_on={}, produces={'return': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/second.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/second.txt'))}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'duration': (1696055304.025182, 1696055304.0267167)}), exc_info=None)], tasks=[TaskWithoutPath(name='task_create_first_file', function=, depends_on={}, produces={'return': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/first.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/first.txt'))}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'duration': (1696055304.0767577, 1696055304.077608)}), TaskWithoutPath(name='task_merge_files', function=, depends_on={'first': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/first.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/first.txt')), 'second': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/second.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/second.txt'))}, produces={'return': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/hello_world.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/hello_world.txt'))}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'duration': (1696055304.123595, 1696055304.1244528)}), TaskWithoutPath(name='task_create_second_file', function= at 0x7f3c1b407d80>, depends_on={}, produces={'return': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/second.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/second.txt'))}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'duration': (1696055304.025182, 1696055304.0267167)})], dag=, resolving_dependencies_report=None, execution_reports=[ExecutionReport(task=TaskWithoutPath(name='task_create_second_file', function= at 0x7f3c1b407d80>, depends_on={}, produces={'return': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/second.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/second.txt'))}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'duration': (1696055304.025182, 1696055304.0267167)}), outcome=, exc_info=None, sections=[]), ExecutionReport(task=TaskWithoutPath(name='task_create_first_file', function=, depends_on={}, produces={'return': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/first.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/first.txt'))}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'duration': (1696055304.0767577, 1696055304.077608)}), outcome=, exc_info=None, sections=[]), ExecutionReport(task=TaskWithoutPath(name='task_merge_files', function=, depends_on={'first': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/first.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/first.txt')), 'second': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/second.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/second.txt'))}, produces={'return': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/hello_world.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/hello_world.txt'))}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'duration': (1696055304.123595, 1696055304.1244528)}), outcome=, exc_info=None, sections=[])], exit_code=, collection_start=1696055303.989013, collection_end=1696055303.9959698, execution_start=1696055304.0121965, execution_end=1696055304.207084, n_tasks_failed=0, scheduler=TopologicalSorter(dag=, priorities={'task_create_first_file': 0, 'task_merge_files': 0, 'task_create_second_file': 0}, _dag_backup=, _is_prepared=True, _nodes_out=set()), should_stop=False, warnings=[])" + "Session(config={'pm': , 'markers': {'depends_on': 'Add dependencies to a task. See this tutorial for more information: [link https://bit.ly/3JlxylS]https://bit.ly/3JlxylS[/].', 'filterwarnings': 'Add a filter for a warning to a task.', 'persist': 'Prevent execution of a task if all products exist and even if something has changed (dependencies, source file, products). This decorator might be useful for expensive tasks where only the formatting of the file has changed. The state of the files which have changed will also be remembered and another run will skip the task with success.', 'produces': 'Add products to a task. See this tutorial for more information: [link https://bit.ly/3JlxylS]https://bit.ly/3JlxylS[/].', 'skip': 'Skip a task and all its dependent tasks.', 'skip_ancestor_failed': 'Internal decorator applied to tasks if any of its preceding tasks failed.', 'skip_unchanged': 'Internal decorator applied to tasks which have already been executed and have not been changed.', 'skipif': 'Skip a task and all its dependent tasks if a condition is met.', 'task': 'Mark a function as a task regardless of its name. Or mark tasks which are repeated in a loop. See this tutorial for more information: [link https://bit.ly/3DWrXS3]https://bit.ly/3DWrXS3[/].', 'try_first': 'Try to execute a task a early as possible.', 'try_last': 'Try to execute a task a late as possible.'}, 'config': None, 'database_url': sqlite:////home/tobia/git/pytask/.pytask/.pytask.sqlite3, 'editor_url_scheme': 'file', 'export': <_ExportFormats.NO: 'no'>, 'ignore': ['.codecov.yml', '.gitignore', '.pre-commit-config.yaml', '.readthedocs.yml', '.readthedocs.yaml', 'readthedocs.yml', 'readthedocs.yaml', 'environment.yml', 'pyproject.toml', 'tox.ini', '.git/*', '.venv/*', '*.egg-info/*', '.ipynb_checkpoints/*', '.mypy_cache/*', '.nox/*', '.tox/*', '_build/*', '__pycache__/*', 'build/*', 'dist/*', 'pytest_cache/*'], 'paths': [], 'layout': 'dot', 'output_path': 'dag.pdf', 'rank_direction': <_RankDirection.TB: 'TB'>, 'expression': '', 'marker_expression': '', 'nodes': False, 'strict_markers': False, 'directories': False, 'exclude': [None, '.git/*'], 'mode': <_CleanMode.DRY_RUN: 'dry-run'>, 'quiet': False, 'capture': , 'debug_pytask': False, 'disable_warnings': False, 'dry_run': False, 'force': False, 'max_failures': inf, 'n_entries_in_table': 15, 'pdb': False, 'pdbcls': None, 's': False, 'show_capture': True, 'show_errors_immediately': False, 'show_locals': False, 'show_traceback': True, 'sort_table': True, 'trace': False, 'verbose': 1, 'stop_after_first_failure': False, 'check_casing_of_paths': True, 'pdb_cls': '', 'tasks': [, , at 0x7f3c1b407d80>], 'task_files': ['task_*.py'], 'command': 'build', 'root': PosixPath('/home/tobia/git/pytask'), 'filterwarnings': []}, hook=, collection_reports=[CollectionReport(outcome=, node=TaskWithoutPath(name='task_create_first_file', function=, depends_on={}, produces={'return': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/first.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/first.txt'))}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'duration': (1696055304.0767577, 1696055304.077608)}), exc_info=None), CollectionReport(outcome=, node=TaskWithoutPath(name='task_merge_files', function=, depends_on={'first': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/first.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/first.txt')), 'second': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/second.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/second.txt'))}, produces={'return': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/hello_world.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/hello_world.txt'))}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'duration': (1696055304.123595, 1696055304.1244528)}), exc_info=None), CollectionReport(outcome=, node=TaskWithoutPath(name='task_create_second_file', function= at 0x7f3c1b407d80>, depends_on={}, produces={'return': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/second.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/second.txt'))}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'duration': (1696055304.025182, 1696055304.0267167)}), exc_info=None)], tasks=[TaskWithoutPath(name='task_create_first_file', function=, depends_on={}, produces={'return': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/first.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/first.txt'))}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'duration': (1696055304.0767577, 1696055304.077608)}), TaskWithoutPath(name='task_merge_files', function=, depends_on={'first': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/first.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/first.txt')), 'second': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/second.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/second.txt'))}, produces={'return': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/hello_world.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/hello_world.txt'))}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'duration': (1696055304.123595, 1696055304.1244528)}), TaskWithoutPath(name='task_create_second_file', function= at 0x7f3c1b407d80>, depends_on={}, produces={'return': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/second.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/second.txt'))}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'duration': (1696055304.025182, 1696055304.0267167)})], dag=, resolving_dependencies_report=None, execution_reports=[ExecutionReport(task=TaskWithoutPath(name='task_create_second_file', function= at 0x7f3c1b407d80>, depends_on={}, produces={'return': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/second.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/second.txt'))}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'duration': (1696055304.025182, 1696055304.0267167)}), outcome=, exc_info=None, sections=[]), ExecutionReport(task=TaskWithoutPath(name='task_create_first_file', function=, depends_on={}, produces={'return': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/first.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/first.txt'))}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'duration': (1696055304.0767577, 1696055304.077608)}), outcome=, exc_info=None, sections=[]), ExecutionReport(task=TaskWithoutPath(name='task_merge_files', function=, depends_on={'first': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/first.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/first.txt')), 'second': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/second.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/second.txt'))}, produces={'return': PathNode(name='/home/tobia/git/pytask/docs/source/how_to_guides/hello_world.txt', path=PosixPath('/home/tobia/git/pytask/docs/source/how_to_guides/hello_world.txt'))}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'duration': (1696055304.123595, 1696055304.1244528)}), outcome=, exc_info=None, sections=[])], exit_code=, collection_start=1696055303.989013, collection_end=1696055303.9959698, execution_start=1696055304.0121965, execution_end=1696055304.207084, n_tasks_failed=0, scheduler=TopologicalSorter(dag=, priorities={'task_create_first_file': 0, 'task_merge_files': 0, 'task_create_second_file': 0}, _dag_backup=, _is_prepared=True, _nodes_out=set()), should_stop=False, warnings=[])" ] }, "execution_count": 4, diff --git a/docs/source/how_to_guides/how_to_write_a_plugin.md b/docs/source/how_to_guides/how_to_write_a_plugin.md index dc372b86..6fc024a0 100644 --- a/docs/source/how_to_guides/how_to_write_a_plugin.md +++ b/docs/source/how_to_guides/how_to_write_a_plugin.md @@ -32,27 +32,26 @@ This section explains some steps which are required for all plugins. pytask discovers plugins via `setuptools` entry-points. Following the approach advocated for by [setuptools_scm](https://github.com/pypa/setuptools_scm), the entry-point is -specified in `setup.cfg`. +specified in `pyproject.toml`. -```cfg -# Content of setup.cfg +```toml +[project] +name = "pytask-plugin" -[metadata] -name = pytask-plugin +[tool.setuptools.package-dir] +"" = "src" -[options.packages.find] -where = src +[tool.setuptools.packages.find] +where = ["src"] +namespaces = false -[options.entry_points] -pytask = - pytask_plugin = pytask_plugin.plugin +[project.entry-points.pytask] +pytask_plugin = "pytask_plugin.plugin" ``` -For `setuptools_scm` you also need a `pyproject.toml` with the following content. +For `setuptools_scm` you also need the following additions in `pyproject.toml`. ```toml -# Content of pyproject.toml - [build-system] requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.0"] @@ -60,13 +59,13 @@ requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.0"] write_to = "src/pytask_plugin/_version.py" ``` -For a complete example with `setuptools_scm` and `setup.cfg` see the -[pytask-parallel repo](https://github.com/pytask-dev/pytask-parallel/blob/main/setup.cfg). +For a complete example with `setuptools_scm` and `pyproject.toml` see the +[pytask-parallel repo](https://github.com/pytask-dev/pytask-parallel/blob/main/pyproject.toml). The entry-point for pytask is called `"pytask"` and points to a module called `pytask_plugin.plugin`. -### plugin.py +### `plugin.py` `plugin.py` is the entrypoint for pytask to your package. You can put all of your hook implementations in this module, but it is recommended to imitate the structure of pytask diff --git a/docs/source/how_to_guides/index.md b/docs/source/how_to_guides/index.md index 5b5fb660..3d2ea37c 100644 --- a/docs/source/how_to_guides/index.md +++ b/docs/source/how_to_guides/index.md @@ -12,6 +12,7 @@ specific tasks with pytask. maxdepth: 1 --- migrating_from_scripts_to_pytask +interfaces_for_dependencies_products functional_interface capture_warnings how_to_influence_build_order diff --git a/docs/source/how_to_guides/interfaces_for_dependencies_products.md b/docs/source/how_to_guides/interfaces_for_dependencies_products.md new file mode 100644 index 00000000..774240cb --- /dev/null +++ b/docs/source/how_to_guides/interfaces_for_dependencies_products.md @@ -0,0 +1,114 @@ +# Interfaces for dependencies and products + +Different interfaces exist for dependencies and products, and it might not be obvious +when to use what. This guide gives you an overview of the different strengths of each +approach. + +## Legend + +- ✅ = True +- ❌ = False +- ➖ = Does not apply + +## Dependencies + +In general, pytask regards everything as a task dependency if it is not marked as a +product. Thus, you can also think of the following examples as how to inject values into +a task. When we talk about products later, the same interfaces will be used. + +| | `def task(arg: ... = ...)` | `Annotated[..., value]` | `@task(kwargs=...)` | `@pytask.mark.depends_on(...)` | +| --------------------------------------- | :------------------------: | :---------------------: | :-----------------: | :----------------------------: | +| Not deprecated | ✅ | ✅ | ✅ | ❌ | +| No type annotations required | ✅ | ❌ | ✅ | ✅ | +| Flexible choice of argument name | ✅ | ✅ | ✅ | ❌ | +| Supports third-party functions as tasks | ❌ | ❌ | ✅ | ❌ | + +(default-argument)= + +### Default argument + +You can pass a value to a task as a default argument. + +```{literalinclude} ../../../docs_src/how_to_guides/interfaces/dependencies_default.py +``` + +(annotation)= + +### Annotation with value + +It is possible to include the value in the type annotation. + +It is especially helpful if you pass a {class}`~pytask.PNode` to the task. If you passed +a node as the default argument, type checkers like mypy would expect the node to enter +the task, but the value injected into the task depends on the nodes +{meth}`~pytask.PNode.load` method. For a {class}`~pytask.PathNode` + +```{literalinclude} ../../../docs_src/how_to_guides/interfaces/dependencies_annotation.py +``` + +(task-kwargs)= + +### `@task(kwargs=...)` + +You can use the `kwargs` argument of the {func}`@task ` decorator to pass a +dictionary. It applies to dependencies and products alike. + +```{literalinclude} ../../../docs_src/how_to_guides/interfaces/dependencies_task_kwargs.py +``` + +## Products + +| | `def task(arg: Annotated[..., Product] = ...)` | `Annotated[..., value, Product]` | `produces` | `@task(produces=...)` | `def task() -> Annotated[..., value]` | `@pytask.mark.produces(...)` | +| --------------------------------------------------------- | :--------------------------------------------: | :------------------------------: | :--------: | :-------------------: | :-----------------------------------: | :--------------------------: | +| Not deprecated | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | +| No type annotations required | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | +| Flexible choice of argument name | ✅ | ✅ | ❌ | ✅ | ➖ | ❌ | +| Supports third-party functions as tasks | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | +| Allows to pass custom node while preserving type of value | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | + +### `Product` annotation + +The syntax is the same as {ref}`default-argument`, but the {class}`~pytask.Product` +annotation turns the argument into a task product. + +```{literalinclude} ../../../docs_src/how_to_guides/interfaces/products_annotation.py +``` + +### `Product` annotation with value + +The syntax is the same as {ref}`annotation`, but the {class}`~pytask.Product` annotation +turns the argument into a task product. + +```{literalinclude} ../../../docs_src/how_to_guides/interfaces/products_annotation_with_pnode.py +``` + +### `produces` + +Without using any type annotation, you can use `produces` as a magical argument name to +treat every value passed to it as a task product. + +```{literalinclude} ../../../docs_src/how_to_guides/interfaces/products_produces.py +``` + +(return-annotation)= + +### Return annotation + +You can also add a node or a value that will be parsed to a node to the annotation of +the return type. It allows us to treat the returns of the task function as products. + +```{literalinclude} ../../../docs_src/how_to_guides/interfaces/products_return_annotation.py +``` + +(task-produces)= + +### `@task(produces=...)` + +In situations where the task return is the product like {ref}`return-annotation`, but +you cannot modify the type annotation of the return, use the argument `produces` of the +{func}`@task ` decorator. + +Pass the node or value you otherwise include in the type annotation to `produces`. + +```{literalinclude} ../../../docs_src/how_to_guides/interfaces/products_task_produces.py +``` diff --git a/docs/source/how_to_guides/writing_custom_nodes.md b/docs/source/how_to_guides/writing_custom_nodes.md index 8d1c22af..e49deb9a 100644 --- a/docs/source/how_to_guides/writing_custom_nodes.md +++ b/docs/source/how_to_guides/writing_custom_nodes.md @@ -9,10 +9,10 @@ your own to improve your workflows. ## Use-case -A typical task operation is to load data like a {class}`pandas.DataFrame` from a pickle +A typical task operation is to load data like a {class}`~pandas.DataFrame` from a pickle file, transform it, and store it on disk. The usual way would be to use paths to point -to inputs and outputs and call {func}`pandas.read_pickle` and -{meth}`pandas.DataFrame.to_pickle`. +to inputs and outputs and call {func}`~pandas.read_pickle` and +{meth}`~pandas.DataFrame.to_pickle`. ```{literalinclude} ../../../docs_src/how_to_guides/writing_custom_nodes_example_1.py ``` @@ -20,7 +20,7 @@ to inputs and outputs and call {func}`pandas.read_pickle` and To remove IO operations from the task and delegate them to pytask, we will write a `PickleNode` that automatically loads and stores Python objects. -And we pass the value to `df` via {obj}`Annotated` to preserve the type hint. +And we pass the value to `df` via {obj}`~typing.Annotated` to preserve the type hint. The result will be the following task. diff --git a/docs/source/tutorials/defining_dependencies_products.md b/docs/source/tutorials/defining_dependencies_products.md index cfb340a4..a7b1de88 100644 --- a/docs/source/tutorials/defining_dependencies_products.md +++ b/docs/source/tutorials/defining_dependencies_products.md @@ -1,44 +1,41 @@ # Defining dependencies and products -To ensure pytask executes all tasks in the correct order, you need to define -dependencies and products for each task. +Tasks have dependencies and products that you must define to run your tasks. + +Defining dependencies and products also serves another purpose. By analyzing them, +pytask determines the order in which to run the tasks. This tutorial offers you different interfaces. If you are comfortable with type -annotations or not afraid to try them, take a look at the tabs named `Python 3.10+` or -`Python 3.8+`. +annotations or not afraid to try them, look at the `Python 3.10+` or `Python 3.8+` tabs. +You find a tutorial on type hints {doc}`here <../type_hints>`. If you want to avoid type annotations for now, look at the tab named `produces`. -The deprecated approaches can be found in the tabs named `Decorators`. - -```{seealso} -An overview on the different interfaces and their strength and weaknesses is given in -{doc}`../explanations/interfaces_for_dependencies_products`. -``` +The `Decorators` tab documents the deprecated approach that should not be used +anymore and will be removed in version v0.5. -First, we focus on how to define products which should already be familiar to you. Then, -we focus on how task dependencies can be declared. +First, we focus on defining products that should already be familiar to you. Then, +we focus on how you can declare task dependencies. -We use the same project layout as before and add a `task_plot_data.py` module. +We use the same project as before and add a `task_plot_data.py` module. ```text my_project -├───pyproject.toml +│ +├───.pytask +│ +├───bld +│ ├────data.pkl +│ └────plot.png │ ├───src │ └───my_project +│ ├────__init__.py │ ├────config.py │ ├────task_data_preparation.py │ └────task_plot_data.py │ -├───setup.py -│ -├───.pytask -│ └────... -│ -└───bld - ├────data.pkl - └────plot.png +└───pyproject.toml ``` ## Products @@ -55,7 +52,7 @@ in `task_data_preparation.py`. :emphasize-lines: 11 ``` -{class}`~pytask.Product` allows to declare an argument as a product. After the +{class}`~pytask.Product` allows marking an argument as a product. After the task has finished, pytask will check whether the file exists. ::: @@ -67,7 +64,7 @@ task has finished, pytask will check whether the file exists. :emphasize-lines: 11 ``` -Using {class}`~pytask.Product` allows to declare an argument as a product. After the +{class}`~pytask.Product` allows marking an argument as a product. After the task has finished, pytask will check whether the file exists. ::: @@ -79,9 +76,9 @@ task has finished, pytask will check whether the file exists. :emphasize-lines: 8 ``` -Tasks can use `produces` as an "magic" argument name. Every value, or in this case path, -passed to this argument is automatically treated as a task product. Here, the path is -given by the default value of the argument. +Tasks can use `produces` as a "magic" argument name. Every value, or in this case path, +passed to this argument is automatically treated as a task product. Here, we pass the +path as the default argument. ::: @@ -97,8 +94,7 @@ This approach is deprecated and will be removed in v0.5 ``` The {func}`@pytask.mark.produces ` marker attaches a product to a -task which is a {class}`pathlib.Path` to file. After the task has finished, pytask will -check whether the file exists. +task. After the task has finished, pytask will check whether the file exists. Add `produces` as an argument of the task function to get access to the same path inside the task function. @@ -113,22 +109,21 @@ beneficial for handling paths conveniently and across platforms. ## Dependencies -Most tasks have dependencies and it is important to specify. Then, pytask ensures that -the dependencies are available before executing the task. +Adding a dependency to a task ensures that the dependency is available before execution. -As an example, we want to extend our project with another task that plots the data that -we generated with `task_create_random_data`. The task is called `task_plot_data` and we -will define it in `task_plot_data.py`. +To show how dependencies work, we extend our project with another task that plots the +data generated with `task_create_random_data`. The task is called `task_plot_data`, and +we will define it in `task_plot_data.py`. ::::{tab-set} :::{tab-item} Python 3.10+ :sync: python310plus -To specify that the task relies on the data set `data.pkl`, you can simply add the path +To specify that the task relies on the data set `data.pkl`, you can add the path to the function signature while choosing any argument name, here `path_to_data`. -pytask assumes that all function arguments that do not have the {class}`~pytask.Product` +pytask assumes that all function arguments that do not have a {class}`~pytask.`Product` annotation are dependencies of the task. ```{literalinclude} ../../../docs_src/tutorials/defining_dependencies_products_dependencies_py310.py @@ -140,7 +135,7 @@ annotation are dependencies of the task. :::{tab-item} Python 3.8+ :sync: python38plus -To specify that the task relies on the data set `data.pkl`, you can simply add the path +To specify that the task relies on the data set `data.pkl`, you can add the path to the function signature while choosing any argument name, here `path_to_data`. pytask assumes that all function arguments that do not have the {class}`~pytask.Product` @@ -155,8 +150,8 @@ annotation are dependencies of the task. :::{tab-item} ​`produces` :sync: produces -To specify that the task relies on the data set `data.pkl`, you can simply add the path -to the function signature while choosing any argument name, here `path_to_data`. +To specify that the task relies on the data set `data.pkl`, you can add the path to the +function signature while choosing any argument name, here `path_to_data`. pytask assumes that all function arguments that are not passed to the argument `produces` are dependencies of the task. @@ -257,9 +252,9 @@ Of course, tasks can have multiple dependencies and products. ```{literalinclude} ../../../docs_src/tutorials/defining_dependencies_products_multiple1_py310.py ``` -You can group your dependencies and product if you prefer not having a function argument -per input. Use dictionaries (recommended), tuples, lists, or more nested structures if -you need. +You can group your dependencies and product if you prefer not to have a function +argument per input. Use dictionaries (recommended), tuples, lists, or more nested +structures if needed. ```{literalinclude} ../../../docs_src/tutorials/defining_dependencies_products_multiple2_py310.py ``` @@ -272,9 +267,9 @@ you need. ```{literalinclude} ../../../docs_src/tutorials/defining_dependencies_products_multiple1_py38.py ``` -You can group your dependencies and product if you prefer not having a function argument -per input. Use dictionaries (recommended), tuples, lists, or more nested structures if -you need. +You can group your dependencies and product if you prefer not to have a function +argument per input. Use dictionaries (recommended), tuples, lists, or more nested +structures if needed. ```{literalinclude} ../../../docs_src/tutorials/defining_dependencies_products_multiple2_py38.py ``` @@ -285,7 +280,7 @@ you need. :sync: produces If your task has multiple products, group them in one container like a dictionary -(recommended), tuples, lists or a more nested structures. +(recommended), tuples, lists, or more nested structures. ```{literalinclude} ../../../docs_src/tutorials/defining_dependencies_products_multiple1_produces.py ``` @@ -349,9 +344,8 @@ Why does pytask recommend dictionaries and convert lists, tuples, or other iterators to dictionaries? First, dictionaries with positions as keys behave very similarly to lists. -Secondly, dictionaries use keys instead of positions that are more verbose and -descriptive and do not assume a fixed ordering. Both attributes are especially desirable -in complex projects. +Secondly, dictionary keys are more descriptive and do not assume a fixed +ordering. Both attributes are especially desirable in complex projects. **Multiple decorators** @@ -410,15 +404,16 @@ def task_fit_model(depends_on, produces): ::: :::: +(after)= + ## Depending on a task -In some situations you want to define a task depending on another task without -specifying the relationship explicitly. +In some situations, you want to define a task depending on another task. -pytask allows you to do that, but you loose features like access to paths which is why +pytask allows you to do that, but you lose features like access to paths, which is why defining dependencies explicitly is always preferred. -There are two modes for it and both use {func}`@task(after=...) `. +There are two modes for it, and both use {func}`@task(after=...) `. First, you can pass the task function or multiple task functions to the decorator. Applied to the tasks from before, we could have written `task_plot_data` as @@ -443,6 +438,15 @@ def task_plot_data(...): You will learn more about expressions in {doc}`selecting_tasks`. +## Further reading + +- There is an additional way to specify products by treating the returns of a task + function as a product. Read {doc}`../how_to_guides/using_task_returns` to learn more + about it. +- An overview of all ways to specify dependencies and products and their strengths and + weaknesses can be found in + {doc}`../how_to_guides/interfaces_for_dependencies_products`. + ## References [^id3]: The official documentation for {mod}`pathlib`. diff --git a/docs/source/tutorials/repeating_tasks_with_different_inputs.md b/docs/source/tutorials/repeating_tasks_with_different_inputs.md index 3ff42a1d..7c3bb2df 100644 --- a/docs/source/tutorials/repeating_tasks_with_different_inputs.md +++ b/docs/source/tutorials/repeating_tasks_with_different_inputs.md @@ -106,7 +106,7 @@ Every task has a unique id that can be used to {doc}`select it standard id combines the path to the module where the task is defined, a double colon, and the name of the task function. Here is an example. -``` +```text ../task_data_preparation.py::task_create_random_data ``` @@ -365,7 +365,7 @@ definition. You won't encounter these problems if you strictly use the below-mentioned interfaces. -Look at this repeated task which runs three times and tries to produce a text file with +Look at this repeated task, which runs three times and tries to produce a text file with some content. ```python diff --git a/docs/source/tutorials/set_up_a_project.md b/docs/source/tutorials/set_up_a_project.md index b31f9c41..074d84a2 100644 --- a/docs/source/tutorials/set_up_a_project.md +++ b/docs/source/tutorials/set_up_a_project.md @@ -1,67 +1,47 @@ # Set up a project -This tutorial shows you how to structure your first project. +Assuming you want to use pytask for a more extensive project, you want +to organize the project as a Python package. This tutorial explains the minimal setup. -Use the -[cookiecutter-pytask-project](https://github.com/pytask-dev/cookiecutter-pytask-project) -template to set up the structure or create the necessary folders and files manually. - -The remaining tutorial will explain the setup. +If you want to use pytask with a collection of scripts, you can skip this lesson +and move to the next section of the tutorials. -## The directory structure +The following directory tree gives an overview of the project's different parts. -The following directory tree is an example of setting up a project. - -``` +```text my_project │ +├───.pytask +│ +├───bld +│ └────... +│ ├───src │ └───my_project +│ ├────__init__.py │ ├────config.py │ └────... │ -├───setup.cfg -│ -├───pyproject.toml -│ -├───.pytask -│ └────... -│ -└───bld - └────... +└───pyproject.toml ``` -### The configuration - -The configuration is defined in `pyproject.toml` in the project's root folder and -contains a `[tool.pytask.ini_options]` section. - -```toml -[tool.pytask.ini_options] -paths = "src/my_project" -``` - -You do not have to add configuration values, but you need the -`[tool.pytask.ini_options]` header. The header alone will signal pytask that this is the -project's root. pytask will store the information it needs across executions in the -`.pytask` folder. - -`paths` allows you to set the location of tasks when you do not pass them explicitly via -the CLI. +You can replicate the directory structure for your project or you start from pytask's +[cookiecutter-pytask-project](https://github.com/pytask-dev/cookiecutter-pytask-project) +template or any other +{doc}`linked template or example project <../how_to_guides/bp_templates_and_projects>`. -### The source directory +## The `src` directory The `src` directory only contains a folder for the project in which the tasks and source -files reside. The nested structure is called the src layout and is the preferred way to -structure Python packages. +files reside. The nested structure is called the "`src` layout" and is the preferred way +to structure Python packages. -It also contains a `config.py` or a similar module to store the project's configuration. -For example, you should define paths pointing to the source and build directory of the -project. +It contains a `config.py` or a similar module to store the project's configuration. You +should define paths pointing to the source and build directory of the project. They +later help to define other paths. ```python # Content of config.py. - from pathlib import Path @@ -69,73 +49,116 @@ SRC = Path(__file__).parent.resolve() BLD = SRC.joinpath("..", "..", "bld").resolve() ``` -### The build directory +:::{seealso} +If you want to know more about the "`src` layout" and why it is NASA-approved, read +[this article by Hynek Schlawack](https://hynek.me/articles/testing-packaging/) or this +[setuptools article](https://setuptools.pypa.io/en/latest/userguide/package_discovery.html#src-layout). +::: + +## The `bld` directory + +The variable `BLD` defines the path to a build directory called `bld`. It is best +practice to store any outputs of the tasks in your project in a different folder than +`src`. -pytask creates the build directory `bld` during the execution for storing the products -of tasks. Delete it to rebuild the entire project. +Whenever you want to regenerate your project, delete the build directory and rerun +pytask. -### Install the project +## `pyproject.toml` -Two files are necessary to turn the source directory into a Python package. It allows -performing imports from `my_project`. E.g., `from my_project.config import SRC`. We also -need `pip >= 21.1`. +The `pyproject.toml` file is the modern configuration file for most Python packages and +apps. It contains -First, we need a `setup.cfg` containing the name, the package version, and the source -code's location. +1. the configuration for our Python package. +2. pytask's configuration. -```ini -# Content of setup.cfg +Let us start with the configuration of the Python package, which contains general +information about the package, like its name and version, the definition of the package +folder, `src`. -[metadata] -name = my_project -version = 0.0.1 +```toml +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[project] +name = "my_project" +version = "0.1.0" -[options] -packages = find: -package_dir = - =src +[tool.setuptools.package-dir] +"" = "src" -[options.packages.find] -where = src +[tool.setuptools.packages.find] +where = ["src"] +namespaces = false ``` -Secondly, extend the `pyproject.toml` with this content: +:::{seealso} +You can find more extensive information about this metadata in the documentation of +[setuptools](https://setuptools.pypa.io/en/latest/userguide/quickstart.html). +::: + +Alongside the package information, we include pytask's configuration under the +`[tool.pytask.ini_options]` section. We only tell pytask to look for tasks in +`src/my_project`. ```toml -# Content of pyproject.toml +[tool.pytask.ini_options] +paths = ["src/my_project"] +``` + +You will learn more about the configuration later in {doc}`tutorial `. + +You can copy the whole content of the `pyproject.toml` here. +
+ +```toml [build-system] -requires = ["setuptools"] +requires = ["setuptools", "setuptools-scm"] build-backend = "setuptools.build_meta" + +[project] +name = "my_project" +version = "0.1.0" + +[tool.setuptools.package-dir] +"" = "src" + +[tool.setuptools.packages.find] +where = ["src"] +namespaces = false + +[tool.pytask.ini_options] +paths = ["src/my_project"] ``` -:::{note} -If you used the -[cookiecutter-pytask-project](https://github.com/pytask-dev/cookiecutter-pytask-project) -template, the two files would look slightly different since -[setuptools_scm](https://github.com/pypa/setuptools_scm) handles your versioning. Do not -change anything and proceed. -::: +
-Now, you can install the package into your environment with +## The `.pytask` directory + +The `.pytask` directory is where pytask stores its information. You do not need to +interact with it. + +## Installation + +At last, you can install the package into your environment with ```console $ pip install -e . ``` -This command will trigger an editable install of the project, which means any changes in -the package's source files are immediately available in the installed version. +This command will trigger an editable install of the project, which is a development +mode and it means any changes in the package's source files are immediately available in +the installed version. Again, setuptools makes +[a good job explaining it](https://setuptools.pypa.io/en/latest/userguide/development_mode.html). :::{important} Do not forget to rerun the editable install every time after you recreate your Python environment. -::: - -## Further Reading -- You can find more examples for structuring a research project in - {doc}`../how_to_guides/bp_templates_and_projects`. -- [This article by Hynek Schlawack](https://hynek.me/articles/testing-packaging/) - explains the `src` layout. -- You find this and more information in the documentation for - [setuptools](https://setuptools.pypa.io/en/latest/userguide/quickstart.html). +Also, do not mix it up with the regular installation command `pip install .` because it +will likely not work. Then, paths defined with the variables `SRC` and `BLD` in +`config.py` will look for files relative to the location where your package is installed +like `/home/user/miniconda3/envs/...` and not in the folder you are working in. +::: diff --git a/docs/source/tutorials/skipping_tasks.md b/docs/source/tutorials/skipping_tasks.md index c291ec8a..7278ee3e 100644 --- a/docs/source/tutorials/skipping_tasks.md +++ b/docs/source/tutorials/skipping_tasks.md @@ -46,7 +46,7 @@ def task_that_takes_really_long_to_run(path: Path = Path("time_intensive_product ... ``` -## Further Reading +## Further reading - {doc}`selecting_tasks`. - {confval}`ignore` on how to ignore task files. diff --git a/docs/source/tutorials/using_a_data_catalog.md b/docs/source/tutorials/using_a_data_catalog.md index 3a9048de..176fcdce 100644 --- a/docs/source/tutorials/using_a_data_catalog.md +++ b/docs/source/tutorials/using_a_data_catalog.md @@ -22,21 +22,19 @@ The project structure is the same as in the previous example with the exception ```text my_project │ -├───pyproject.toml +├───.pytask +│ +├───bld +│ └────plot.png │ ├───src │ └───my_project +│ ├────__init__.py │ ├────config.py │ ├────task_data_preparation.py │ └────task_plot_data.py │ -├───setup.py -│ -├───.pytask -│ └────... -│ -└───bld - └────plot.png +└───pyproject.toml ``` ## The `DataCatalog` diff --git a/docs/source/tutorials/write_a_task.md b/docs/source/tutorials/write_a_task.md index 6bc3c135..1996649a 100644 --- a/docs/source/tutorials/write_a_task.md +++ b/docs/source/tutorials/write_a_task.md @@ -13,20 +13,18 @@ automatically discovers them. ```text my_project │ -├───pyproject.toml +├───.pytask +│ +├───bld +│ └────data.pkl │ ├───src │ └───my_project +│ ├────__init__.py │ ├────config.py │ └────task_data_preparation.py │ -├───setup.py -│ -├───.pytask -│ └────... -│ -└───bld - └────data.pkl +└───pyproject.toml ``` Generally, a task is a function whose name starts with `task_`. Tasks produce outputs @@ -123,6 +121,8 @@ Now, execute pytask to collect tasks in the current and subsequent directories. ```{include} ../_static/md/write-a-task.md ``` +(customize-task-names)= + ## Customize task names Use the {func}`@task ` decorator to mark a function as a diff --git a/docs_src/how_to_guides/interfaces/dependencies_annotation.py b/docs_src/how_to_guides/interfaces/dependencies_annotation.py new file mode 100644 index 00000000..9e8aeb6f --- /dev/null +++ b/docs_src/how_to_guides/interfaces/dependencies_annotation.py @@ -0,0 +1,8 @@ +from pathlib import Path +from typing import Annotated + +from pytask import PathNode + + +def task_example(path: Annotated[Path, PathNode(path=Path("input.txt"))]) -> None: + ... diff --git a/docs_src/how_to_guides/interfaces/dependencies_default.py b/docs_src/how_to_guides/interfaces/dependencies_default.py new file mode 100644 index 00000000..50c8c5b5 --- /dev/null +++ b/docs_src/how_to_guides/interfaces/dependencies_default.py @@ -0,0 +1,5 @@ +from pathlib import Path + + +def task_example(path: Path = Path("input.txt")) -> None: + ... diff --git a/docs_src/how_to_guides/interfaces/dependencies_task_kwargs.py b/docs_src/how_to_guides/interfaces/dependencies_task_kwargs.py new file mode 100644 index 00000000..25cd8fda --- /dev/null +++ b/docs_src/how_to_guides/interfaces/dependencies_task_kwargs.py @@ -0,0 +1,8 @@ +from pathlib import Path + +from pytask import task + + +@task(kwargs={"path": Path("input.txt")}) +def task_example(path: Path) -> None: + ... diff --git a/docs_src/how_to_guides/interfaces/products_annotation.py b/docs_src/how_to_guides/interfaces/products_annotation.py new file mode 100644 index 00000000..b7b519a1 --- /dev/null +++ b/docs_src/how_to_guides/interfaces/products_annotation.py @@ -0,0 +1,8 @@ +from pathlib import Path +from typing import Annotated + +from pytask import Product + + +def task_write_file(path: Annotated[Path, Product] = Path("file.txt")) -> None: + path.touch() diff --git a/docs_src/how_to_guides/interfaces/products_annotation_with_pnode.py b/docs_src/how_to_guides/interfaces/products_annotation_with_pnode.py new file mode 100644 index 00000000..5afad79f --- /dev/null +++ b/docs_src/how_to_guides/interfaces/products_annotation_with_pnode.py @@ -0,0 +1,11 @@ +from pathlib import Path +from typing import Annotated + +from pytask import PathNode +from pytask import Product + + +def task_write_file( + path: Annotated[Path, PathNode(path=Path("file.txt")), Product] +) -> None: + path.touch() diff --git a/docs_src/how_to_guides/interfaces/products_produces.py b/docs_src/how_to_guides/interfaces/products_produces.py new file mode 100644 index 00000000..76e5bce5 --- /dev/null +++ b/docs_src/how_to_guides/interfaces/products_produces.py @@ -0,0 +1,5 @@ +from pathlib import Path + + +def task_write_file(produces: Path = Path("file.txt")) -> None: + produces.touch() diff --git a/docs_src/how_to_guides/interfaces/products_return_annotation.py b/docs_src/how_to_guides/interfaces/products_return_annotation.py new file mode 100644 index 00000000..85a863b7 --- /dev/null +++ b/docs_src/how_to_guides/interfaces/products_return_annotation.py @@ -0,0 +1,6 @@ +from pathlib import Path +from typing import Annotated + + +def task_write_file() -> Annotated[str, Path("file.txt")]: + return "" diff --git a/docs_src/how_to_guides/interfaces/products_task_produces.py b/docs_src/how_to_guides/interfaces/products_task_produces.py new file mode 100644 index 00000000..89bec5b3 --- /dev/null +++ b/docs_src/how_to_guides/interfaces/products_task_produces.py @@ -0,0 +1,8 @@ +from pathlib import Path + +from pytask import task + + +@task(produces=Path("file.txt")) +def task_write_file() -> str: + return "" diff --git a/pyproject.toml b/pyproject.toml index 266d7d16..90c3c605 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,16 +101,6 @@ fix = true ignore = [ "I", # ignore isort "TRY", # ignore tryceratops. - # Numpy docstyle - "D107", - "D203", - "D212", - "D213", - "D402", - "D413", - "D415", - "D416", - "D417", # Others. "ISC001", "S101", # raise errors for asserts. diff --git a/src/_pytask/task_utils.py b/src/_pytask/task_utils.py index 25826de9..a021f65d 100644 --- a/src/_pytask/task_utils.py +++ b/src/_pytask/task_utils.py @@ -55,22 +55,24 @@ def task( ---------- name Use it to override the name of the task that is, by default, the name of the - callable. + task function. Read :ref:`customize-task-names` for more information. after An expression or a task function or a list of task functions that need to be - executed before this task can. + executed before this task can be executed. See :ref:`after` for more + information. id - An id for the task if it is part of a parametrization. Otherwise, an automatic - id will be generated. See - :doc:`this tutorial <../tutorials/repeating_tasks_with_different_inputs>` for - more information. + An id for the task if it is part of a repetition. Otherwise, an automatic id + will be generated. See :ref:`how-to-repeat-a-task-with-different-inputs-the-id` + for more information. kwargs - A dictionary containing keyword arguments which are passed to the task when it - is executed. - produces - Definition of products to parse the function returns and store them. See - :doc:`this how-to guide <../how_to_guides/using_task_returns>` for more + Use a dictionary to pass any keyword arguments to the task function which can be + dependencies or products of the task. Read :ref:`task-kwargs` for more information. + produces + Use this argument if you want to parse the return of the task function as a + product, but you cannot annotate the return of the function. See :doc:`this + how-to guide <../how_to_guides/using_task_returns>` or :ref:`task-produces` for + more information. Examples -------- @@ -78,11 +80,9 @@ def task( .. code-block:: python - from typing import Annotated - from pytask import task + from typing import Annotated from pytask import task - @task - def create_text_file() -> Annotated[str, Path("file.txt")]: + @task def create_text_file() -> Annotated[str, Path("file.txt")]: return "Hello, World!" """ diff --git a/tests/test_collect.py b/tests/test_collect.py index aecfe64b..3e2408cf 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -695,3 +695,24 @@ def __getattr__(self, name): result = runner.invoke(cli, [tmp_path.as_posix()]) assert result.exit_code == ExitCode.OK + + +@pytest.mark.end_to_end() +@pytest.mark.parametrize( + "second_node", ["PythonNode()", "PathNode(path=Path('a.txt'))"] +) +def test_error_with_multiple_dependency_annotations(runner, tmp_path, second_node): + source = f""" + from typing_extensions import Annotated + from pytask import PythonNode, PathNode + from pathlib import Path + + def task_example( + dependency: Annotated[str, PythonNode(), {second_node}] = "hello" + ) -> None: ... + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.COLLECTION_FAILED + assert "Parameter 'dependency' has multiple node annot" in result.output diff --git a/tests/test_execute.py b/tests/test_execute.py index a79dc916..50ed2e11 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -486,27 +486,6 @@ def task_example( assert tmp_path.joinpath("out.txt").read_text() == "world" -@pytest.mark.end_to_end() -@pytest.mark.parametrize( - "second_node", ["PythonNode()", "PathNode(path=Path('a.txt'))"] -) -def test_error_with_multiple_dependency_annotations(runner, tmp_path, second_node): - source = f""" - from typing_extensions import Annotated - from pytask import PythonNode, PathNode - from pathlib import Path - - def task_example( - dependency: Annotated[str, PythonNode(), {second_node}] = "hello" - ) -> None: ... - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - - result = runner.invoke(cli, [tmp_path.as_posix()]) - assert result.exit_code == ExitCode.COLLECTION_FAILED - assert "Parameter 'dependency'" in result.output - - @pytest.mark.end_to_end() def test_return_with_path_annotation_as_return(runner, tmp_path): source = """