diff --git a/.gitignore b/.gitignore index 3fbda029..b08979df 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,5 @@ build dist src/_pytask/_version.py *.pkl + +tests/test_jupyter/*.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8d163715..ac63e1d2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -62,6 +62,7 @@ repos: additional_dependencies: [ attrs>=21.3.0, click, + pluggy, types-setuptools ] pass_filenames: false @@ -100,6 +101,10 @@ repos: docs/source/tutorials/selecting_tasks.md| docs/source/tutorials/set_up_a_project.md )$ +- repo: https://github.com/kynan/nbstripout + rev: 0.6.1 + hooks: + - id: nbstripout - repo: https://github.com/codespell-project/codespell rev: v2.2.5 hooks: diff --git a/docs/source/changes.md b/docs/source/changes.md index d96f40e6..fcdc005c 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -28,9 +28,12 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and - {pull}`408` removes `.value` from `Node` protocol. - {pull}`409` make `.from_annot` an optional feature of nodes. - {pull}`410` allows to pass functions to `PythonNode(hash=...)`. +- {pull}`411` implements a new functional interface and adds experimental support for + defining and running tasks in REPLs or Jupyter notebooks. - {pull}`412` adds protocols for tasks. - {pull}`413` removes scripts to generate `.svg`s. - {pull}`414` allow more ruff rules. +- {pull}`416` removes `.from_annot` again. ## 0.3.2 - 2023-06-07 diff --git a/docs/source/how_to_guides/invoking_pytask_extended.md b/docs/source/how_to_guides/invoking_pytask_extended.md index 8f154a80..75be5918 100644 --- a/docs/source/how_to_guides/invoking_pytask_extended.md +++ b/docs/source/how_to_guides/invoking_pytask_extended.md @@ -10,7 +10,7 @@ Invoke pytask programmatically with import pytask -session = pytask.main({"paths": ...}) +session = pytask.build(paths=...) ``` Pass command line arguments with their long name and hyphens replaced by underscores as diff --git a/docs/source/reference_guides/api.md b/docs/source/reference_guides/api.md index bfd922e2..a39fe280 100644 --- a/docs/source/reference_guides/api.md +++ b/docs/source/reference_guides/api.md @@ -325,7 +325,7 @@ outcome. ```{eval-rst} .. autofunction:: pytask.build_dag -.. autofunction:: pytask.main +.. autofunction:: pytask.build ``` ## Reports diff --git a/docs/source/tutorials/visualizing_the_dag.md b/docs/source/tutorials/visualizing_the_dag.md index 2930bed4..3e365e41 100644 --- a/docs/source/tutorials/visualizing_the_dag.md +++ b/docs/source/tutorials/visualizing_the_dag.md @@ -37,7 +37,7 @@ layouts, which are listed [here](https://graphviz.org/docs/layouts/). The programmatic and interactive interface allows customizing the figure. -Similar to {func}`pytask.main`, there exists {func}`pytask.build_dag` which returns the +Similar to {func}`pytask.build`, there exists {func}`pytask.build_dag` which returns the DAG as a {class}`networkx.DiGraph`. ```python diff --git a/environment.yml b/environment.yml index 528ad279..d5e671a1 100644 --- a/environment.yml +++ b/environment.yml @@ -26,6 +26,7 @@ dependencies: - black - jupyterlab - matplotlib + - nbval - pre-commit - pygraphviz - pytest diff --git a/src/_pytask/build.py b/src/_pytask/build.py index 7585f582..15e1e071 100644 --- a/src/_pytask/build.py +++ b/src/_pytask/build.py @@ -4,9 +4,13 @@ import sys from pathlib import Path from typing import Any +from typing import Callable +from typing import Iterable +from typing import Literal from typing import TYPE_CHECKING import click +from _pytask.capture import CaptureMethod from _pytask.click import ColoredCommand from _pytask.config import hookimpl from _pytask.config_utils import _find_project_root_and_config @@ -26,16 +30,50 @@ if TYPE_CHECKING: + from _pytask.node_protocols import PTask from typing import NoReturn @hookimpl(tryfirst=True) def pytask_extend_command_line_interface(cli: click.Group) -> None: """Extend the command line interface.""" - cli.add_command(build) - - -def main(raw_config: dict[str, Any]) -> Session: # noqa: C901, PLR0912, PLR0915 + cli.add_command(build_command) + + +def build( # noqa: C901, PLR0912, PLR0913, PLR0915 + *, + capture: Literal["fd", "no", "sys", "tee-sys"] | CaptureMethod = CaptureMethod.NO, + check_casing_of_paths: bool = True, + config: Path | None = None, + database_url: str = "", + debug_pytask: bool = False, + disable_warnings: bool = False, + dry_run: bool = False, + editor_url_scheme: Literal["no_link", "file", "vscode", "pycharm"] # noqa: PYI051 + | str = "file", + expression: str = "", + force: bool = False, + ignore: Iterable[str] = (), + marker_expression: str = "", + max_failures: float = float("inf"), + n_entries_in_table: int = 15, + paths: str | Path | Iterable[str | Path] = (), + pdb: bool = False, + pdb_cls: str = "", + s: bool = False, + show_capture: bool = True, + show_errors_immediately: bool = False, + show_locals: bool = False, + show_traceback: bool = True, + sort_table: bool = True, + stop_after_first_failure: bool = False, + strict_markers: bool = False, + tasks: Callable[..., Any] | PTask | Iterable[Callable[..., Any] | PTask] = (), + task_files: str | Iterable[str] = "task_*.py", + trace: bool = False, + verbose: int = 1, + **kwargs: Any, +) -> Session: """Run pytask. This is the main command to run pytask which usually receives kwargs from the @@ -44,13 +82,73 @@ def main(raw_config: dict[str, Any]) -> Session: # noqa: C901, PLR0912, PLR0915 Parameters ---------- - raw_config : dict[str, Any] - A dictionary with options passed to pytask. In general, this dictionary holds - the information passed via the command line interface. + capture + The capture method for stdout and stderr. + check_casing_of_paths + Whether errors should be raised when file names have different casings. + config + A path to the configuration file. + database_url + An URL to the database that tracks the status of tasks. + debug_pytask + Whether debug information should be shown. + disable_warnings + Whether warnings should be disabled and not displayed. + dry_run + Whether a dry-run should be performed that shows which tasks need to be rerun. + editor_url_scheme + An url scheme that allows to click on task names, node names and filenames and + jump right into you preferred edior to the right line. + expression + Same as ``-k`` on the command line. Select tasks via expressions on task ids. + force + Run tasks even though they would be skipped since nothing has changed. + ignore + A pattern to ignore files or directories. Refer to ``pathlib.Path.match`` + for more info. + marker_expression + Same as ``-m`` on the command line. Select tasks via marker expressions. + max_failures + Stop after some failures. + n_entries_in_table + How many entries to display in the table during the execution. Tasks which are + running are always displayed. + paths + A path or collection of paths where pytask looks for the configuration and + tasks. + pdb + Start the interactive debugger on errors. + pdb_cls + Start a custom debugger on errors. For example: + ``--pdbcls=IPython.terminal.debugger:TerminalPdb`` + s + Shortcut for ``pytask.build(capture"no")``. + show_capture + Choose which captured output should be shown for failed tasks. + show_errors_immediately + Show errors with tracebacks as soon as the task fails. + show_locals + Show local variables in tracebacks. + show_traceback + Choose whether tracebacks should be displayed or not. + sort_table + Sort the table of tasks at the end of the execution. + stop_after_first_failure + Stop after the first failure. + strict_markers + Raise errors for unknown markers. + tasks + A task or a collection of tasks that is passed to ``pytask.build(tasks=...)``. + task_files + A pattern to describe modules that contain tasks. + trace + Enter debugger in the beginning of each task. + verbose + Make pytask verbose (>= 0) or quiet (= 0). Returns ------- - session : _pytask.session.Session + session : pytask.Session The session captures all the information of the current run. """ @@ -61,6 +159,39 @@ def main(raw_config: dict[str, Any]) -> Session: # noqa: C901, PLR0912, PLR0915 pm.register(cli) pm.hook.pytask_add_hooks(pm=pm) + raw_config = { + "capture": capture, + "check_casing_of_paths": check_casing_of_paths, + "config": config, + "database_url": database_url, + "debug_pytask": debug_pytask, + "disable_warnings": disable_warnings, + "dry_run": dry_run, + "editor_url_scheme": editor_url_scheme, + "expression": expression, + "force": force, + "ignore": ignore, + "marker_expression": marker_expression, + "max_failures": max_failures, + "n_entries_in_table": n_entries_in_table, + "paths": paths, + "pdb": pdb, + "pdb_cls": pdb_cls, + "s": s, + "show_capture": show_capture, + "show_errors_immediately": show_errors_immediately, + "show_locals": show_locals, + "show_traceback": show_traceback, + "sort_table": sort_table, + "stop_after_first_failure": stop_after_first_failure, + "strict_markers": strict_markers, + "tasks": tasks, + "task_files": task_files, + "trace": trace, + "verbose": verbose, + **kwargs, + } + # If someone called the programmatic interface, we need to do some parsing. if "command" not in raw_config: raw_config["command"] = "build" @@ -97,9 +228,9 @@ def main(raw_config: dict[str, Any]) -> Session: # noqa: C901, PLR0912, PLR0915 raw_config = {**raw_config, **config_from_file} - config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config) + config_ = pm.hook.pytask_configure(pm=pm, raw_config=raw_config) - session = Session.from_config(config) + session = Session.from_config(config_) except (ConfigurationError, Exception): exc_info = sys.exc_info() @@ -137,7 +268,7 @@ def main(raw_config: dict[str, Any]) -> Session: # noqa: C901, PLR0912, PLR0915 return session -@click.command(cls=ColoredCommand) +@click.command(cls=ColoredCommand, name="build") @click.option( "--debug-pytask", is_flag=True, @@ -161,13 +292,13 @@ def main(raw_config: dict[str, Any]) -> Session: # noqa: C901, PLR0912, PLR0915 "--show-errors-immediately", is_flag=True, default=False, - help="Print errors with tracebacks as soon as the task fails.", + help="Show errors with tracebacks as soon as the task fails.", ) @click.option( "--show-traceback/--show-no-traceback", type=bool, default=True, - help=("Choose whether tracebacks should be displayed or not."), + help="Choose whether tracebacks should be displayed or not.", ) @click.option( "--dry-run", type=bool, is_flag=True, default=False, help="Perform a dry-run." @@ -179,7 +310,7 @@ def main(raw_config: dict[str, Any]) -> Session: # noqa: C901, PLR0912, PLR0915 default=False, help="Execute a task even if it succeeded successfully before.", ) -def build(**raw_config: Any) -> NoReturn: +def build_command(**raw_config: Any) -> NoReturn: """Collect tasks, execute them and report the results. The default command. pytask collects tasks from the given paths or the @@ -187,5 +318,5 @@ def build(**raw_config: Any) -> NoReturn: """ raw_config["command"] = "build" - session = main(raw_config) + session = build(**raw_config) sys.exit(session.exit_code) diff --git a/src/_pytask/capture.py b/src/_pytask/capture.py index b99bf97f..11c42365 100644 --- a/src/_pytask/capture.py +++ b/src/_pytask/capture.py @@ -50,7 +50,7 @@ from _pytask.node_protocols import PTask -class _CaptureMethod(enum.Enum): +class CaptureMethod(enum.Enum): FD = "fd" NO = "no" SYS = "sys" @@ -63,8 +63,8 @@ def pytask_extend_command_line_interface(cli: click.Group) -> None: additional_parameters = [ click.Option( ["--capture"], - type=EnumChoice(_CaptureMethod), - default=_CaptureMethod.FD, + type=EnumChoice(CaptureMethod), + default=CaptureMethod.FD, help="Per task capturing method.", ), click.Option( @@ -77,7 +77,7 @@ def pytask_extend_command_line_interface(cli: click.Group) -> None: ["--show-capture"], type=EnumChoice(ShowCapture), default=ShowCapture.ALL, - help=("Choose which captured output should be shown for failed tasks."), + help="Choose which captured output should be shown for failed tasks.", ), ] cli.commands["build"].params.extend(additional_parameters) @@ -90,8 +90,11 @@ def pytask_parse_config(config: dict[str, Any]) -> None: Note that, ``-s`` is a shortcut for ``--capture=no``. """ + if isinstance(config["capture"], str): + config["capture"] = CaptureMethod(config["capture"]) + if config["s"]: - config["capture"] = _CaptureMethod.NO + config["capture"] = CaptureMethod.NO @hookimpl @@ -642,20 +645,20 @@ def readouterr(self) -> CaptureResult[AnyStr]: return CaptureResult(out, err) # type: ignore -def _get_multicapture(method: _CaptureMethod) -> MultiCapture[str]: +def _get_multicapture(method: CaptureMethod) -> MultiCapture[str]: """Set up the MultiCapture class with the passed method. For each valid method, the function instantiates the :class:`MultiCapture` class with the specified buffers for ``stdin``, ``stdout``, and ``stderr``. """ - if method == _CaptureMethod.FD: + if method == CaptureMethod.FD: return MultiCapture(in_=FDCapture(0), out=FDCapture(1), err=FDCapture(2)) - if method == _CaptureMethod.SYS: + if method == CaptureMethod.SYS: return MultiCapture(in_=SysCapture(0), out=SysCapture(1), err=SysCapture(2)) - if method == _CaptureMethod.NO: + if method == CaptureMethod.NO: return MultiCapture(in_=None, out=None, err=None) - if method == _CaptureMethod.TEE_SYS: + if method == CaptureMethod.TEE_SYS: return MultiCapture( in_=None, out=SysCapture(1, tee=True), err=SysCapture(2, tee=True) ) @@ -679,7 +682,7 @@ class CaptureManager: """ - def __init__(self, method: _CaptureMethod) -> None: + def __init__(self, method: CaptureMethod) -> None: self._method = method self._capturing: MultiCapture[str] | None = None diff --git a/src/_pytask/clean.py b/src/_pytask/clean.py index d9a69dc3..22d8cc7b 100644 --- a/src/_pytask/clean.py +++ b/src/_pytask/clean.py @@ -101,7 +101,7 @@ def clean(**raw_config: Any) -> NoReturn: # noqa: C901, PLR0912, PLR0915 raw_config["command"] = "clean" try: - # Duplication of the same mechanism in :func:`pytask.main.main`. + # Duplication of the same mechanism in :func:`pytask.build`. pm = get_plugin_manager() from _pytask import cli diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index 0f48ec28..95b43e29 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -19,13 +19,17 @@ from _pytask.console import console from _pytask.console import create_summary_panel from _pytask.console import format_task_name +from _pytask.console import get_file +from _pytask.console import is_jupyter from _pytask.exceptions import CollectionError from _pytask.mark_utils import has_mark -from _pytask.node_protocols import Node +from _pytask.node_protocols import PNode +from _pytask.node_protocols import PPathNode from _pytask.node_protocols import PTask from _pytask.nodes import PathNode from _pytask.nodes import PythonNode from _pytask.nodes import Task +from _pytask.nodes import TaskWithoutPath from _pytask.outcomes import CollectionOutcome from _pytask.outcomes import count_outcomes from _pytask.path import find_case_sensitive_path @@ -33,6 +37,7 @@ from _pytask.report import CollectionReport from _pytask.shared import find_duplicates from _pytask.shared import reduce_node_name +from _pytask.task_utils import task as task_decorator from _pytask.traceback import render_exc_info from rich.text import Text @@ -47,6 +52,13 @@ def pytask_collect(session: Session) -> bool: session.collection_start = time.time() _collect_from_paths(session) + _collect_from_tasks(session) + + session.tasks.extend( + i.node + for i in session.collection_reports + if i.outcome == CollectionOutcome.SUCCESS and isinstance(i.node, PTask) + ) try: session.hook.pytask_collect_modify_tasks(session=session, tasks=session.tasks) @@ -76,10 +88,55 @@ def _collect_from_paths(session: Session) -> None: if reports: session.collection_reports.extend(reports) - session.tasks.extend( - i.node for i in reports if i.outcome == CollectionOutcome.SUCCESS + + +def _collect_from_tasks(session: Session) -> None: + """Collect tasks from user provided tasks via the functional interface.""" + for raw_task in session.config.get("tasks", ()): + if isinstance(raw_task, PTask): + report = session.hook.pytask_collect_task_protocol( + session=session, + reports=session.collection_reports, + path=None, + name=None, + obj=raw_task, ) + if callable(raw_task): + if not hasattr(raw_task, "pytask_meta"): + raw_task = task_decorator()(raw_task) # noqa: PLW2901 + + try: + path = get_file(raw_task) + except (TypeError, OSError): + path = None + else: + if path.name == "": + path = None # pragma: no cover + + # Detect whether a path is defined in a Jupyter notebook. + if is_jupyter() and "ipykernel" in path.as_posix() and path.suffix == ".py": + path = None # pragma: no cover + + name = raw_task.pytask_meta.name + + # When a task is not a PTask and a callable, set arbitrary values and it will + # pass without errors and not collected. + else: + name = "" + path = None + + report = session.hook.pytask_collect_task_protocol( + session=session, + reports=session.collection_reports, + path=path, + name=name, + obj=raw_task, + ) + + if report is not None: + session.collection_reports.append(report) + @hookimpl def pytask_ignore_collect(path: Path, config: dict[str, Any]) -> bool: @@ -136,7 +193,7 @@ def pytask_collect_file( @hookimpl def pytask_collect_task_protocol( - session: Session, path: Path, name: str, obj: Any + session: Session, path: Path | None, name: str, obj: Any ) -> CollectionReport | None: """Start protocol for collecting a task.""" try: @@ -163,7 +220,7 @@ def pytask_collect_task_protocol( @hookimpl(trylast=True) def pytask_collect_task( session: Session, path: Path, name: str, obj: Any -) -> Task | None: +) -> PTask | None: """Collect a task which is a function. There is some discussion on how to detect functions in this thread: @@ -172,9 +229,11 @@ def pytask_collect_task( """ if (name.startswith("task_") or has_mark(obj, "task")) and callable(obj): - dependencies = parse_dependencies_from_task_function(session, path, name, obj) - - products = parse_products_from_task_function(session, path, name, obj) + path_nodes = Path.cwd() if path is None else path.parent + dependencies = parse_dependencies_from_task_function( + session, path_nodes, name, obj + ) + products = parse_products_from_task_function(session, path_nodes, name, obj) markers = obj.pytask_meta.markers if hasattr(obj, "pytask_meta") else [] @@ -182,6 +241,14 @@ def pytask_collect_task( # e.g. due to pytask_meta, in different layers of the wrapping. unwrapped = inspect.unwrap(obj) + if path is None: + return TaskWithoutPath( + name=name, + function=unwrapped, + depends_on=dependencies, + produces=products, + markers=markers, + ) return Task( base_name=name, path=path, @@ -190,24 +257,35 @@ def pytask_collect_task( produces=products, markers=markers, ) + if isinstance(obj, PTask): + return obj return None -_TEMPLATE_ERROR: str = ( - "The provided path of the dependency/product in the marker is\n\n{}\n\n, but the " - "path of the file on disk is\n\n{}\n\nCase-sensitive file systems would raise an " - "error because the upper and lower case format of the paths does not match.\n\n" - "Please, align the names to ensure reproducibility on case-sensitive file systems " - "(often Linux or macOS) or disable this error with 'check_casing_of_paths = false' " - " in your pytask configuration file.\n\n" - "Hint: If parts of the path preceding your project directory are not properly " - "formatted, check whether you need to call `.resolve()` on `SRC`, `BLD` or other " - "paths created from the `__file__` attribute of a module." -) +_TEMPLATE_ERROR: str = """\ +The provided path of the dependency/product is + +{} + +, but the path of the file on disk is + +{} + +Case-sensitive file systems would raise an error because the upper and lower case \ +format of the paths does not match. + +Please, align the names to ensure reproducibility on case-sensitive file systems \ +(often Linux or macOS) or disable this error with 'check_casing_of_paths = false' in \ +your pytask configuration file. + +Hint: If parts of the path preceding your project directory are not properly \ +formatted, check whether you need to call `.resolve()` on `SRC`, `BLD` or other paths \ +created from the `__file__` attribute of a module. +""" @hookimpl(trylast=True) -def pytask_collect_node(session: Session, path: Path, node_info: NodeInfo) -> Node: +def pytask_collect_node(session: Session, path: Path, node_info: NodeInfo) -> PNode: """Collect a node of a task as a :class:`pytask.nodes.PathNode`. Strings are assumed to be paths. This might be a strict assumption, but since this @@ -217,6 +295,13 @@ def pytask_collect_node(session: Session, path: Path, node_info: NodeInfo) -> No ``trylast=True`` might be necessary if other plugins try to parse strings themselves like a plugin for downloading files which depends on URLs given as strings. + Parameters + ---------- + path + The path helps if the path of the node is given relative to the task. The path + either points to the parent directory of the task module or to the current + working directory for tasks defined in the REPL or in Jupyter notebooks. + """ node = node_info.value @@ -226,26 +311,29 @@ def pytask_collect_node(session: Session, path: Path, node_info: NodeInfo) -> No node.name = node_info.arg_name + suffix return node - if isinstance(node, Node): + if isinstance(node, PPathNode) and not node.path.is_absolute(): + node.path = path.joinpath(node.path) + + # ``normpath`` removes ``../`` from the path which is necessary for the casing + # check which will fail since ``.resolves()`` also normalizes a path. + node.path = Path(os.path.normpath(node.path)) + _raise_error_if_casing_of_path_is_wrong( + node.path, session.config["check_casing_of_paths"] + ) + + if isinstance(node, PNode): return node if isinstance(node, Path): if not node.is_absolute(): - node = path.parent.joinpath(node) + node = path.joinpath(node) # ``normpath`` removes ``../`` from the path which is necessary for the casing # check which will fail since ``.resolves()`` also normalizes a path. node = Path(os.path.normpath(node)) - - if ( - not IS_FILE_SYSTEM_CASE_SENSITIVE - and session.config["check_casing_of_paths"] - and sys.platform == "win32" - ): - case_sensitive_path = find_case_sensitive_path(node, "win32") - if str(node) != str(case_sensitive_path): - raise ValueError(_TEMPLATE_ERROR.format(node, case_sensitive_path)) - + _raise_error_if_casing_of_path_is_wrong( + node, session.config["check_casing_of_paths"] + ) return PathNode.from_path(node) suffix = "-" + "-".join(map(str, node_info.path)) if node_info.path else "" @@ -253,6 +341,20 @@ def pytask_collect_node(session: Session, path: Path, node_info: NodeInfo) -> No return PythonNode(value=node, name=node_name) +def _raise_error_if_casing_of_path_is_wrong( + path: Path, check_casing_of_paths: bool +) -> None: + """Raise an error if the path does not have the correct casing.""" + if ( + not IS_FILE_SYSTEM_CASE_SENSITIVE + and sys.platform == "win32" + and check_casing_of_paths + ): + case_sensitive_path = find_case_sensitive_path(path, "win32") + if str(path) != str(case_sensitive_path): + raise ValueError(_TEMPLATE_ERROR.format(path, case_sensitive_path)) + + def _not_ignored_paths( paths: Iterable[Path], session: Session ) -> Generator[Path, None, None]: diff --git a/src/_pytask/collect_command.py b/src/_pytask/collect_command.py index 73428ef7..7ebeb56f 100644 --- a/src/_pytask/collect_command.py +++ b/src/_pytask/collect_command.py @@ -56,7 +56,7 @@ def collect(**raw_config: Any | None) -> NoReturn: raw_config["command"] = "collect" try: - # Duplication of the same mechanism in :func:`pytask.main.main`. + # Duplication of the same mechanism in :func:`pytask.build`. pm = get_plugin_manager() from _pytask import cli diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py index ee26af25..de138d39 100644 --- a/src/_pytask/collect_utils.py +++ b/src/_pytask/collect_utils.py @@ -1,7 +1,6 @@ """Contains utility functions for :mod:`_pytask.collect`.""" from __future__ import annotations -import functools import itertools import uuid import warnings @@ -17,8 +16,7 @@ from _pytask.mark_utils import has_mark from _pytask.mark_utils import remove_marks from _pytask.models import NodeInfo -from _pytask.node_protocols import Node -from _pytask.nodes import ProductType +from _pytask.node_protocols import PNode from _pytask.nodes import PythonNode from _pytask.shared import find_duplicates from _pytask.task_utils import parse_keyword_arguments_from_signature_defaults @@ -26,6 +24,7 @@ from _pytask.tree_util import tree_leaves from _pytask.tree_util import tree_map from _pytask.tree_util import tree_map_with_path +from _pytask.typing import ProductType from attrs import define from attrs import field from typing_extensions import Annotated @@ -259,41 +258,51 @@ def parse_dependencies_from_task_function( parameters_with_product_annot = _find_args_with_product_annotation(obj) parameters_with_node_annot = _find_args_with_node_annotation(obj) + # Complete kwargs with node annotations, when no value is given by kwargs. + for name in list(parameters_with_node_annot): + if name not in kwargs: + kwargs[name] = parameters_with_node_annot.pop(name) + else: + msg = ( + f"The value for the parameter {name!r} is defined twice in " + "'@pytask.mark.task(kwargs=...)' and in the type annotation. Choose " + "only one option." + ) + raise ValueError(msg) + for parameter_name, value in kwargs.items(): - if parameter_name in parameters_with_product_annot: + if ( + parameter_name in parameters_with_product_annot + or parameter_name == "return" + ): continue if parameter_name == "depends_on": continue - partialed_evolve = functools.partial( - _evolve_instance, - instance_from_annot=parameters_with_node_annot.get(parameter_name), - ) - nodes = tree_map_with_path( lambda p, x: _collect_dependency( session, path, name, - NodeInfo(parameter_name, p, partialed_evolve(x)), # noqa: B023 + NodeInfo(parameter_name, p, x), # noqa: B023 ), value, ) # If all nodes are python nodes, we simplify the parameter value and store it in - # one node. + # one node. If it is a node, we keep it. are_all_nodes_python_nodes_without_hash = all( isinstance(x, PythonNode) and not x.hash for x in tree_leaves(nodes) ) - if not isinstance(nodes, Node) and are_all_nodes_python_nodes_without_hash: + if not isinstance(nodes, PNode) and are_all_nodes_python_nodes_without_hash: dependencies[parameter_name] = PythonNode(value=value, name=parameter_name) else: dependencies[parameter_name] = nodes return dependencies -def _find_args_with_node_annotation(func: Callable[..., Any]) -> dict[str, Node]: +def _find_args_with_node_annotation(func: Callable[..., Any]) -> dict[str, PNode]: """Find args with node annotations.""" annotations = get_annotations(func, eval_str=True) metas = { @@ -341,7 +350,7 @@ def task_example(produces: Annotated[..., Product]): """ -def parse_products_from_task_function( +def parse_products_from_task_function( # noqa: C901 session: Session, path: Path, name: str, obj: Any ) -> dict[str, Any]: """Parse products from task function. @@ -393,24 +402,40 @@ def parse_products_from_task_function( if parameters_with_product_annot: has_annotation = True + for parameter_name in parameters_with_product_annot: - if parameter_name in kwargs: - partialed_evolve = functools.partial( - _evolve_instance, - instance_from_annot=parameters_with_node_annot.get(parameter_name), + if ( + parameter_name not in kwargs + and parameter_name not in parameters_with_node_annot + ): + continue + + if ( + parameter_name in kwargs + and parameter_name in parameters_with_node_annot + ): + msg = ( + f"The value for the parameter {name!r} is defined twice in " + "'@pytask.mark.task(kwargs=...)' and in the type annotation. " + "Choose only one option." ) + raise ValueError(msg) - collected_products = tree_map_with_path( - lambda p, x: _collect_product( - session, - path, - name, - NodeInfo(parameter_name, p, partialed_evolve(x)), # noqa: B023 - is_string_allowed=False, - ), - kwargs[parameter_name], - ) - out = {parameter_name: collected_products} + value = kwargs.get(parameter_name) or parameters_with_node_annot.get( + parameter_name + ) + + collected_products = tree_map_with_path( + lambda p, x: _collect_product( + session, + path, + name, + NodeInfo(parameter_name, p, x), # noqa: B023 + is_string_allowed=False, + ), + value, + ) + out = {parameter_name: collected_products} if "return" in parameters_with_node_annot: has_return = True @@ -490,7 +515,7 @@ def _find_args_with_product_annotation(func: Callable[..., Any]) -> list[str]: def _collect_decorator_node( session: Session, path: Path, name: str, node_info: NodeInfo -) -> Node: +) -> PNode: """Collect nodes for a task. Raises @@ -528,7 +553,7 @@ def _collect_decorator_node( def _collect_dependency( session: Session, path: Path, name: str, node_info: NodeInfo -) -> Node: +) -> PNode: """Collect nodes for a task. Raises @@ -556,7 +581,7 @@ def _collect_product( task_name: str, node_info: NodeInfo, is_string_allowed: bool = False, -) -> Node: +) -> PNode: """Collect products for a task. Defining products with strings is only allowed when using the decorator. Parameter @@ -593,19 +618,3 @@ def _collect_product( raise NodeNotCollectedError(msg) return collected_node - - -def _evolve_instance(x: Any, instance_from_annot: Node | None) -> Any: - """Evolve a value to a node if it is given by annotations.""" - if not instance_from_annot: - return x - - if not hasattr(instance_from_annot, "from_annot"): - msg = ( - f"The node {instance_from_annot!r} does not define '.from_annot' which is " - f"necessary to complete the node with the value {x!r}." - ) - raise AttributeError(msg) - - instance_from_annot.from_annot(x) # type: ignore[attr-defined] - return instance_from_annot diff --git a/src/_pytask/console.py b/src/_pytask/console.py index 7261533a..5bbea462 100644 --- a/src/_pytask/console.py +++ b/src/_pytask/console.py @@ -38,6 +38,8 @@ "console", "format_task_name", "format_strings_as_flat_tree", + "get_file", + "is_jupyter", "render_to_string", "unify_styles", ] @@ -179,7 +181,7 @@ def create_url_style_for_task( try: info = { - "path": _get_file(task_function), + "path": get_file(task_function), "line_number": _get_source_lines(task_function), } except (OSError, TypeError): @@ -200,7 +202,7 @@ def create_url_style_for_path(path: Path, edtior_url_scheme: str) -> Style: ) -def _get_file( +def get_file( function: Callable[..., Any], skipped_paths: list[Path] | None = None ) -> Path: """Get path to module where the function is defined. @@ -214,12 +216,12 @@ def _get_file( skipped_paths = _SKIPPED_PATHS if isinstance(function, functools.partial): - return _get_file(function.func) + return get_file(function.func) if ( hasattr(function, "__wrapped__") and Path(inspect.getsourcefile(function)) in skipped_paths ): - return _get_file(function.__wrapped__) + return get_file(function.__wrapped__) return Path(inspect.getsourcefile(function)) @@ -285,3 +287,26 @@ def create_summary_panel( if counts[outcome_enum.FAIL] else outcome_enum.SUCCESS.style, ) + + +def is_jupyter() -> bool: # pragma: no cover + """Check if we're running in a Jupyter notebook. + + Copied from rich. + + """ + try: + get_ipython # type: ignore[name-defined] # noqa: B018 + except NameError: + return False + ipython = get_ipython() # type: ignore[name-defined] # noqa: F821 + shell = ipython.__class__.__name__ + if ( + "google.colab" in str(ipython.__class__) + or os.getenv("DATABRICKS_RUNTIME_VERSION") + or shell == "ZMQInteractiveShell" + ): + return True # Jupyter notebook or qtconsole + if shell == "TerminalInteractiveShell": + return False # Terminal running IPython + return False # Other type (?) diff --git a/src/_pytask/dag.py b/src/_pytask/dag.py index fbc3cd66..1dfde8d6 100644 --- a/src/_pytask/dag.py +++ b/src/_pytask/dag.py @@ -24,7 +24,7 @@ from _pytask.mark_utils import get_marks from _pytask.mark_utils import has_mark from _pytask.node_protocols import MetaNode -from _pytask.node_protocols import Node +from _pytask.node_protocols import PNode from _pytask.node_protocols import PPathNode from _pytask.node_protocols import PTask from _pytask.node_protocols import PTaskWithPath @@ -242,7 +242,7 @@ def _check_if_root_nodes_are_available(dag: nx.DiGraph) -> None: def _check_if_tasks_are_skipped( - node: Node, dag: nx.DiGraph, is_task_skipped: dict[str, bool] + node: PNode, dag: nx.DiGraph, is_task_skipped: dict[str, bool] ) -> tuple[bool, dict[str, bool]]: """Check for a given node whether it is only used by skipped tasks.""" are_all_tasks_skipped = [] diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py index 294ae929..77391aa4 100644 --- a/src/_pytask/hookspecs.py +++ b/src/_pytask/hookspecs.py @@ -15,13 +15,12 @@ if TYPE_CHECKING: from _pytask.node_protocols import MetaNode from _pytask.models import NodeInfo - from _pytask.node_protocols import Node + from _pytask.node_protocols import PNode import click from _pytask.node_protocols import PTask import networkx as nx - import pathlib + from pathlib import Path from _pytask.session import Session - from _pytask.nodes import Task from _pytask.outcomes import CollectionOutcome from _pytask.outcomes import TaskOutcome from _pytask.reports import CollectionReport @@ -119,7 +118,7 @@ def pytask_collect(session: Session) -> Any: @hookspec(firstresult=True) -def pytask_ignore_collect(path: pathlib.Path, config: dict[str, Any]) -> bool: +def pytask_ignore_collect(path: Path, config: dict[str, Any]) -> bool: """Ignore collected path. This hook is indicates for each directory and file whether it should be ignored. @@ -139,7 +138,7 @@ def pytask_collect_modify_tasks(session: Session, tasks: list[PTask]) -> None: @hookspec(firstresult=True) def pytask_collect_file_protocol( - session: Session, path: pathlib.Path, reports: list[CollectionReport] + session: Session, path: Path, reports: list[CollectionReport] ) -> list[CollectionReport]: """Start protocol to collect files. @@ -151,7 +150,7 @@ def pytask_collect_file_protocol( @hookspec def pytask_collect_file( - session: Session, path: pathlib.Path, reports: list[CollectionReport] + session: Session, path: Path, reports: list[CollectionReport] ) -> list[CollectionReport] | None: """Collect tasks from a file. @@ -167,22 +166,22 @@ def pytask_collect_file_log(session: Session, reports: list[CollectionReport]) - @hookspec(firstresult=True) def pytask_collect_task_protocol( - session: Session, path: pathlib.Path, name: str, obj: Any + session: Session, path: Path | None, name: str, obj: Any ) -> CollectionReport | None: """Start protocol to collect tasks.""" @hookspec def pytask_collect_task_setup( - session: Session, path: pathlib.Path, name: str, obj: Any + session: Session, path: Path | None, name: str, obj: Any ) -> None: """Steps before collecting a task.""" @hookspec(firstresult=True) def pytask_collect_task( - session: Session, path: pathlib.Path, name: str, obj: Any -) -> Task: + session: Session, path: Path | None, name: str, obj: Any +) -> PTask: """Collect a single task.""" @@ -197,8 +196,8 @@ def pytask_collect_task_teardown(session: Session, task: PTask) -> None: @hookspec(firstresult=True) def pytask_collect_node( - session: Session, path: pathlib.Path, node_info: NodeInfo -) -> Node | None: + session: Session, path: Path, node_info: NodeInfo +) -> PNode | None: """Collect a node which is a dependency or a product of a task.""" diff --git a/src/_pytask/mark/__init__.py b/src/_pytask/mark/__init__.py index 9cd1184b..afb3badc 100644 --- a/src/_pytask/mark/__init__.py +++ b/src/_pytask/mark/__init__.py @@ -50,7 +50,7 @@ def markers(**raw_config: Any) -> NoReturn: raw_config["command"] = "markers" try: - # Duplication of the same mechanism in :func:`pytask.main.main`. + # Duplication of the same mechanism in :func:`pytask.build`. pm = get_plugin_manager() from _pytask import cli diff --git a/src/_pytask/node_protocols.py b/src/_pytask/node_protocols.py index c2328b7b..8eee22c9 100644 --- a/src/_pytask/node_protocols.py +++ b/src/_pytask/node_protocols.py @@ -14,11 +14,14 @@ from _pytask.mark import Mark +__all__ = ["MetaNode", "PNode", "PPathNode", "PTask", "PTaskWithPath"] + + @runtime_checkable class MetaNode(Protocol): """Protocol for an intersection between nodes and tasks.""" - name: str | None + name: str """The name of node that must be unique.""" @abstractmethod @@ -33,7 +36,7 @@ def state(self) -> str | None: @runtime_checkable -class Node(MetaNode, Protocol): +class PNode(MetaNode, Protocol): """Protocol for nodes.""" def load(self) -> Any: @@ -46,7 +49,7 @@ def save(self, value: Any) -> Any: @runtime_checkable -class PPathNode(Node, Protocol): +class PPathNode(PNode, Protocol): """Nodes with paths. Nodes with paths receive special handling when it comes to printing their names. @@ -61,8 +64,8 @@ class PTask(MetaNode, Protocol): """Protocol for nodes.""" name: str - depends_on: PyTree[Node] - produces: PyTree[Node] + depends_on: PyTree[PNode] + produces: PyTree[PNode] markers: list[Mark] report_sections: list[tuple[str, str, str]] attributes: dict[Any, Any] diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py index 831542ee..a82f42a1 100644 --- a/src/_pytask/nodes.py +++ b/src/_pytask/nodes.py @@ -1,16 +1,18 @@ -"""Deals with nodes which are dependencies or products of a task.""" +"""Contains implementations of tasks and nodes following the node protocols.""" from __future__ import annotations import functools import hashlib -from pathlib import Path +import inspect +from pathlib import Path # noqa: TCH003 from typing import Any from typing import Callable from typing import TYPE_CHECKING -from _pytask.node_protocols import MetaNode -from _pytask.node_protocols import Node +from _pytask.node_protocols import PNode from _pytask.node_protocols import PPathNode +from _pytask.node_protocols import PTask +from _pytask.node_protocols import PTaskWithPath from attrs import define from attrs import field @@ -20,25 +22,55 @@ from _pytask.mark import Mark -__all__ = ["PathNode", "Product", "Task"] +__all__ = ["PathNode", "PythonNode", "Task", "TaskWithoutPath"] -@define(frozen=True) -class ProductType: - """A class to mark products.""" +@define(kw_only=True) +class TaskWithoutPath(PTask): + """The class for tasks without a source file. + + Tasks may have no source file because + - they are dynamically created in a REPL. + - they are created in a Jupyter notebook. + + """ + name: str + """The base name of the task.""" + function: Callable[..., Any] + """The task function.""" + depends_on: PyTree[PNode] = field(factory=dict) + """A list of dependencies of task.""" + produces: PyTree[PNode] = field(factory=dict) + """A list of products of task.""" + markers: list[Mark] = field(factory=list) + """A list of markers attached to the task function.""" + report_sections: list[tuple[str, str, str]] = field(factory=list) + """Reports with entries for when, what, and content.""" + attributes: dict[Any, Any] = field(factory=dict) + """A dictionary to store additional information of the task.""" -Product = ProductType() -"""ProductType: A singleton to mark products in annotations.""" + def state(self) -> str | None: + """Return the state of the node.""" + try: + source = inspect.getsource(self.function) + except OSError: + return None + else: + return hashlib.sha256(source.encode()).hexdigest() + + def execute(self, **kwargs: Any) -> None: + """Execute the task.""" + return self.function(**kwargs) @define(kw_only=True) -class Task(MetaNode): +class Task(PTaskWithPath): """The class for tasks which are Python functions.""" base_name: str """The base name of the task.""" - path: Path + path: Path | None """Path to the file where the task was defined.""" function: Callable[..., Any] """The task function.""" @@ -46,9 +78,9 @@ class Task(MetaNode): """The name of the task.""" display_name: str | None = field(default=None, init=False) """The shortest uniquely identifiable name for task for display.""" - depends_on: PyTree[Node] = field(factory=dict) + depends_on: PyTree[PNode] = field(factory=dict) """A list of dependencies of task.""" - produces: PyTree[Node] = field(factory=dict) + produces: PyTree[PNode] = field(factory=dict) """A list of products of task.""" markers: list[Mark] = field(factory=list) """A list of markers attached to the task function.""" @@ -60,7 +92,10 @@ class Task(MetaNode): def __attrs_post_init__(self: Task) -> None: """Change class after initialization.""" if self.name is None: - self.name = self.path.as_posix() + "::" + self.base_name + if self.path is None: + self.name = self.base_name + else: + self.name = self.path.as_posix() + "::" + self.base_name if self.display_name is None: self.display_name = self.name @@ -80,30 +115,11 @@ def execute(self, **kwargs: Any) -> None: class PathNode(PPathNode): """The class for a node which is a path.""" - name: str = "" + name: str """Name of the node which makes it identifiable in the DAG.""" - path: Path | None = None + path: Path """The path to the file.""" - def from_annot(self, value: Path) -> None: - """Set path and if other attributes are not set, set sensible defaults. - - Use it, if you want to control the name of the node. - - .. codeblock: python - - def task_example(value: Annotated[Any, PathNode(name="value")]): - ... - - - """ - if not isinstance(value, Path): - msg = "'value' must be a 'pathlib.Path'." - raise TypeError(msg) - if not self.name: - self.name = value.as_posix() - self.path = value - @classmethod @functools.lru_cache def from_path(cls, path: Path) -> PathNode: @@ -143,7 +159,7 @@ def save(self, value: bytes | str) -> None: @define(kw_only=True) -class PythonNode(Node): +class PythonNode(PNode): """The class for a node which is a Python object.""" name: str = "" @@ -161,21 +177,6 @@ def save(self, value: Any) -> None: """Save the value.""" self.value = value - def from_annot(self, value: Any) -> None: - """Set the value from a function annotation. - - Use it, if you want to add information on how a node handles an argument while - keeping the type of the value unrelated to pytask. For example, the node could - be hashed. - - .. codeblock: python - - def task_example(value: Annotated[Any, PythonNode(hash=True)]): - ... - - """ - self.value = value - def state(self) -> str | None: """Calculate state of the node. diff --git a/src/_pytask/parameters.py b/src/_pytask/parameters.py index 32baae42..833be73b 100644 --- a/src/_pytask/parameters.py +++ b/src/_pytask/parameters.py @@ -36,8 +36,8 @@ type=str, multiple=True, help=( - "A pattern to ignore files or directories. Refer to pathlib.Path.match for " - "more info." + "A pattern to ignore files or directories. Refer to 'pathlib.Path.match' " + "for more info." ), default=[], ) diff --git a/src/_pytask/path.py b/src/_pytask/path.py index 71c80b50..6a913fb9 100644 --- a/src/_pytask/path.py +++ b/src/_pytask/path.py @@ -10,9 +10,7 @@ from typing import Sequence -def relative_to( - path: str | Path, source: str | Path, include_source: bool = True -) -> Path: +def relative_to(path: Path, source: Path, include_source: bool = True) -> Path: """Make a path relative to another path. In contrast to :meth:`pathlib.Path.relative_to`, this function allows to keep the @@ -23,23 +21,21 @@ def relative_to( The default behavior of :mod:`pathlib` is to exclude the source path from the relative path. - >>> relative_to("folder/file.py", "folder", False).as_posix() + >>> relative_to(Path("folder/file.py"), Path("folder"), False).as_posix() 'file.py' To provide relative locations to users, it is sometimes more helpful to provide the source as an orientation. - >>> relative_to("folder/file.py", "folder").as_posix() + >>> relative_to(Path("folder/file.py"), Path("folder")).as_posix() 'folder/file.py' """ - source_name = Path(source).name if include_source else "" - return Path(source_name, Path(path).relative_to(source)) + source_name = source.name if include_source else "" + return Path(source_name, path.relative_to(source)) -def find_closest_ancestor( - path: str | Path, potential_ancestors: Sequence[str | Path] -) -> Path: +def find_closest_ancestor(path: Path, potential_ancestors: Sequence[Path]) -> Path: """Find the closest ancestor of a path. In case a path is the path to the task file itself, we return the path. @@ -60,14 +56,8 @@ def find_closest_ancestor( 'folder/subfolder' """ - if isinstance(path, str): - path = Path(path) - closest_ancestor = None for ancestor in potential_ancestors: - if isinstance(ancestor, str): - ancestor = Path(ancestor) # noqa: PLW2901 - if ancestor == path: closest_ancestor = path break @@ -90,11 +80,11 @@ def find_closest_ancestor( def find_common_ancestor_of_nodes(*names: str) -> Path: """Find the common ancestor from task names and nodes.""" - cleaned_names = [name.split("::")[0] for name in names] + cleaned_names = [Path(name.split("::")[0]) for name in names] return find_common_ancestor(*cleaned_names) -def find_common_ancestor(*paths: str | Path) -> Path: +def find_common_ancestor(*paths: Path) -> Path: """Find a common ancestor of many paths.""" return Path(os.path.commonpath(paths)) diff --git a/src/_pytask/profile.py b/src/_pytask/profile.py index 0dc5c079..5fc2f7ce 100644 --- a/src/_pytask/profile.py +++ b/src/_pytask/profile.py @@ -115,7 +115,7 @@ def profile(**raw_config: Any) -> NoReturn: raw_config["command"] = "profile" try: - # Duplication of the same mechanism in :func:`pytask.main.main`. + # Duplication of the same mechanism in :func:`pytask.build`. pm = get_plugin_manager() from _pytask import cli diff --git a/src/_pytask/session.py b/src/_pytask/session.py index 1b6e4f09..6f80e45a 100644 --- a/src/_pytask/session.py +++ b/src/_pytask/session.py @@ -16,7 +16,7 @@ import networkx as nx from _pytask.report import CollectionReport from _pytask.report import ExecutionReport - from _ptytask.report import DagReport + from _pytask.report import DagReport @define diff --git a/src/_pytask/shared.py b/src/_pytask/shared.py index f07a1b22..2685a5b6 100644 --- a/src/_pytask/shared.py +++ b/src/_pytask/shared.py @@ -63,7 +63,7 @@ def parse_paths(x: Any | None) -> list[Path] | None: return out -def reduce_node_name(node: MetaNode, paths: Sequence[str | Path]) -> str: +def reduce_node_name(node: MetaNode, paths: Sequence[Path]) -> str: """Reduce the node name. The whole name of the node - which includes the drive letter - can be very long @@ -86,7 +86,7 @@ def reduce_node_name(node: MetaNode, paths: Sequence[str | Path]) -> str: def reduce_names_of_multiple_nodes( - names: list[str], dag: nx.DiGraph, paths: Sequence[str | Path] + names: list[str], dag: nx.DiGraph, paths: Sequence[Path] ) -> list[str]: """Reduce the names of multiple nodes in the DAG.""" short_names = [] diff --git a/src/_pytask/typing.py b/src/_pytask/typing.py new file mode 100644 index 00000000..e568c7ee --- /dev/null +++ b/src/_pytask/typing.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from attr import define + + +__all__ = ["Product", "ProductType"] + + +@define(frozen=True) +class ProductType: + """A class to mark products.""" + + +Product = ProductType() +"""ProductType: A singleton to mark products in annotations.""" diff --git a/src/pytask/__init__.py b/src/pytask/__init__.py index d626d4dd..4c4f255a 100644 --- a/src/pytask/__init__.py +++ b/src/pytask/__init__.py @@ -2,7 +2,7 @@ from __future__ import annotations from _pytask import __version__ -from _pytask.build import main +from _pytask.build import build from _pytask.click import ColoredCommand from _pytask.click import ColoredGroup from _pytask.click import EnumChoice @@ -37,8 +37,11 @@ from _pytask.models import CollectionMetadata from _pytask.models import NodeInfo from _pytask.node_protocols import MetaNode +from _pytask.node_protocols import PNode +from _pytask.node_protocols import PPathNode +from _pytask.node_protocols import PTask +from _pytask.node_protocols import PTaskWithPath from _pytask.nodes import PathNode -from _pytask.nodes import Product from _pytask.nodes import PythonNode from _pytask.nodes import Task from _pytask.outcomes import CollectionOutcome @@ -59,6 +62,7 @@ from _pytask.traceback import remove_internal_traceback_frames_from_exc_info from _pytask.traceback import remove_traceback_from_exc_info from _pytask.traceback import render_exc_info +from _pytask.typing import Product from _pytask.warnings_utils import parse_warning_filter from _pytask.warnings_utils import warning_record_to_str from _pytask.warnings_utils import WarningReport @@ -92,8 +96,13 @@ "NodeInfo", "NodeNotCollectedError", "NodeNotFoundError", + "PathNode", "Persisted", + "PNode", + "PPathNode", "Product", + "PTask", + "PTaskWithPath", "PytaskError", "PythonNode", "ResolvingDependenciesError", @@ -107,6 +116,7 @@ "TaskOutcome", "WarningReport", "__version__", + "build", "build_dag", "check_for_optional_program", "cli", @@ -120,7 +130,6 @@ "has_mark", "hookimpl", "import_optional_dependency", - "main", "mark", "parse_nodes", "parse_warning_filter", diff --git a/tests/test_capture.py b/tests/test_capture.py index 0cda066a..b078ab9b 100644 --- a/tests/test_capture.py +++ b/tests/test_capture.py @@ -13,9 +13,9 @@ import pytest from _pytask import capture -from _pytask.capture import _CaptureMethod from _pytask.capture import _get_multicapture from _pytask.capture import CaptureManager +from _pytask.capture import CaptureMethod from _pytask.capture import CaptureResult from _pytask.capture import MultiCapture from pytask import cli @@ -99,7 +99,7 @@ def TeeStdCapture( # noqa: N802 @pytest.mark.end_to_end() class TestCaptureManager: @pytest.mark.parametrize( - "method", [_CaptureMethod.NO, _CaptureMethod.SYS, _CaptureMethod.FD] + "method", [CaptureMethod.NO, CaptureMethod.SYS, CaptureMethod.FD] ) def test_capturing_basic_api(self, method): capouter = StdCaptureFD() @@ -116,7 +116,7 @@ def test_capturing_basic_api(self, method): print("hello") capman.suspend() out, err = capman.read() - if method == _CaptureMethod.NO: + if method == CaptureMethod.NO: assert old == (sys.stdout, sys.stderr, sys.stdin) else: assert not out @@ -124,7 +124,7 @@ def test_capturing_basic_api(self, method): print("hello") capman.suspend() out, err = capman.read() - if method != _CaptureMethod.NO: + if method != CaptureMethod.NO: assert out == "hello\n" capman.stop_capturing() finally: @@ -133,7 +133,7 @@ def test_capturing_basic_api(self, method): def test_init_capturing(self): capouter = StdCaptureFD() try: - capman = CaptureManager(_CaptureMethod.FD) + capman = CaptureManager(CaptureMethod.FD) capman.start_capturing() pytest.raises(AssertionError, capman.start_capturing) capman.stop_capturing() @@ -750,7 +750,7 @@ def test_fdcapture_invalid_fd_without_fd_reuse(self, tmp_path): @pytest.mark.unit() def test__get_multicapture() -> None: - assert isinstance(_get_multicapture(_CaptureMethod.NO), MultiCapture) + assert isinstance(_get_multicapture(CaptureMethod.NO), MultiCapture) pytest.raises(ValueError, _get_multicapture, "unknown").match( r"^unknown capturing method: 'unknown'" ) diff --git a/tests/test_collect.py b/tests/test_collect.py index 0c56db3c..ee1fa07c 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -10,10 +10,10 @@ from _pytask.collect import pytask_collect_node from _pytask.exceptions import NodeNotCollectedError from _pytask.models import NodeInfo +from pytask import build from pytask import cli from pytask import CollectionOutcome from pytask import ExitCode -from pytask import main from pytask import Session from pytask import Task @@ -31,7 +31,7 @@ def task_write_text(depends_on, produces): tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").write_text("Relative paths work.") - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.collection_reports[0].outcome == CollectionOutcome.SUCCESS assert tmp_path.joinpath("out.txt").read_text() == "Relative paths work." @@ -49,7 +49,7 @@ def task_with_non_path_dependency(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.COLLECTION_FAILED assert session.collection_reports[0].outcome == CollectionOutcome.FAIL @@ -70,7 +70,7 @@ def task_with_non_path_dependency(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.COLLECTION_FAILED assert session.collection_reports[0].outcome == CollectionOutcome.FAIL @@ -114,7 +114,7 @@ def task_1(depends_on, produces): def test_collect_same_task_different_ways(tmp_path, path_extension): tmp_path.joinpath("task_module.py").write_text("def task_passes(): pass") - session = main({"paths": tmp_path.joinpath(path_extension)}) + session = build(paths=tmp_path.joinpath(path_extension)) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 1 @@ -141,7 +141,7 @@ def test_collect_files_w_custom_file_name_pattern( for file in task_files: tmp_path.joinpath(file).write_text("def task_example(): pass") - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert len(session.tasks) == expected_collected_tasks @@ -181,13 +181,12 @@ def test_pytask_collect_node(session, path, node_info, expected): ) def test_pytask_collect_node_raises_error_if_path_is_not_correctly_cased(tmp_path): session = Session({"check_casing_of_paths": True}, None) - task_path = tmp_path / "task_example.py" real_node = tmp_path / "text.txt" real_node.touch() collected_node = tmp_path / "TeXt.TxT" with pytest.raises(Exception, match="The provided path of"): - pytask_collect_node(session, task_path, NodeInfo("", (), collected_node)) + pytask_collect_node(session, tmp_path, NodeInfo("", (), collected_node)) @pytest.mark.unit() @@ -196,7 +195,6 @@ def test_pytask_collect_node_does_not_raise_error_if_path_is_not_normalized( tmp_path, is_absolute ): session = Session({"check_casing_of_paths": True}, None) - task_path = tmp_path / "task_example.py" real_node = tmp_path / "text.txt" collected_node = Path("..", tmp_path.name, "text.txt") @@ -205,7 +203,7 @@ def test_pytask_collect_node_does_not_raise_error_if_path_is_not_normalized( with warnings.catch_warnings(record=True) as record: result = pytask_collect_node( - session, task_path, NodeInfo("", (), collected_node) + session, tmp_path, NodeInfo("", (), collected_node) ) assert not record @@ -264,7 +262,7 @@ def task_example(path_in = Path("in.txt"), produces = Path("out.txt")): tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").write_text("hello") - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 1 @@ -278,7 +276,7 @@ def test_collect_tasks_from_modules_with_the_same_name(tmp_path): tmp_path.joinpath("b").mkdir() tmp_path.joinpath("a", "task_module.py").write_text("def task_a(): pass") tmp_path.joinpath("b", "task_module.py").write_text("def task_a(): pass") - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert len(session.collection_reports) == 2 assert all( report.outcome == CollectionOutcome.SUCCESS @@ -306,7 +304,7 @@ def task_my_task(): pass """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) outcome = session.collection_reports[0].outcome assert outcome == CollectionOutcome.SUCCESS @@ -368,7 +366,7 @@ def task_example( ... """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.COLLECTION_FAILED assert len(session.tasks) == 0 @@ -395,7 +393,7 @@ def task_example( tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("input_1.txt").touch() tmp_path.joinpath("input_2.txt").touch() - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.COLLECTION_FAILED assert len(session.tasks) == 0 @@ -430,16 +428,77 @@ def test_setting_name_for_path_node_via_annotation(tmp_path): from pathlib import Path from typing_extensions import Annotated from pytask import Product, PathNode - from typing import Any def task_example( - path: Annotated[Path, Product, PathNode(name="product")] = Path("out.txt"), + path: Annotated[Path, Product, PathNode(path=Path("out.txt"), name="product")], ) -> None: path.write_text("text") """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": [tmp_path]}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK product = session.tasks[0].produces["path"] assert product.name == "product" + + +@pytest.mark.end_to_end() +def test_error_when_dependency_is_defined_in_kwargs_and_annotation(runner, tmp_path): + source = """ + import pytask + from pathlib import Path + from typing_extensions import Annotated + from pytask import Product, PathNode + from pytask import PythonNode + + @pytask.mark.task(kwargs={"in_": "world"}) + def task_example( + in_: Annotated[str, PythonNode(name="string", value="hello")], + path: Annotated[Path, Product, PathNode(path=Path("out.txt"), name="product")], + ) -> None: + path.write_text(in_) + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert "ValueError: The value for the parameter 'in_'" in result.output + + +@pytest.mark.end_to_end() +def test_error_when_product_is_defined_in_kwargs_and_annotation(runner, tmp_path): + source = """ + import pytask + from pathlib import Path + from typing_extensions import Annotated + from pytask import Product, PathNode + + node = PathNode(path=Path("out.txt"), name="product") + + @pytask.mark.task(kwargs={"path": node}) + def task_example(path: Annotated[Path, Product, node]) -> None: + path.write_text("text") + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert "ValueError: The value for the parameter 'path'" in result.output + + +@pytest.mark.end_to_end() +def test_relative_path_of_path_node(runner, tmp_path): + source = """ + from pathlib import Path + from typing_extensions import Annotated + from pytask import Product, PathNode + + def task_example( + path: Annotated[Path, Product, PathNode(path=Path("out.txt"), name="product")], + ) -> None: + path.write_text("text") + """ + tmp_path.joinpath("subfolder").mkdir() + tmp_path.joinpath("subfolder", "task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert tmp_path.joinpath("subfolder", "out.txt").exists() diff --git a/tests/test_collect_command.py b/tests/test_collect_command.py index c0c16c46..86a48386 100644 --- a/tests/test_collect_command.py +++ b/tests/test_collect_command.py @@ -357,7 +357,7 @@ def function(depends_on, produces): # noqa: ARG001 @pytest.mark.unit() def test_print_collected_tasks_without_nodes(capsys): dictionary = { - "task_path.py": [ + Path("task_path.py"): [ Task( base_name="function", path=Path("task_path.py"), @@ -380,7 +380,7 @@ def test_print_collected_tasks_without_nodes(capsys): @pytest.mark.unit() def test_print_collected_tasks_with_nodes(capsys): dictionary = { - "task_path.py": [ + Path("task_path.py"): [ Task( base_name="function", path=Path("task_path.py"), @@ -529,7 +529,6 @@ def state(self): def load(self): ... def save(self, value): ... - def from_annot(self, value): ... def task_example( data = CustomNode("custom", "text"), @@ -556,7 +555,6 @@ def test_node_protocol_for_custom_nodes_with_paths(runner, tmp_path): class PickleFile: name: str path: Path - value: Path def state(self): return str(self.path.stat().st_mtime) @@ -570,13 +568,10 @@ def save(self, value): with self.path.open("wb") as f: pickle.dump(value, f) - def from_annot(self, value): ... - - _PATH = Path(__file__).parent.joinpath("in.pkl") def task_example( - data = PickleFile(_PATH.as_posix(), _PATH, _PATH), + data = PickleFile(_PATH.as_posix(), _PATH), out: Annotated[Path, Product] = Path("out.txt"), ) -> None: out.write_text(data) @@ -598,7 +593,7 @@ def test_setting_name_for_python_node_via_annotation(runner, tmp_path): from typing import Any def task_example( - input: Annotated[str, PythonNode(name="node-name")] = "text", + input: Annotated[str, PythonNode(name="node-name", value="text")], path: Annotated[Path, Product] = Path("out.txt"), ) -> None: path.write_text(input) diff --git a/tests/test_collect_utils.py b/tests/test_collect_utils.py index d2728023..19cd36b3 100644 --- a/tests/test_collect_utils.py +++ b/tests/test_collect_utils.py @@ -12,9 +12,9 @@ from _pytask.collect_utils import _find_args_with_product_annotation from _pytask.collect_utils import _merge_dictionaries from _pytask.collect_utils import _Placeholder +from _pytask.typing import Product # noqa: TCH002 from pytask import depends_on from pytask import produces -from pytask import Product from typing_extensions import Annotated # noqa: TCH002 diff --git a/tests/test_config.py b/tests/test_config.py index 130f73b1..ed45baec 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -3,13 +3,13 @@ import textwrap import pytest +from pytask import build from pytask import ExitCode -from pytask import main @pytest.mark.end_to_end() def test_debug_pytask(capsys, tmp_path): - session = main({"paths": tmp_path, "debug_pytask": True}) + session = build(paths=tmp_path, debug_pytask=True) assert session.exit_code == ExitCode.OK @@ -31,7 +31,7 @@ def test_pass_config_to_cli(tmp_path): """ tmp_path.joinpath("pyproject.toml").write_text(textwrap.dedent(config)) - session = main({"config": tmp_path.joinpath("pyproject.toml"), "paths": tmp_path}) + session = build(config=tmp_path.joinpath("pyproject.toml"), paths=tmp_path) assert session.exit_code == ExitCode.OK assert "elton" in session.config["markers"] @@ -55,7 +55,7 @@ def test_passing_paths_via_configuration_file(tmp_path, file_or_folder): "def task_passes(): pass" ) - session = main({"config": tmp_path.joinpath("pyproject.toml")}) + session = build(config=tmp_path.joinpath("pyproject.toml")) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 1 diff --git a/tests/test_console.py b/tests/test_console.py index f4ba0048..22b61b84 100644 --- a/tests/test_console.py +++ b/tests/test_console.py @@ -4,12 +4,12 @@ from pathlib import Path import pytest -from _pytask.console import _get_file from _pytask.console import _get_source_lines from _pytask.console import create_summary_panel from _pytask.console import create_url_style_for_path from _pytask.console import create_url_style_for_task from _pytask.console import format_task_name +from _pytask.console import get_file from _pytask.console import render_to_string from pytask import CollectionOutcome from pytask import console @@ -163,7 +163,7 @@ def test_format_task_id( ], ) def test_get_file(task_func, skipped_paths, expected): - result = _get_file(task_func, skipped_paths) + result = get_file(task_func, skipped_paths) assert result == expected diff --git a/tests/test_debugging.py b/tests/test_debugging.py index 6687bd0d..aece1cd7 100644 --- a/tests/test_debugging.py +++ b/tests/test_debugging.py @@ -473,7 +473,7 @@ def test_set_trace_is_returned_after_pytask_finishes(tmp_path): import pytask def test_function(): - pytask.main({{"paths": "{tmp_path.as_posix()}"}}) + pytask.build(paths={tmp_path.as_posix()!r}) breakpoint() """ tmp_path.joinpath("test_example.py").write_text(textwrap.dedent(source)) diff --git a/tests/test_execute.py b/tests/test_execute.py index 442c471a..7848b62a 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -8,12 +8,13 @@ import textwrap from pathlib import Path +import pytask import pytest -from _pytask.capture import _CaptureMethod +from _pytask.capture import CaptureMethod from _pytask.exceptions import NodeNotFoundError +from pytask import build from pytask import cli from pytask import ExitCode -from pytask import main from pytask import TaskOutcome @@ -46,7 +47,7 @@ def task_example(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.FAILED assert len(session.execution_reports) == 1 @@ -103,7 +104,7 @@ def task_3(depends_on): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.FAILED assert sum(i.outcome == TaskOutcome.SUCCESS for i in session.execution_reports) == 2 @@ -138,7 +139,7 @@ def task_example(depends_on, produces): for dependency in dependencies: tmp_path.joinpath(dependency).touch() - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK @@ -157,7 +158,7 @@ def task_example(depends_on, produces): tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").write_text("Here I am. Once again.") - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert tmp_path.joinpath("out.txt").read_text() == "Here I am. Once again." @@ -243,7 +244,7 @@ def task_example(depends_on, produces): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK @@ -257,7 +258,7 @@ def task_3(): raise Exception """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path, "max_failures": n_failures}) + session = build(paths=tmp_path, max_failures=n_failures) assert len(session.tasks) == 3 assert len(session.execution_reports) == n_failures @@ -273,9 +274,7 @@ def task_3(): raise Exception """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main( - {"paths": tmp_path, "stop_after_first_failure": stop_after_first_failure} - ) + session = build(paths=tmp_path, stop_after_first_failure=stop_after_first_failure) assert len(session.tasks) == 3 assert len(session.execution_reports) == 1 if stop_after_first_failure else 3 @@ -296,7 +295,7 @@ def task_y(): pass """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert session.execution_reports[0].task.name.endswith("task_z") @@ -390,9 +389,9 @@ def task_example(): @pytest.mark.end_to_end() def test_task_executed_with_force_although_unchanged(tmp_path): tmp_path.joinpath("task_module.py").write_text("def task_example(): pass") - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.execution_reports[0].outcome == TaskOutcome.SUCCESS - session = main({"paths": tmp_path, "force": True}) + session = build(paths=tmp_path, force=True) assert session.execution_reports[0].outcome == TaskOutcome.SUCCESS @@ -436,7 +435,7 @@ def task_example(path_to_file: Annotated[Path, Product] = Path("out.txt")) -> No """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path, "capture": _CaptureMethod.NO}) + session = build(paths=tmp_path, capture=CaptureMethod.NO) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 1 @@ -459,7 +458,7 @@ def task_example( """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path, "capture": _CaptureMethod.NO}) + session = build(paths=tmp_path, capture=CaptureMethod.NO) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 1 @@ -472,7 +471,7 @@ def task_example( "definition", [ " = PythonNode(value=data['dependency'], hash=True)", - ": Annotated[Any, PythonNode(hash=True)] = data['dependency']", + ": Annotated[Any, PythonNode(value=data['dependency'], hash=True)]", ], ) def test_task_with_hashed_python_node(runner, tmp_path, definition): @@ -534,8 +533,10 @@ def test_error_with_multiple_different_dep_annotations(runner, tmp_path): from pytask import Product, PythonNode, PathNode from typing import Any + annotation = Annotated[Any, PythonNode(), PathNode(name="a", path=Path("a.txt"))] + def task_example( - dependency: Annotated[Any, PythonNode(), PathNode()] = "hello", + dependency: annotation = "hello", path: Annotated[Path, Product] = Path("out.txt") ) -> None: path.write_text(dependency) @@ -617,9 +618,8 @@ def test_return_with_custom_type_annotation_as_return(runner, tmp_path): @attrs.define class PickleNode: - name: str = "" - path: Path | None = None - value: None = None + name: str + path: Path def state(self) -> str | None: if self.path.exists(): @@ -632,8 +632,6 @@ def load(self) -> Any: def save(self, value: Any) -> None: self.path.write_bytes(pickle.dumps(value)) - def from_annot(self, value: Any) -> None: ... - node = PickleNode("pickled_data", Path(__file__).parent.joinpath("data.pkl")) def task_example() -> Annotated[int, node]: @@ -706,3 +704,70 @@ def task_example() -> Annotated[Dict[str, str], nodes]: tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) result = runner.invoke(cli, [tmp_path.as_posix()]) assert result.exit_code == ExitCode.OK + + +@pytest.mark.end_to_end() +def test_execute_tasks_and_pass_values_only_by_python_nodes(runner, tmp_path): + source = """ + from _pytask.nodes import PathNode + from pytask import PythonNode + from typing_extensions import Annotated + from pathlib import Path + + + node_text = PythonNode(name="text") + + + def task_create_text() -> Annotated[int, node_text]: + return "This is the text." + + node_file = PathNode.from_path(Path(__file__).parent.joinpath("file.txt")) + + def task_create_file(text: Annotated[int, node_text]) -> Annotated[str, node_file]: + return text + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert tmp_path.joinpath("file.txt").read_text() == "This is the text." + + +@pytest.mark.end_to_end() +@pytest.mark.xfail(sys.platform == "win32", reason="Decoding issues in Gitlab Actions.") +def test_execute_tasks_via_functional_api(tmp_path): + source = """ + from pytask import PathNode + import pytask + from pytask import PythonNode + from typing_extensions import Annotated + from pathlib import Path + + + node_text = PythonNode(name="text", hash=True) + + def create_text() -> Annotated[int, node_text]: + return "This is the text." + + node_file = PathNode.from_path(Path(__file__).parent.joinpath("file.txt")) + + def create_file(text: Annotated[int, node_text]) -> Annotated[str, node_file]: + return text + + if __name__ == "__main__": + session = pytask.build(tasks=[create_file, create_text]) + + assert len(session.tasks) == 2 + assert len(session.dag.nodes) == 4 + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + result = subprocess.run( + ("python", tmp_path.joinpath("task_module.py").as_posix()), check=False + ) + assert result.returncode == ExitCode.OK + assert tmp_path.joinpath("file.txt").read_text() == "This is the text." + + +@pytest.mark.end_to_end() +def test_pass_non_task_to_functional_api_that_are_ignored(): + session = pytask.build(tasks=None) + assert len(session.tasks) == 0 diff --git a/tests/test_ignore.py b/tests/test_ignore.py index 7c95483d..48c5af53 100644 --- a/tests/test_ignore.py +++ b/tests/test_ignore.py @@ -5,8 +5,8 @@ import pytest from _pytask.collect import pytask_ignore_collect from _pytask.config import _IGNORED_FOLDERS +from pytask import build from pytask import ExitCode -from pytask import main @pytest.mark.end_to_end() @@ -16,7 +16,7 @@ def test_ignore_default_paths(tmp_path, ignored_folder): tmp_path.joinpath(folder).mkdir() tmp_path.joinpath(folder, "task_module.py").write_text("def task_d(): pass") - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 0 @@ -32,7 +32,7 @@ def test_ignore_paths(tmp_path, ignore, new_line): ) tmp_path.joinpath("pyproject.toml").write_text(config) - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 0 if ignore else len(session.tasks) == 1 diff --git a/tests/test_jupyter/test_functional_interface.ipynb b/tests/test_jupyter/test_functional_interface.ipynb new file mode 100644 index 00000000..2b7be985 --- /dev/null +++ b/tests/test_jupyter/test_functional_interface.ipynb @@ -0,0 +1,71 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "12bc75b1", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "\n", + "import pytask\n", + "\n", + "from pytask import ExitCode\n", + "from pytask import PathNode\n", + "from pytask import PythonNode\n", + "from typing_extensions import Annotated" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29ac7311", + "metadata": {}, + "outputs": [], + "source": [ + "node_text = PythonNode(name=\"text\", hash=True)\n", + "\n", + "def create_text() -> Annotated[int, node_text]:\n", + " return \"This is the text.\"\n", + "\n", + "node_file = PathNode.from_path(Path(\"file.txt\").resolve())\n", + "\n", + "def create_file(text: Annotated[int, node_text]) -> Annotated[str, node_file]:\n", + " return text" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "738c9418", + "metadata": {}, + "outputs": [], + "source": [ + "session = pytask.build(tasks=[create_file, create_text])\n", + "assert session.exit_code == ExitCode.OK" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/test_jupyter/test_functional_interface_w_relative_path.ipynb b/tests/test_jupyter/test_functional_interface_w_relative_path.ipynb new file mode 100644 index 00000000..d942576d --- /dev/null +++ b/tests/test_jupyter/test_functional_interface_w_relative_path.ipynb @@ -0,0 +1,71 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "12bc75b1", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "\n", + "import pytask\n", + "\n", + "from pytask import ExitCode\n", + "from pytask import PathNode\n", + "from pytask import PythonNode\n", + "from typing_extensions import Annotated" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29ac7311", + "metadata": {}, + "outputs": [], + "source": [ + "node_text = PythonNode(name=\"text\", hash=True)\n", + "\n", + "def create_text() -> Annotated[int, node_text]:\n", + " return \"This is the text.\"\n", + "\n", + "node_file = PathNode(name=\"product\", path=Path(\"file.txt\"))\n", + "\n", + "def create_file(text: Annotated[int, node_text]) -> Annotated[str, node_file]:\n", + " return text" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "738c9418", + "metadata": {}, + "outputs": [], + "source": [ + "session = pytask.build(tasks=[create_file, create_text])\n", + "assert session.exit_code == ExitCode.OK" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/test_mark.py b/tests/test_mark.py index a1fb7773..f9ccf93a 100644 --- a/tests/test_mark.py +++ b/tests/test_mark.py @@ -6,9 +6,9 @@ import pytask import pytest +from pytask import build from pytask import cli from pytask import ExitCode -from pytask import main from pytask import MarkGenerator @@ -109,7 +109,7 @@ def task_two(): """ ) ) - session = main({"paths": tmp_path, "marker_expression": expr}) + session = build(paths=tmp_path, marker_expression=expr) tasks_that_run = [ report.task.name.rsplit("::")[1] @@ -152,7 +152,7 @@ def task_no_2(): """ ) ) - session = main({"paths": tmp_path, "expression": expr}) + session = build(paths=tmp_path, expression=expr) assert session.exit_code == ExitCode.OK tasks_that_run = [ @@ -184,7 +184,7 @@ def task_func(arg=arg): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path, "expression": expr}) + session = build(paths=tmp_path, expression=expr) assert session.exit_code == ExitCode.OK tasks_that_run = [ @@ -233,7 +233,7 @@ def test_keyword_option_wrong_arguments( tmp_path.joinpath("task_module.py").write_text( textwrap.dedent("def task_func(arg): pass") ) - session = main({"paths": tmp_path, option: expr}) + session = build(paths=tmp_path, **{option: expr}) assert session.exit_code == ExitCode.DAG_FAILED captured = capsys.readouterr() diff --git a/tests/test_mark_cli.py b/tests/test_mark_cli.py index c1754f27..23c0f54b 100644 --- a/tests/test_mark_cli.py +++ b/tests/test_mark_cli.py @@ -3,9 +3,9 @@ import textwrap import pytest +from pytask import build from pytask import cli from pytask import ExitCode -from pytask import main @pytest.mark.end_to_end() @@ -48,5 +48,5 @@ def test_marker_names(tmp_path, marker_name): markers = ['{marker_name}'] """ tmp_path.joinpath("pyproject.toml").write_text(textwrap.dedent(toml)) - session = main({"paths": tmp_path, "markers": True}) + session = build(paths=tmp_path, markers=True) assert session.exit_code == ExitCode.CONFIGURATION_FAILED diff --git a/tests/test_node_protocols.py b/tests/test_node_protocols.py index 02ce9867..6fafd14f 100644 --- a/tests/test_node_protocols.py +++ b/tests/test_node_protocols.py @@ -28,9 +28,6 @@ def load(self): def save(self, value): self.value = value - def from_annot(self, value): ... - - def task_example( data = CustomNode("custom", "text"), out: Annotated[Path, Product] = Path("out.txt"), @@ -70,9 +67,6 @@ def save(self, value): with self.path.open("wb") as f: pickle.dump(value, f) - def from_annot(self, value): ... - - _PATH = Path(__file__).parent.joinpath("in.pkl") def task_example( diff --git a/tests/test_path.py b/tests/test_path.py index e55be2f7..a1cbe3cf 100644 --- a/tests/test_path.py +++ b/tests/test_path.py @@ -23,7 +23,6 @@ @pytest.mark.parametrize( ("path", "source", "include_source", "expected"), [ - ("src/hello.py", "src", True, Path("src/hello.py")), (Path("src/hello.py"), Path("src"), True, Path("src/hello.py")), (Path("src/hello.py"), Path("src"), False, Path("hello.py")), ], @@ -37,7 +36,6 @@ def test_relative_to(path, source, include_source, expected): @pytest.mark.parametrize( ("path", "potential_ancestors", "expected"), [ - ("src/task.py", ["src", "bld"], Path("src")), (Path("src/task.py"), [Path("src"), Path("bld")], Path("src")), (Path("tasks/task.py"), [Path("src"), Path("bld")], None), (Path("src/ts/task.py"), [Path("src"), Path("src/ts")], Path("src/ts")), diff --git a/tests/test_persist.py b/tests/test_persist.py index 07115584..9dd3f4f1 100644 --- a/tests/test_persist.py +++ b/tests/test_persist.py @@ -5,10 +5,10 @@ import pytest from _pytask.database_utils import State from _pytask.persist import pytask_execute_task_process_report +from pytask import build from pytask import create_database from pytask import DatabaseSession from pytask import ExitCode -from pytask import main from pytask import Persisted from pytask import SkippedUnchanged from pytask import TaskOutcome @@ -20,7 +20,7 @@ class DummyClass: @pytest.mark.end_to_end() def test_persist_marker_is_set(tmp_path): - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert "persist" in session.config["markers"] @@ -46,7 +46,7 @@ def task_dummy(depends_on, produces): tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").write_text("I'm not the reason you care.") - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert len(session.execution_reports) == 1 @@ -56,7 +56,7 @@ def task_dummy(depends_on, produces): tmp_path.joinpath("out.txt").write_text("Never again in despair.") - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert len(session.execution_reports) == 1 @@ -72,7 +72,7 @@ def task_dummy(depends_on, produces): modification_time = session.get(State, (task_id, node_id)).modification_time assert float(modification_time) == tmp_path.joinpath("out.txt").stat().st_mtime - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert len(session.execution_reports) == 1 @@ -97,7 +97,7 @@ def task_dummy(depends_on, produces): "They say oh my god I see the way you shine." ) - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert len(session.execution_reports) == 1 diff --git a/tests/test_profile.py b/tests/test_profile.py index fe241aa5..a516e0e8 100644 --- a/tests/test_profile.py +++ b/tests/test_profile.py @@ -8,10 +8,10 @@ from _pytask.cli import cli from _pytask.profile import _to_human_readable_size from _pytask.profile import Runtime +from pytask import build from pytask import create_database from pytask import DatabaseSession from pytask import ExitCode -from pytask import main @pytest.mark.end_to_end() @@ -22,7 +22,7 @@ def task_example(): time.sleep(2) """ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 1 diff --git a/tests/test_shared.py b/tests/test_shared.py index 9383b79c..2abbcfe9 100644 --- a/tests/test_shared.py +++ b/tests/test_shared.py @@ -5,7 +5,7 @@ import pytest from _pytask.outcomes import ExitCode from _pytask.shared import find_duplicates -from pytask import main +from pytask import build @pytest.mark.unit() @@ -27,7 +27,7 @@ def test_parse_markers(tmp_path): """ tmp_path.joinpath("pyproject.toml").write_text(textwrap.dedent(toml)) - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert "a1" in session.config["markers"] diff --git a/tests/test_skipping.py b/tests/test_skipping.py index bb18d0fa..65384779 100644 --- a/tests/test_skipping.py +++ b/tests/test_skipping.py @@ -6,9 +6,9 @@ import pytest from _pytask.skipping import pytask_execute_task_setup +from pytask import build from pytask import cli from pytask import ExitCode -from pytask import main from pytask import Mark from pytask import Session from pytask import Skipped @@ -30,10 +30,10 @@ def task_dummy(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.execution_reports[0].outcome == TaskOutcome.SUCCESS - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert isinstance(session.execution_reports[0].exc_info[1], SkippedUnchanged) @@ -50,12 +50,12 @@ def task_dummy(depends_on, produces): tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").write_text("Original content of in.txt.") - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.execution_reports[0].outcome == TaskOutcome.SUCCESS assert tmp_path.joinpath("out.txt").read_text() == "Original content of in.txt." - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.execution_reports[0].outcome == TaskOutcome.SKIP_UNCHANGED assert isinstance(session.execution_reports[0].exc_info[1], SkippedUnchanged) @@ -77,7 +77,7 @@ def task_second(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.execution_reports[0].outcome == TaskOutcome.FAIL assert isinstance(session.execution_reports[0].exc_info[1], Exception) @@ -101,7 +101,7 @@ def task_second(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.execution_reports[0].outcome == TaskOutcome.SKIP assert isinstance(session.execution_reports[0].exc_info[1], Skipped) @@ -124,7 +124,7 @@ def task_first(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.execution_reports[0].outcome == TaskOutcome.SKIP assert isinstance(session.execution_reports[0].exc_info[1], Skipped) @@ -173,7 +173,7 @@ def task_second(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) node = session.collection_reports[0].node assert len(node.markers) == 1 assert node.markers[0].name == "skipif" @@ -203,7 +203,7 @@ def task_second(depends_on): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) node = session.collection_reports[0].node assert len(node.markers) == 1 @@ -234,7 +234,7 @@ def task_second(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) node = session.collection_reports[0].node assert len(node.markers) == 2 assert node.markers[0].name == "skipif" diff --git a/tests/test_task.py b/tests/test_task.py index 7bf9157b..228e1e3c 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -3,9 +3,9 @@ import textwrap import pytest +from pytask import build from pytask import cli from pytask import ExitCode -from pytask import main @pytest.mark.end_to_end() @@ -23,7 +23,7 @@ def {func_name}(produces): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK @@ -434,7 +434,7 @@ def task_func(i=i): pass """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.COLLECTION_FAILED assert isinstance(session.collection_reports[0].exc_info[1], ValueError) @@ -471,9 +471,7 @@ class Args(NamedTuple): @pytask.mark.task(kwargs=args) def task_example( - path_in: Path, - arg: Annotated[str, PythonNode(hash=True)], - path_out: Annotated[Path, Product] + path_in: Path, arg: str, path_out: Annotated[Path, Product] ) -> None: path_out.write_text(path_in.read_text() + " " + arg) """ diff --git a/tests/test_tree_util.py b/tests/test_tree_util.py index 2f39b1a7..107552cc 100644 --- a/tests/test_tree_util.py +++ b/tests/test_tree_util.py @@ -7,8 +7,8 @@ from _pytask.outcomes import ExitCode from _pytask.tree_util import tree_map from _pytask.tree_util import tree_structure +from pytask import build from pytask import cli -from pytask import main @pytest.mark.end_to_end() @@ -40,7 +40,7 @@ def task_example(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == exit_code diff --git a/tests/test_warnings.py b/tests/test_warnings.py index c5b2e763..a5a43f2b 100644 --- a/tests/test_warnings.py +++ b/tests/test_warnings.py @@ -5,9 +5,9 @@ import textwrap import pytest +from pytask import build from pytask import cli from pytask import ExitCode -from pytask import main @pytest.mark.end_to_end() @@ -46,7 +46,7 @@ def task_example(): """ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path, "disable_warnings": disable_warnings}) + session = build(paths=tmp_path, disable_warnings=disable_warnings) assert session.exit_code == ExitCode.OK if disable_warnings: