diff --git a/docs/source/changes.md b/docs/source/changes.md index 866452ea..719370a1 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -5,6 +5,10 @@ chronological order. Releases follow [semantic versioning](https://semver.org/) releases are available on [PyPI](https://pypi.org/project/pytask) and [Anaconda.org](https://anaconda.org/conda-forge/pytask). +## 0.5.1 - 2024-xx-xx + +- {pull}`617` fixes an interaction with provisional nodes and `@mark.persist`. + ## 0.5.0 - 2024-05-26 - {pull}`548` fixes the type hints for {meth}`~pytask.Task.execute` and diff --git a/requirements-dev.lock b/requirements-dev.lock index df7f9104..2506d7d1 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -6,65 +6,115 @@ # features: [] # all-features: true # with-sources: false +# generate-hashes: false -e file:. +aiohttp==3.9.5 + # via coiled + # via pytask +aiosignal==1.3.1 + # via aiohttp alabaster==0.7.16 # via sphinx +anyio==4.4.0 + # via httpx apeye==1.4.1 # via sphinx-toolbox apeye-core==1.1.5 # via apeye asttokens==2.4.1 # via stack-data +async-timeout==4.0.3 + # via aiohttp attrs==23.2.0 + # via aiohttp # via jsonschema + # via jupyter-cache # via pytask # via referencing autodocsumm==0.2.12 # via sphinx-toolbox +autopep8==2.2.0 + # via nbqa babel==2.14.0 # via sphinx +backoff==2.2.1 + # via coiled +bcrypt==4.1.3 + # via paramiko beautifulsoup4==4.12.3 # via furo - # via nbconvert # via sphinx-toolbox -bleach==6.1.0 - # via nbconvert +boto3==1.34.122 + # via coiled +botocore==1.34.122 + # via boto3 + # via s3transfer cachecontrol==0.14.0 # via sphinx-toolbox +cachetools==5.3.3 + # via tox certifi==2024.2.2 + # via httpcore + # via httpx # via requests +cffi==1.16.0 + # via cryptography + # via pynacl +chardet==5.2.0 + # via tox charset-normalizer==3.3.2 # via requests click==8.1.7 # via click-default-group + # via coiled + # via dask + # via distributed + # via jupyter-cache # via pytask # via sphinx-click click-default-group==1.2.4 # via pytask +cloudpickle==3.0.0 + # via dask + # via distributed +coiled==1.28.0 + # via pytask +colorama==0.4.6 + # via tox comm==0.2.2 # via ipykernel + # via ipywidgets contourpy==1.2.1 # via matplotlib coverage==7.4.4 # via pytest-cov +cryptography==42.0.8 + # via paramiko cssutils==2.10.2 # via dict2css cycler==0.12.1 # via matplotlib +dask==2024.5.2 + # via coiled + # via distributed debugpy==1.8.1 # via ipykernel decorator==5.1.1 + # via fabric # via ipython deepdiff==7.0.1 # via pytask -defusedxml==0.7.1 - # via nbconvert +deprecated==1.2.14 + # via fabric dict2css==0.3.0.post1 # via sphinx-toolbox +distlib==0.3.8 + # via virtualenv +distributed==2024.5.2 + # via coiled docutils==0.20.1 # via myst-parser - # via nbsphinx # via sphinx # via sphinx-click # via sphinx-prompt @@ -75,68 +125,125 @@ domdf-python-tools==3.8.0.post2 # via apeye-core # via dict2css # via sphinx-toolbox +exceptiongroup==1.2.1 + # via anyio + # via ipython + # via pytest execnet==2.1.1 # via pytest-xdist executing==2.0.1 # via stack-data +fabric==3.2.2 + # via coiled fastjsonschema==2.19.1 # via nbformat filelock==3.13.4 # via cachecontrol + # via coiled # via sphinx-toolbox + # via tox + # via virtualenv fonttools==4.51.0 # via matplotlib +frozenlist==1.4.1 + # via aiohttp + # via aiosignal fsspec==2024.3.1 + # via dask # via universal-pathlib furo==2024.1.29 # via pytask +gilknocker==0.4.1 + # via coiled greenlet==3.0.3 # via sqlalchemy +h11==0.14.0 + # via httpcore +h2==4.1.0 + # via httpx +hpack==4.0.0 + # via h2 html5lib==1.1 # via sphinx-toolbox +httpcore==1.0.5 + # via httpx +httpx==0.27.0 + # via coiled + # via pytask +hyperframe==6.0.1 + # via h2 idna==3.7 + # via anyio # via apeye-core + # via httpx # via requests + # via yarl imagesize==1.4.1 # via sphinx +importlib-metadata==7.1.0 + # via coiled + # via dask + # via jupyter-cache + # via jupyter-client + # via myst-nb + # via sphinx +importlib-resources==6.4.0 + # via matplotlib iniconfig==2.0.0 # via pytest +invoke==2.2.0 + # via coiled + # via fabric ipykernel==6.29.4 + # via myst-nb # via nbmake -ipython==8.23.0 +ipython==8.18.1 # via ipykernel + # via ipywidgets + # via myst-nb + # via nbqa # via pytask +ipywidgets==8.1.3 + # via coiled jedi==0.19.1 # via ipython jinja2==3.1.3 + # via distributed # via myst-parser - # via nbconvert - # via nbsphinx # via sphinx # via sphinx-jinja2-compat +jmespath==1.0.1 + # via boto3 + # via botocore + # via coiled +jsondiff==2.0.0 + # via coiled jsonschema==4.21.1 # via nbformat jsonschema-specifications==2023.12.1 # via jsonschema +jupyter-cache==1.0.0 + # via myst-nb jupyter-client==8.6.1 # via ipykernel # via nbclient jupyter-core==5.7.2 # via ipykernel # via jupyter-client - # via nbconvert # via nbformat -jupyterlab-pygments==0.3.0 - # via nbconvert +jupyterlab-widgets==3.0.11 + # via ipywidgets kiwisolver==1.4.5 # via matplotlib +locket==1.0.0 + # via distributed + # via partd markdown-it-py==3.0.0 # via mdit-py-plugins # via myst-parser # via rich markupsafe==2.1.5 # via jinja2 - # via nbconvert # via sphinx-jinja2-compat matplotlib==3.8.4 # via pytask @@ -147,32 +254,40 @@ mdit-py-plugins==0.4.0 # via myst-parser mdurl==0.1.2 # via markdown-it-py -mistune==3.0.2 - # via nbconvert msgpack==1.0.8 # via cachecontrol + # via distributed +multidict==6.0.5 + # via aiohttp + # via yarl +mypy==1.10.0 + # via pytask +mypy-extensions==1.0.0 + # via mypy +myst-nb==1.1.0 + # via pytask myst-parser==2.0.0 + # via myst-nb # via pytask natsort==8.4.0 # via domdf-python-tools nbclient==0.6.8 - # via nbconvert + # via jupyter-cache + # via myst-nb # via nbmake -nbconvert==7.16.3 - # via nbsphinx nbformat==5.10.4 + # via jupyter-cache + # via myst-nb # via nbclient - # via nbconvert # via nbmake - # via nbsphinx nbmake==1.5.3 # via pytask -nbsphinx==0.9.3 +nbqa==1.8.5 # via pytask nest-asyncio==1.6.0 # via ipykernel # via nbclient -networkx==3.3 +networkx==3.2.1 # via pytask numpy==1.26.4 # via contourpy @@ -182,46 +297,73 @@ optree==0.11.0 ordered-set==4.1.0 # via deepdiff packaging==24.0 + # via coiled + # via dask + # via distributed # via ipykernel # via matplotlib - # via nbconvert + # via pip-requirements-parser + # via pyproject-api # via pytask # via pytest # via sphinx -pandocfilters==1.5.1 - # via nbconvert + # via tox + # via tox-uv +paramiko==3.4.0 + # via coiled + # via fabric parso==0.8.4 # via jedi +partd==1.4.2 + # via dask pexpect==4.9.0 # via ipython # via pytask pillow==10.3.0 # via matplotlib +pip==24.0 + # via coiled +pip-requirements-parser==32.0.1 + # via coiled platformdirs==4.2.0 # via apeye # via jupyter-core + # via tox + # via virtualenv pluggy==1.4.0 # via pytask # via pytest + # via tox +prometheus-client==0.20.0 + # via coiled prompt-toolkit==3.0.43 # via ipython psutil==5.9.8 + # via distributed # via ipykernel ptyprocess==0.7.0 # via pexpect pure-eval==0.2.2 # via stack-data +pycodestyle==2.11.1 + # via autopep8 +pycparser==2.22 + # via cffi pygments==2.17.2 # via furo # via ipython - # via nbconvert # via nbmake # via rich # via sphinx # via sphinx-prompt # via sphinx-tabs +pynacl==1.5.0 + # via paramiko pyparsing==3.1.2 # via matplotlib + # via pip-requirements-parser +pyproject-api==1.6.1 + # via tox pytest==8.1.1 # via nbmake # via pytask @@ -233,9 +375,14 @@ pytest-cov==5.0.0 pytest-xdist==3.5.0 # via pytask python-dateutil==2.9.0.post0 + # via botocore # via jupyter-client # via matplotlib pyyaml==6.0.1 + # via dask + # via distributed + # via jupyter-cache + # via myst-nb # via myst-parser pyzmq==25.1.2 # via ipykernel @@ -248,6 +395,7 @@ requests==2.31.0 # via cachecontrol # via sphinx rich==13.7.1 + # via coiled # via pytask rpds-py==0.18.0 # via jsonschema @@ -256,20 +404,28 @@ ruamel-yaml==0.18.6 # via sphinx-toolbox ruamel-yaml-clib==0.2.8 # via ruamel-yaml +s3transfer==0.10.1 + # via boto3 +setuptools==70.0.0 + # via coiled six==1.16.0 # via asttokens - # via bleach # via html5lib # via python-dateutil +sniffio==1.3.1 + # via anyio + # via httpx snowballstemmer==2.2.0 # via sphinx +sortedcontainers==2.4.0 + # via distributed soupsieve==2.5 # via beautifulsoup4 sphinx==7.2.6 # via autodocsumm # via furo + # via myst-nb # via myst-parser - # via nbsphinx # via pytask # via sphinx-autodoc-typehints # via sphinx-basic-ng @@ -313,41 +469,89 @@ sphinxcontrib-serializinghtml==1.1.10 sphinxext-opengraph==0.9.1 # via pytask sqlalchemy==2.0.29 + # via jupyter-cache # via pytask stack-data==0.6.3 # via ipython syrupy==4.6.1 # via pytask tabulate==0.9.0 + # via jupyter-cache + # via pytask # via sphinx-toolbox -tinycss2==1.2.1 - # via nbconvert +tblib==3.0.0 + # via distributed +tokenize-rt==5.2.0 + # via nbqa +toml==0.10.2 + # via coiled +tomli==2.0.1 + # via autopep8 + # via coverage + # via mypy + # via nbqa + # via pyproject-api + # via pytask + # via pytest + # via tox +toolz==0.12.1 + # via dask + # via distributed + # via partd tornado==6.4 + # via distributed # via ipykernel # via jupyter-client +tox==4.15.1 + # via tox-uv +tox-uv==1.9.0 +tqdm==4.66.4 + # via pytask traitlets==5.14.2 # via comm # via ipykernel # via ipython + # via ipywidgets # via jupyter-client # via jupyter-core # via matplotlib-inline # via nbclient - # via nbconvert # via nbformat - # via nbsphinx typing-extensions==4.11.0 + # via anyio + # via coiled # via domdf-python-tools + # via ipython + # via mypy + # via myst-nb # via optree # via sphinx-toolbox # via sqlalchemy universal-pathlib==0.2.2 # via pytask -urllib3==2.2.1 +urllib3==1.26.18 + # via botocore + # via distributed # via requests +uv==0.2.9 + # via tox-uv +virtualenv==20.26.2 + # via tox wcwidth==0.2.13 # via prompt-toolkit + # via tabulate webencodings==0.5.1 - # via bleach # via html5lib - # via tinycss2 +wheel==0.43.0 + # via coiled +widgetsnbextension==4.0.11 + # via ipywidgets +wrapt==1.16.0 + # via deprecated +yarl==1.9.4 + # via aiohttp +zict==3.0.0 + # via distributed +zipp==3.19.2 + # via importlib-metadata + # via importlib-resources diff --git a/requirements.lock b/requirements.lock index df7f9104..eeefc4f2 100644 --- a/requirements.lock +++ b/requirements.lock @@ -6,65 +6,107 @@ # features: [] # all-features: true # with-sources: false +# generate-hashes: false -e file:. +aiohttp==3.9.5 + # via coiled + # via pytask +aiosignal==1.3.1 + # via aiohttp alabaster==0.7.16 # via sphinx +anyio==4.4.0 + # via httpx apeye==1.4.1 # via sphinx-toolbox apeye-core==1.1.5 # via apeye asttokens==2.4.1 # via stack-data +async-timeout==4.0.3 + # via aiohttp attrs==23.2.0 + # via aiohttp # via jsonschema + # via jupyter-cache # via pytask # via referencing autodocsumm==0.2.12 # via sphinx-toolbox +autopep8==2.2.0 + # via nbqa babel==2.14.0 # via sphinx +backoff==2.2.1 + # via coiled +bcrypt==4.1.3 + # via paramiko beautifulsoup4==4.12.3 # via furo - # via nbconvert # via sphinx-toolbox -bleach==6.1.0 - # via nbconvert +boto3==1.34.122 + # via coiled +botocore==1.34.122 + # via boto3 + # via s3transfer cachecontrol==0.14.0 # via sphinx-toolbox certifi==2024.2.2 + # via httpcore + # via httpx # via requests +cffi==1.16.0 + # via cryptography + # via pynacl charset-normalizer==3.3.2 # via requests click==8.1.7 # via click-default-group + # via coiled + # via dask + # via distributed + # via jupyter-cache # via pytask # via sphinx-click click-default-group==1.2.4 # via pytask +cloudpickle==3.0.0 + # via dask + # via distributed +coiled==1.28.0 + # via pytask comm==0.2.2 # via ipykernel + # via ipywidgets contourpy==1.2.1 # via matplotlib coverage==7.4.4 # via pytest-cov +cryptography==42.0.8 + # via paramiko cssutils==2.10.2 # via dict2css cycler==0.12.1 # via matplotlib +dask==2024.5.2 + # via coiled + # via distributed debugpy==1.8.1 # via ipykernel decorator==5.1.1 + # via fabric # via ipython deepdiff==7.0.1 # via pytask -defusedxml==0.7.1 - # via nbconvert +deprecated==1.2.14 + # via fabric dict2css==0.3.0.post1 # via sphinx-toolbox +distributed==2024.5.2 + # via coiled docutils==0.20.1 # via myst-parser - # via nbsphinx # via sphinx # via sphinx-click # via sphinx-prompt @@ -75,68 +117,123 @@ domdf-python-tools==3.8.0.post2 # via apeye-core # via dict2css # via sphinx-toolbox +exceptiongroup==1.2.1 + # via anyio + # via ipython + # via pytest execnet==2.1.1 # via pytest-xdist executing==2.0.1 # via stack-data +fabric==3.2.2 + # via coiled fastjsonschema==2.19.1 # via nbformat filelock==3.13.4 # via cachecontrol + # via coiled # via sphinx-toolbox fonttools==4.51.0 # via matplotlib +frozenlist==1.4.1 + # via aiohttp + # via aiosignal fsspec==2024.3.1 + # via dask # via universal-pathlib furo==2024.1.29 # via pytask +gilknocker==0.4.1 + # via coiled greenlet==3.0.3 # via sqlalchemy +h11==0.14.0 + # via httpcore +h2==4.1.0 + # via httpx +hpack==4.0.0 + # via h2 html5lib==1.1 # via sphinx-toolbox +httpcore==1.0.5 + # via httpx +httpx==0.27.0 + # via coiled + # via pytask +hyperframe==6.0.1 + # via h2 idna==3.7 + # via anyio # via apeye-core + # via httpx # via requests + # via yarl imagesize==1.4.1 # via sphinx +importlib-metadata==7.1.0 + # via coiled + # via dask + # via jupyter-cache + # via jupyter-client + # via myst-nb + # via sphinx +importlib-resources==6.4.0 + # via matplotlib iniconfig==2.0.0 # via pytest +invoke==2.2.0 + # via coiled + # via fabric ipykernel==6.29.4 + # via myst-nb # via nbmake -ipython==8.23.0 +ipython==8.18.1 # via ipykernel + # via ipywidgets + # via myst-nb + # via nbqa # via pytask +ipywidgets==8.1.3 + # via coiled jedi==0.19.1 # via ipython jinja2==3.1.3 + # via distributed # via myst-parser - # via nbconvert - # via nbsphinx # via sphinx # via sphinx-jinja2-compat +jmespath==1.0.1 + # via boto3 + # via botocore + # via coiled +jsondiff==2.0.0 + # via coiled jsonschema==4.21.1 # via nbformat jsonschema-specifications==2023.12.1 # via jsonschema +jupyter-cache==1.0.0 + # via myst-nb jupyter-client==8.6.1 # via ipykernel # via nbclient jupyter-core==5.7.2 # via ipykernel # via jupyter-client - # via nbconvert # via nbformat -jupyterlab-pygments==0.3.0 - # via nbconvert +jupyterlab-widgets==3.0.11 + # via ipywidgets kiwisolver==1.4.5 # via matplotlib +locket==1.0.0 + # via distributed + # via partd markdown-it-py==3.0.0 # via mdit-py-plugins # via myst-parser # via rich markupsafe==2.1.5 # via jinja2 - # via nbconvert # via sphinx-jinja2-compat matplotlib==3.8.4 # via pytask @@ -147,32 +244,40 @@ mdit-py-plugins==0.4.0 # via myst-parser mdurl==0.1.2 # via markdown-it-py -mistune==3.0.2 - # via nbconvert msgpack==1.0.8 # via cachecontrol + # via distributed +multidict==6.0.5 + # via aiohttp + # via yarl +mypy==1.10.0 + # via pytask +mypy-extensions==1.0.0 + # via mypy +myst-nb==1.1.0 + # via pytask myst-parser==2.0.0 + # via myst-nb # via pytask natsort==8.4.0 # via domdf-python-tools nbclient==0.6.8 - # via nbconvert + # via jupyter-cache + # via myst-nb # via nbmake -nbconvert==7.16.3 - # via nbsphinx nbformat==5.10.4 + # via jupyter-cache + # via myst-nb # via nbclient - # via nbconvert # via nbmake - # via nbsphinx nbmake==1.5.3 # via pytask -nbsphinx==0.9.3 +nbqa==1.8.5 # via pytask nest-asyncio==1.6.0 # via ipykernel # via nbclient -networkx==3.3 +networkx==3.2.1 # via pytask numpy==1.26.4 # via contourpy @@ -182,46 +287,65 @@ optree==0.11.0 ordered-set==4.1.0 # via deepdiff packaging==24.0 + # via coiled + # via dask + # via distributed # via ipykernel # via matplotlib - # via nbconvert + # via pip-requirements-parser # via pytask # via pytest # via sphinx -pandocfilters==1.5.1 - # via nbconvert +paramiko==3.4.0 + # via coiled + # via fabric parso==0.8.4 # via jedi +partd==1.4.2 + # via dask pexpect==4.9.0 # via ipython # via pytask pillow==10.3.0 # via matplotlib +pip==24.0 + # via coiled +pip-requirements-parser==32.0.1 + # via coiled platformdirs==4.2.0 # via apeye # via jupyter-core pluggy==1.4.0 # via pytask # via pytest +prometheus-client==0.20.0 + # via coiled prompt-toolkit==3.0.43 # via ipython psutil==5.9.8 + # via distributed # via ipykernel ptyprocess==0.7.0 # via pexpect pure-eval==0.2.2 # via stack-data +pycodestyle==2.11.1 + # via autopep8 +pycparser==2.22 + # via cffi pygments==2.17.2 # via furo # via ipython - # via nbconvert # via nbmake # via rich # via sphinx # via sphinx-prompt # via sphinx-tabs +pynacl==1.5.0 + # via paramiko pyparsing==3.1.2 # via matplotlib + # via pip-requirements-parser pytest==8.1.1 # via nbmake # via pytask @@ -233,9 +357,14 @@ pytest-cov==5.0.0 pytest-xdist==3.5.0 # via pytask python-dateutil==2.9.0.post0 + # via botocore # via jupyter-client # via matplotlib pyyaml==6.0.1 + # via dask + # via distributed + # via jupyter-cache + # via myst-nb # via myst-parser pyzmq==25.1.2 # via ipykernel @@ -248,6 +377,7 @@ requests==2.31.0 # via cachecontrol # via sphinx rich==13.7.1 + # via coiled # via pytask rpds-py==0.18.0 # via jsonschema @@ -256,20 +386,28 @@ ruamel-yaml==0.18.6 # via sphinx-toolbox ruamel-yaml-clib==0.2.8 # via ruamel-yaml +s3transfer==0.10.1 + # via boto3 +setuptools==70.0.0 + # via coiled six==1.16.0 # via asttokens - # via bleach # via html5lib # via python-dateutil +sniffio==1.3.1 + # via anyio + # via httpx snowballstemmer==2.2.0 # via sphinx +sortedcontainers==2.4.0 + # via distributed soupsieve==2.5 # via beautifulsoup4 sphinx==7.2.6 # via autodocsumm # via furo + # via myst-nb # via myst-parser - # via nbsphinx # via pytask # via sphinx-autodoc-typehints # via sphinx-basic-ng @@ -313,41 +451,80 @@ sphinxcontrib-serializinghtml==1.1.10 sphinxext-opengraph==0.9.1 # via pytask sqlalchemy==2.0.29 + # via jupyter-cache # via pytask stack-data==0.6.3 # via ipython syrupy==4.6.1 # via pytask tabulate==0.9.0 + # via jupyter-cache + # via pytask # via sphinx-toolbox -tinycss2==1.2.1 - # via nbconvert +tblib==3.0.0 + # via distributed +tokenize-rt==5.2.0 + # via nbqa +toml==0.10.2 + # via coiled +tomli==2.0.1 + # via autopep8 + # via coverage + # via mypy + # via nbqa + # via pytask + # via pytest +toolz==0.12.1 + # via dask + # via distributed + # via partd tornado==6.4 + # via distributed # via ipykernel # via jupyter-client +tqdm==4.66.4 + # via pytask traitlets==5.14.2 # via comm # via ipykernel # via ipython + # via ipywidgets # via jupyter-client # via jupyter-core # via matplotlib-inline # via nbclient - # via nbconvert # via nbformat - # via nbsphinx typing-extensions==4.11.0 + # via anyio + # via coiled # via domdf-python-tools + # via ipython + # via mypy + # via myst-nb # via optree # via sphinx-toolbox # via sqlalchemy universal-pathlib==0.2.2 # via pytask -urllib3==2.2.1 +urllib3==1.26.18 + # via botocore + # via distributed # via requests wcwidth==0.2.13 # via prompt-toolkit + # via tabulate webencodings==0.5.1 - # via bleach # via html5lib - # via tinycss2 +wheel==0.43.0 + # via coiled +widgetsnbextension==4.0.11 + # via ipywidgets +wrapt==1.16.0 + # via deprecated +yarl==1.9.4 + # via aiohttp +zict==3.0.0 + # via distributed +zipp==3.19.2 + # via importlib-metadata + # via importlib-resources diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index 055a80d5..99427927 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -383,6 +383,17 @@ def pytask_collect_node( # noqa: C901, PLR0912 if isinstance(node, DirectoryNode): if node.root_dir is None: node.root_dir = path + + if not node.root_dir.is_absolute(): + node.root_dir = path.joinpath(node.root_dir) + + # ``normpath`` removes ``../`` from the path which is necessary for the + # casing check which will fail since ``.resolves()`` also normalizes a path. + node.root_dir = Path(os.path.normpath(node.root_dir)) + _raise_error_if_casing_of_path_is_wrong( + node.root_dir, session.config["check_casing_of_paths"] + ) + if ( not node.name or node.name == node.root_dir.joinpath(node.pattern).as_posix() diff --git a/src/_pytask/persist.py b/src/_pytask/persist.py index 7cb272f0..59e06fab 100644 --- a/src/_pytask/persist.py +++ b/src/_pytask/persist.py @@ -36,7 +36,8 @@ def pytask_parse_config(config: dict[str, Any]) -> None: def pytask_execute_task_setup(session: Session, task: PTask) -> None: """Exit persisting tasks early. - The decorator needs to be set and all nodes need to exist. + This check needs to run after the same hook implementation that resolves provisional + dependencies and before the same hook implementation for skipping tasks. """ if has_mark(task, "persist"): diff --git a/src/_pytask/provisional.py b/src/_pytask/provisional.py index eec120ea..96e450b7 100644 --- a/src/_pytask/provisional.py +++ b/src/_pytask/provisional.py @@ -32,9 +32,13 @@ from _pytask.session import Session -@hookimpl +@hookimpl(tryfirst=True) def pytask_execute_task_setup(session: Session, task: PTask) -> None: - """Collect provisional nodes and parse them.""" + """Collect provisional nodes and parse them. + + Provisional nodes need to be resolved before the same hook in persist. + + """ task.depends_on = tree_map_with_path( # type: ignore[assignment] lambda p, x: collect_provisional_nodes(session, task, x, p), task.depends_on ) diff --git a/tests/test_execute.py b/tests/test_execute.py index 92b8f5c0..6bea5223 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -921,226 +921,6 @@ def func(path): path.touch() assert tmp_path.joinpath("out.txt").exists() -@pytest.mark.end_to_end() -def test_task_that_produces_provisional_path_node(tmp_path): - source = """ - from typing_extensions import Annotated - from pytask import DirectoryNode, Product - from pathlib import Path - - def task_example( - root_path: Annotated[Path, DirectoryNode(pattern="*.txt"), Product] - ): - root_path.joinpath("a.txt").write_text("Hello, ") - root_path.joinpath("b.txt").write_text("World!") - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - - session = build(paths=tmp_path) - - assert session.exit_code == ExitCode.OK - assert len(session.tasks) == 1 - assert len(session.tasks[0].produces["root_path"]) == 2 - - # Rexecution does skip the task. - session = build(paths=tmp_path) - assert session.execution_reports[0].outcome == TaskOutcome.SKIP_UNCHANGED - - -@pytest.mark.end_to_end() -def test_task_that_depends_on_relative_provisional_path_node(tmp_path): - source = """ - from typing_extensions import Annotated - from pytask import DirectoryNode - from pathlib import Path - - def task_example( - paths = DirectoryNode(pattern="[ab].txt") - ) -> Annotated[str, Path("merged.txt")]: - path_dict = {path.stem: path for path in paths} - return path_dict["a"].read_text() + path_dict["b"].read_text() - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - tmp_path.joinpath("a.txt").write_text("Hello, ") - tmp_path.joinpath("b.txt").write_text("World!") - - session = build(paths=tmp_path) - - assert session.exit_code == ExitCode.OK - assert len(session.tasks) == 1 - assert len(session.tasks[0].depends_on["paths"]) == 2 - - -@pytest.mark.end_to_end() -def test_task_that_depends_on_provisional_path_node_with_root_dir(tmp_path): - source = """ - from typing_extensions import Annotated - from pytask import DirectoryNode - from pathlib import Path - - root_dir = Path(__file__).parent / "subfolder" - - def task_example( - paths = DirectoryNode(root_dir=root_dir, pattern="[ab].txt") - ) -> Annotated[str, Path(__file__).parent.joinpath("merged.txt")]: - path_dict = {path.stem: path for path in paths} - return path_dict["a"].read_text() + path_dict["b"].read_text() - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - tmp_path.joinpath("subfolder").mkdir() - tmp_path.joinpath("subfolder", "a.txt").write_text("Hello, ") - tmp_path.joinpath("subfolder", "b.txt").write_text("World!") - - session = build(paths=tmp_path) - - assert session.exit_code == ExitCode.OK - assert len(session.tasks) == 1 - assert len(session.tasks[0].depends_on["paths"]) == 2 - - -@pytest.mark.end_to_end() -def test_task_that_depends_on_provisional_task(runner, tmp_path): - source = """ - from typing_extensions import Annotated - from pytask import DirectoryNode, task - from pathlib import Path - - def task_produces() -> Annotated[None, DirectoryNode(pattern="[ab].txt")]: - path = Path(__file__).parent - path.joinpath("a.txt").write_text("Hello, ") - path.joinpath("b.txt").write_text("World!") - - @task(after=task_produces) - def task_depends( - paths = DirectoryNode(pattern="[ab].txt") - ) -> Annotated[str, Path(__file__).parent.joinpath("merged.txt")]: - path_dict = {path.stem: path for path in paths} - return path_dict["a"].read_text() + path_dict["b"].read_text() - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - - result = runner.invoke(cli, [tmp_path.as_posix()]) - assert result.exit_code == ExitCode.OK - assert "2 Collected tasks" in result.output - assert "2 Succeeded" in result.output - - -@pytest.mark.end_to_end() -def test_gracefully_fail_when_dag_raises_error(runner, tmp_path): - source = """ - from typing_extensions import Annotated - from pytask import DirectoryNode, task - from pathlib import Path - - def task_produces() -> Annotated[None, DirectoryNode(pattern="*.txt")]: - path = Path(__file__).parent - path.joinpath("a.txt").write_text("Hello, ") - path.joinpath("b.txt").write_text("World!") - - @task(after=task_produces) - def task_depends( - paths = DirectoryNode(pattern="[ab].txt") - ) -> Annotated[str, Path(__file__).parent.joinpath("merged.txt")]: - path_dict = {path.stem: path for path in paths} - return path_dict["a"].read_text() + path_dict["b"].read_text() - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - - result = runner.invoke(cli, [tmp_path.as_posix()]) - assert result.exit_code == ExitCode.OK - - result = runner.invoke(cli, [tmp_path.as_posix()]) - assert result.exit_code == ExitCode.FAILED - assert "There are some tasks which produce" in result.output - - -@pytest.mark.end_to_end() -def test_provisional_task_generation(runner, tmp_path): - source = """ - from typing_extensions import Annotated - from pytask import DirectoryNode, task - from pathlib import Path - - def task_produces() -> Annotated[None, DirectoryNode(pattern="[ab].txt")]: - path = Path(__file__).parent - path.joinpath("a.txt").write_text("Hello, ") - path.joinpath("b.txt").write_text("World!") - - @task(after=task_produces, is_generator=True) - def task_depends( - paths = DirectoryNode(pattern="[ab].txt") - ): - for path in paths: - - @task - def task_copy( - path: Path = path - ) -> Annotated[str, path.with_name(path.stem + "-copy.txt")]: - return path.read_text() - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - - result = runner.invoke(cli, [tmp_path.as_posix()]) - assert result.exit_code == ExitCode.OK - assert "4 Collected tasks" in result.output - assert "4 Succeeded" in result.output - assert tmp_path.joinpath("a-copy.txt").exists() - assert tmp_path.joinpath("b-copy.txt").exists() - - -@pytest.mark.end_to_end() -def test_gracefully_fail_when_task_generator_raises_error(runner, tmp_path): - source = """ - from typing_extensions import Annotated - from pytask import DirectoryNode, task, Product - from pathlib import Path - - @task(is_generator=True) - def task_example( - root_dir: Annotated[Path, DirectoryNode(pattern="[a].txt"), Product] - ) -> ...: - raise Exception - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - - result = runner.invoke(cli, [tmp_path.as_posix()]) - assert result.exit_code == ExitCode.FAILED - assert "1 Collected task" in result.output - assert "1 Failed" in result.output - - -@pytest.mark.end_to_end() -def test_use_provisional_node_as_product_in_generator_without_rerun(runner, tmp_path): - source = """ - from typing_extensions import Annotated - from pytask import DirectoryNode, task, Product - from pathlib import Path - - @task(is_generator=True) - def task_example( - root_dir: Annotated[Path, DirectoryNode(pattern="[ab].txt"), Product] - ) -> ...: - for path in (root_dir / "a.txt", root_dir / "b.txt"): - - @task - def create_file() -> Annotated[Path, path]: - return "content" - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - - result = runner.invoke(cli, [tmp_path.as_posix()]) - assert result.exit_code == ExitCode.OK - assert "3 Collected task" in result.output - assert "3 Succeeded" in result.output - - # No rerun. - result = runner.invoke(cli, [tmp_path.as_posix()]) - assert result.exit_code == ExitCode.OK - assert "3 Collected task" in result.output - assert "1 Succeeded" in result.output - assert "2 Skipped because unchanged" in result.output - - @pytest.mark.end_to_end() def test_download_file(runner, tmp_path): source = """ diff --git a/tests/test_provisional.py b/tests/test_provisional.py new file mode 100644 index 00000000..5cf11913 --- /dev/null +++ b/tests/test_provisional.py @@ -0,0 +1,270 @@ +from __future__ import annotations + +import textwrap + +import pytest +from pytask import ExitCode +from pytask import TaskOutcome +from pytask import build +from pytask import cli + + +@pytest.mark.end_to_end() +def test_task_that_produces_provisional_path_node(tmp_path): + source = """ + from typing_extensions import Annotated + from pytask import DirectoryNode, Product + from pathlib import Path + + def task_example( + root_path: Annotated[Path, DirectoryNode(pattern="*.txt"), Product] + ): + root_path.joinpath("a.txt").write_text("Hello, ") + root_path.joinpath("b.txt").write_text("World!") + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + session = build(paths=tmp_path) + + assert session.exit_code == ExitCode.OK + assert len(session.tasks) == 1 + assert len(session.tasks[0].produces["root_path"]) == 2 + + # Rexecution does skip the task. + session = build(paths=tmp_path) + assert session.execution_reports[0].outcome == TaskOutcome.SKIP_UNCHANGED + + +@pytest.mark.end_to_end() +def test_task_that_depends_on_relative_provisional_path_node(tmp_path): + source = """ + from typing_extensions import Annotated + from pytask import DirectoryNode + from pathlib import Path + + def task_example( + paths = DirectoryNode(pattern="[ab].txt") + ) -> Annotated[str, Path("merged.txt")]: + path_dict = {path.stem: path for path in paths} + return path_dict["a"].read_text() + path_dict["b"].read_text() + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + tmp_path.joinpath("a.txt").write_text("Hello, ") + tmp_path.joinpath("b.txt").write_text("World!") + + session = build(paths=tmp_path) + + assert session.exit_code == ExitCode.OK + assert len(session.tasks) == 1 + assert len(session.tasks[0].depends_on["paths"]) == 2 + + +@pytest.mark.end_to_end() +def test_task_that_depends_on_provisional_path_node_with_absolute_root_dir(tmp_path): + source = """ + from typing_extensions import Annotated + from pytask import DirectoryNode + from pathlib import Path + + root_dir = Path(__file__).parent / "subfolder" + + def task_example( + paths = DirectoryNode(root_dir=root_dir, pattern="[ab].txt") + ) -> Annotated[str, Path(__file__).parent.joinpath("merged.txt")]: + path_dict = {path.stem: path for path in paths} + return path_dict["a"].read_text() + path_dict["b"].read_text() + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + tmp_path.joinpath("subfolder").mkdir() + tmp_path.joinpath("subfolder", "a.txt").write_text("Hello, ") + tmp_path.joinpath("subfolder", "b.txt").write_text("World!") + + session = build(paths=tmp_path) + + assert session.exit_code == ExitCode.OK + assert len(session.tasks) == 1 + assert len(session.tasks[0].depends_on["paths"]) == 2 + + +@pytest.mark.end_to_end() +def test_task_that_depends_on_provisional_path_node_with_relative_root_dir(tmp_path): + source = """ + from typing_extensions import Annotated + from pytask import DirectoryNode + from pathlib import Path + + def task_example( + paths = DirectoryNode(root_dir=Path("subfolder"), pattern="[ab].txt") + ) -> Annotated[str, Path(__file__).parent.joinpath("merged.txt")]: + path_dict = {path.stem: path for path in paths} + return path_dict["a"].read_text() + path_dict["b"].read_text() + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + tmp_path.joinpath("subfolder").mkdir() + tmp_path.joinpath("subfolder", "a.txt").write_text("Hello, ") + tmp_path.joinpath("subfolder", "b.txt").write_text("World!") + + session = build(paths=tmp_path) + + assert session.exit_code == ExitCode.OK + assert len(session.tasks) == 1 + assert len(session.tasks[0].depends_on["paths"]) == 2 + + +@pytest.mark.end_to_end() +def test_task_that_depends_on_provisional_task(runner, tmp_path): + source = """ + from typing_extensions import Annotated + from pytask import DirectoryNode, task + from pathlib import Path + + def task_produces() -> Annotated[None, DirectoryNode(pattern="[ab].txt")]: + path = Path(__file__).parent + path.joinpath("a.txt").write_text("Hello, ") + path.joinpath("b.txt").write_text("World!") + + @task(after=task_produces) + def task_depends( + paths = DirectoryNode(pattern="[ab].txt") + ) -> Annotated[str, Path(__file__).parent.joinpath("merged.txt")]: + path_dict = {path.stem: path for path in paths} + return path_dict["a"].read_text() + path_dict["b"].read_text() + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert "2 Collected tasks" in result.output + assert "2 Succeeded" in result.output + + +@pytest.mark.end_to_end() +def test_gracefully_fail_when_dag_raises_error(runner, tmp_path): + source = """ + from typing_extensions import Annotated + from pytask import DirectoryNode, task + from pathlib import Path + + def task_produces() -> Annotated[None, DirectoryNode(pattern="*.txt")]: + path = Path(__file__).parent + path.joinpath("a.txt").write_text("Hello, ") + path.joinpath("b.txt").write_text("World!") + + @task(after=task_produces) + def task_depends( + paths = DirectoryNode(pattern="[ab].txt") + ) -> Annotated[str, Path(__file__).parent.joinpath("merged.txt")]: + path_dict = {path.stem: path for path in paths} + return path_dict["a"].read_text() + path_dict["b"].read_text() + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.FAILED + assert "There are some tasks which produce" in result.output + + +@pytest.mark.end_to_end() +def test_provisional_task_generation(runner, tmp_path): + source = """ + from typing_extensions import Annotated + from pytask import DirectoryNode, task + from pathlib import Path + + def task_produces() -> Annotated[None, DirectoryNode(pattern="[ab].txt")]: + path = Path(__file__).parent + path.joinpath("a.txt").write_text("Hello, ") + path.joinpath("b.txt").write_text("World!") + + @task(after=task_produces, is_generator=True) + def task_depends( + paths = DirectoryNode(pattern="[ab].txt") + ): + for path in paths: + + @task + def task_copy( + path: Path = path + ) -> Annotated[str, path.with_name(path.stem + "-copy.txt")]: + return path.read_text() + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert "4 Collected tasks" in result.output + assert "4 Succeeded" in result.output + assert tmp_path.joinpath("a-copy.txt").exists() + assert tmp_path.joinpath("b-copy.txt").exists() + + +@pytest.mark.end_to_end() +def test_gracefully_fail_when_task_generator_raises_error(runner, tmp_path): + source = """ + from typing_extensions import Annotated + from pytask import DirectoryNode, task, Product + from pathlib import Path + + @task(is_generator=True) + def task_example( + root_dir: Annotated[Path, DirectoryNode(pattern="[a].txt"), Product] + ) -> ...: + raise Exception + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.FAILED + assert "1 Collected task" in result.output + assert "1 Failed" in result.output + + +@pytest.mark.end_to_end() +def test_use_provisional_node_as_product_in_generator_without_rerun(runner, tmp_path): + source = """ + from typing_extensions import Annotated + from pytask import DirectoryNode, task, Product + from pathlib import Path + + @task(is_generator=True) + def task_example( + root_dir: Annotated[Path, DirectoryNode(pattern="[ab].txt"), Product] + ) -> ...: + for path in (root_dir / "a.txt", root_dir / "b.txt"): + + @task + def create_file() -> Annotated[Path, path]: + return "content" + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert "3 Collected task" in result.output + assert "3 Succeeded" in result.output + + # No rerun. + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert "3 Collected task" in result.output + assert "1 Succeeded" in result.output + assert "2 Skipped because unchanged" in result.output + + +def test_provisional_nodes_are_resolved_before_persist(runner, tmp_path): + source = """ + from pytask import DirectoryNode, mark + from pathlib import Path + + @mark.persist + def task_example(path = DirectoryNode(root_dir=Path("files"), pattern="*.py")): ... + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + tmp_path.joinpath("files").mkdir() + tmp_path.joinpath("files", "a.py").write_text("a = 1") + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK