From 10cd0da5e74c0b4dbe5203d610c7b8034d7d3462 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 12 Jul 2024 16:35:52 +0200 Subject: [PATCH 1/5] Fix issue with rerunning tasks via pytask.build. --- docs/source/changes.md | 2 ++ pyproject.toml | 5 ++++- src/_pytask/collect.py | 5 ++++- tests/test_execute.py | 35 ++++++++++++++++++++++++++++++++++- 4 files changed, 44 insertions(+), 3 deletions(-) diff --git a/docs/source/changes.md b/docs/source/changes.md index d4ce3a5d..f20672bd 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -13,6 +13,8 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and - {pull}`619` makes coiled an optional import for tests. Thanks to {user}`erooke`. - {pull}`620` makes tests more flexible about their location. Thanks to {user}`erooke`. - {pull}`621` fixes the pull requests template. +- {pull}`626` resolves an issue with rerunning tasks via the programmatic API. Closes + {issue}`625`. Thanks to @noppelmax for the issue! ## 0.5.0 - 2024-05-26 diff --git a/pyproject.toml b/pyproject.toml index 8a11a25c..3bb7539a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -90,7 +90,10 @@ build-backend = "hatchling.build" [tool.rye] managed = true -dev-dependencies = ["tox-uv>=1.7.0"] +dev-dependencies = [ + "tox-uv>=1.7.0", + "pandas>=2.2.2", +] [tool.rye.scripts] clean-docs = { cmd = "rm -rf docs/build" } diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index 99427927..abb97d44 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -7,6 +7,7 @@ import os import sys import time +from contextlib import suppress from pathlib import Path from typing import TYPE_CHECKING from typing import Any @@ -114,7 +115,9 @@ def _collect_from_tasks(session: Session) -> None: name = raw_task.pytask_meta.name if has_mark(raw_task, "task"): - COLLECTED_TASKS[path].remove(raw_task) + # See #625. + with suppress(ValueError): + COLLECTED_TASKS[path].remove(raw_task) # When a task is not a callable, it can be anything or a PTask. Set arbitrary # values and it will pass without errors and not collected. diff --git a/tests/test_execute.py b/tests/test_execute.py index 6bea5223..ae039cc3 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -261,7 +261,15 @@ def task_example(): @pytest.mark.end_to_end() def test_task_executed_with_force_although_unchanged(tmp_path): - tmp_path.joinpath("task_module.py").write_text("def task_example(): pass") + source = """ + from pytask import task + + def task_example(): pass + + @task + def task_example_2(): pass + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) session = build(paths=tmp_path) assert session.execution_reports[0].outcome == TaskOutcome.SUCCESS session = build(paths=tmp_path, force=True) @@ -625,6 +633,31 @@ def create_file( assert tmp_path.joinpath("file.txt").read_text() == "This is the text." +@pytest.mark.end_to_end() +def test_execute_tasks_multiple_times_via_api(tmp_path): + """See #625.""" + source = """ + import pathlib + from typing import Annotated + from pytask import build, task + import sys + + @task + def task1() -> None: pass + def task2() -> None: pass + + if __name__ == "__main__": + session1 = build(tasks=[task1, task2]) + session2 = build(tasks=[task1, task2]) + sys.exit(session2.exit_code) + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + result = subprocess.run( + ("python", tmp_path.joinpath("task_module.py").as_posix()), check=False + ) + assert result.returncode == ExitCode.OK + + @pytest.mark.end_to_end() def test_pass_non_task_to_functional_api_that_are_ignored(): session = pytask.build(tasks=None) From 465d9f79f1dc3b3ed1ad66769122fc37b3da9b9a Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 12 Jul 2024 16:47:47 +0200 Subject: [PATCH 2/5] Skip test in GA for Windows and fix import. --- tests/test_execute.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_execute.py b/tests/test_execute.py index ae039cc3..4a4a78a6 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import os import pickle import re import subprocess @@ -634,11 +635,15 @@ def create_file( @pytest.mark.end_to_end() +@pytest.mark.xfail( + sys.platform == "win32" and os.environ.get("CI"), + reason="Wrong python interpreter used in Github Actions.", +) def test_execute_tasks_multiple_times_via_api(tmp_path): """See #625.""" source = """ import pathlib - from typing import Annotated + from typing_extensions import Annotated from pytask import build, task import sys From e3b451a2c13234c37a650c69c1a4393076060a57 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 12 Jul 2024 16:55:03 +0200 Subject: [PATCH 3/5] Fix. --- tests/test_execute.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_execute.py b/tests/test_execute.py index 4a4a78a6..f5bce080 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -635,9 +635,9 @@ def create_file( @pytest.mark.end_to_end() -@pytest.mark.xfail( - sys.platform == "win32" and os.environ.get("CI"), - reason="Wrong python interpreter used in Github Actions.", +@pytest.mark.skipif( + sys.platform == "win32" and os.environ.get("CI") == "true", + reason="Windows does not pick up the right Python interpreter.", ) def test_execute_tasks_multiple_times_via_api(tmp_path): """See #625.""" From 3fec635292fc00e842806586ec77060562e19764 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 14 Jul 2024 16:38:33 +0200 Subject: [PATCH 4/5] FIx. --- src/_pytask/collect.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index f293a12a..96276fa5 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -115,7 +115,9 @@ def _collect_from_tasks(session: Session) -> None: name = raw_task.pytask_meta.name if has_mark(raw_task, "task"): - # See #625. + # When tasks with @task are passed to the programmatic interface multiple + # times, they are deleted from ``COLLECTED_TASKS`` in the first iteration + # and are missing in the later. See #625. with suppress(ValueError): COLLECTED_TASKS[path].remove(raw_task) From 798ca7a99c0ed236d52a8313f7ba49a2ce3610a4 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 14 Jul 2024 16:45:09 +0200 Subject: [PATCH 5/5] Remove dev dep. --- pyproject.toml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3bb7539a..8a11a25c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -90,10 +90,7 @@ build-backend = "hatchling.build" [tool.rye] managed = true -dev-dependencies = [ - "tox-uv>=1.7.0", - "pandas>=2.2.2", -] +dev-dependencies = ["tox-uv>=1.7.0"] [tool.rye.scripts] clean-docs = { cmd = "rm -rf docs/build" }