From 7d0ad18fcbb74e9a848ae9b9f56f5d3c6ad5fea2 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 3 Apr 2022 00:39:02 +0200 Subject: [PATCH 1/3] Add a counter under the execution table to show completed tasks. --- docs/source/changes.md | 2 ++ src/_pytask/live.py | 45 +++++++++++++++++++++++++++++++++++------- tests/test_live.py | 6 ++++++ 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/docs/source/changes.md b/docs/source/changes.md index 31739581..ab4755ef 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -34,6 +34,8 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and - {pull}`245` adds choices on the command line to the help pages as metavars and show defaults. - {pull}`246` formalizes choices for {class}`click.Choice` to {class}`enum.Enum`. +- {pull}`248` adds a counter at the bottom of the execution table to show how many tasks + have been processed. ## 0.1.9 - 2022-02-23 diff --git a/src/_pytask/live.py b/src/_pytask/live.py index 9203bae5..ccbe4ccf 100644 --- a/src/_pytask/live.py +++ b/src/_pytask/live.py @@ -5,6 +5,7 @@ from typing import Dict from typing import Generator from typing import List +from typing import Union import attr import click @@ -16,9 +17,11 @@ from _pytask.outcomes import TaskOutcome from _pytask.report import CollectionReport from _pytask.report import ExecutionReport +from _pytask.session import Session from _pytask.shared import get_first_non_none_value from rich.live import Live from rich.status import Status +from rich.style import Style from rich.table import Table from rich.text import Text @@ -83,10 +86,16 @@ def pytask_post_parse(config: dict[str, Any]) -> None: config["verbose"], config["editor_url_scheme"], ) - config["pm"].register(live_execution) + config["pm"].register(live_execution, "live_execution") live_collection = LiveCollection(live_manager) - config["pm"].register(live_collection) + config["pm"].register(live_collection, "live_collection") + + +@hookimpl(tryfirst=True) +def pytask_execute_build(session: Session) -> None: + live_execution = session.config["pm"].get_plugin("live_execution") + live_execution._n_tasks = len(session.tasks) @attr.s(eq=False) @@ -148,6 +157,7 @@ class LiveExecution: _editor_url_scheme = attr.ib(type=str) _running_tasks = attr.ib(factory=dict, type=Dict[str, Task]) _reports = attr.ib(factory=list, type=List[Dict[str, Any]]) + _n_tasks = attr.ib(default="x", type=Union[int, str]) @hookimpl(hookwrapper=True) def pytask_execute_build(self) -> Generator[None, None, None]: @@ -156,7 +166,9 @@ def pytask_execute_build(self) -> Generator[None, None, None]: self._live_manager.start() yield self._live_manager.stop(transient=True) - table = self._generate_table(reduce_table=False, sort_table=True) + table = self._generate_table( + reduce_table=False, sort_table=True, add_caption=False + ) if table is not None: console.print(table) @@ -172,7 +184,9 @@ def pytask_execute_task_log_end(self, report: ExecutionReport) -> bool: self.update_reports(report) return True - def _generate_table(self, reduce_table: bool, sort_table: bool) -> Table | None: + def _generate_table( + self, reduce_table: bool, sort_table: bool, add_caption: bool + ) -> Table | None: """Generate the table. First, display all completed tasks and, then, all running tasks. @@ -210,7 +224,19 @@ def _generate_table(self, reduce_table: bool, sort_table: bool) -> Table | None: relevant_reports, key=lambda report: report["name"] ) - table = Table() + if add_caption: + caption_kwargs = { + "caption": Text( + f"Completed: {len(self._reports)}/{self._n_tasks}", + style=Style(dim=True, italic=False), + ), + "caption_justify": "right", + "caption_style": None, + } + else: + caption_kwargs = {} + + table = Table(**caption_kwargs) table.add_column("Task", overflow="fold") table.add_column("Outcome") for report in relevant_reports: @@ -237,10 +263,15 @@ def _generate_table(self, reduce_table: bool, sort_table: bool) -> Table | None: return table def _update_table( - self, reduce_table: bool = True, sort_table: bool = False + self, + reduce_table: bool = True, + sort_table: bool = False, + add_caption: bool = True, ) -> None: """Regenerate the table.""" - table = self._generate_table(reduce_table=reduce_table, sort_table=sort_table) + table = self._generate_table( + reduce_table=reduce_table, sort_table=sort_table, add_caption=add_caption + ) self._live_manager.update(table) def update_running_tasks(self, new_running_task: Task) -> None: diff --git a/tests/test_live.py b/tests/test_live.py index 1a62f746..e1183867 100644 --- a/tests/test_live.py +++ b/tests/test_live.py @@ -77,6 +77,7 @@ def test_live_execution_sequentially(capsys, tmp_path): assert "Outcome" not in captured.out assert "task_module.py::task_example" not in captured.out assert "running" not in captured.out + assert "Completed: 0/x" not in captured.out live_manager.resume() live_manager.start() @@ -88,6 +89,7 @@ def test_live_execution_sequentially(capsys, tmp_path): assert "Outcome" in captured.out assert "task_module.py::task_example" in captured.out assert "running" in captured.out + assert "Completed: 0/x" in captured.out live_manager.start() @@ -104,6 +106,7 @@ def test_live_execution_sequentially(capsys, tmp_path): assert "task_module.py::task_example" in captured.out assert "running" not in captured.out assert TaskOutcome.SUCCESS.symbol in captured.out + assert "Completed: 1/x" in captured.out @pytest.mark.unit @@ -160,6 +163,7 @@ def test_live_execution_displays_subset_of_table(capsys, tmp_path, n_entries_in_ live_manager = LiveManager() live = LiveExecution(live_manager, n_entries_in_table, 1, "no_link") + live._n_tasks = 2 live_manager.start() live.update_running_tasks(running_task) @@ -170,6 +174,7 @@ def test_live_execution_displays_subset_of_table(capsys, tmp_path, n_entries_in_ assert "Outcome" in captured.out assert "::task_running" in captured.out assert " running " in captured.out + assert "Completed: 0/2" in captured.out completed_task = Task(base_name="task_completed", path=path, function=lambda x: x) completed_task.short_name = "task_module.py::task_completed" @@ -188,6 +193,7 @@ def test_live_execution_displays_subset_of_table(capsys, tmp_path, n_entries_in_ assert "Outcome" in captured.out assert "::task_running" in captured.out assert " running " in captured.out + assert "Completed: 1/2" in captured.out if n_entries_in_table == 1: assert "task_module.py::task_completed" not in captured.out From 3f32f2439b38d426176e94e102d5b368d7d90eac Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 3 Apr 2022 00:50:51 +0200 Subject: [PATCH 2/3] Refactored private public attributes for LiveExecution. --- src/_pytask/live.py | 40 ++++++++++++++++++++-------------------- tests/test_live.py | 30 +++++++++++++++++++++++++----- 2 files changed, 45 insertions(+), 25 deletions(-) diff --git a/src/_pytask/live.py b/src/_pytask/live.py index ccbe4ccf..a3c21729 100644 --- a/src/_pytask/live.py +++ b/src/_pytask/live.py @@ -81,10 +81,10 @@ def pytask_post_parse(config: dict[str, Any]) -> None: if config["verbose"] >= 1: live_execution = LiveExecution( - live_manager, - config["n_entries_in_table"], - config["verbose"], - config["editor_url_scheme"], + live_manager=live_manager, + n_entries_in_table=config["n_entries_in_table"], + verbose=config["verbose"], + editor_url_scheme=config["editor_url_scheme"], ) config["pm"].register(live_execution, "live_execution") @@ -95,7 +95,7 @@ def pytask_post_parse(config: dict[str, Any]) -> None: @hookimpl(tryfirst=True) def pytask_execute_build(session: Session) -> None: live_execution = session.config["pm"].get_plugin("live_execution") - live_execution._n_tasks = len(session.tasks) + live_execution.n_tasks = len(session.tasks) @attr.s(eq=False) @@ -147,25 +147,25 @@ def is_started(self) -> None: return self._live.is_started -@attr.s(eq=False) +@attr.s(eq=False, kw_only=True) class LiveExecution: """A class for managing the table displaying task progress during the execution.""" - _live_manager = attr.ib(type=LiveManager) - _n_entries_in_table = attr.ib(type=int) - _verbose = attr.ib(type=int) - _editor_url_scheme = attr.ib(type=str) - _running_tasks = attr.ib(factory=dict, type=Dict[str, Task]) + live_manager = attr.ib(type=LiveManager) + n_entries_in_table = attr.ib(type=int) + verbose = attr.ib(type=int) + editor_url_scheme = attr.ib(type=str) + n_tasks = attr.ib(default="x", type=Union[int, str]) _reports = attr.ib(factory=list, type=List[Dict[str, Any]]) - _n_tasks = attr.ib(default="x", type=Union[int, str]) + _running_tasks = attr.ib(factory=dict, type=Dict[str, Task]) @hookimpl(hookwrapper=True) def pytask_execute_build(self) -> Generator[None, None, None]: """Wrap the execution with the live manager and yield a complete table at the end.""" - self._live_manager.start() + self.live_manager.start() yield - self._live_manager.stop(transient=True) + self.live_manager.stop(transient=True) table = self._generate_table( reduce_table=False, sort_table=True, add_caption=False ) @@ -195,9 +195,9 @@ def _generate_table( if more entries are requested, the list is filled up with completed tasks. """ - n_reports_to_display = self._n_entries_in_table - len(self._running_tasks) + n_reports_to_display = self.n_entries_in_table - len(self._running_tasks) - if self._verbose < 2: + if self.verbose < 2: reports = [ report for report in self._reports @@ -227,7 +227,7 @@ def _generate_table( if add_caption: caption_kwargs = { "caption": Text( - f"Completed: {len(self._reports)}/{self._n_tasks}", + f"Completed: {len(self._reports)}/{self.n_tasks}", style=Style(dim=True, italic=False), ), "caption_justify": "right", @@ -243,7 +243,7 @@ def _generate_table( table.add_row( format_task_id( report["task"], - editor_url_scheme=self._editor_url_scheme, + editor_url_scheme=self.editor_url_scheme, short_name=True, ), Text(report["outcome"].symbol, style=report["outcome"].style), @@ -251,7 +251,7 @@ def _generate_table( for task in self._running_tasks.values(): table.add_row( format_task_id( - task, editor_url_scheme=self._editor_url_scheme, short_name=True + task, editor_url_scheme=self.editor_url_scheme, short_name=True ), "running", ) @@ -272,7 +272,7 @@ def _update_table( table = self._generate_table( reduce_table=reduce_table, sort_table=sort_table, add_caption=add_caption ) - self._live_manager.update(table) + self.live_manager.update(table) def update_running_tasks(self, new_running_task: Task) -> None: """Add a new running task.""" diff --git a/tests/test_live.py b/tests/test_live.py index e1183867..603dfdf5 100644 --- a/tests/test_live.py +++ b/tests/test_live.py @@ -65,7 +65,12 @@ def test_live_execution_sequentially(capsys, tmp_path): task.short_name = "task_module.py::task_example" live_manager = LiveManager() - live = LiveExecution(live_manager, 20, 1, "no_link") + live = LiveExecution( + live_manager=live_manager, + n_entries_in_table=20, + verbose=1, + editor_url_scheme="no_link", + ) live_manager.start() live.update_running_tasks(task) @@ -118,7 +123,12 @@ def test_live_execution_displays_skips_and_persists(capsys, tmp_path, verbose, o task.short_name = "task_module.py::task_example" live_manager = LiveManager() - live = LiveExecution(live_manager, 20, verbose, "no_link") + live = LiveExecution( + live_manager=live_manager, + n_entries_in_table=20, + verbose=verbose, + editor_url_scheme="no_link", + ) live_manager.start() live.update_running_tasks(task) @@ -162,8 +172,13 @@ def test_live_execution_displays_subset_of_table(capsys, tmp_path, n_entries_in_ running_task.short_name = "task_module.py::task_running" live_manager = LiveManager() - live = LiveExecution(live_manager, n_entries_in_table, 1, "no_link") - live._n_tasks = 2 + live = LiveExecution( + live_manager=live_manager, + n_entries_in_table=n_entries_in_table, + verbose=1, + editor_url_scheme="no_link", + n_tasks=2, + ) live_manager.start() live.update_running_tasks(running_task) @@ -210,7 +225,12 @@ def test_live_execution_skips_do_not_crowd_out_displayed_tasks(capsys, tmp_path) task.short_name = "task_module.py::task_example" live_manager = LiveManager() - live = LiveExecution(live_manager, 20, 1, "no_link") + live = LiveExecution( + live_manager=live_manager, + n_entries_in_table=20, + verbose=1, + editor_url_scheme="no_link", + ) live_manager.start() live.update_running_tasks(task) From 764cff75d4c055655f685045c471d0e36f2cbe76 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 3 Apr 2022 00:53:27 +0200 Subject: [PATCH 3/3] Refactored private public attributes for LiveCollection. --- src/_pytask/live.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/_pytask/live.py b/src/_pytask/live.py index a3c21729..61a3eb04 100644 --- a/src/_pytask/live.py +++ b/src/_pytask/live.py @@ -88,7 +88,7 @@ def pytask_post_parse(config: dict[str, Any]) -> None: ) config["pm"].register(live_execution, "live_execution") - live_collection = LiveCollection(live_manager) + live_collection = LiveCollection(live_manager=live_manager) config["pm"].register(live_collection, "live_collection") @@ -292,18 +292,18 @@ def update_reports(self, new_report: ExecutionReport) -> None: self._update_table() -@attr.s(eq=False) +@attr.s(eq=False, kw_only=True) class LiveCollection: """A class for managing the live status during the collection.""" - _live_manager = attr.ib(type=LiveManager) + live_manager = attr.ib(type=LiveManager) _n_collected_tasks = attr.ib(default=0, type=int) _n_errors = attr.ib(default=0, type=int) @hookimpl(hookwrapper=True) def pytask_collect(self) -> Generator[None, None, None]: - """Start the status of the cllection.""" - self._live_manager.start() + """Start the status of the collection.""" + self.live_manager.start() yield @hookimpl @@ -315,7 +315,7 @@ def pytask_collect_file_log(self, reports: list[CollectionReport]) -> None: @hookimpl(hookwrapper=True) def pytask_collect_log(self) -> Generator[None, None, None]: """Stop the live display when all tasks have been collected.""" - self._live_manager.stop(transient=True) + self.live_manager.stop(transient=True) yield def _update_statistics(self, reports: list[CollectionReport]) -> None: @@ -331,7 +331,7 @@ def _update_statistics(self, reports: list[CollectionReport]) -> None: def _update_status(self) -> None: """Update the status.""" status = self._generate_status() - self._live_manager.update(status) + self.live_manager.update(status) def _generate_status(self) -> Status: """Generate the status."""