From 1374f934e49611f3960b7e2e5b547f83cafaae27 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 16 Jan 2023 20:37:25 +0000 Subject: [PATCH 1/3] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pygrep-hooks: v1.9.0 → v1.10.0](https://github.com/pre-commit/pygrep-hooks/compare/v1.9.0...v1.10.0) - [github.com/charliermarsh/ruff-pre-commit: v0.0.215 → v0.0.223](https://github.com/charliermarsh/ruff-pre-commit/compare/v0.0.215...v0.0.223) - [github.com/dosisod/refurb: v1.9.1 → v1.10.0](https://github.com/dosisod/refurb/compare/v1.9.1...v1.10.0) --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 56f6a44..27786df 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: - id: debug-statements - id: end-of-file-fixer - repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.9.0 # Use the ref you want to point at + rev: v1.10.0 # Use the ref you want to point at hooks: - id: python-check-blanket-noqa - id: python-check-mock-methods @@ -36,11 +36,11 @@ repos: hooks: - id: black - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.215 + rev: v0.0.223 hooks: - id: ruff - repo: https://github.com/dosisod/refurb - rev: v1.9.1 + rev: v1.10.0 hooks: - id: refurb args: [--ignore, FURB126] From 64c7c6bdfd551e7daf34f6ba619b523dfaea81dc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 16 Jan 2023 20:40:06 +0000 Subject: [PATCH 2/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/pytask_parallel/execute.py | 28 ++++++++++++++-------------- tests/test_config.py | 2 +- tests/test_execute.py | 14 +++++++------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/pytask_parallel/execute.py b/src/pytask_parallel/execute.py index c780833..ad76c2c 100644 --- a/src/pytask_parallel/execute.py +++ b/src/pytask_parallel/execute.py @@ -78,21 +78,21 @@ def pytask_execute_build(session: Session) -> bool | None: for task_name in ready_tasks: task = session.dag.nodes[task_name]["task"] session.hook.pytask_execute_task_log_start( - session=session, task=task + session=session, task=task, ) try: session.hook.pytask_execute_task_setup( - session=session, task=task + session=session, task=task, ) except Exception: # noqa: BLE001 report = ExecutionReport.from_task_and_exception( - task, sys.exc_info() + task, sys.exc_info(), ) newly_collected_reports.append(report) session.scheduler.done(task_name) else: running_tasks[task_name] = session.hook.pytask_execute_task( - session=session, task=task + session=session, task=task, ) sleeper.reset() @@ -112,8 +112,8 @@ def pytask_execute_build(session: Session) -> bool | None: task = session.dag.nodes[task_name]["task"] newly_collected_reports.append( ExecutionReport.from_task_and_exception( - task, exc_info - ) + task, exc_info, + ), ) running_tasks.pop(task_name) session.scheduler.done(task_name) @@ -121,11 +121,11 @@ def pytask_execute_build(session: Session) -> bool | None: task = session.dag.nodes[task_name]["task"] try: session.hook.pytask_execute_task_teardown( - session=session, task=task + session=session, task=task, ) except Exception: # noqa: BLE001 report = ExecutionReport.from_task_and_exception( - task, sys.exc_info() + task, sys.exc_info(), ) else: report = ExecutionReport.from_task(task) @@ -138,10 +138,10 @@ def pytask_execute_build(session: Session) -> bool | None: for report in newly_collected_reports: session.hook.pytask_execute_task_process_report( - session=session, report=report + session=session, report=report, ) session.hook.pytask_execute_task_log_end( - session=session, task=task, report=report + session=session, task=task, report=report, ) reports.append(report) @@ -234,7 +234,7 @@ def _unserialize_and_execute_task( except Exception: # noqa: BLE001 exc_info = sys.exc_info() processed_exc_info = _process_exception( - exc_info, show_locals, console_options + exc_info, show_locals, console_options, ) else: processed_exc_info = None @@ -247,7 +247,7 @@ def _unserialize_and_execute_task( message=warning_record_to_str(warning_message), fs_location=fs_location, id_=task_short_name, - ) + ), ) return warning_reports, processed_exc_info @@ -281,13 +281,13 @@ def pytask_execute_task(session: Session, task: Task) -> Future[Any] | None: if session.config["n_workers"] > 1: kwargs = _create_kwargs_for_task(task) return session.config["_parallel_executor"].submit( - _mock_processes_for_threads, func=task.execute, **kwargs + _mock_processes_for_threads, func=task.execute, **kwargs, ) return None def _mock_processes_for_threads( - func: Callable[..., Any], **kwargs: Any + func: Callable[..., Any], **kwargs: Any, ) -> tuple[list[Any], tuple[type[BaseException], BaseException, TracebackType] | None]: """Mock execution function such that it returns the same as for processes. diff --git a/tests/test_config.py b/tests/test_config.py index 817713f..7c7b9a4 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -40,7 +40,7 @@ def test_interplay_between_debugging_and_parallel(tmp_path, pdb, n_workers, expe ], ) def test_reading_values_from_config_file( - tmp_path, configuration_option, value, exit_code + tmp_path, configuration_option, value, exit_code, ): config = f""" [tool.pytask.ini_options] diff --git a/tests/test_execute.py b/tests/test_execute.py index 959838c..10bafbb 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -48,7 +48,7 @@ def task_2(produces): tmp_path.joinpath("out_2.txt").unlink() session = main( - {"paths": tmp_path, "n_workers": 2, "parallel_backend": parallel_backend} + {"paths": tmp_path, "n_workers": 2, "parallel_backend": parallel_backend}, ) assert session.exit_code == ExitCode.OK @@ -124,7 +124,7 @@ def myfunc(): } with PARALLEL_BACKENDS[parallel_backend]( - max_workers=session.config["n_workers"] + max_workers=session.config["n_workers"], ) as executor: session.config["_parallel_executor"] = executor @@ -163,7 +163,7 @@ def task_3(): time.sleep(3) "n_workers": 2, "parallel_backend": parallel_backend, "max_failures": 1, - } + }, ) assert session.exit_code == ExitCode.FAILED @@ -203,7 +203,7 @@ def task_5(): tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) session = main( - {"paths": tmp_path, "parallel_backend": parallel_backend, "n_workers": 2} + {"paths": tmp_path, "parallel_backend": parallel_backend, "n_workers": 2}, ) assert session.exit_code == ExitCode.OK @@ -217,7 +217,7 @@ def task_5(): @pytest.mark.parametrize("parallel_backend", PARALLEL_BACKENDS) @pytest.mark.parametrize("show_locals", [True, False]) def test_rendering_of_tracebacks_with_rich( - runner, tmp_path, parallel_backend, show_locals + runner, tmp_path, parallel_backend, show_locals, ): source = """ import pytask @@ -259,7 +259,7 @@ def task_example(produces): tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) session = main( - {"paths": tmp_path, "parallel_backend": parallel_backend, "n_workers": 2} + {"paths": tmp_path, "parallel_backend": parallel_backend, "n_workers": 2}, ) assert session.exit_code == ExitCode.OK @@ -286,7 +286,7 @@ def task_example(produces): tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) result = runner.invoke( - cli, [tmp_path.as_posix(), "-n", "2", "--parallel-backend", parallel_backend] + cli, [tmp_path.as_posix(), "-n", "2", "--parallel-backend", parallel_backend], ) assert result.exit_code == ExitCode.OK From 240913ee6ca11498b7fc9128f791c74189d07e93 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sat, 21 Jan 2023 13:58:19 +0100 Subject: [PATCH 3/3] Fix. --- pyproject.toml | 3 ++- src/pytask_parallel/config.py | 2 +- src/pytask_parallel/execute.py | 28 ++++++++++++++-------------- tests/test_config.py | 2 +- tests/test_execute.py | 14 +++++++------- 5 files changed, 25 insertions(+), 24 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3393b3b..6f3d7ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,11 +51,12 @@ extend-ignore = [ "EM", # flake8-errmsg "ANN401", # flake8-annotate typing.Any "PD", # pandas-vet + "COM812", # trailing comma missing, but black takes care of that ] [tool.ruff.per-file-ignores] -"tests/*" = ["D", "ANN"] +"tests/*" = ["D", "ANN", "PLR2004"] [tool.ruff.pydocstyle] diff --git a/src/pytask_parallel/config.py b/src/pytask_parallel/config.py index 6c4af23..632026d 100644 --- a/src/pytask_parallel/config.py +++ b/src/pytask_parallel/config.py @@ -12,7 +12,7 @@ @hookimpl def pytask_parse_config(config: dict[str, Any]) -> None: """Parse the configuration.""" - if config["n_workers"] == "auto": + if config["n_workers"] == "auto": # noqa: PLR2004 config["n_workers"] = max(os.cpu_count() - 1, 1) if ( diff --git a/src/pytask_parallel/execute.py b/src/pytask_parallel/execute.py index ad76c2c..c780833 100644 --- a/src/pytask_parallel/execute.py +++ b/src/pytask_parallel/execute.py @@ -78,21 +78,21 @@ def pytask_execute_build(session: Session) -> bool | None: for task_name in ready_tasks: task = session.dag.nodes[task_name]["task"] session.hook.pytask_execute_task_log_start( - session=session, task=task, + session=session, task=task ) try: session.hook.pytask_execute_task_setup( - session=session, task=task, + session=session, task=task ) except Exception: # noqa: BLE001 report = ExecutionReport.from_task_and_exception( - task, sys.exc_info(), + task, sys.exc_info() ) newly_collected_reports.append(report) session.scheduler.done(task_name) else: running_tasks[task_name] = session.hook.pytask_execute_task( - session=session, task=task, + session=session, task=task ) sleeper.reset() @@ -112,8 +112,8 @@ def pytask_execute_build(session: Session) -> bool | None: task = session.dag.nodes[task_name]["task"] newly_collected_reports.append( ExecutionReport.from_task_and_exception( - task, exc_info, - ), + task, exc_info + ) ) running_tasks.pop(task_name) session.scheduler.done(task_name) @@ -121,11 +121,11 @@ def pytask_execute_build(session: Session) -> bool | None: task = session.dag.nodes[task_name]["task"] try: session.hook.pytask_execute_task_teardown( - session=session, task=task, + session=session, task=task ) except Exception: # noqa: BLE001 report = ExecutionReport.from_task_and_exception( - task, sys.exc_info(), + task, sys.exc_info() ) else: report = ExecutionReport.from_task(task) @@ -138,10 +138,10 @@ def pytask_execute_build(session: Session) -> bool | None: for report in newly_collected_reports: session.hook.pytask_execute_task_process_report( - session=session, report=report, + session=session, report=report ) session.hook.pytask_execute_task_log_end( - session=session, task=task, report=report, + session=session, task=task, report=report ) reports.append(report) @@ -234,7 +234,7 @@ def _unserialize_and_execute_task( except Exception: # noqa: BLE001 exc_info = sys.exc_info() processed_exc_info = _process_exception( - exc_info, show_locals, console_options, + exc_info, show_locals, console_options ) else: processed_exc_info = None @@ -247,7 +247,7 @@ def _unserialize_and_execute_task( message=warning_record_to_str(warning_message), fs_location=fs_location, id_=task_short_name, - ), + ) ) return warning_reports, processed_exc_info @@ -281,13 +281,13 @@ def pytask_execute_task(session: Session, task: Task) -> Future[Any] | None: if session.config["n_workers"] > 1: kwargs = _create_kwargs_for_task(task) return session.config["_parallel_executor"].submit( - _mock_processes_for_threads, func=task.execute, **kwargs, + _mock_processes_for_threads, func=task.execute, **kwargs ) return None def _mock_processes_for_threads( - func: Callable[..., Any], **kwargs: Any, + func: Callable[..., Any], **kwargs: Any ) -> tuple[list[Any], tuple[type[BaseException], BaseException, TracebackType] | None]: """Mock execution function such that it returns the same as for processes. diff --git a/tests/test_config.py b/tests/test_config.py index 7c7b9a4..817713f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -40,7 +40,7 @@ def test_interplay_between_debugging_and_parallel(tmp_path, pdb, n_workers, expe ], ) def test_reading_values_from_config_file( - tmp_path, configuration_option, value, exit_code, + tmp_path, configuration_option, value, exit_code ): config = f""" [tool.pytask.ini_options] diff --git a/tests/test_execute.py b/tests/test_execute.py index 10bafbb..959838c 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -48,7 +48,7 @@ def task_2(produces): tmp_path.joinpath("out_2.txt").unlink() session = main( - {"paths": tmp_path, "n_workers": 2, "parallel_backend": parallel_backend}, + {"paths": tmp_path, "n_workers": 2, "parallel_backend": parallel_backend} ) assert session.exit_code == ExitCode.OK @@ -124,7 +124,7 @@ def myfunc(): } with PARALLEL_BACKENDS[parallel_backend]( - max_workers=session.config["n_workers"], + max_workers=session.config["n_workers"] ) as executor: session.config["_parallel_executor"] = executor @@ -163,7 +163,7 @@ def task_3(): time.sleep(3) "n_workers": 2, "parallel_backend": parallel_backend, "max_failures": 1, - }, + } ) assert session.exit_code == ExitCode.FAILED @@ -203,7 +203,7 @@ def task_5(): tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) session = main( - {"paths": tmp_path, "parallel_backend": parallel_backend, "n_workers": 2}, + {"paths": tmp_path, "parallel_backend": parallel_backend, "n_workers": 2} ) assert session.exit_code == ExitCode.OK @@ -217,7 +217,7 @@ def task_5(): @pytest.mark.parametrize("parallel_backend", PARALLEL_BACKENDS) @pytest.mark.parametrize("show_locals", [True, False]) def test_rendering_of_tracebacks_with_rich( - runner, tmp_path, parallel_backend, show_locals, + runner, tmp_path, parallel_backend, show_locals ): source = """ import pytask @@ -259,7 +259,7 @@ def task_example(produces): tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) session = main( - {"paths": tmp_path, "parallel_backend": parallel_backend, "n_workers": 2}, + {"paths": tmp_path, "parallel_backend": parallel_backend, "n_workers": 2} ) assert session.exit_code == ExitCode.OK @@ -286,7 +286,7 @@ def task_example(produces): tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) result = runner.invoke( - cli, [tmp_path.as_posix(), "-n", "2", "--parallel-backend", parallel_backend], + cli, [tmp_path.as_posix(), "-n", "2", "--parallel-backend", parallel_backend] ) assert result.exit_code == ExitCode.OK