From 75c1e015838032e2c127f09649b60da0f0cb6048 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sat, 7 Oct 2023 19:07:30 +0200 Subject: [PATCH 1/7] Better error message. --- CHANGES.md | 2 ++ src/pytask_parallel/execute.py | 14 ++++++++++++++ tests/test_execute.py | 13 +++++++++++++ 3 files changed, 29 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 3674815..5be415d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,6 +12,8 @@ releases are available on [PyPI](https://pypi.org/project/pytask-parallel) and - {pull}`66` deactivates parallelization for dry-runs. - {pull}`67` fixes parallelization with partialed task functions. - {pull}`68` updates dependencies and syntaxes. +- {pull}`69` raises more informative error message when `breakpoint()` was uses when + parallelizing with processes or loky. ## 0.3.1 - 2023-05-27 diff --git a/src/pytask_parallel/execute.py b/src/pytask_parallel/execute.py index b54dd24..9239374 100644 --- a/src/pytask_parallel/execute.py +++ b/src/pytask_parallel/execute.py @@ -210,6 +210,14 @@ def pytask_execute_task(session: Session, task: PTask) -> Future[Any] | None: return None +def _raise_exception_on_breakpoint(*args: Any, **kwargs: Any) -> None: # noqa: ARG001 + raise RuntimeError( + "You cannot use 'breakpoint()' while parallelizing the execution of tasks with " + "pytask-parallel. Please, remove the breakpoint or run the task without " + "parallelization to debug it." + ) + + def _execute_task( # noqa: PLR0913 task: PTask, kwargs: dict[str, Any], @@ -226,6 +234,12 @@ def _execute_task( # noqa: PLR0913 """ __tracebackhide__ = True + # Path sys.breakpointhook to intercept any call of breakpoint() in a subprocess and + # print a better exception message. + import sys + + sys.breakpointhook = _raise_exception_on_breakpoint + with warnings.catch_warnings(record=True) as log: # mypy can't infer that record=True means log is not None; help it. assert log is not None diff --git a/tests/test_execute.py b/tests/test_execute.py index f24e62b..59c6812 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -268,3 +268,16 @@ def test_parallel_execution_is_deactivated(runner, tmp_path, flag, parallel_back ) assert result.exit_code == ExitCode.OK assert "Started 2 workers" not in result.output + + +@pytest.mark.end_to_end() +@pytest.mark.parametrize( + "parallel_backend", [i for i in PARALLEL_BACKENDS if i != ParallelBackend.THREADS] +) +def test_raise_error_on_breakpoint(runner, tmp_path, parallel_backend): + tmp_path.joinpath("task_example.py").write_text("def task_example(): breakpoint()") + result = runner.invoke( + cli, [tmp_path.as_posix(), "-n 2", "--parallel-backend", parallel_backend] + ) + assert result.exit_code == ExitCode.FAILED + assert "You cannot use 'breakpoint()'" in result.output From 098ef777d9f790f00253fd9e5c3bb558a909e2bb Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sat, 7 Oct 2023 19:23:56 +0200 Subject: [PATCH 2/7] fix also set_trace. --- .pre-commit-config.yaml | 1 - src/pytask_parallel/execute.py | 27 ++++++++++++++++++--------- tests/test_execute.py | 5 +++-- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9f012c2..ae64077 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,6 @@ repos: - id: check-merge-conflict - id: check-vcs-permalinks - id: check-yaml - - id: debug-statements - id: end-of-file-fixer - id: fix-byte-order-marker - id: mixed-line-ending diff --git a/src/pytask_parallel/execute.py b/src/pytask_parallel/execute.py index 9239374..70886e3 100644 --- a/src/pytask_parallel/execute.py +++ b/src/pytask_parallel/execute.py @@ -212,12 +212,26 @@ def pytask_execute_task(session: Session, task: PTask) -> Future[Any] | None: def _raise_exception_on_breakpoint(*args: Any, **kwargs: Any) -> None: # noqa: ARG001 raise RuntimeError( - "You cannot use 'breakpoint()' while parallelizing the execution of tasks with " - "pytask-parallel. Please, remove the breakpoint or run the task without " - "parallelization to debug it." + "You cannot use 'breakpoint()' or 'pdb.set_trace()' while parallelizing the " + "execution of tasks with pytask-parallel. Please, remove the breakpoint or run " + "the task without parallelization to debug it." ) +def _patch_set_trace_and_breakpoint() -> None: + """Patch :func:`pdb.set_trace` and :func:`breakpoint`. + + Patch sys.breakpointhook to intercept any call of breakpoint() and pdb.set_trace in + a subprocess and print a better exception message. + + """ + import pdb # noqa: T100 + import sys + + pdb.set_trace = _raise_exception_on_breakpoint + sys.breakpointhook = _raise_exception_on_breakpoint + + def _execute_task( # noqa: PLR0913 task: PTask, kwargs: dict[str, Any], @@ -233,12 +247,7 @@ def _execute_task( # noqa: PLR0913 """ __tracebackhide__ = True - - # Path sys.breakpointhook to intercept any call of breakpoint() in a subprocess and - # print a better exception message. - import sys - - sys.breakpointhook = _raise_exception_on_breakpoint + _patch_set_trace_and_breakpoint() with warnings.catch_warnings(record=True) as log: # mypy can't infer that record=True means log is not None; help it. diff --git a/tests/test_execute.py b/tests/test_execute.py index 59c6812..49ec3d1 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -271,11 +271,12 @@ def test_parallel_execution_is_deactivated(runner, tmp_path, flag, parallel_back @pytest.mark.end_to_end() +@pytest.mark.parametrize("code", ["breakpoint()", "import pdb; pdb.set_trace()"]) @pytest.mark.parametrize( "parallel_backend", [i for i in PARALLEL_BACKENDS if i != ParallelBackend.THREADS] ) -def test_raise_error_on_breakpoint(runner, tmp_path, parallel_backend): - tmp_path.joinpath("task_example.py").write_text("def task_example(): breakpoint()") +def test_raise_error_on_breakpoint(runner, tmp_path, code, parallel_backend): + tmp_path.joinpath("task_example.py").write_text(f"def task_example(): {code}") result = runner.invoke( cli, [tmp_path.as_posix(), "-n 2", "--parallel-backend", parallel_backend] ) From cb951a64a9413718dc5b3d48151ab55cc9539bc1 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sat, 7 Oct 2023 19:26:50 +0200 Subject: [PATCH 3/7] fix. --- CHANGES.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5be415d..912235e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,8 +11,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask-parallel) and - {pull}`64` aligns pytask-parallel with pytask v0.4.0rc2. - {pull}`66` deactivates parallelization for dry-runs. - {pull}`67` fixes parallelization with partialed task functions. -- {pull}`68` updates dependencies and syntaxes. -- {pull}`69` raises more informative error message when `breakpoint()` was uses when +- {pull}`68` raises more informative error message when `breakpoint()` was uses when parallelizing with processes or loky. ## 0.3.1 - 2023-05-27 From 57e3966422ea1ff49beb385852a41620d7e638c9 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sat, 7 Oct 2023 19:29:28 +0200 Subject: [PATCH 4/7] fix. --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 912235e..861fb9f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,7 +5,7 @@ chronological order. Releases follow [semantic versioning](https://semver.org/) releases are available on [PyPI](https://pypi.org/project/pytask-parallel) and [Anaconda.org](https://anaconda.org/conda-forge/pytask-parallel). -## 0.4.0 - 2023-xx-xx +## 0.4.0 - 2023-10-07 - {pull}`62` deprecates Python 3.7. - {pull}`64` aligns pytask-parallel with pytask v0.4.0rc2. From 9e69a2732d1d21dacb5087f2c459340b7ee018e5 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sat, 7 Oct 2023 20:22:23 +0200 Subject: [PATCH 5/7] use main release.! --- .pre-commit-config.yaml | 2 +- environment.yml | 2 +- setup.cfg | 2 +- tox.ini | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ae64077..4cae91e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -73,7 +73,7 @@ repos: additional_dependencies: [ cloudpickle, optree, - pytask==0.4.0rc2, + pytask==0.4.0, rich, types-attrs, types-click, diff --git a/environment.yml b/environment.yml index c22fb80..c2757db 100644 --- a/environment.yml +++ b/environment.yml @@ -12,7 +12,7 @@ dependencies: - toml # Package dependencies - - pytask>=0.4.0rc4 + - pytask>=0.4.0 - cloudpickle - loky - optree diff --git a/setup.cfg b/setup.cfg index 445c432..1a63f49 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,7 @@ install_requires = loky optree>=0.9.0 pluggy>=1.0.0 - pytask>=0.4.0rc2 + pytask>=0.4.0 rich python_requires = >=3.8 include_package_data = True diff --git a/tox.ini b/tox.ini index c6a0a69..74d23e9 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ conda_deps = cloudpickle loky - pytask >=0.4.0rc2 + pytask >=0.4.0 pytest pytest-cov pytest-xdist From d6ab0dbf930c3a0e81ca507a0186bc7b20300288 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sat, 7 Oct 2023 20:29:02 +0200 Subject: [PATCH 6/7] Switch to normal python tox. --- .github/workflows/main.yml | 31 ++++++++----------------------- .pre-commit-config.yaml | 3 ++- setup.cfg | 1 - tox.ini | 17 +++++++---------- 4 files changed, 17 insertions(+), 35 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5a7b6a6..22c12d9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -27,22 +27,16 @@ jobs: fail-fast: false matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v4 - - uses: mamba-org/setup-micromamba@v1 + - uses: actions/setup-python@v4 with: - environment-name: gha-testing - condarc: | - channels: - - nodefaults - - conda-forge - create-args: >- - python=${{ matrix.python-version }} - mamba - tox-conda - cache-environment: true + python-version: ${{ matrix.python-version }} + cache: pip + allow-prereleases: true + - run: pip install tox - name: Install core dependencies. shell: bash -l {0} @@ -55,24 +49,15 @@ jobs: run: tox -e pytest -- tests -m "unit or (not integration and not end_to_end)" --cov=./ --cov-report=xml -n auto - name: Upload coverage report for unit tests and doctests. - if: runner.os == 'Linux' && matrix.python-version == '3.9' + if: runner.os == 'Linux' && matrix.python-version == '3.10' shell: bash -l {0} run: bash <(curl -s https://codecov.io/bash) -F unit -c - # - name: Run integration tests. - # shell: bash -l {0} - # run: tox -e pytest -- tests -m integration --cov=./ --cov-report=xml -n auto - - # - name: Upload coverage reports of integration tests. - # if: runner.os == 'Linux' && matrix.python-version == '3.9' - # shell: bash -l {0} - # run: bash <(curl -s https://codecov.io/bash) -F integration -c - - name: Run end-to-end tests. shell: bash -l {0} run: tox -e pytest -- tests -m end_to_end --cov=./ --cov-report=xml -n auto - name: Upload coverage reports of end-to-end tests. - if: runner.os == 'Linux' && matrix.python-version == '3.9' + if: runner.os == 'Linux' && matrix.python-version == '3.10' shell: bash -l {0} run: bash <(curl -s https://codecov.io/bash) -F end_to_end -c diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4cae91e..6d46d48 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -71,8 +71,9 @@ repos: --ignore-missing-imports, ] additional_dependencies: [ + attrs, cloudpickle, - optree, + loky, pytask==0.4.0, rich, types-attrs, diff --git a/setup.cfg b/setup.cfg index 1a63f49..e2aa791 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,6 @@ install_requires = click cloudpickle loky - optree>=0.9.0 pluggy>=1.0.0 pytask>=0.4.0 rich diff --git a/tox.ini b/tox.ini index 74d23e9..7fd4378 100644 --- a/tox.ini +++ b/tox.ini @@ -5,20 +5,17 @@ envlist = pytest usedevelop = true [testenv:pytest] -conda_channels = - conda-forge/label/pytask_rc - conda-forge - nodefaults -conda_deps = +deps = + # pytest + pytest + pytest-cov + pytest-xdist setuptools_scm toml + # Package + pytask >=0.4.0 cloudpickle loky - pytask >=0.4.0 - pytest - pytest-cov - pytest-xdist commands = - pip install --no-deps -e . pytest {posargs} From 93093ab13f1743d3afc7d4b4a7f0d105ecba9607 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sat, 7 Oct 2023 20:30:56 +0200 Subject: [PATCH 7/7] fix. --- .github/workflows/main.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 22c12d9..a948cef 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -38,10 +38,6 @@ jobs: allow-prereleases: true - run: pip install tox - - name: Install core dependencies. - shell: bash -l {0} - run: conda install -c conda-forge tox-conda coverage - # Unit, integration, and end-to-end tests. - name: Run unit tests and doctests.