diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5a7b6a6..a948cef 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -27,26 +27,16 @@ jobs: fail-fast: false matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v4 - - uses: mamba-org/setup-micromamba@v1 + - uses: actions/setup-python@v4 with: - environment-name: gha-testing - condarc: | - channels: - - nodefaults - - conda-forge - create-args: >- - python=${{ matrix.python-version }} - mamba - tox-conda - cache-environment: true - - - name: Install core dependencies. - shell: bash -l {0} - run: conda install -c conda-forge tox-conda coverage + python-version: ${{ matrix.python-version }} + cache: pip + allow-prereleases: true + - run: pip install tox # Unit, integration, and end-to-end tests. @@ -55,24 +45,15 @@ jobs: run: tox -e pytest -- tests -m "unit or (not integration and not end_to_end)" --cov=./ --cov-report=xml -n auto - name: Upload coverage report for unit tests and doctests. - if: runner.os == 'Linux' && matrix.python-version == '3.9' + if: runner.os == 'Linux' && matrix.python-version == '3.10' shell: bash -l {0} run: bash <(curl -s https://codecov.io/bash) -F unit -c - # - name: Run integration tests. - # shell: bash -l {0} - # run: tox -e pytest -- tests -m integration --cov=./ --cov-report=xml -n auto - - # - name: Upload coverage reports of integration tests. - # if: runner.os == 'Linux' && matrix.python-version == '3.9' - # shell: bash -l {0} - # run: bash <(curl -s https://codecov.io/bash) -F integration -c - - name: Run end-to-end tests. shell: bash -l {0} run: tox -e pytest -- tests -m end_to_end --cov=./ --cov-report=xml -n auto - name: Upload coverage reports of end-to-end tests. - if: runner.os == 'Linux' && matrix.python-version == '3.9' + if: runner.os == 'Linux' && matrix.python-version == '3.10' shell: bash -l {0} run: bash <(curl -s https://codecov.io/bash) -F end_to_end -c diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9f012c2..6d46d48 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,6 @@ repos: - id: check-merge-conflict - id: check-vcs-permalinks - id: check-yaml - - id: debug-statements - id: end-of-file-fixer - id: fix-byte-order-marker - id: mixed-line-ending @@ -72,9 +71,10 @@ repos: --ignore-missing-imports, ] additional_dependencies: [ + attrs, cloudpickle, - optree, - pytask==0.4.0rc2, + loky, + pytask==0.4.0, rich, types-attrs, types-click, diff --git a/CHANGES.md b/CHANGES.md index 3674815..861fb9f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,13 +5,14 @@ chronological order. Releases follow [semantic versioning](https://semver.org/) releases are available on [PyPI](https://pypi.org/project/pytask-parallel) and [Anaconda.org](https://anaconda.org/conda-forge/pytask-parallel). -## 0.4.0 - 2023-xx-xx +## 0.4.0 - 2023-10-07 - {pull}`62` deprecates Python 3.7. - {pull}`64` aligns pytask-parallel with pytask v0.4.0rc2. - {pull}`66` deactivates parallelization for dry-runs. - {pull}`67` fixes parallelization with partialed task functions. -- {pull}`68` updates dependencies and syntaxes. +- {pull}`68` raises more informative error message when `breakpoint()` was uses when + parallelizing with processes or loky. ## 0.3.1 - 2023-05-27 diff --git a/environment.yml b/environment.yml index c22fb80..c2757db 100644 --- a/environment.yml +++ b/environment.yml @@ -12,7 +12,7 @@ dependencies: - toml # Package dependencies - - pytask>=0.4.0rc4 + - pytask>=0.4.0 - cloudpickle - loky - optree diff --git a/setup.cfg b/setup.cfg index 445c432..e2aa791 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,9 +28,8 @@ install_requires = click cloudpickle loky - optree>=0.9.0 pluggy>=1.0.0 - pytask>=0.4.0rc2 + pytask>=0.4.0 rich python_requires = >=3.8 include_package_data = True diff --git a/src/pytask_parallel/execute.py b/src/pytask_parallel/execute.py index b54dd24..70886e3 100644 --- a/src/pytask_parallel/execute.py +++ b/src/pytask_parallel/execute.py @@ -210,6 +210,28 @@ def pytask_execute_task(session: Session, task: PTask) -> Future[Any] | None: return None +def _raise_exception_on_breakpoint(*args: Any, **kwargs: Any) -> None: # noqa: ARG001 + raise RuntimeError( + "You cannot use 'breakpoint()' or 'pdb.set_trace()' while parallelizing the " + "execution of tasks with pytask-parallel. Please, remove the breakpoint or run " + "the task without parallelization to debug it." + ) + + +def _patch_set_trace_and_breakpoint() -> None: + """Patch :func:`pdb.set_trace` and :func:`breakpoint`. + + Patch sys.breakpointhook to intercept any call of breakpoint() and pdb.set_trace in + a subprocess and print a better exception message. + + """ + import pdb # noqa: T100 + import sys + + pdb.set_trace = _raise_exception_on_breakpoint + sys.breakpointhook = _raise_exception_on_breakpoint + + def _execute_task( # noqa: PLR0913 task: PTask, kwargs: dict[str, Any], @@ -225,6 +247,7 @@ def _execute_task( # noqa: PLR0913 """ __tracebackhide__ = True + _patch_set_trace_and_breakpoint() with warnings.catch_warnings(record=True) as log: # mypy can't infer that record=True means log is not None; help it. diff --git a/tests/test_execute.py b/tests/test_execute.py index f24e62b..49ec3d1 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -268,3 +268,17 @@ def test_parallel_execution_is_deactivated(runner, tmp_path, flag, parallel_back ) assert result.exit_code == ExitCode.OK assert "Started 2 workers" not in result.output + + +@pytest.mark.end_to_end() +@pytest.mark.parametrize("code", ["breakpoint()", "import pdb; pdb.set_trace()"]) +@pytest.mark.parametrize( + "parallel_backend", [i for i in PARALLEL_BACKENDS if i != ParallelBackend.THREADS] +) +def test_raise_error_on_breakpoint(runner, tmp_path, code, parallel_backend): + tmp_path.joinpath("task_example.py").write_text(f"def task_example(): {code}") + result = runner.invoke( + cli, [tmp_path.as_posix(), "-n 2", "--parallel-backend", parallel_backend] + ) + assert result.exit_code == ExitCode.FAILED + assert "You cannot use 'breakpoint()'" in result.output diff --git a/tox.ini b/tox.ini index c6a0a69..7fd4378 100644 --- a/tox.ini +++ b/tox.ini @@ -5,20 +5,17 @@ envlist = pytest usedevelop = true [testenv:pytest] -conda_channels = - conda-forge/label/pytask_rc - conda-forge - nodefaults -conda_deps = +deps = + # pytest + pytest + pytest-cov + pytest-xdist setuptools_scm toml + # Package + pytask >=0.4.0 cloudpickle loky - pytask >=0.4.0rc2 - pytest - pytest-cov - pytest-xdist commands = - pip install --no-deps -e . pytest {posargs}