Deprecate INI configuration. (#50)

Author: tobiasraabe

.pre-commit-config.yaml

@@ -93,7 +93,6 @@ repos:
             types-setuptools
         ]
         pass_filenames: false
-        language_version: "3.9"
 -   repo: https://github.com/mgedmin/check-manifest
     rev: "0.49"
     hooks:

CHANGES.md

@@ -5,6 +5,10 @@ chronological order. Releases follow [semantic versioning](https://semver.org/)
 releases are available on [PyPI](https://pypi.org/project/pytask-parallel) and
 [Anaconda.org](https://anaconda.org/conda-forge/pytask-parallel).
 
+## 0.3.0 - 2023-xx-xx
+
+- {pull}`50` deprecates INI configurations and aligns the package with pytask v0.3.
+
 ## 0.2.1 - 2022-08-19
 
 - {pull}`43` adds docformatter.

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2020-2021 Tobias Raabe
+Copyright 2020 Tobias Raabe
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this
 software and associated documentation files (the "Software"), to deal in the Software

MANIFEST.in

@@ -7,3 +7,5 @@ exclude tox.ini
 
 include README.md
 include LICENSE
+
+recursive-include src py.typed

README.md

@@ -5,7 +5,7 @@
 [![image](https://img.shields.io/conda/vn/conda-forge/pytask-parallel.svg)](https://anaconda.org/conda-forge/pytask-parallel)
 [![image](https://img.shields.io/conda/pn/conda-forge/pytask-parallel.svg)](https://anaconda.org/conda-forge/pytask-parallel)
 [![PyPI - License](https://img.shields.io/pypi/l/pytask-parallel)](https://pypi.org/project/pytask-parallel)
-[![image](https://img.shields.io/github/workflow/status/pytask-dev/pytask-parallel/Continuous%20Integration%20Workflow/main)](https://github.com/pytask-dev/pytask-parallel/actions?query=branch%3Amain)
+[![image](https://img.shields.io/github/actions/workflow/status/pytask-dev/pytask-parallel/main.yml?branch=main)](https://github.com/pytask-dev/pytask-parallel/actions?query=branch%3Amain)
 [![image](https://codecov.io/gh/pytask-dev/pytask-parallel/branch/main/graph/badge.svg)](https://codecov.io/gh/pytask-dev/pytask-parallel)
 [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/pytask-dev/pytask-parallel/main.svg)](https://results.pre-commit.ci/latest/github/pytask-dev/pytask-parallel/main)
 [![image](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)

environment.yml

@@ -16,7 +16,7 @@ dependencies:
   - conda-verify
 
   # Package dependencies
-  - pytask >=0.2.0
+  - pytask >=0.3
   - cloudpickle
   - loky
   - pybaum >=0.1.1

setup.cfg

@@ -28,7 +28,7 @@ install_requires =
     cloudpickle
     loky
     pybaum>=0.1.1
-    pytask>=0.2
+    pytask>=0.3
 python_requires = >=3.7
 include_package_data = True
 package_dir = =src

src/pytask_parallel/backends.py

@@ -3,19 +3,42 @@
 
 from concurrent.futures import ProcessPoolExecutor
 from concurrent.futures import ThreadPoolExecutor
+from enum import Enum
 
 
-PARALLEL_BACKENDS = {
-    "processes": ProcessPoolExecutor,
-    "threads": ThreadPoolExecutor,
-}
-
-PARALLEL_BACKENDS_DEFAULT = "processes"
-
 try:
     from loky import get_reusable_executor
+
 except ImportError:
-    pass
+
+    class ParallelBackendChoices(str, Enum):
+        PROCESSES = "processes"
+        THREADS = "threads"
+
+    PARALLEL_BACKENDS_DEFAULT = ParallelBackendChoices.PROCESSES
+
+    PARALLEL_BACKENDS = {
+        ParallelBackendChoices.PROCESSES: ProcessPoolExecutor,
+        ParallelBackendChoices.THREADS: ThreadPoolExecutor,
+    }
+
 else:
-    PARALLEL_BACKENDS["loky"] = get_reusable_executor
-    PARALLEL_BACKENDS_DEFAULT = "loky"
+
+    class ParallelBackendChoices(str, Enum):  # type: ignore[no-redef]
+        PROCESSES = "processes"
+        THREADS = "threads"
+        LOKY = "loky"
+
+    PARALLEL_BACKENDS_DEFAULT = ParallelBackendChoices.PROCESSES
+
+    PARALLEL_BACKENDS = {
+        ParallelBackendChoices.PROCESSES: ProcessPoolExecutor,
+        ParallelBackendChoices.THREADS: ThreadPoolExecutor,
+        ParallelBackendChoices.LOKY: (  # type: ignore[attr-defined]
+            get_reusable_executor
+        ),
+    }
+
+    PARALLEL_BACKENDS_DEFAULT = (
+        ParallelBackendChoices.LOKY  # type: ignore[attr-defined]
+    )

src/pytask_parallel/build.py

@@ -2,9 +2,10 @@
 from __future__ import annotations
 
 import click
+from pytask import EnumChoice
 from pytask import hookimpl
-from pytask_parallel.backends import PARALLEL_BACKENDS
 from pytask_parallel.backends import PARALLEL_BACKENDS_DEFAULT
+from pytask_parallel.backends import ParallelBackendChoices
 
 
 @hookimpl
@@ -15,19 +16,16 @@ def pytask_extend_command_line_interface(cli: click.Group) -> None:
             ["-n", "--n-workers"],
             help=(
                 "Max. number of pytask_parallel tasks. Integer >= 1 or 'auto' which is "
-                "os.cpu_count() - 1.  [default: 1 (no parallelization)]"
+                "os.cpu_count() - 1."
             ),
             metavar="[INTEGER|auto]",
-            default=None,
+            default=1,
         ),
         click.Option(
             ["--parallel-backend"],
-            type=click.Choice(PARALLEL_BACKENDS),
-            help=(
-                "Backend for the parallelization.  "
-                f"[default: {PARALLEL_BACKENDS_DEFAULT}]"
-            ),
-            default=None,
+            type=EnumChoice(ParallelBackendChoices),
+            help="Backend for the parallelization.",
+            default=PARALLEL_BACKENDS_DEFAULT,
         ),
     ]
     cli.commands["build"].params.extend(additional_parameters)

src/pytask_parallel/config.py

@@ -1,63 +1,38 @@
 """Configure pytask."""
 from __future__ import annotations
 
+import enum
 import os
 from typing import Any
-from typing import Callable
 
 from pytask import hookimpl
-from pytask_parallel.backends import PARALLEL_BACKENDS_DEFAULT
-from pytask_parallel.callbacks import n_workers_callback
-from pytask_parallel.callbacks import parallel_backend_callback
+from pytask_parallel.backends import ParallelBackendChoices
 
 
 @hookimpl
-def pytask_parse_config(
-    config: dict[str, Any],
-    config_from_cli: dict[str, Any],
-    config_from_file: dict[str, Any],
-) -> None:
+def pytask_parse_config(config: dict[str, Any]) -> None:
     """Parse the configuration."""
-    config["n_workers"] = _get_first_non_none_value(
-        config_from_cli,
-        config_from_file,
-        key="n_workers",
-        default=1,
-        callback=n_workers_callback,
-    )
     if config["n_workers"] == "auto":
         config["n_workers"] = max(os.cpu_count() - 1, 1)
 
-    config["delay"] = 0.1
+    if (
+        isinstance(config["parallel_backend"], str)
+        and config["parallel_backend"] in ParallelBackendChoices._value2member_map_
+    ):
+        config["parallel_backend"] = ParallelBackendChoices(config["parallel_backend"])
+    elif (
+        isinstance(config["parallel_backend"], enum.Enum)
+        and config["parallel_backend"] in ParallelBackendChoices
+    ):
+        pass
+    else:
+        raise ValueError("Invalid value for 'parallel_backend'.")
 
-    config["parallel_backend"] = _get_first_non_none_value(
-        config_from_cli,
-        config_from_file,
-        key="parallel_backend",
-        default=PARALLEL_BACKENDS_DEFAULT,
-        callback=parallel_backend_callback,
-    )
+    config["delay"] = 0.1
 
 
 @hookimpl
 def pytask_post_parse(config: dict[str, Any]) -> None:
     """Disable parallelization if debugging is enabled."""
     if config["pdb"] or config["trace"]:
         config["n_workers"] = 1
-
-
-def _get_first_non_none_value(
-    *configs: dict[str, Any],
-    key: str,
-    default: Any | None = None,
-    callback: Callable[..., Any] | None = None,
-) -> Any:
-    """Get the first non-None value for a key from a list of dictionaries.
-
-    This function allows to prioritize information from many configurations by changing
-    the order of the inputs while also providing a default.
-
-    """
-    callback = (lambda x: x) if callback is None else callback  # noqa: E731
-    processed_values = (callback(config.get(key)) for config in configs)
-    return next((value for value in processed_values if value is not None), default)

src/pytask_parallel/execute.py

@@ -19,31 +19,25 @@
 from pytask import get_marks
 from pytask import hookimpl
 from pytask import Mark
+from pytask import parse_warning_filter
 from pytask import remove_internal_traceback_frames_from_exc_info
 from pytask import Session
 from pytask import Task
+from pytask import warning_record_to_str
+from pytask import WarningReport
 from pytask_parallel.backends import PARALLEL_BACKENDS
+from pytask_parallel.backends import ParallelBackendChoices
 from rich.console import ConsoleOptions
 from rich.traceback import Traceback
 
-# Can be removed if pinned to pytask >= 0.2.6.
-try:
-    from pytask import parse_warning_filter
-    from pytask import warning_record_to_str
-    from pytask import WarningReport
-except ImportError:
-    from _pytask.warnings import parse_warning_filter
-    from _pytask.warnings import warning_record_to_str
-    from _pytask.warnings_utils import WarningReport
-
 
 @hookimpl
 def pytask_post_parse(config: dict[str, Any]) -> None:
     """Register the parallel backend."""
-    if config["parallel_backend"] in ("loky", "processes"):
-        config["pm"].register(ProcessesNameSpace)
-    elif config["parallel_backend"] in ("threads",):
+    if config["parallel_backend"] == ParallelBackendChoices.THREADS:
         config["pm"].register(DefaultBackendNameSpace)
+    else:
+        config["pm"].register(ProcessesNameSpace)
 
 
 @hookimpl(tryfirst=True)
@@ -66,7 +60,7 @@ def pytask_execute_build(session: Session) -> bool | None:
 
         with parallel_backend(max_workers=session.config["n_workers"]) as executor:
 
-            session.executor = executor
+            session.config["_parallel_executor"] = executor
             sleeper = _Sleeper()
 
             while session.scheduler.is_active():
@@ -189,7 +183,7 @@ def pytask_execute_task(session: Session, task: Task) -> Future[Any] | None:
             bytes_function = cloudpickle.dumps(task)
             bytes_kwargs = cloudpickle.dumps(kwargs)
 
-            return session.executor.submit(
+            return session.config["_parallel_executor"].submit(
                 _unserialize_and_execute_task,
                 bytes_function=bytes_function,
                 bytes_kwargs=bytes_kwargs,
@@ -285,7 +279,7 @@ def pytask_execute_task(session: Session, task: Task) -> Future[Any] | None:
         """
         if session.config["n_workers"] > 1:
             kwargs = _create_kwargs_for_task(task)
-            return session.executor.submit(
+            return session.config["_parallel_executor"].submit(
                 _mock_processes_for_threads, func=task.execute, **kwargs
             )
         else:

src/pytask_parallel/py.typed

@@ -6,7 +6,7 @@
 import pytest
 from pytask import ExitCode
 from pytask import main
-from pytask_parallel.backends import PARALLEL_BACKENDS
+from pytask_parallel.backends import ParallelBackendChoices
 
 
 @pytest.mark.end_to_end
@@ -26,9 +26,6 @@ def test_interplay_between_debugging_and_parallel(tmp_path, pdb, n_workers, expe
 
 
 @pytest.mark.end_to_end
-@pytest.mark.parametrize(
-    "config_file", ["pytask.ini", "tox.ini", "setup.cfg", "pyproject.toml"]
-)
 @pytest.mark.parametrize(
     "configuration_option, value, exit_code",
     [
@@ -38,31 +35,20 @@ def test_interplay_between_debugging_and_parallel(tmp_path, pdb, n_workers, expe
         ("parallel_backend", "unknown_backend", ExitCode.CONFIGURATION_FAILED),
     ]
     + [
-        ("parallel_backend", parallel_backend, ExitCode.OK)
-        for parallel_backend in PARALLEL_BACKENDS
+        ("parallel_backend", parallel_backend.value, ExitCode.OK)
+        for parallel_backend in ParallelBackendChoices
     ],
 )
 def test_reading_values_from_config_file(
-    tmp_path, capsys, config_file, configuration_option, value, exit_code
+    tmp_path, configuration_option, value, exit_code
 ):
-    if config_file == "pyproject.toml":
-        config = f"""
-        [tool.pytask.ini_options]
-        {configuration_option} = {value!r}
-        """
-    else:
-        config = f"""
-        [pytask]
-        {configuration_option} = {value}
-        """
-    tmp_path.joinpath(config_file).write_text(textwrap.dedent(config))
+    config = f"""
+    [tool.pytask.ini_options]
+    {configuration_option} = {value!r}
+    """
+    tmp_path.joinpath("pyproject.toml").write_text(textwrap.dedent(config))
 
     session = main({"paths": tmp_path})
-    captured = capsys.readouterr()
-    if config_file == "pyproject.toml":
-        assert "WARNING" not in captured.out
-    else:
-        assert "WARNING" in captured.out
 
     assert session.exit_code == exit_code
     if value == "auto":

tests/test_config.py

@@ -126,7 +126,7 @@ def myfunc():
     with PARALLEL_BACKENDS[parallel_backend](
         max_workers=session.config["n_workers"]
     ) as executor:
-        session.executor = executor
+        session.config["_parallel_executor"] = executor
 
         backend_name_space = {
             "processes": ProcessesNameSpace,

tests/test_execute.py

@@ -1,16 +1,14 @@
 [tox]
 envlist = pytest
-skipsdist = True
-skip_missing_interpreters = True
 
 [testenv]
-basepython = python
+usedevelop = true
 
 [testenv:pytest]
 conda_deps =
     cloudpickle
     loky
-    pytask >=0.1.0
+    pytask >=0.3.0
     pytest
     pytest-cov
     pytest-xdist
@@ -21,10 +19,6 @@ commands =
     pip install --no-deps -e .
     pytest {posargs}
 
-[doc8]
-ignore = D002, D004
-max-line-length = 89
-
 [flake8]
 docstring-convention = numpy
 ignore =