Skip to content

Commit c91d008

Browse files
tobiasraabetobiasraabe
authored
Deprecate INI configuration. (#50)
1 parent c22187b commit c91d008

File tree

15 files changed

+88
-113
lines changed

15 files changed

+88
-113
lines changed

‎.pre-commit-config.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,6 @@ repos:
9393
types-setuptools
9494
]
9595
pass_filenames: false
96-
language_version: "3.9"
9796
- repo: https://github.com/mgedmin/check-manifest
9897
rev: "0.49"
9998
hooks:

‎CHANGES.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@ chronological order. Releases follow [semantic versioning](https://semver.org/)
55
releases are available on [PyPI](https://pypi.org/project/pytask-parallel) and
66
[Anaconda.org](https://anaconda.org/conda-forge/pytask-parallel).
77

8+
## 0.3.0 - 2023-xx-xx
9+
10+
- {pull}`50` deprecates INI configurations and aligns the package with pytask v0.3.
11+
812
## 0.2.1 - 2022-08-19
913

1014
- {pull}`43` adds docformatter.

‎LICENSE‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
Copyright 2020-2021 Tobias Raabe
1+
Copyright 2020 Tobias Raabe
22

33
Permission is hereby granted, free of charge, to any person obtaining a copy of this
44
software and associated documentation files (the "Software"), to deal in the Software

‎MANIFEST.in‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,5 @@ exclude tox.ini
77

88
include README.md
99
include LICENSE
10+
11+
recursive-include src py.typed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
[![image](https://img.shields.io/conda/vn/conda-forge/pytask-parallel.svg)](https://anaconda.org/conda-forge/pytask-parallel)
66
[![image](https://img.shields.io/conda/pn/conda-forge/pytask-parallel.svg)](https://anaconda.org/conda-forge/pytask-parallel)
77
[![PyPI - License](https://img.shields.io/pypi/l/pytask-parallel)](https://pypi.org/project/pytask-parallel)
8-
[![image](https://img.shields.io/github/workflow/status/pytask-dev/pytask-parallel/Continuous%20Integration%20Workflow/main)](https://github.com/pytask-dev/pytask-parallel/actions?query=branch%3Amain)
8+
[![image](https://img.shields.io/github/actions/workflow/status/pytask-dev/pytask-parallel/main.yml?branch=main)](https://github.com/pytask-dev/pytask-parallel/actions?query=branch%3Amain)
99
[![image](https://codecov.io/gh/pytask-dev/pytask-parallel/branch/main/graph/badge.svg)](https://codecov.io/gh/pytask-dev/pytask-parallel)
1010
[![pre-commit.ci status](https://results.pre-commit.ci/badge/github/pytask-dev/pytask-parallel/main.svg)](https://results.pre-commit.ci/latest/github/pytask-dev/pytask-parallel/main)
1111
[![image](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)

‎environment.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ dependencies:
1616
- conda-verify
1717

1818
# Package dependencies
19-
- pytask >=0.2.0
19+
- pytask >=0.3
2020
- cloudpickle
2121
- loky
2222
- pybaum >=0.1.1

‎setup.cfg‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ install_requires =
2828
cloudpickle
2929
loky
3030
pybaum>=0.1.1
31-
pytask>=0.2
31+
pytask>=0.3
3232
python_requires = >=3.7
3333
include_package_data = True
3434
package_dir = =src

‎src/pytask_parallel/backends.py‎

Lines changed: 33 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,42 @@
33

44
from concurrent.futures import ProcessPoolExecutor
55
from concurrent.futures import ThreadPoolExecutor
6+
from enum import Enum
67

78

8-
PARALLEL_BACKENDS = {
9-
"processes": ProcessPoolExecutor,
10-
"threads": ThreadPoolExecutor,
11-
}
12-
13-
PARALLEL_BACKENDS_DEFAULT = "processes"
14-
159
try:
1610
from loky import get_reusable_executor
11+
1712
except ImportError:
18-
pass
13+
14+
class ParallelBackendChoices(str, Enum):
15+
PROCESSES = "processes"
16+
THREADS = "threads"
17+
18+
PARALLEL_BACKENDS_DEFAULT = ParallelBackendChoices.PROCESSES
19+
20+
PARALLEL_BACKENDS = {
21+
ParallelBackendChoices.PROCESSES: ProcessPoolExecutor,
22+
ParallelBackendChoices.THREADS: ThreadPoolExecutor,
23+
}
24+
1925
else:
20-
PARALLEL_BACKENDS["loky"] = get_reusable_executor
21-
PARALLEL_BACKENDS_DEFAULT = "loky"
26+
27+
class ParallelBackendChoices(str, Enum): # type: ignore[no-redef]
28+
PROCESSES = "processes"
29+
THREADS = "threads"
30+
LOKY = "loky"
31+
32+
PARALLEL_BACKENDS_DEFAULT = ParallelBackendChoices.PROCESSES
33+
34+
PARALLEL_BACKENDS = {
35+
ParallelBackendChoices.PROCESSES: ProcessPoolExecutor,
36+
ParallelBackendChoices.THREADS: ThreadPoolExecutor,
37+
ParallelBackendChoices.LOKY: ( # type: ignore[attr-defined]
38+
get_reusable_executor
39+
),
40+
}
41+
42+
PARALLEL_BACKENDS_DEFAULT = (
43+
ParallelBackendChoices.LOKY # type: ignore[attr-defined]
44+
)

‎src/pytask_parallel/build.py‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,10 @@
22
from __future__ import annotations
33

44
import click
5+
from pytask import EnumChoice
56
from pytask import hookimpl
6-
from pytask_parallel.backends import PARALLEL_BACKENDS
77
from pytask_parallel.backends import PARALLEL_BACKENDS_DEFAULT
8+
from pytask_parallel.backends import ParallelBackendChoices
89

910

1011
@hookimpl
@@ -15,19 +16,16 @@ def pytask_extend_command_line_interface(cli: click.Group) -> None:
1516
["-n", "--n-workers"],
1617
help=(
1718
"Max. number of pytask_parallel tasks. Integer >= 1 or 'auto' which is "
18-
"os.cpu_count() - 1. [default: 1 (no parallelization)]"
19+
"os.cpu_count() - 1."
1920
),
2021
metavar="[INTEGER|auto]",
21-
default=None,
22+
default=1,
2223
),
2324
click.Option(
2425
["--parallel-backend"],
25-
type=click.Choice(PARALLEL_BACKENDS),
26-
help=(
27-
"Backend for the parallelization. "
28-
f"[default: {PARALLEL_BACKENDS_DEFAULT}]"
29-
),
30-
default=None,
26+
type=EnumChoice(ParallelBackendChoices),
27+
help="Backend for the parallelization.",
28+
default=PARALLEL_BACKENDS_DEFAULT,
3129
),
3230
]
3331
cli.commands["build"].params.extend(additional_parameters)

‎src/pytask_parallel/config.py‎

Lines changed: 16 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1,63 +1,38 @@
11
"""Configure pytask."""
22
from __future__ import annotations
33

4+
import enum
45
import os
56
from typing import Any
6-
from typing import Callable
77

88
from pytask import hookimpl
9-
from pytask_parallel.backends import PARALLEL_BACKENDS_DEFAULT
10-
from pytask_parallel.callbacks import n_workers_callback
11-
from pytask_parallel.callbacks import parallel_backend_callback
9+
from pytask_parallel.backends import ParallelBackendChoices
1210

1311

1412
@hookimpl
15-
def pytask_parse_config(
16-
config: dict[str, Any],
17-
config_from_cli: dict[str, Any],
18-
config_from_file: dict[str, Any],
19-
) -> None:
13+
def pytask_parse_config(config: dict[str, Any]) -> None:
2014
"""Parse the configuration."""
21-
config["n_workers"] = _get_first_non_none_value(
22-
config_from_cli,
23-
config_from_file,
24-
key="n_workers",
25-
default=1,
26-
callback=n_workers_callback,
27-
)
2815
if config["n_workers"] == "auto":
2916
config["n_workers"] = max(os.cpu_count() - 1, 1)
3017

31-
config["delay"] = 0.1
18+
if (
19+
isinstance(config["parallel_backend"], str)
20+
and config["parallel_backend"] in ParallelBackendChoices._value2member_map_
21+
):
22+
config["parallel_backend"] = ParallelBackendChoices(config["parallel_backend"])
23+
elif (
24+
isinstance(config["parallel_backend"], enum.Enum)
25+
and config["parallel_backend"] in ParallelBackendChoices
26+
):
27+
pass
28+
else:
29+
raise ValueError("Invalid value for 'parallel_backend'.")
3230

33-
config["parallel_backend"] = _get_first_non_none_value(
34-
config_from_cli,
35-
config_from_file,
36-
key="parallel_backend",
37-
default=PARALLEL_BACKENDS_DEFAULT,
38-
callback=parallel_backend_callback,
39-
)
31+
config["delay"] = 0.1
4032

4133

4234
@hookimpl
4335
def pytask_post_parse(config: dict[str, Any]) -> None:
4436
"""Disable parallelization if debugging is enabled."""
4537
if config["pdb"] or config["trace"]:
4638
config["n_workers"] = 1
47-
48-
49-
def _get_first_non_none_value(
50-
*configs: dict[str, Any],
51-
key: str,
52-
default: Any | None = None,
53-
callback: Callable[..., Any] | None = None,
54-
) -> Any:
55-
"""Get the first non-None value for a key from a list of dictionaries.
56-
57-
This function allows to prioritize information from many configurations by changing
58-
the order of the inputs while also providing a default.
59-
60-
"""
61-
callback = (lambda x: x) if callback is None else callback # noqa: E731
62-
processed_values = (callback(config.get(key)) for config in configs)
63-
return next((value for value in processed_values if value is not None), default)

0 commit comments

Comments
 (0)