Skip to content

Release v0.0.3. #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Sep 13, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .conda/meta.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ requirements:
run:
- python >=3.6
- cloudpickle
- pytask >=0.0.5
- pytask >=0.0.6

test:
requires:
Expand All @@ -32,6 +32,8 @@ test:
commands:
- pytask --version
- pytask --help
- pytask clean
- pytask markers

- pytest tests

Expand Down
11 changes: 8 additions & 3 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,11 @@ repos:
hooks:
- id: reorder-python-imports
- repo: https://github.com/psf/black
rev: 19.10b0
rev: 20.8b1
hooks:
- id: black
- repo: https://github.com/asottile/blacken-docs
rev: v1.7.0
rev: v1.8.0
hooks:
- id: blacken-docs
additional_dependencies: [black]
Expand All @@ -48,9 +48,14 @@ repos:
Pygments,
]
- repo: https://github.com/PyCQA/doc8
rev: 0.8.1rc3
rev: 0.9.0a1
hooks:
- id: doc8
- repo: https://github.com/econchick/interrogate
rev: 1.3.1
hooks:
- id: interrogate
args: [-v, --fail-under=40, src, tests]
- repo: https://github.com/codespell-project/codespell
rev: v1.17.1
hooks:
Expand Down
10 changes: 8 additions & 2 deletions CHANGES.rst
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,17 @@ chronological order. Releases follow `semantic versioning <https://semver.org/>`
all releases are available on `Anaconda.org <https://anaconda.org/pytask/pytask-parallel>`_.


0.0.3 - 2020-09-12
------------------

- :gh:`3` align the program with pytask v0.0.6.


0.0.2 - 2020-08-12
------------------

- :gh:`2` prepares the plugin for pytask v0.0.5.
- :gh:`3` better parsing and callbacks.
- :gh:`1` prepares the plugin for pytask v0.0.5.
- :gh:`2` better parsing and callbacks.


0.0.1 - 2020-07-17
Expand Down
2 changes: 1 addition & 1 deletion environment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ dependencies:
- conda-verify

# Package dependencies
- pytask >= 0.0.5
- pytask >= 0.0.6
- cloudpickle

# Misc
Expand Down
2 changes: 1 addition & 1 deletion setup.cfg
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[bumpversion]
current_version = 0.0.2
current_version = 0.0.3
parse = (?P<major>\d+)\.(?P<minor>\d+)(\.(?P<patch>\d+))(\-?((dev)?(?P<dev>\d+))?)
serialize =
{major}.{minor}.{patch}dev{dev}
Expand Down
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

setup(
name="pytask-parallel",
version="0.0.2",
version="0.0.3",
packages=find_packages(where="src"),
package_dir={"": "src"},
entry_points={"pytask": ["pytask_parallel = pytask_parallel.plugin"]},
Expand Down
2 changes: 1 addition & 1 deletion src/pytask_parallel/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "0.0.2"
__version__ = "0.0.3"
13 changes: 7 additions & 6 deletions src/pytask_parallel/cli.py → src/pytask_parallel/build.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
"""Extend the build command."""
import click
from _pytask.config import hookimpl
from pytask_parallel.callbacks import delay_click_callback
from pytask_parallel.callbacks import n_workers_click_callback


@hookimpl
def pytask_add_parameters_to_cli(command):
def pytask_extend_command_line_interface(cli):
"""Extend the command line interface."""
additional_parameters = [
click.Option(
["-n", "--n-workers"],
Expand All @@ -14,12 +14,13 @@ def pytask_add_parameters_to_cli(command):
"os.cpu_count() - 1. [default: 1 (no parallelization)]"
),
metavar="[INTEGER|auto]",
callback=n_workers_click_callback,
default=None,
),
click.Option(
["--parallel-backend"],
type=click.Choice(["processes", "threads"]),
help="Backend for the parallelization. [default: processes]",
default=None,
),
click.Option(
["--delay"],
Expand All @@ -28,7 +29,7 @@ def pytask_add_parameters_to_cli(command):
"(seconds)]"
),
metavar="NUMBER > 0",
callback=delay_click_callback,
default=None,
),
]
command.params.extend(additional_parameters)
cli.commands["build"].params.extend(additional_parameters)
51 changes: 17 additions & 34 deletions src/pytask_parallel/callbacks.py
Original file line number Diff line number Diff line change
@@ -1,59 +1,42 @@
import click


def n_workers_click_callback(ctx, name, value): # noqa: U100
return n_workers_callback(value)
"""Validate command line inputs and configuration values."""


def n_workers_callback(value):
error_occurred = False
"""Validate the n-workers option."""
if value == "auto":
pass
elif value is None:
pass
elif value is None or value == "None":
value = None
elif isinstance(value, int) and 1 <= value:
pass
elif isinstance(value, str) and value.isdigit():
value = int(value)
else:
try:
value = int(value)
except ValueError:
error_occurred = True
else:
if value < 1:
error_occurred = True

if error_occurred:
raise click.UsageError("n-processes can either be an integer >= 1 or 'auto'.")
raise ValueError("n_processes can either be an integer >= 1, 'auto' or None.")

return value


def parallel_backend_callback(value):
"""Validate the input for the parallel backend."""
if value == "None":
value = None
if value not in ["processes", "threads", None]:
raise click.UsageError("parallel_backend has to be 'processes' or 'threads'.")
raise ValueError("parallel_backend has to be 'processes' or 'threads'.")
return value


def delay_click_callback(ctx, name, value): # noqa: U100
return delay_callback(value)


def delay_callback(value):
error_occurred = False
if isinstance(value, float) and 0 < value:
pass
elif value is None:
pass
"""Validate the delay option."""
if value is None or value == "None":
value = None
else:
try:
value = float(value)
except ValueError:
error_occurred = True
else:
if value < 0:
error_occurred = True
pass

if error_occurred:
raise click.UsageError("delay has to be a number greater than 0.")
if not (isinstance(value, float) and value > 0):
raise ValueError("delay has to be a number greater than 0.")

return value
17 changes: 12 additions & 5 deletions src/pytask_parallel/config.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,17 @@
"""Configure pytask."""
import os

from _pytask.config import hookimpl
from _pytask.shared import get_first_not_none_value
from _pytask.shared import get_first_non_none_value
from pytask_parallel.callbacks import delay_callback
from pytask_parallel.callbacks import n_workers_callback
from pytask_parallel.callbacks import parallel_backend_callback


@hookimpl
def pytask_parse_config(config, config_from_cli, config_from_file):
config["n_workers"] = get_first_not_none_value(
"""Parse the configuration."""
config["n_workers"] = get_first_non_none_value(
config_from_cli,
config_from_file,
key="n_workers",
Expand All @@ -18,11 +21,15 @@ def pytask_parse_config(config, config_from_cli, config_from_file):
if config["n_workers"] == "auto":
config["n_workers"] = max(os.cpu_count() - 1, 1)

config["delay"] = get_first_not_none_value(
config_from_cli, config_from_file, key="delay", default=0.1, callback=float
config["delay"] = get_first_non_none_value(
config_from_cli,
config_from_file,
key="delay",
default=0.1,
callback=delay_callback,
)

config["parallel_backend"] = get_first_not_none_value(
config["parallel_backend"] = get_first_non_none_value(
config_from_cli,
config_from_file,
key="parallel_backend",
Expand Down
29 changes: 29 additions & 0 deletions src/pytask_parallel/execute.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

@hookimpl
def pytask_post_parse(config):
"""Register the parallel backend."""
if config["parallel_backend"] == "processes":
config["pm"].register(ProcessesNameSpace)
elif config["parallel_backend"] == "threads":
Expand All @@ -25,6 +26,7 @@ def pytask_post_parse(config):

@hookimpl(tryfirst=True)
def pytask_execute_create_scheduler(session):
"""Create the scheduler."""
if session.config["n_workers"] > 1:
task_names = {task.name for task in session.tasks}
task_dict = {
Expand All @@ -41,6 +43,16 @@ def pytask_execute_create_scheduler(session):

@hookimpl(tryfirst=True)
def pytask_execute_build(session):
"""Execute tasks with a parallel backend.

There are three phases while the scheduler has tasks which need to be executed.

1. Take all ready tasks, set up their execution and submit them.
2. For all tasks which are running, find those which have finished and turn them
into a report.
3. Process all reports and report the result on the command line.

"""
if session.config["n_workers"] > 1:
reports = []
running_tasks = {}
Expand Down Expand Up @@ -124,18 +136,35 @@ def pytask_execute_build(session):
class ProcessesNameSpace:
@hookimpl(tryfirst=True)
def pytask_execute_task(session, task): # noqa: N805
"""Execute a task.

Take a task, pickle it and send the bytes over to another process.

"""
if session.config["n_workers"] > 1:
bytes_ = cloudpickle.dumps(task)
return session.executor.submit(unserialize_and_execute_task, bytes_)


def unserialize_and_execute_task(bytes_):
"""Unserialize and execute task.

This function receives bytes and unpickles them to a task which is them execute in a
spawned process or thread.

"""
task = cloudpickle.loads(bytes_)
task.execute()


class ThreadsNameSpace:
@hookimpl(tryfirst=True)
def pytask_execute_task(session, task): # noqa: N805
"""Execute a task.

Since threads share their memory, it is not necessary to pickle and unpickle the
task.

"""
if session.config["n_workers"] > 1:
return session.executor.submit(task.execute)
6 changes: 4 additions & 2 deletions src/pytask_parallel/plugin.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
"""Entry-point for the plugin."""
from _pytask.config import hookimpl
from pytask_parallel import cli
from pytask_parallel import build
from pytask_parallel import config
from pytask_parallel import execute


@hookimpl
def pytask_add_hooks(pm):
pm.register(cli)
"""Register plugins."""
pm.register(build)
pm.register(config)
pm.register(execute)
7 changes: 7 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
import pytest
from click.testing import CliRunner


@pytest.fixture()
def runner():
return CliRunner()
Loading