Release v0.0.3. (#3)

Author: tobiasraabe

.conda/meta.yaml

@@ -21,7 +21,7 @@ requirements:
   run:
     - python >=3.6
     - cloudpickle
-    - pytask >=0.0.5
+    - pytask >=0.0.6
 
 test:
   requires:
@@ -32,6 +32,8 @@ test:
   commands:
     - pytask --version
     - pytask --help
+    - pytask clean
+    - pytask markers
 
     - pytest tests
 

.pre-commit-config.yaml

@@ -19,11 +19,11 @@ repos:
     hooks:
     -   id: reorder-python-imports
 -   repo: https://github.com/psf/black
-    rev: 19.10b0
+    rev: 20.8b1
     hooks:
     -   id: black
 -   repo: https://github.com/asottile/blacken-docs
-    rev: v1.7.0
+    rev: v1.8.0
     hooks:
     -   id: blacken-docs
         additional_dependencies: [black]
@@ -48,9 +48,14 @@ repos:
             Pygments,
         ]
 -   repo: https://github.com/PyCQA/doc8
-    rev: 0.8.1rc3
+    rev: 0.9.0a1
     hooks:
     -   id: doc8
+-   repo: https://github.com/econchick/interrogate
+    rev: 1.3.1
+    hooks:
+    -   id: interrogate
+        args: [-v, --fail-under=40, src, tests]
 - repo: https://github.com/codespell-project/codespell
   rev: v1.17.1
   hooks:

CHANGES.rst

@@ -6,11 +6,17 @@ chronological order. Releases follow `semantic versioning <https://semver.org/>`
 all releases are available on `Anaconda.org <https://anaconda.org/pytask/pytask-parallel>`_.
 
 
+0.0.3 - 2020-09-12
+------------------
+
+- :gh:`3` align the program with pytask v0.0.6.
+
+
 0.0.2 - 2020-08-12
 ------------------
 
-- :gh:`2` prepares the plugin for pytask v0.0.5.
-- :gh:`3` better parsing and callbacks.
+- :gh:`1` prepares the plugin for pytask v0.0.5.
+- :gh:`2` better parsing and callbacks.
 
 
 0.0.1 - 2020-07-17

environment.yml

@@ -12,7 +12,7 @@ dependencies:
   - conda-verify
 
   # Package dependencies
-  - pytask >= 0.0.5
+  - pytask >= 0.0.6
   - cloudpickle
 
   # Misc

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.0.2
+current_version = 0.0.3
 parse = (?P<major>\d+)\.(?P<minor>\d+)(\.(?P<patch>\d+))(\-?((dev)?(?P<dev>\d+))?)
 serialize = 
 	{major}.{minor}.{patch}dev{dev}

setup.py

@@ -3,7 +3,7 @@
 
 setup(
     name="pytask-parallel",
-    version="0.0.2",
+    version="0.0.3",
     packages=find_packages(where="src"),
     package_dir={"": "src"},
     entry_points={"pytask": ["pytask_parallel = pytask_parallel.plugin"]},

src/pytask_parallel/__init__.py

@@ -1 +1 @@
-__version__ = "0.0.2"
+__version__ = "0.0.3"

src/pytask_parallel/cli.py renamed to src/pytask_parallel/build.py

@@ -1,11 +1,11 @@
+"""Extend the build command."""
 import click
 from _pytask.config import hookimpl
-from pytask_parallel.callbacks import delay_click_callback
-from pytask_parallel.callbacks import n_workers_click_callback
 
 
 @hookimpl
-def pytask_add_parameters_to_cli(command):
+def pytask_extend_command_line_interface(cli):
+    """Extend the command line interface."""
     additional_parameters = [
         click.Option(
             ["-n", "--n-workers"],
@@ -14,12 +14,13 @@ def pytask_add_parameters_to_cli(command):
                 "os.cpu_count() - 1.  [default: 1 (no parallelization)]"
             ),
             metavar="[INTEGER|auto]",
-            callback=n_workers_click_callback,
+            default=None,
         ),
         click.Option(
             ["--parallel-backend"],
             type=click.Choice(["processes", "threads"]),
             help="Backend for the parallelization.  [default: processes]",
+            default=None,
         ),
         click.Option(
             ["--delay"],
@@ -28,7 +29,7 @@ def pytask_add_parameters_to_cli(command):
                 "(seconds)]"
             ),
             metavar="NUMBER > 0",
-            callback=delay_click_callback,
+            default=None,
         ),
     ]
-    command.params.extend(additional_parameters)
+    cli.commands["build"].params.extend(additional_parameters)

src/pytask_parallel/callbacks.py

@@ -1,59 +1,42 @@
-import click
-
-
-def n_workers_click_callback(ctx, name, value):  # noqa: U100
-    return n_workers_callback(value)
+"""Validate command line inputs and configuration values."""
 
 
 def n_workers_callback(value):
-    error_occurred = False
+    """Validate the n-workers option."""
     if value == "auto":
         pass
-    elif value is None:
-        pass
+    elif value is None or value == "None":
+        value = None
     elif isinstance(value, int) and 1 <= value:
         pass
+    elif isinstance(value, str) and value.isdigit():
+        value = int(value)
     else:
-        try:
-            value = int(value)
-        except ValueError:
-            error_occurred = True
-        else:
-            if value < 1:
-                error_occurred = True
-
-    if error_occurred:
-        raise click.UsageError("n-processes can either be an integer >= 1 or 'auto'.")
+        raise ValueError("n_processes can either be an integer >= 1, 'auto' or None.")
 
     return value
 
 
 def parallel_backend_callback(value):
+    """Validate the input for the parallel backend."""
+    if value == "None":
+        value = None
     if value not in ["processes", "threads", None]:
-        raise click.UsageError("parallel_backend has to be 'processes' or 'threads'.")
+        raise ValueError("parallel_backend has to be 'processes' or 'threads'.")
     return value
 
 
-def delay_click_callback(ctx, name, value):  # noqa: U100
-    return delay_callback(value)
-
-
 def delay_callback(value):
-    error_occurred = False
-    if isinstance(value, float) and 0 < value:
-        pass
-    elif value is None:
-        pass
+    """Validate the delay option."""
+    if value is None or value == "None":
+        value = None
     else:
         try:
             value = float(value)
         except ValueError:
-            error_occurred = True
-        else:
-            if value < 0:
-                error_occurred = True
+            pass
 
-    if error_occurred:
-        raise click.UsageError("delay has to be a number greater than 0.")
+        if not (isinstance(value, float) and value > 0):
+            raise ValueError("delay has to be a number greater than 0.")
 
     return value

src/pytask_parallel/config.py

@@ -1,14 +1,17 @@
+"""Configure pytask."""
 import os
 
 from _pytask.config import hookimpl
-from _pytask.shared import get_first_not_none_value
+from _pytask.shared import get_first_non_none_value
+from pytask_parallel.callbacks import delay_callback
 from pytask_parallel.callbacks import n_workers_callback
 from pytask_parallel.callbacks import parallel_backend_callback
 
 
 @hookimpl
 def pytask_parse_config(config, config_from_cli, config_from_file):
-    config["n_workers"] = get_first_not_none_value(
+    """Parse the configuration."""
+    config["n_workers"] = get_first_non_none_value(
         config_from_cli,
         config_from_file,
         key="n_workers",
@@ -18,11 +21,15 @@ def pytask_parse_config(config, config_from_cli, config_from_file):
     if config["n_workers"] == "auto":
         config["n_workers"] = max(os.cpu_count() - 1, 1)
 
-    config["delay"] = get_first_not_none_value(
-        config_from_cli, config_from_file, key="delay", default=0.1, callback=float
+    config["delay"] = get_first_non_none_value(
+        config_from_cli,
+        config_from_file,
+        key="delay",
+        default=0.1,
+        callback=delay_callback,
     )
 
-    config["parallel_backend"] = get_first_not_none_value(
+    config["parallel_backend"] = get_first_non_none_value(
         config_from_cli,
         config_from_file,
         key="parallel_backend",

src/pytask_parallel/execute.py

@@ -17,6 +17,7 @@
 
 @hookimpl
 def pytask_post_parse(config):
+    """Register the parallel backend."""
     if config["parallel_backend"] == "processes":
         config["pm"].register(ProcessesNameSpace)
     elif config["parallel_backend"] == "threads":
@@ -25,6 +26,7 @@ def pytask_post_parse(config):
 
 @hookimpl(tryfirst=True)
 def pytask_execute_create_scheduler(session):
+    """Create the scheduler."""
     if session.config["n_workers"] > 1:
         task_names = {task.name for task in session.tasks}
         task_dict = {
@@ -41,6 +43,16 @@ def pytask_execute_create_scheduler(session):
 
 @hookimpl(tryfirst=True)
 def pytask_execute_build(session):
+    """Execute tasks with a parallel backend.
+
+    There are three phases while the scheduler has tasks which need to be executed.
+
+    1. Take all ready tasks, set up their execution and submit them.
+    2. For all tasks which are running, find those which have finished and turn them
+       into a report.
+    3. Process all reports and report the result on the command line.
+
+    """
     if session.config["n_workers"] > 1:
         reports = []
         running_tasks = {}
@@ -124,18 +136,35 @@ def pytask_execute_build(session):
 class ProcessesNameSpace:
     @hookimpl(tryfirst=True)
     def pytask_execute_task(session, task):  # noqa: N805
+        """Execute a task.
+
+        Take a task, pickle it and send the bytes over to another process.
+
+        """
         if session.config["n_workers"] > 1:
             bytes_ = cloudpickle.dumps(task)
             return session.executor.submit(unserialize_and_execute_task, bytes_)
 
 
 def unserialize_and_execute_task(bytes_):
+    """Unserialize and execute task.
+
+    This function receives bytes and unpickles them to a task which is them execute in a
+    spawned process or thread.
+
+    """
     task = cloudpickle.loads(bytes_)
     task.execute()
 
 
 class ThreadsNameSpace:
     @hookimpl(tryfirst=True)
     def pytask_execute_task(session, task):  # noqa: N805
+        """Execute a task.
+
+        Since threads share their memory, it is not necessary to pickle and unpickle the
+        task.
+
+        """
         if session.config["n_workers"] > 1:
             return session.executor.submit(task.execute)

src/pytask_parallel/plugin.py

@@ -1,11 +1,13 @@
+"""Entry-point for the plugin."""
 from _pytask.config import hookimpl
-from pytask_parallel import cli
+from pytask_parallel import build
 from pytask_parallel import config
 from pytask_parallel import execute
 
 
 @hookimpl
 def pytask_add_hooks(pm):
-    pm.register(cli)
+    """Register plugins."""
+    pm.register(build)
     pm.register(config)
     pm.register(execute)

tests/conftest.py

@@ -0,0 +1,7 @@
+import pytest
+from click.testing import CliRunner
+
+
+@pytest.fixture()
+def runner():
+    return CliRunner()