Schedule all ready tasks. (#99)

Author: tobiasraabe

docs/source/changes.md

@@ -19,6 +19,9 @@ releases are available on [PyPI](https://pypi.org/project/pytask-parallel) and
   or processes automatically.
 - {pull}`96` handles local paths with remote executors. `PathNode`s are not supported as
   dependencies or products (except for return annotations).
+- {pull}`99` changes that all tasks that are ready are being scheduled. It improves
+  interactions with adaptive scaling. {issue}`98` does handle the resulting issues: no
+  strong adherence to priorities, no pending status.
 
 ## 0.4.1 - 2024-01-12
 

src/pytask_parallel/execute.py

@@ -63,13 +63,7 @@ def pytask_execute_build(session: Session) -> bool | None:  # noqa: C901, PLR091
         while session.scheduler.is_active():
             try:
                 newly_collected_reports = []
-                n_new_tasks = session.config["n_workers"] - len(running_tasks)
-
-                ready_tasks = (
-                    list(session.scheduler.get_ready(n_new_tasks))
-                    if n_new_tasks >= 1
-                    else []
-                )
+                ready_tasks = list(session.scheduler.get_ready(10_000))
 
                 for task_name in ready_tasks:
                     task = session.dag.nodes[task_name]["task"]

tests/test_execute.py

@@ -78,6 +78,7 @@ def task_2(path: Annotated[Path, Product] = Path("out_2.txt")):
 
 
 @pytest.mark.end_to_end()
+@pytest.mark.skip(reason="See #98")
 @pytest.mark.parametrize("parallel_backend", _IMPLEMENTED_BACKENDS)
 def test_stop_execution_when_max_failures_is_reached(tmp_path, parallel_backend):
     source = """
@@ -106,6 +107,7 @@ def task_3(): time.sleep(3)
 
 
 @pytest.mark.end_to_end()
+@pytest.mark.skip(reason="See #98")
 @pytest.mark.parametrize("parallel_backend", _IMPLEMENTED_BACKENDS)
 def test_task_priorities(tmp_path, parallel_backend):
     source = """