Refactor Interface

Author: pre-commit-ci[bot]

executorlib/__init__.py

@@ -40,31 +40,31 @@ class Executor:
         max_workers (int): for backwards compatibility with the standard library, max_workers also defines the number of
                            cores which can be used in parallel - just like the max_cores parameter. Using max_cores is
                            recommended, as computers have a limited number of compute cores.
+        backend (str): Switch between the different backends "flux", "local" or "slurm". The default is "local".
         max_cores (int): defines the number cores which can be used in parallel
         cores_per_worker (int): number of MPI cores to be used for each function call
         threads_per_core (int): number of OpenMP threads to be used for each function call
         gpus_per_worker (int): number of GPUs per worker - defaults to 0
-        oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and SLURM only) - default False
         cwd (str/None): current working directory where the parallel python task is executed
+        openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and SLURM only) - default False
+        slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
+        flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux
+        flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only)
+        flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function.
         conda_environment_name (str): name of the conda environment to initialize
         conda_environment_path (str): path of the conda environment to initialize
-        executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux
         hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
                                       context of an HPC cluster this essential to be able to communicate to an
                                       Executor running on a different compute node within the same allocation. And
                                       in principle any computer should be able to resolve that their own hostname
                                       points to the same address as localhost. Still MacOS >= 12 seems to disable
                                       this look up for security reasons. So on MacOS it is required to set this
                                       option to true
-        backend (str): Switch between the different backends "flux", "local" or "slurm". Alternatively, when "auto"
-                       is selected (the default) the available backend is determined automatically.
         block_allocation (boolean): To accelerate the submission of a series of python functions with the same resource
                                     requirements, executorlib supports block allocation. In this case all resources have
                                     to be defined on the executor, rather than during the submission of the individual
                                     function.
         init_function (None): optional function to preset arguments for functions which are submitted later
-        command_line_argument_lst (list): Additional command line arguments for the srun call (SLURM only)
-        pmi (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only)
         disable_dependencies (boolean): Disable resolving future objects during the submission.
         refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked.
         plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For
@@ -94,21 +94,22 @@ class Executor:
     def __init__(
         self,
         max_workers: int = 1,
+        backend: str = "auto",
         max_cores: int = 1,
         cores_per_worker: int = 1,
         threads_per_core: int = 1,
         gpus_per_worker: int = 0,
-        oversubscribe: bool = False,
         cwd: Optional[str] = None,
+        openmpi_oversubscribe: bool = False,
+        slurm_cmd_args: list[str] = [],
+        flux_executor=None,
+        flux_executor_pmi_mode: Optional[str] = None,
+        flux_executor_nesting: bool = False,
         conda_environment_name: Optional[str] = None,
         conda_environment_path: Optional[str] = None,
-        executor=None,
         hostname_localhost: bool = False,
-        backend: str = "auto",
         block_allocation: bool = True,
         init_function: Optional[callable] = None,
-        command_line_argument_lst: list[str] = [],
-        pmi: Optional[str] = None,
         disable_dependencies: bool = False,
         refresh_rate: float = 0.01,
         plot_dependency_graph: bool = False,
@@ -119,22 +120,22 @@ def __init__(
     def __new__(
         cls,
         max_workers: int = 1,
+        backend: str = "auto",
         max_cores: int = 1,
         cores_per_worker: int = 1,
         threads_per_core: int = 1,
         gpus_per_worker: int = 0,
-        oversubscribe: bool = False,
         cwd: Optional[str] = None,
+        openmpi_oversubscribe: bool = False,
+        slurm_cmd_args: list[str] = [],
+        flux_executor=None,
+        flux_executor_pmi_mode: Optional[str] = None,
+        flux_executor_nesting: bool = False,
         conda_environment_name: Optional[str] = None,
         conda_environment_path: Optional[str] = None,
-        executor=None,
         hostname_localhost: bool = False,
-        backend: str = "auto",
-        block_allocation: bool = False,
+        block_allocation: bool = True,
         init_function: Optional[callable] = None,
-        command_line_argument_lst: list[str] = [],
-        pmi: Optional[str] = None,
-        nested_flux_executor: bool = False,
         disable_dependencies: bool = False,
         refresh_rate: float = 0.01,
         plot_dependency_graph: bool = False,
@@ -151,32 +152,31 @@ def __new__(
             max_workers (int): for backwards compatibility with the standard library, max_workers also defines the
                                number of cores which can be used in parallel - just like the max_cores parameter. Using
                                max_cores is recommended, as computers have a limited number of compute cores.
+            backend (str): Switch between the different backends "flux", "local" or "slurm". The default is "local".
             max_cores (int): defines the number cores which can be used in parallel
             cores_per_worker (int): number of MPI cores to be used for each function call
             threads_per_core (int): number of OpenMP threads to be used for each function call
             gpus_per_worker (int): number of GPUs per worker - defaults to 0
-            oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and SLURM only) - default False
+            openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and SLURM only) - default False
+            slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
             cwd (str/None): current working directory where the parallel python task is executed
+            flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux
+            flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only)
+            flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function.
             conda_environment_name (str): name of the conda environment to initialize
             conda_environment_path (str): path of the conda environment to initialize
-            executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux
             hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
                                       context of an HPC cluster this essential to be able to communicate to an
                                       Executor running on a different compute node within the same allocation. And
                                       in principle any computer should be able to resolve that their own hostname
                                       points to the same address as localhost. Still MacOS >= 12 seems to disable
                                       this look up for security reasons. So on MacOS it is required to set this
                                       option to true
-            backend (str): Switch between the different backends "flux", "local" or "slurm". Alternatively, when "auto"
-                           is selected (the default) the available backend is determined automatically.
             block_allocation (boolean): To accelerate the submission of a series of python functions with the same
                                         resource requirements, executorlib supports block allocation. In this case all
                                         resources have to be defined on the executor, rather than during the submission
                                         of the individual function.
             init_function (None): optional function to preset arguments for functions which are submitted later
-            command_line_argument_lst (list): Additional command line arguments for the srun call (SLURM only)
-            pmi (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only)
-            nested_flux_executor (bool): Provide hierarchically nested Flux job scheduler inside the submitted function.
             disable_dependencies (boolean): Disable resolving future objects during the submission.
             refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked.
             plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For
@@ -186,22 +186,22 @@ def __new__(
         if not disable_dependencies:
             return ExecutorWithDependencies(
                 max_workers=max_workers,
+                backend=backend,
                 max_cores=max_cores,
                 cores_per_worker=cores_per_worker,
                 threads_per_core=threads_per_core,
                 gpus_per_worker=gpus_per_worker,
-                oversubscribe=oversubscribe,
                 cwd=cwd,
+                openmpi_oversubscribe=openmpi_oversubscribe,
+                slurm_cmd_args=slurm_cmd_args,
+                flux_executor=flux_executor,
+                flux_executor_pmi_mode=flux_executor_pmi_mode,
+                flux_executor_nesting=flux_executor_nesting,
                 conda_environment_name=conda_environment_name,
                 conda_environment_path=conda_environment_path,
-                executor=executor,
                 hostname_localhost=hostname_localhost,
-                backend=backend,
                 block_allocation=block_allocation,
                 init_function=init_function,
-                command_line_argument_lst=command_line_argument_lst,
-                pmi=pmi,
-                nested_flux_executor=nested_flux_executor,
                 refresh_rate=refresh_rate,
                 plot_dependency_graph=plot_dependency_graph,
             )
@@ -210,20 +210,20 @@ def __new__(
             _check_refresh_rate(refresh_rate=refresh_rate)
             return create_executor(
                 max_workers=max_workers,
+                backend=backend,
                 max_cores=max_cores,
                 cores_per_worker=cores_per_worker,
                 threads_per_core=threads_per_core,
                 gpus_per_worker=gpus_per_worker,
-                oversubscribe=oversubscribe,
                 cwd=cwd,
+                openmpi_oversubscribe=openmpi_oversubscribe,
+                slurm_cmd_args=slurm_cmd_args,
+                flux_executor=flux_executor,
+                flux_executor_pmi_mode=flux_executor_pmi_mode,
+                flux_executor_nesting=flux_executor_nesting,
                 conda_environment_name=conda_environment_name,
                 conda_environment_path=conda_environment_path,
-                executor=executor,
                 hostname_localhost=hostname_localhost,
-                backend=backend,
                 block_allocation=block_allocation,
                 init_function=init_function,
-                command_line_argument_lst=command_line_argument_lst,
-                pmi=pmi,
-                nested_flux_executor=nested_flux_executor,
             )