From 97a0920462561c7dfe5f4c696d1eb930ab7d0dbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 1 Feb 2025 12:11:42 +0100 Subject: [PATCH 01/28] [major] Refactor the Executor interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the backend parameter from the initialization of the Executor class with individual executor classes for the individual backends: * `Executor(backend="local")` becomes `from executorlib import LocalExecutor§ * `Executor(backend="flux_allocation")` becomes `from executorlib import FluxAllocationExecutor§ * `Executor(backend="flux_submission")` becomes `from executorlib import FluxSubmissionExecutor§ * `Executor(backend="slurm_allocation")` becomes "SlurmAllocationExecutor" * `Executor(backend="slurm_submission")` becomes `from executorlib import SlurmSubmissionExecutor` This has two advantages: On the one hand it is less error prone to mistyping the backend name, as the user can use auto completion to import the right module. On the other hand it is more consistent with the standard library which defines the `ProcessPoolExecutor` and the `ThreadPoolExecutor`, rather than a `backend` parameter. --- executorlib/__init__.py | 246 +---- executorlib/interfaces.py | 1005 ++++++++++++++++++++ notebooks/1-local.ipynb | 36 +- notebooks/2-hpc-submission.ipynb | 15 +- notebooks/3-hpc-allocation.ipynb | 51 +- notebooks/4-developer.ipynb | 8 +- tests/test_cache_executor_interactive.py | 4 +- tests/test_cache_executor_pysqa_flux.py | 5 +- tests/test_dependencies_executor.py | 23 +- tests/test_executor_backend_flux.py | 26 +- tests/test_executor_backend_mpi.py | 23 +- tests/test_executor_backend_mpi_noblock.py | 20 +- tests/test_integration_pyiron_workflow.py | 14 +- tests/test_shell_executor.py | 10 +- tests/test_shell_interactive.py | 4 +- 15 files changed, 1113 insertions(+), 377 deletions(-) create mode 100644 executorlib/interfaces.py diff --git a/executorlib/__init__.py b/executorlib/__init__.py index cedd6c2d..2c99197e 100644 --- a/executorlib/__init__.py +++ b/executorlib/__init__.py @@ -1,248 +1,6 @@ -from typing import Callable, Optional - +from executorlib.interfaces import LocalExecutor, FluxAllocationExecutor, FluxSubmissionExecutor, SlurmAllocationExecutor, SlurmSubmissionExecutor from executorlib._version import get_versions as _get_versions -from executorlib.interactive.executor import ( - ExecutorWithDependencies as _ExecutorWithDependencies, -) -from executorlib.interactive.executor import create_executor as _create_executor -from executorlib.standalone.inputcheck import ( - check_plot_dependency_graph as _check_plot_dependency_graph, -) -from executorlib.standalone.inputcheck import ( - check_pysqa_config_directory as _check_pysqa_config_directory, -) -from executorlib.standalone.inputcheck import ( - check_refresh_rate as _check_refresh_rate, -) + __version__ = _get_versions()["version"] __all__: list = [] - - -class Executor: - """ - The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or - preferable the flux framework for distributing python functions within a given resource allocation. In contrast to - the mpi4py.futures.MPIPoolExecutor the executorlib.Executor can be executed in a serial python process and does not - require the python script to be executed with MPI. It is even possible to execute the executorlib.Executor directly - in an interactive Jupyter notebook. - - Args: - max_workers (int): for backwards compatibility with the standard library, max_workers also defines the number of - cores which can be used in parallel - just like the max_cores parameter. Using max_cores is - recommended, as computers have a limited number of compute cores. - backend (str): Switch between the different backends "flux", "local" or "slurm". The default is "local". - cache_directory (str, optional): The directory to store cache files. Defaults to "cache". - max_cores (int): defines the number cores which can be used in parallel - resource_dict (dict): A dictionary of resources required by the task. With the following keys: - - cores (int): number of MPI cores to be used for each function call - - threads_per_core (int): number of OpenMP threads to be used for each function call - - gpus_per_core (int): number of GPUs per worker - defaults to 0 - - cwd (str/None): current working directory where the parallel python task is executed - - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and - SLURM only) - default False - - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only) - flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux - flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only) - flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function. - flux_log_files (bool, optional): Write flux stdout and stderr files. Defaults to False. - pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend). - hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the - context of an HPC cluster this essential to be able to communicate to an - Executor running on a different compute node within the same allocation. And - in principle any computer should be able to resolve that their own hostname - points to the same address as localhost. Still MacOS >= 12 seems to disable - this look up for security reasons. So on MacOS it is required to set this - option to true - block_allocation (boolean): To accelerate the submission of a series of python functions with the same resource - requirements, executorlib supports block allocation. In this case all resources have - to be defined on the executor, rather than during the submission of the individual - function. - init_function (None): optional function to preset arguments for functions which are submitted later - disable_dependencies (boolean): Disable resolving future objects during the submission. - refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. - plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For - debugging purposes and to get an overview of the specified dependencies. - plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. - - Examples: - ``` - >>> import numpy as np - >>> from executorlib import Executor - >>> - >>> def calc(i, j, k): - >>> from mpi4py import MPI - >>> size = MPI.COMM_WORLD.Get_size() - >>> rank = MPI.COMM_WORLD.Get_rank() - >>> return np.array([i, j, k]), size, rank - >>> - >>> def init_k(): - >>> return {"k": 3} - >>> - >>> with Executor(cores=2, init_function=init_k) as p: - >>> fs = p.submit(calc, 2, j=4) - >>> print(fs.result()) - [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] - ``` - """ - - def __init__( - self, - max_workers: Optional[int] = None, - backend: str = "local", - cache_directory: Optional[str] = None, - max_cores: Optional[int] = None, - resource_dict: Optional[dict] = None, - flux_executor=None, - flux_executor_pmi_mode: Optional[str] = None, - flux_executor_nesting: bool = False, - flux_log_files: bool = False, - pysqa_config_directory: Optional[str] = None, - hostname_localhost: Optional[bool] = None, - block_allocation: bool = False, - init_function: Optional[Callable] = None, - disable_dependencies: bool = False, - refresh_rate: float = 0.01, - plot_dependency_graph: bool = False, - plot_dependency_graph_filename: Optional[str] = None, - ): - # Use __new__() instead of __init__(). This function is only implemented to enable auto-completion. - pass - - def __new__( - cls, - max_workers: Optional[int] = None, - backend: str = "local", - cache_directory: Optional[str] = None, - max_cores: Optional[int] = None, - resource_dict: Optional[dict] = None, - flux_executor=None, - flux_executor_pmi_mode: Optional[str] = None, - flux_executor_nesting: bool = False, - flux_log_files: bool = False, - pysqa_config_directory: Optional[str] = None, - hostname_localhost: Optional[bool] = None, - block_allocation: bool = False, - init_function: Optional[Callable] = None, - disable_dependencies: bool = False, - refresh_rate: float = 0.01, - plot_dependency_graph: bool = False, - plot_dependency_graph_filename: Optional[str] = None, - ): - """ - Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor, - executorlib.slurm.PySlurmExecutor or executorlib.flux.PyFluxExecutor depending on which backend is available. The - executorlib.flux.PyFluxExecutor is the preferred choice while the executorlib.mpi.PyMPIExecutor is primarily used - for development and testing. The executorlib.flux.PyFluxExecutor requires flux-core from the flux-framework to be - installed and in addition flux-sched to enable GPU scheduling. Finally, the executorlib.slurm.PySlurmExecutor - requires the SLURM workload manager to be installed on the system. - - Args: - max_workers (int): for backwards compatibility with the standard library, max_workers also defines the - number of cores which can be used in parallel - just like the max_cores parameter. Using - max_cores is recommended, as computers have a limited number of compute cores. - backend (str): Switch between the different backends "flux", "local" or "slurm". The default is "local". - cache_directory (str, optional): The directory to store cache files. Defaults to "cache". - max_cores (int): defines the number cores which can be used in parallel - resource_dict (dict): A dictionary of resources required by the task. With the following keys: - - cores (int): number of MPI cores to be used for each function call - - threads_per_core (int): number of OpenMP threads to be used for each function call - - gpus_per_core (int): number of GPUs per worker - defaults to 0 - - cwd (str/None): current working directory where the parallel python task is executed - - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI - and SLURM only) - default False - - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM - only) - flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux - flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only) - flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function. - flux_log_files (bool, optional): Write flux stdout and stderr files. Defaults to False. - pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend). - hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the - context of an HPC cluster this essential to be able to communicate to an - Executor running on a different compute node within the same allocation. And - in principle any computer should be able to resolve that their own hostname - points to the same address as localhost. Still MacOS >= 12 seems to disable - this look up for security reasons. So on MacOS it is required to set this - option to true - block_allocation (boolean): To accelerate the submission of a series of python functions with the same - resource requirements, executorlib supports block allocation. In this case all - resources have to be defined on the executor, rather than during the submission - of the individual function. - init_function (None): optional function to preset arguments for functions which are submitted later - disable_dependencies (boolean): Disable resolving future objects during the submission. - refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. - plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For - debugging purposes and to get an overview of the specified dependencies. - plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. - - """ - default_resource_dict: dict = { - "cores": 1, - "threads_per_core": 1, - "gpus_per_core": 0, - "cwd": None, - "openmpi_oversubscribe": False, - "slurm_cmd_args": [], - } - if resource_dict is None: - resource_dict = {} - resource_dict.update( - {k: v for k, v in default_resource_dict.items() if k not in resource_dict} - ) - if "_submission" in backend and not plot_dependency_graph: - from executorlib.cache.executor import create_file_executor - - return create_file_executor( - max_workers=max_workers, - backend=backend, - max_cores=max_cores, - cache_directory=cache_directory, - resource_dict=resource_dict, - flux_executor=flux_executor, - flux_executor_pmi_mode=flux_executor_pmi_mode, - flux_executor_nesting=flux_executor_nesting, - flux_log_files=flux_log_files, - pysqa_config_directory=pysqa_config_directory, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, - disable_dependencies=disable_dependencies, - ) - elif not disable_dependencies: - _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) - return _ExecutorWithDependencies( - max_workers=max_workers, - backend=backend, - cache_directory=cache_directory, - max_cores=max_cores, - resource_dict=resource_dict, - flux_executor=flux_executor, - flux_executor_pmi_mode=flux_executor_pmi_mode, - flux_executor_nesting=flux_executor_nesting, - flux_log_files=flux_log_files, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, - refresh_rate=refresh_rate, - plot_dependency_graph=plot_dependency_graph, - plot_dependency_graph_filename=plot_dependency_graph_filename, - ) - else: - _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) - _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) - _check_refresh_rate(refresh_rate=refresh_rate) - return _create_executor( - max_workers=max_workers, - backend=backend, - cache_directory=cache_directory, - max_cores=max_cores, - resource_dict=resource_dict, - flux_executor=flux_executor, - flux_executor_pmi_mode=flux_executor_pmi_mode, - flux_executor_nesting=flux_executor_nesting, - flux_log_files=flux_log_files, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, - ) diff --git a/executorlib/interfaces.py b/executorlib/interfaces.py new file mode 100644 index 00000000..0c03ac70 --- /dev/null +++ b/executorlib/interfaces.py @@ -0,0 +1,1005 @@ +from typing import Callable, Optional + +from executorlib._version import get_versions as _get_versions +from executorlib.interactive.executor import ( + ExecutorWithDependencies as _ExecutorWithDependencies, +) +from executorlib.interactive.executor import create_executor as _create_executor +from executorlib.standalone.inputcheck import ( + check_plot_dependency_graph as _check_plot_dependency_graph, +) +from executorlib.standalone.inputcheck import ( + check_pysqa_config_directory as _check_pysqa_config_directory, +) +from executorlib.standalone.inputcheck import ( + check_refresh_rate as _check_refresh_rate, +) + +__version__ = _get_versions()["version"] +__all__: list = [] + + +class LocalExecutor: + """ + The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or + preferable the flux framework for distributing python functions within a given resource allocation. In contrast to + the mpi4py.futures.MPIPoolExecutor the executorlib.Executor can be executed in a serial python process and does not + require the python script to be executed with MPI. It is even possible to execute the executorlib.Executor directly + in an interactive Jupyter notebook. + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the number of + cores which can be used in parallel - just like the max_cores parameter. Using max_cores is + recommended, as computers have a limited number of compute cores. + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + max_cores (int): defines the number cores which can be used in parallel + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and + SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only) + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same resource + requirements, executorlib supports block allocation. In this case all resources have + to be defined on the executor, rather than during the submission of the individual + function. + init_function (None): optional function to preset arguments for functions which are submitted later + disable_dependencies (boolean): Disable resolving future objects during the submission. + refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. + plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For + debugging purposes and to get an overview of the specified dependencies. + plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + + Examples: + ``` + >>> import numpy as np + >>> from executorlib.interfaces import LocalExecutor + >>> + >>> def calc(i, j, k): + >>> from mpi4py import MPI + >>> size = MPI.COMM_WORLD.Get_size() + >>> rank = MPI.COMM_WORLD.Get_rank() + >>> return np.array([i, j, k]), size, rank + >>> + >>> def init_k(): + >>> return {"k": 3} + >>> + >>> with LocalExecutor(cores=2, init_function=init_k) as p: + >>> fs = p.submit(calc, 2, j=4) + >>> print(fs.result()) + [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] + ``` + """ + + def __init__( + self, + max_workers: Optional[int] = None, + cache_directory: Optional[str] = None, + max_cores: Optional[int] = None, + resource_dict: Optional[dict] = None, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, + disable_dependencies: bool = False, + refresh_rate: float = 0.01, + plot_dependency_graph: bool = False, + plot_dependency_graph_filename: Optional[str] = None, + ): + # Use __new__() instead of __init__(). This function is only implemented to enable auto-completion. + pass + + def __new__( + cls, + max_workers: Optional[int] = None, + cache_directory: Optional[str] = None, + max_cores: Optional[int] = None, + resource_dict: Optional[dict] = None, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, + disable_dependencies: bool = False, + refresh_rate: float = 0.01, + plot_dependency_graph: bool = False, + plot_dependency_graph_filename: Optional[str] = None, + ): + """ + Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor, + executorlib.slurm.PySlurmExecutor or executorlib.flux.PyFluxExecutor depending on which backend is available. The + executorlib.flux.PyFluxExecutor is the preferred choice while the executorlib.mpi.PyMPIExecutor is primarily used + for development and testing. The executorlib.flux.PyFluxExecutor requires flux-core from the flux-framework to be + installed and in addition flux-sched to enable GPU scheduling. Finally, the executorlib.slurm.PySlurmExecutor + requires the SLURM workload manager to be installed on the system. + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the + number of cores which can be used in parallel - just like the max_cores parameter. Using + max_cores is recommended, as computers have a limited number of compute cores. + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + max_cores (int): defines the number cores which can be used in parallel + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI + and SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM + only) + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same + resource requirements, executorlib supports block allocation. In this case all + resources have to be defined on the executor, rather than during the submission + of the individual function. + init_function (None): optional function to preset arguments for functions which are submitted later + disable_dependencies (boolean): Disable resolving future objects during the submission. + refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. + plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For + debugging purposes and to get an overview of the specified dependencies. + plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + + """ + default_resource_dict: dict = { + "cores": 1, + "threads_per_core": 1, + "gpus_per_core": 0, + "cwd": None, + "openmpi_oversubscribe": False, + "slurm_cmd_args": [], + } + if resource_dict is None: + resource_dict = {} + resource_dict.update( + {k: v for k, v in default_resource_dict.items() if k not in resource_dict} + ) + if not disable_dependencies: + return _ExecutorWithDependencies( + max_workers=max_workers, + backend="local", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + refresh_rate=refresh_rate, + plot_dependency_graph=plot_dependency_graph, + plot_dependency_graph_filename=plot_dependency_graph_filename, + ) + else: + _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) + _check_refresh_rate(refresh_rate=refresh_rate) + return _create_executor( + max_workers=max_workers, + backend="local", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + ) + + +class SlurmSubmissionExecutor: + """ + The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or + preferable the flux framework for distributing python functions within a given resource allocation. In contrast to + the mpi4py.futures.MPIPoolExecutor the executorlib.Executor can be executed in a serial python process and does not + require the python script to be executed with MPI. It is even possible to execute the executorlib.Executor directly + in an interactive Jupyter notebook. + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the number of + cores which can be used in parallel - just like the max_cores parameter. Using max_cores is + recommended, as computers have a limited number of compute cores. + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + max_cores (int): defines the number cores which can be used in parallel + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and + SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only) + pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend). + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same resource + requirements, executorlib supports block allocation. In this case all resources have + to be defined on the executor, rather than during the submission of the individual + function. + init_function (None): optional function to preset arguments for functions which are submitted later + disable_dependencies (boolean): Disable resolving future objects during the submission. + refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. + plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For + debugging purposes and to get an overview of the specified dependencies. + plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + + Examples: + ``` + >>> import numpy as np + >>> from executorlib.interfaces import SlurmSubmissionExecutor + >>> + >>> def calc(i, j, k): + >>> from mpi4py import MPI + >>> size = MPI.COMM_WORLD.Get_size() + >>> rank = MPI.COMM_WORLD.Get_rank() + >>> return np.array([i, j, k]), size, rank + >>> + >>> def init_k(): + >>> return {"k": 3} + >>> + >>> with SlurmSubmissionExecutor(cores=2, init_function=init_k) as p: + >>> fs = p.submit(calc, 2, j=4) + >>> print(fs.result()) + [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] + ``` + """ + + def __init__( + self, + max_workers: Optional[int] = None, + cache_directory: Optional[str] = None, + max_cores: Optional[int] = None, + resource_dict: Optional[dict] = None, + pysqa_config_directory: Optional[str] = None, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, + disable_dependencies: bool = False, + refresh_rate: float = 0.01, + plot_dependency_graph: bool = False, + plot_dependency_graph_filename: Optional[str] = None, + ): + # Use __new__() instead of __init__(). This function is only implemented to enable auto-completion. + pass + + def __new__( + cls, + max_workers: Optional[int] = None, + cache_directory: Optional[str] = None, + max_cores: Optional[int] = None, + resource_dict: Optional[dict] = None, + pysqa_config_directory: Optional[str] = None, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, + disable_dependencies: bool = False, + refresh_rate: float = 0.01, + plot_dependency_graph: bool = False, + plot_dependency_graph_filename: Optional[str] = None, + ): + """ + Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor, + executorlib.slurm.PySlurmExecutor or executorlib.flux.PyFluxExecutor depending on which backend is available. The + executorlib.flux.PyFluxExecutor is the preferred choice while the executorlib.mpi.PyMPIExecutor is primarily used + for development and testing. The executorlib.flux.PyFluxExecutor requires flux-core from the flux-framework to be + installed and in addition flux-sched to enable GPU scheduling. Finally, the executorlib.slurm.PySlurmExecutor + requires the SLURM workload manager to be installed on the system. + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the + number of cores which can be used in parallel - just like the max_cores parameter. Using + max_cores is recommended, as computers have a limited number of compute cores. + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + max_cores (int): defines the number cores which can be used in parallel + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI + and SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM + only) + pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend). + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same + resource requirements, executorlib supports block allocation. In this case all + resources have to be defined on the executor, rather than during the submission + of the individual function. + init_function (None): optional function to preset arguments for functions which are submitted later + disable_dependencies (boolean): Disable resolving future objects during the submission. + refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. + plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For + debugging purposes and to get an overview of the specified dependencies. + plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + + """ + default_resource_dict: dict = { + "cores": 1, + "threads_per_core": 1, + "gpus_per_core": 0, + "cwd": None, + "openmpi_oversubscribe": False, + "slurm_cmd_args": [], + } + if resource_dict is None: + resource_dict = {} + resource_dict.update( + {k: v for k, v in default_resource_dict.items() if k not in resource_dict} + ) + if not plot_dependency_graph: + from executorlib.cache.executor import create_file_executor + + return create_file_executor( + max_workers=max_workers, + backend="slurm_submission", + max_cores=max_cores, + cache_directory=cache_directory, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + pysqa_config_directory=pysqa_config_directory, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + disable_dependencies=disable_dependencies, + ) + elif not disable_dependencies: + _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) + return _ExecutorWithDependencies( + max_workers=max_workers, + backend="slurm_submission", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + refresh_rate=refresh_rate, + plot_dependency_graph=plot_dependency_graph, + plot_dependency_graph_filename=plot_dependency_graph_filename, + ) + else: + _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) + _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) + _check_refresh_rate(refresh_rate=refresh_rate) + return _create_executor( + max_workers=max_workers, + backend="slurm_submission", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + ) + + +class SlurmAllocationExecutor: + """ + The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or + preferable the flux framework for distributing python functions within a given resource allocation. In contrast to + the mpi4py.futures.MPIPoolExecutor the executorlib.Executor can be executed in a serial python process and does not + require the python script to be executed with MPI. It is even possible to execute the executorlib.Executor directly + in an interactive Jupyter notebook. + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the number of + cores which can be used in parallel - just like the max_cores parameter. Using max_cores is + recommended, as computers have a limited number of compute cores. + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + max_cores (int): defines the number cores which can be used in parallel + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and + SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only) + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same resource + requirements, executorlib supports block allocation. In this case all resources have + to be defined on the executor, rather than during the submission of the individual + function. + init_function (None): optional function to preset arguments for functions which are submitted later + disable_dependencies (boolean): Disable resolving future objects during the submission. + refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. + plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For + debugging purposes and to get an overview of the specified dependencies. + plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + + Examples: + ``` + >>> import numpy as np + >>> from executorlib import Executor + >>> + >>> def calc(i, j, k): + >>> from mpi4py import MPI + >>> size = MPI.COMM_WORLD.Get_size() + >>> rank = MPI.COMM_WORLD.Get_rank() + >>> return np.array([i, j, k]), size, rank + >>> + >>> def init_k(): + >>> return {"k": 3} + >>> + >>> with Executor(cores=2, init_function=init_k) as p: + >>> fs = p.submit(calc, 2, j=4) + >>> print(fs.result()) + [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] + ``` + """ + + def __init__( + self, + max_workers: Optional[int] = None, + cache_directory: Optional[str] = None, + max_cores: Optional[int] = None, + resource_dict: Optional[dict] = None, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, + disable_dependencies: bool = False, + refresh_rate: float = 0.01, + plot_dependency_graph: bool = False, + plot_dependency_graph_filename: Optional[str] = None, + ): + # Use __new__() instead of __init__(). This function is only implemented to enable auto-completion. + pass + + def __new__( + cls, + max_workers: Optional[int] = None, + cache_directory: Optional[str] = None, + max_cores: Optional[int] = None, + resource_dict: Optional[dict] = None, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, + disable_dependencies: bool = False, + refresh_rate: float = 0.01, + plot_dependency_graph: bool = False, + plot_dependency_graph_filename: Optional[str] = None, + ): + """ + Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor, + executorlib.slurm.PySlurmExecutor or executorlib.flux.PyFluxExecutor depending on which backend is available. The + executorlib.flux.PyFluxExecutor is the preferred choice while the executorlib.mpi.PyMPIExecutor is primarily used + for development and testing. The executorlib.flux.PyFluxExecutor requires flux-core from the flux-framework to be + installed and in addition flux-sched to enable GPU scheduling. Finally, the executorlib.slurm.PySlurmExecutor + requires the SLURM workload manager to be installed on the system. + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the + number of cores which can be used in parallel - just like the max_cores parameter. Using + max_cores is recommended, as computers have a limited number of compute cores. + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + max_cores (int): defines the number cores which can be used in parallel + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI + and SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM + only) + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same + resource requirements, executorlib supports block allocation. In this case all + resources have to be defined on the executor, rather than during the submission + of the individual function. + init_function (None): optional function to preset arguments for functions which are submitted later + disable_dependencies (boolean): Disable resolving future objects during the submission. + refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. + plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For + debugging purposes and to get an overview of the specified dependencies. + plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + + """ + default_resource_dict: dict = { + "cores": 1, + "threads_per_core": 1, + "gpus_per_core": 0, + "cwd": None, + "openmpi_oversubscribe": False, + "slurm_cmd_args": [], + } + if resource_dict is None: + resource_dict = {} + resource_dict.update( + {k: v for k, v in default_resource_dict.items() if k not in resource_dict} + ) + if not disable_dependencies: + return _ExecutorWithDependencies( + max_workers=max_workers, + backend="slurm_allocation", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + refresh_rate=refresh_rate, + plot_dependency_graph=plot_dependency_graph, + plot_dependency_graph_filename=plot_dependency_graph_filename, + ) + else: + _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) + _check_refresh_rate(refresh_rate=refresh_rate) + return _create_executor( + max_workers=max_workers, + backend="slurm_allocation", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + ) + + +class FluxAllocationExecutor: + """ + The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or + preferable the flux framework for distributing python functions within a given resource allocation. In contrast to + the mpi4py.futures.MPIPoolExecutor the executorlib.Executor can be executed in a serial python process and does not + require the python script to be executed with MPI. It is even possible to execute the executorlib.Executor directly + in an interactive Jupyter notebook. + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the number of + cores which can be used in parallel - just like the max_cores parameter. Using max_cores is + recommended, as computers have a limited number of compute cores. + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + max_cores (int): defines the number cores which can be used in parallel + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and + SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only) + flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux + flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only) + flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function. + flux_log_files (bool, optional): Write flux stdout and stderr files. Defaults to False. + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same resource + requirements, executorlib supports block allocation. In this case all resources have + to be defined on the executor, rather than during the submission of the individual + function. + init_function (None): optional function to preset arguments for functions which are submitted later + disable_dependencies (boolean): Disable resolving future objects during the submission. + refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. + plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For + debugging purposes and to get an overview of the specified dependencies. + plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + + Examples: + ``` + >>> import numpy as np + >>> from executorlib.interfaces import FluxAllocationExecutor + >>> + >>> def calc(i, j, k): + >>> from mpi4py import MPI + >>> size = MPI.COMM_WORLD.Get_size() + >>> rank = MPI.COMM_WORLD.Get_rank() + >>> return np.array([i, j, k]), size, rank + >>> + >>> def init_k(): + >>> return {"k": 3} + >>> + >>> with Executor(cores=2, init_function=init_k) as p: + >>> fs = p.submit(calc, 2, j=4) + >>> print(fs.result()) + [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] + ``` + """ + + def __init__( + self, + max_workers: Optional[int] = None, + cache_directory: Optional[str] = None, + max_cores: Optional[int] = None, + resource_dict: Optional[dict] = None, + flux_executor=None, + flux_executor_pmi_mode: Optional[str] = None, + flux_executor_nesting: bool = False, + flux_log_files: bool = False, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, + disable_dependencies: bool = False, + refresh_rate: float = 0.01, + plot_dependency_graph: bool = False, + plot_dependency_graph_filename: Optional[str] = None, + ): + # Use __new__() instead of __init__(). This function is only implemented to enable auto-completion. + pass + + def __new__( + cls, + max_workers: Optional[int] = None, + cache_directory: Optional[str] = None, + max_cores: Optional[int] = None, + resource_dict: Optional[dict] = None, + flux_executor=None, + flux_executor_pmi_mode: Optional[str] = None, + flux_executor_nesting: bool = False, + flux_log_files: bool = False, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, + disable_dependencies: bool = False, + refresh_rate: float = 0.01, + plot_dependency_graph: bool = False, + plot_dependency_graph_filename: Optional[str] = None, + ): + """ + Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor, + executorlib.slurm.PySlurmExecutor or executorlib.flux.PyFluxExecutor depending on which backend is available. The + executorlib.flux.PyFluxExecutor is the preferred choice while the executorlib.mpi.PyMPIExecutor is primarily used + for development and testing. The executorlib.flux.PyFluxExecutor requires flux-core from the flux-framework to be + installed and in addition flux-sched to enable GPU scheduling. Finally, the executorlib.slurm.PySlurmExecutor + requires the SLURM workload manager to be installed on the system. + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the + number of cores which can be used in parallel - just like the max_cores parameter. Using + max_cores is recommended, as computers have a limited number of compute cores. + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + max_cores (int): defines the number cores which can be used in parallel + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI + and SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM + only) + flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux + flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only) + flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function. + flux_log_files (bool, optional): Write flux stdout and stderr files. Defaults to False. + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same + resource requirements, executorlib supports block allocation. In this case all + resources have to be defined on the executor, rather than during the submission + of the individual function. + init_function (None): optional function to preset arguments for functions which are submitted later + disable_dependencies (boolean): Disable resolving future objects during the submission. + refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. + plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For + debugging purposes and to get an overview of the specified dependencies. + plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + + """ + default_resource_dict: dict = { + "cores": 1, + "threads_per_core": 1, + "gpus_per_core": 0, + "cwd": None, + "openmpi_oversubscribe": False, + "slurm_cmd_args": [], + } + if resource_dict is None: + resource_dict = {} + resource_dict.update( + {k: v for k, v in default_resource_dict.items() if k not in resource_dict} + ) + if not disable_dependencies: + return _ExecutorWithDependencies( + max_workers=max_workers, + backend="flux_allocation", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=flux_executor, + flux_executor_pmi_mode=flux_executor_pmi_mode, + flux_executor_nesting=flux_executor_nesting, + flux_log_files=flux_log_files, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + refresh_rate=refresh_rate, + plot_dependency_graph=plot_dependency_graph, + plot_dependency_graph_filename=plot_dependency_graph_filename, + ) + else: + _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) + _check_refresh_rate(refresh_rate=refresh_rate) + return _create_executor( + max_workers=max_workers, + backend="flux_allocation", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=flux_executor, + flux_executor_pmi_mode=flux_executor_pmi_mode, + flux_executor_nesting=flux_executor_nesting, + flux_log_files=flux_log_files, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + ) + + +class FluxSubmissionExecutor: + """ + The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or + preferable the flux framework for distributing python functions within a given resource allocation. In contrast to + the mpi4py.futures.MPIPoolExecutor the executorlib.Executor can be executed in a serial python process and does not + require the python script to be executed with MPI. It is even possible to execute the executorlib.Executor directly + in an interactive Jupyter notebook. + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the number of + cores which can be used in parallel - just like the max_cores parameter. Using max_cores is + recommended, as computers have a limited number of compute cores. + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + max_cores (int): defines the number cores which can be used in parallel + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and + SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only) + pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend). + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same resource + requirements, executorlib supports block allocation. In this case all resources have + to be defined on the executor, rather than during the submission of the individual + function. + init_function (None): optional function to preset arguments for functions which are submitted later + disable_dependencies (boolean): Disable resolving future objects during the submission. + refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. + plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For + debugging purposes and to get an overview of the specified dependencies. + plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + + Examples: + ``` + >>> import numpy as np + >>> from executorlib.interfaces import FluxSubmissionExecutor + >>> + >>> def calc(i, j, k): + >>> from mpi4py import MPI + >>> size = MPI.COMM_WORLD.Get_size() + >>> rank = MPI.COMM_WORLD.Get_rank() + >>> return np.array([i, j, k]), size, rank + >>> + >>> def init_k(): + >>> return {"k": 3} + >>> + >>> with Executor(cores=2, init_function=init_k) as p: + >>> fs = p.submit(calc, 2, j=4) + >>> print(fs.result()) + [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] + ``` + """ + + def __init__( + self, + max_workers: Optional[int] = None, + cache_directory: Optional[str] = None, + max_cores: Optional[int] = None, + resource_dict: Optional[dict] = None, + pysqa_config_directory: Optional[str] = None, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, + disable_dependencies: bool = False, + refresh_rate: float = 0.01, + plot_dependency_graph: bool = False, + plot_dependency_graph_filename: Optional[str] = None, + ): + # Use __new__() instead of __init__(). This function is only implemented to enable auto-completion. + pass + + def __new__( + cls, + max_workers: Optional[int] = None, + cache_directory: Optional[str] = None, + max_cores: Optional[int] = None, + resource_dict: Optional[dict] = None, + pysqa_config_directory: Optional[str] = None, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, + disable_dependencies: bool = False, + refresh_rate: float = 0.01, + plot_dependency_graph: bool = False, + plot_dependency_graph_filename: Optional[str] = None, + ): + """ + Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor, + executorlib.slurm.PySlurmExecutor or executorlib.flux.PyFluxExecutor depending on which backend is available. The + executorlib.flux.PyFluxExecutor is the preferred choice while the executorlib.mpi.PyMPIExecutor is primarily used + for development and testing. The executorlib.flux.PyFluxExecutor requires flux-core from the flux-framework to be + installed and in addition flux-sched to enable GPU scheduling. Finally, the executorlib.slurm.PySlurmExecutor + requires the SLURM workload manager to be installed on the system. + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the + number of cores which can be used in parallel - just like the max_cores parameter. Using + max_cores is recommended, as computers have a limited number of compute cores. + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + max_cores (int): defines the number cores which can be used in parallel + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI + and SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM + only) + pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend). + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same + resource requirements, executorlib supports block allocation. In this case all + resources have to be defined on the executor, rather than during the submission + of the individual function. + init_function (None): optional function to preset arguments for functions which are submitted later + disable_dependencies (boolean): Disable resolving future objects during the submission. + refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. + plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For + debugging purposes and to get an overview of the specified dependencies. + plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + + """ + default_resource_dict: dict = { + "cores": 1, + "threads_per_core": 1, + "gpus_per_core": 0, + "cwd": None, + "openmpi_oversubscribe": False, + "slurm_cmd_args": [], + } + if resource_dict is None: + resource_dict = {} + resource_dict.update( + {k: v for k, v in default_resource_dict.items() if k not in resource_dict} + ) + if not plot_dependency_graph: + from executorlib.cache.executor import create_file_executor + + return create_file_executor( + max_workers=max_workers, + backend="flux_submission", + max_cores=max_cores, + cache_directory=cache_directory, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + pysqa_config_directory=pysqa_config_directory, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + disable_dependencies=disable_dependencies, + ) + elif not disable_dependencies: + _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) + return _ExecutorWithDependencies( + max_workers=max_workers, + backend="flux_submission", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + refresh_rate=refresh_rate, + plot_dependency_graph=plot_dependency_graph, + plot_dependency_graph_filename=plot_dependency_graph_filename, + ) + else: + _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) + _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) + _check_refresh_rate(refresh_rate=refresh_rate) + return _create_executor( + max_workers=max_workers, + backend="flux_submission", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + ) diff --git a/notebooks/1-local.ipynb b/notebooks/1-local.ipynb index 861491f3..9e75169f 100644 --- a/notebooks/1-local.ipynb +++ b/notebooks/1-local.ipynb @@ -26,9 +26,7 @@ "id": "b1907f12-7378-423b-9b83-1b65fc0a20f5", "metadata": {}, "outputs": [], - "source": [ - "from executorlib import Executor" - ] + "source": "from executorlib import LocalExecutor" }, { "cell_type": "markdown", @@ -56,7 +54,7 @@ ], "source": [ "%%time\n", - "with Executor(backend=\"local\") as exe:\n", + "with LocalExecutor() as exe:\n", " future = exe.submit(sum, [1, 1])\n", " print(future.result())" ] @@ -87,7 +85,7 @@ ], "source": [ "%%time\n", - "with Executor(backend=\"local\") as exe:\n", + "with LocalExecutor() as exe:\n", " future_lst = [exe.submit(sum, [i, i]) for i in range(2, 5)]\n", " print([f.result() for f in future_lst])" ] @@ -118,7 +116,7 @@ ], "source": [ "%%time\n", - "with Executor(backend=\"local\") as exe:\n", + "with LocalExecutor() as exe:\n", " results = exe.map(sum, [[5, 5], [6, 6], [7, 7]])\n", " print(list(results))" ] @@ -191,7 +189,7 @@ } ], "source": [ - "with Executor(backend=\"local\") as exe:\n", + "with LocalExecutor() as exe:\n", " fs = exe.submit(calc_mpi, 3, resource_dict={\"cores\": 2})\n", " print(fs.result())" ] @@ -219,7 +217,7 @@ } ], "source": [ - "with Executor(backend=\"local\", resource_dict={\"cores\": 2}) as exe:\n", + "with LocalExecutor(resource_dict={\"cores\": 2}) as exe:\n", " fs = exe.submit(calc_mpi, 3)\n", " print(fs.result())" ] @@ -285,7 +283,7 @@ } ], "source": [ - "with Executor(backend=\"local\") as exe:\n", + "with LocalExecutor() as exe:\n", " fs = exe.submit(calc_with_threads, 3, resource_dict={\"threads_per_core\": 2})\n", " print(fs.result())" ] @@ -313,7 +311,7 @@ } ], "source": [ - "with Executor(backend=\"local\", resource_dict={\"threads_per_core\": 2}) as exe:\n", + "with LocalExecutor(resource_dict={\"threads_per_core\": 2}) as exe:\n", " fs = exe.submit(calc_with_threads, 3)\n", " print(fs.result())" ] @@ -364,7 +362,7 @@ ], "source": [ "%%time\n", - "with Executor(max_workers=2, backend=\"local\", block_allocation=True) as exe:\n", + "with LocalExecutor(max_workers=2, block_allocation=True) as exe:\n", " future = exe.submit(sum, [1, 1])\n", " print(future.result())" ] @@ -457,8 +455,8 @@ } ], "source": [ - "with Executor(\n", - " backend=\"local\", resource_dict={\"cores\": 2}, block_allocation=True\n", + "with LocalExecutor(\n", + " resource_dict={\"cores\": 2}, block_allocation=True\n", ") as exe:\n", " fs = exe.submit(calc_mpi, 3)\n", " print(fs.result())" @@ -517,8 +515,8 @@ } ], "source": [ - "with Executor(\n", - " backend=\"local\", init_function=init_function, block_allocation=True\n", + "with LocalExecutor(\n", + " init_function=init_function, block_allocation=True\n", ") as exe:\n", " fs = exe.submit(calc_with_preload, 2, j=5)\n", " print(fs.result())" @@ -561,7 +559,7 @@ ], "source": [ "%%time\n", - "with Executor(backend=\"local\", cache_directory=\"./cache\") as exe:\n", + "with LocalExecutor(cache_directory=\"./cache\") as exe:\n", " future_lst = [exe.submit(sum, [i, i]) for i in range(1, 4)]\n", " print([f.result() for f in future_lst])" ] @@ -594,7 +592,7 @@ ], "source": [ "%%time\n", - "with Executor(backend=\"local\", cache_directory=\"./cache\") as exe:\n", + "with LocalExecutor(cache_directory=\"./cache\") as exe:\n", " future_lst = [exe.submit(sum, [i, i]) for i in range(1, 4)]\n", " print([f.result() for f in future_lst])" ] @@ -687,7 +685,7 @@ } ], "source": [ - "with Executor(backend=\"local\") as exe:\n", + "with LocalExecutor() as exe:\n", " future = 0\n", " for i in range(1, 4):\n", " future = exe.submit(calc_add, i, future)\n", @@ -815,7 +813,7 @@ } ], "source": [ - "with Executor(backend=\"local\", plot_dependency_graph=True) as exe:\n", + "with LocalExecutor(plot_dependency_graph=True) as exe:\n", " future = 0\n", " for i in range(1, 4):\n", " future = exe.submit(calc_add, i, future)\n", diff --git a/notebooks/2-hpc-submission.ipynb b/notebooks/2-hpc-submission.ipynb index f0998333..e62b4f0a 100644 --- a/notebooks/2-hpc-submission.ipynb +++ b/notebooks/2-hpc-submission.ipynb @@ -28,7 +28,7 @@ "metadata": {}, "source": [ "```python\n", - "from executorlib import Executor\n", + "from executorlib import SlurmSubmissionExecutor\n", "```" ] }, @@ -46,7 +46,7 @@ "metadata": {}, "source": [ "```python\n", - "with Executor(backend=\"slurm_submission\", cache_directory=\"./cache\") as exe:\n", + "with SlurmSubmissionExecutor(cache_directory=\"./cache\") as exe:\n", " future_lst = [exe.submit(sum, [i, i]) for i in range(1, 4)]\n", " print([f.result() for f in future_lst])\n", "```" @@ -87,7 +87,7 @@ "{{command}}\n", "\"\"\"\n", "\n", - "with Executor(backend=\"slurm_submission\", cache_directory=\"./cache\") as exe:\n", + "with SlurmSubmissionExecutor(cache_directory=\"./cache\") as exe:\n", " future = exe.submit(\n", " sum, [4, 4], \n", " resource_dict={\n", @@ -141,7 +141,9 @@ "metadata": {}, "source": [ "```python\n", - "with Executor(backend=\"flux_submission\", cache_directory=\"./cache\") as exe:\n", + "from executorlib import FluxSubmissionExecutor\n", + "\n", + "with FluxSubmissionExecutor(cache_directory=\"./cache\") as exe:\n", " future = 0\n", " for i in range(4, 8):\n", " future = exe.submit(add_funct, i, future)\n", @@ -181,7 +183,7 @@ "metadata": {}, "source": [ "```python\n", - "with Executor(backend=\"flux_submission\", cache_directory=\"./cache\") as exe:\n", + "with FluxSubmissionExecutor(cache_directory=\"./cache\") as exe:\n", " fs = exe.submit(calc, 3, resource_dict={\"cores\": 2})\n", " print(fs.result())\n", "```" @@ -205,8 +207,7 @@ "```\n", "\n", "```python\n", - "with Executor(\n", - " backend=\"flux_submission\",\n", + "with FluxSubmissionExecutor(\n", " cache_directory=\"./cache\",\n", " resource_dict={\"gpus_per_core\": 1}\n", ") as exe:\n", diff --git a/notebooks/3-hpc-allocation.ipynb b/notebooks/3-hpc-allocation.ipynb index 1d4ef3cf..8375d2fd 100644 --- a/notebooks/3-hpc-allocation.ipynb +++ b/notebooks/3-hpc-allocation.ipynb @@ -31,9 +31,7 @@ "id": "133b751f-0925-4d11-99f0-3f8dd9360b54", "metadata": {}, "outputs": [], - "source": [ - "from executorlib import Executor" - ] + "source": "from executorlib import SlurmAllocationExecutor" }, { "cell_type": "markdown", @@ -41,7 +39,7 @@ "metadata": {}, "source": [ "```python\n", - "with Executor(backend=\"slurm_allocation\") as exe:\n", + "with SlurmAllocationExecutor() as exe:\n", " future = exe.submit(sum, [1, 1])\n", " print(future.result())\n", "```" @@ -111,7 +109,9 @@ } ], "source": [ - "with Executor(backend=\"flux_allocation\", flux_executor_pmi_mode=\"pmix\") as exe:\n", + "from executorlib import FluxAllocationExecutor\n", + "\n", + "with FluxAllocationExecutor(flux_executor_pmi_mode=\"pmix\") as exe:\n", " fs = exe.submit(calc_mpi, 3, resource_dict={\"cores\": 2})\n", " print(fs.result())" ] @@ -162,8 +162,7 @@ } ], "source": [ - "with Executor(\n", - " backend=\"flux_allocation\",\n", + "with FluxAllocationExecutor(\n", " flux_executor_pmi_mode=\"pmix\",\n", " max_workers=2,\n", " init_function=init_function,\n", @@ -218,7 +217,7 @@ } ], "source": [ - "with Executor(backend=\"flux_allocation\", flux_executor_pmi_mode=\"pmix\") as exe:\n", + "with FluxAllocationExecutor(flux_executor_pmi_mode=\"pmix\") as exe:\n", " future = 0\n", " for i in range(1, 4):\n", " future = exe.submit(add_funct, i, future)\n", @@ -249,8 +248,8 @@ } ], "source": [ - "with Executor(\n", - " backend=\"flux_allocation\", flux_executor_pmi_mode=\"pmix\", cache_directory=\"./cache\"\n", + "with FluxAllocationExecutor(\n", + " flux_executor_pmi_mode=\"pmix\", cache_directory=\"./cache\"\n", ") as exe:\n", " future_lst = [exe.submit(sum, [i, i]) for i in range(1, 4)]\n", " print([f.result() for f in future_lst])" @@ -300,9 +299,9 @@ "outputs": [], "source": [ "def calc_nested():\n", - " from executorlib import Executor\n", + " from executorlib import FluxAllocationExecutor\n", "\n", - " with Executor(backend=\"flux_allocation\", flux_executor_pmi_mode=\"pmix\") as exe:\n", + " with FluxAllocationExecutor(flux_executor_pmi_mode=\"pmix\") as exe:\n", " fs = exe.submit(sum, [1, 1])\n", " return fs.result()" ] @@ -322,8 +321,8 @@ } ], "source": [ - "with Executor(\n", - " backend=\"flux_allocation\", flux_executor_pmi_mode=\"pmix\", flux_executor_nesting=True\n", + "with FluxAllocationExecutor(\n", + " flux_executor_pmi_mode=\"pmix\", flux_executor_nesting=True\n", ") as exe:\n", " fs = exe.submit(calc_nested)\n", " print(fs.result())" @@ -378,18 +377,18 @@ "output_type": "stream", "text": [ " JOBID USER NAME ST NTASKS NNODES TIME INFO\n", - "\u001b[01;32m ƒDqBpVYK jan python CD 1 1 0.695s fedora\n", - "\u001b[0;0m\u001b[01;32m ƒDxdEtYf jan python CD 1 1 0.225s fedora\n", - "\u001b[0;0m\u001b[01;32m ƒDVahzPq jan python CD 1 1 0.254s fedora\n", - "\u001b[0;0m\u001b[01;32m ƒDSsZJXH jan python CD 1 1 0.316s fedora\n", - "\u001b[0;0m\u001b[01;32m ƒDSu3Hod jan python CD 1 1 0.277s fedora\n", - "\u001b[0;0m\u001b[01;32m ƒDFbkmFD jan python CD 1 1 0.247s fedora\n", - "\u001b[0;0m\u001b[01;32m ƒD9eKeas jan python CD 1 1 0.227s fedora\n", - "\u001b[0;0m\u001b[01;32m ƒD3iNXCs jan python CD 1 1 0.224s fedora\n", - "\u001b[0;0m\u001b[01;32m ƒCoZ3P5q jan python CD 1 1 0.261s fedora\n", - "\u001b[0;0m\u001b[01;32m ƒCoXZPoV jan python CD 1 1 0.261s fedora\n", - "\u001b[0;0m\u001b[01;32m ƒCZ1URjd jan python CD 2 1 0.360s fedora\n", - "\u001b[0;0m" + "\u001B[01;32m ƒDqBpVYK jan python CD 1 1 0.695s fedora\n", + "\u001B[0;0m\u001B[01;32m ƒDxdEtYf jan python CD 1 1 0.225s fedora\n", + "\u001B[0;0m\u001B[01;32m ƒDVahzPq jan python CD 1 1 0.254s fedora\n", + "\u001B[0;0m\u001B[01;32m ƒDSsZJXH jan python CD 1 1 0.316s fedora\n", + "\u001B[0;0m\u001B[01;32m ƒDSu3Hod jan python CD 1 1 0.277s fedora\n", + "\u001B[0;0m\u001B[01;32m ƒDFbkmFD jan python CD 1 1 0.247s fedora\n", + "\u001B[0;0m\u001B[01;32m ƒD9eKeas jan python CD 1 1 0.227s fedora\n", + "\u001B[0;0m\u001B[01;32m ƒD3iNXCs jan python CD 1 1 0.224s fedora\n", + "\u001B[0;0m\u001B[01;32m ƒCoZ3P5q jan python CD 1 1 0.261s fedora\n", + "\u001B[0;0m\u001B[01;32m ƒCoXZPoV jan python CD 1 1 0.261s fedora\n", + "\u001B[0;0m\u001B[01;32m ƒCZ1URjd jan python CD 2 1 0.360s fedora\n", + "\u001B[0;0m" ] } ], diff --git a/notebooks/4-developer.ipynb b/notebooks/4-developer.ipynb index 5bda8e1a..b86a34ed 100644 --- a/notebooks/4-developer.ipynb +++ b/notebooks/4-developer.ipynb @@ -56,9 +56,7 @@ "id": "83515b16-c4d5-4b02-acd7-9e1eb57fd335", "metadata": {}, "outputs": [], - "source": [ - "from executorlib import Executor" - ] + "source": "from executorlib import LocalExecutor" }, { "cell_type": "code", @@ -93,7 +91,7 @@ } ], "source": [ - "with Executor(backend=\"local\") as exe:\n", + "with LocalExecutor() as exe:\n", " future = exe.submit(\n", " execute_shell_command,\n", " [\"echo\", \"test\"],\n", @@ -250,7 +248,7 @@ } ], "source": [ - "with Executor(\n", + "with LocalExecutor(\n", " max_workers=1,\n", " init_function=init_process,\n", " block_allocation=True,\n", diff --git a/tests/test_cache_executor_interactive.py b/tests/test_cache_executor_interactive.py index 35f08a5c..efd46780 100644 --- a/tests/test_cache_executor_interactive.py +++ b/tests/test_cache_executor_interactive.py @@ -2,7 +2,7 @@ import shutil import unittest -from executorlib import Executor +from executorlib import LocalExecutor try: from executorlib.standalone.hdf import get_cache_data @@ -18,7 +18,7 @@ class TestCacheFunctions(unittest.TestCase): def test_cache_data(self): cache_directory = "./cache" - with Executor(backend="local", cache_directory=cache_directory) as exe: + with LocalExecutor(cache_directory=cache_directory) as exe: future_lst = [exe.submit(sum, [i, i]) for i in range(1, 4)] result_lst = [f.result() for f in future_lst] diff --git a/tests/test_cache_executor_pysqa_flux.py b/tests/test_cache_executor_pysqa_flux.py index 827fd455..3e8f7c87 100644 --- a/tests/test_cache_executor_pysqa_flux.py +++ b/tests/test_cache_executor_pysqa_flux.py @@ -3,7 +3,7 @@ import unittest import shutil -from executorlib import Executor +from executorlib import FluxSubmissionExecutor from executorlib.standalone.serialize import cloudpickle_register try: @@ -32,8 +32,7 @@ def mpi_funct(i): ) class TestCacheExecutorPysqa(unittest.TestCase): def test_executor(self): - with Executor( - backend="flux_submission", + with FluxSubmissionExecutor( resource_dict={"cores": 2, "cwd": "cache"}, block_allocation=False, cache_directory="cache", diff --git a/tests/test_dependencies_executor.py b/tests/test_dependencies_executor.py index 5e5949cf..7fae712b 100644 --- a/tests/test_dependencies_executor.py +++ b/tests/test_dependencies_executor.py @@ -4,7 +4,7 @@ from time import sleep from queue import Queue -from executorlib import Executor +from executorlib import LocalExecutor from executorlib.interactive.executor import create_executor from executorlib.interactive.shared import execute_tasks_with_dependencies from executorlib.standalone.plot import generate_nodes_and_edges @@ -46,7 +46,7 @@ def raise_error(): class TestExecutorWithDependencies(unittest.TestCase): def test_executor(self): - with Executor(max_cores=1, backend="local") as exe: + with LocalExecutor(max_cores=1) as exe: cloudpickle_register(ind=1) future_1 = exe.submit(add_function, 1, parameter_2=2) future_2 = exe.submit(add_function, 1, parameter_2=future_1) @@ -57,9 +57,8 @@ def test_executor(self): "graphviz is not installed, so the plot_dependency_graph tests are skipped.", ) def test_executor_dependency_plot(self): - with Executor( + with LocalExecutor( max_cores=1, - backend="local", plot_dependency_graph=True, ) as exe: cloudpickle_register(ind=1) @@ -84,9 +83,8 @@ def test_executor_dependency_plot(self): ) def test_executor_dependency_plot_filename(self): graph_file = os.path.join(os.path.dirname(__file__), "test.png") - with Executor( + with LocalExecutor( max_cores=1, - backend="local", plot_dependency_graph=False, plot_dependency_graph_filename=graph_file, ) as exe: @@ -158,7 +156,7 @@ def test_dependency_steps(self): def test_many_to_one(self): length = 5 parameter = 1 - with Executor(max_cores=2, backend="local") as exe: + with LocalExecutor(max_cores=2) as exe: cloudpickle_register(ind=1) future_lst = exe.submit( generate_tasks, @@ -190,9 +188,8 @@ def test_many_to_one(self): def test_many_to_one_plot(self): length = 5 parameter = 1 - with Executor( + with LocalExecutor( max_cores=2, - backend="local", plot_dependency_graph=True, ) as exe: cloudpickle_register(ind=1) @@ -236,24 +233,24 @@ def test_many_to_one_plot(self): class TestExecutorErrors(unittest.TestCase): def test_block_allocation_false_one_worker(self): with self.assertRaises(RuntimeError): - with Executor(max_cores=1, backend="local", block_allocation=False) as exe: + with LocalExecutor(max_cores=1, block_allocation=False) as exe: cloudpickle_register(ind=1) _ = exe.submit(raise_error) def test_block_allocation_true_one_worker(self): with self.assertRaises(RuntimeError): - with Executor(max_cores=1, backend="local", block_allocation=True) as exe: + with LocalExecutor(max_cores=1, block_allocation=True) as exe: cloudpickle_register(ind=1) _ = exe.submit(raise_error) def test_block_allocation_false_two_workers(self): with self.assertRaises(RuntimeError): - with Executor(max_cores=2, backend="local", block_allocation=False) as exe: + with LocalExecutor(max_cores=2, block_allocation=False) as exe: cloudpickle_register(ind=1) _ = exe.submit(raise_error) def test_block_allocation_true_two_workers(self): with self.assertRaises(RuntimeError): - with Executor(max_cores=2, backend="local", block_allocation=True) as exe: + with LocalExecutor(max_cores=2, block_allocation=True) as exe: cloudpickle_register(ind=1) _ = exe.submit(raise_error) diff --git a/tests/test_executor_backend_flux.py b/tests/test_executor_backend_flux.py index d94508e8..8ce476f0 100644 --- a/tests/test_executor_backend_flux.py +++ b/tests/test_executor_backend_flux.py @@ -3,7 +3,7 @@ import numpy as np -from executorlib import Executor +from executorlib import FluxAllocationExecutor try: @@ -44,10 +44,9 @@ def setUp(self): self.executor = flux.job.FluxExecutor() def test_flux_executor_serial(self): - with Executor( + with FluxAllocationExecutor( max_cores=2, flux_executor=self.executor, - backend="flux_allocation", block_allocation=True, ) as exe: fs_1 = exe.submit(calc, 1) @@ -58,11 +57,10 @@ def test_flux_executor_serial(self): self.assertTrue(fs_2.done()) def test_flux_executor_threads(self): - with Executor( + with FluxAllocationExecutor( max_cores=1, resource_dict={"threads_per_core": 2}, flux_executor=self.executor, - backend="flux_allocation", block_allocation=True, ) as exe: fs_1 = exe.submit(calc, 1) @@ -73,11 +71,10 @@ def test_flux_executor_threads(self): self.assertTrue(fs_2.done()) def test_flux_executor_parallel(self): - with Executor( + with FluxAllocationExecutor( max_cores=2, resource_dict={"cores": 2}, flux_executor=self.executor, - backend="flux_allocation", block_allocation=True, flux_executor_pmi_mode=pmi, ) as exe: @@ -86,11 +83,10 @@ def test_flux_executor_parallel(self): self.assertTrue(fs_1.done()) def test_single_task(self): - with Executor( + with FluxAllocationExecutor( max_cores=2, resource_dict={"cores": 2}, flux_executor=self.executor, - backend="flux_allocation", block_allocation=True, flux_executor_pmi_mode=pmi, ) as p: @@ -105,11 +101,10 @@ def test_output_files_cwd(self): os.makedirs(dirname, exist_ok=True) file_stdout = os.path.join(dirname, "flux.out") file_stderr = os.path.join(dirname, "flux.err") - with Executor( + with FluxAllocationExecutor( max_cores=1, resource_dict={"cores": 1, "cwd": dirname}, flux_executor=self.executor, - backend="flux_allocation", block_allocation=True, flux_log_files=True, ) as p: @@ -126,11 +121,10 @@ def test_output_files_cwd(self): def test_output_files_abs(self): file_stdout = os.path.abspath("flux.out") file_stderr = os.path.abspath("flux.err") - with Executor( + with FluxAllocationExecutor( max_cores=1, resource_dict={"cores": 1}, flux_executor=self.executor, - backend="flux_allocation", block_allocation=True, flux_log_files=True, ) as p: @@ -145,12 +139,11 @@ def test_output_files_abs(self): os.remove(file_stderr) def test_internal_memory(self): - with Executor( + with FluxAllocationExecutor( max_cores=1, resource_dict={"cores": 1}, init_function=set_global, flux_executor=self.executor, - backend="flux_allocation", block_allocation=True, ) as p: f = p.submit(get_global) @@ -160,10 +153,9 @@ def test_internal_memory(self): def test_validate_max_workers(self): with self.assertRaises(ValueError): - Executor( + FluxAllocationExecutor( max_workers=10, resource_dict={"cores": 10, "threads_per_core": 10}, flux_executor=self.executor, - backend="flux_allocation", block_allocation=True, ) diff --git a/tests/test_executor_backend_mpi.py b/tests/test_executor_backend_mpi.py index e0b4a3c5..2ccf4770 100644 --- a/tests/test_executor_backend_mpi.py +++ b/tests/test_executor_backend_mpi.py @@ -4,7 +4,7 @@ import time import unittest -from executorlib import Executor +from executorlib import LocalExecutor from executorlib.standalone.serialize import cloudpickle_register @@ -34,7 +34,7 @@ def mpi_funct_sleep(i): class TestExecutorBackend(unittest.TestCase): def test_meta_executor_serial(self): - with Executor(max_cores=2, backend="local", block_allocation=True) as exe: + with LocalExecutor(max_cores=2, block_allocation=True) as exe: cloudpickle_register(ind=1) fs_1 = exe.submit(calc, 1) fs_2 = exe.submit(calc, 2) @@ -44,7 +44,7 @@ def test_meta_executor_serial(self): self.assertTrue(fs_2.done()) def test_meta_executor_single(self): - with Executor(max_cores=1, backend="local", block_allocation=True) as exe: + with LocalExecutor(max_cores=1, block_allocation=True) as exe: cloudpickle_register(ind=1) fs_1 = exe.submit(calc, 1) fs_2 = exe.submit(calc, 2) @@ -55,7 +55,7 @@ def test_meta_executor_single(self): def test_oversubscribe(self): with self.assertRaises(ValueError): - with Executor(max_cores=1, backend="local", block_allocation=True) as exe: + with LocalExecutor(max_cores=1, block_allocation=True) as exe: cloudpickle_register(ind=1) fs_1 = exe.submit(calc, 1, resource_dict={"cores": 2}) @@ -63,10 +63,9 @@ def test_oversubscribe(self): skip_mpi4py_test, "mpi4py is not installed, so the mpi4py tests are skipped." ) def test_meta_executor_parallel(self): - with Executor( + with LocalExecutor( max_workers=2, resource_dict={"cores": 2}, - backend="local", block_allocation=True, ) as exe: cloudpickle_register(ind=1) @@ -76,10 +75,9 @@ def test_meta_executor_parallel(self): def test_errors(self): with self.assertRaises(TypeError): - Executor( + LocalExecutor( max_cores=1, resource_dict={"cores": 1, "gpus_per_core": 1}, - backend="local", ) @@ -91,10 +89,9 @@ def tearDown(self): skip_mpi4py_test, "mpi4py is not installed, so the mpi4py tests are skipped." ) def test_meta_executor_parallel_cache(self): - with Executor( + with LocalExecutor( max_workers=2, resource_dict={"cores": 2}, - backend="local", block_allocation=True, cache_directory="./cache", ) as exe: @@ -117,10 +114,9 @@ class TestWorkingDirectory(unittest.TestCase): def test_output_files_cwd(self): dirname = os.path.abspath(os.path.dirname(__file__)) os.makedirs(dirname, exist_ok=True) - with Executor( + with LocalExecutor( max_cores=1, resource_dict={"cores": 1, "cwd": dirname}, - backend="local", block_allocation=True, ) as p: output = p.map(calc, [1, 2, 3]) @@ -135,9 +131,8 @@ def test_validate_max_workers(self): os.environ["SLURM_NTASKS"] = "6" os.environ["SLURM_CPUS_PER_TASK"] = "4" with self.assertRaises(ValueError): - Executor( + LocalExecutor( max_workers=10, resource_dict={"cores": 10, "threads_per_core": 10}, - backend="slurm_allocation", block_allocation=True, ) diff --git a/tests/test_executor_backend_mpi_noblock.py b/tests/test_executor_backend_mpi_noblock.py index 47ea2bb3..e9c27091 100644 --- a/tests/test_executor_backend_mpi_noblock.py +++ b/tests/test_executor_backend_mpi_noblock.py @@ -1,6 +1,6 @@ import unittest -from executorlib import Executor +from executorlib import LocalExecutor from executorlib.standalone.serialize import cloudpickle_register @@ -14,9 +14,8 @@ def resource_dict(resource_dict): class TestExecutorBackend(unittest.TestCase): def test_meta_executor_serial_with_dependencies(self): - with Executor( + with LocalExecutor( max_cores=2, - backend="local", block_allocation=False, disable_dependencies=True, ) as exe: @@ -29,9 +28,8 @@ def test_meta_executor_serial_with_dependencies(self): self.assertTrue(fs_2.done()) def test_meta_executor_serial_without_dependencies(self): - with Executor( + with LocalExecutor( max_cores=2, - backend="local", block_allocation=False, disable_dependencies=False, ) as exe: @@ -44,9 +42,8 @@ def test_meta_executor_serial_without_dependencies(self): self.assertTrue(fs_2.done()) def test_meta_executor_single(self): - with Executor( + with LocalExecutor( max_cores=1, - backend="local", block_allocation=False, ) as exe: cloudpickle_register(ind=1) @@ -59,25 +56,22 @@ def test_meta_executor_single(self): def test_errors(self): with self.assertRaises(TypeError): - Executor( + LocalExecutor( max_cores=1, resource_dict={ "cores": 1, "gpus_per_core": 1, }, - backend="local", ) with self.assertRaises(ValueError): - with Executor( + with LocalExecutor( max_cores=1, - backend="local", block_allocation=False, ) as exe: exe.submit(resource_dict, resource_dict={}) with self.assertRaises(ValueError): - with Executor( + with LocalExecutor( max_cores=1, - backend="local", block_allocation=True, ) as exe: exe.submit(resource_dict, resource_dict={}) diff --git a/tests/test_integration_pyiron_workflow.py b/tests/test_integration_pyiron_workflow.py index 5eb4ba86..dad81894 100644 --- a/tests/test_integration_pyiron_workflow.py +++ b/tests/test_integration_pyiron_workflow.py @@ -12,7 +12,7 @@ from typing import Callable import unittest -from executorlib import Executor +from executorlib import LocalExecutor from executorlib.standalone.serialize import cloudpickle_register @@ -75,7 +75,7 @@ def slowly_returns_dynamic(dynamic_arg): return dynamic_arg dynamic_dynamic = slowly_returns_dynamic() - executor = Executor(block_allocation=True, max_workers=1) + executor = LocalExecutor(block_allocation=True, max_workers=1) cloudpickle_register(ind=1) dynamic_object = does_nothing() fs = executor.submit(dynamic_dynamic.run, dynamic_object) @@ -105,7 +105,7 @@ def slowly_returns_42(): self.assertIsNone( dynamic_42.result, msg="Just a sanity check that the test is set up right" ) - executor = Executor(block_allocation=True, max_workers=1) + executor = LocalExecutor(block_allocation=True, max_workers=1) cloudpickle_register(ind=1) fs = executor.submit(dynamic_42.run) fs.add_done_callback(dynamic_42.process_result) @@ -136,7 +136,7 @@ def returns_42(): dynamic_42.running, msg="Sanity check that the test starts in the expected condition", ) - executor = Executor(block_allocation=True, max_workers=1) + executor = LocalExecutor(block_allocation=True, max_workers=1) cloudpickle_register(ind=1) fs = executor.submit(dynamic_42.run) fs.add_done_callback(dynamic_42.process_result) @@ -160,7 +160,7 @@ def raise_error(): raise RuntimeError re = raise_error() - executor = Executor(block_allocation=True, max_workers=1) + executor = LocalExecutor(block_allocation=True, max_workers=1) cloudpickle_register(ind=1) fs = executor.submit(re.run) with self.assertRaises( @@ -190,7 +190,7 @@ def slowly_returns_dynamic(): return inside_variable dynamic_dynamic = slowly_returns_dynamic() - executor = Executor(block_allocation=True, max_workers=1) + executor = LocalExecutor(block_allocation=True, max_workers=1) cloudpickle_register(ind=1) fs = executor.submit(dynamic_dynamic.run) self.assertIsInstance( @@ -219,7 +219,7 @@ def slow(): return fortytwo f = slow() - executor = Executor(block_allocation=True, max_workers=1) + executor = LocalExecutor(block_allocation=True, max_workers=1) cloudpickle_register(ind=1) fs = executor.submit(f.run) self.assertEqual( diff --git a/tests/test_shell_executor.py b/tests/test_shell_executor.py index dbdb9aea..00f1ca53 100644 --- a/tests/test_shell_executor.py +++ b/tests/test_shell_executor.py @@ -3,7 +3,7 @@ import queue import unittest -from executorlib import Executor +from executorlib import LocalExecutor from executorlib.standalone.serialize import cloudpickle_register from executorlib.interactive.shared import execute_parallel_tasks from executorlib.standalone.interactive.spawner import MpiExecSpawner @@ -83,7 +83,7 @@ def test_broken_executable(self): ) def test_shell_static_executor_args(self): - with Executor(max_workers=1) as exe: + with LocalExecutor(max_workers=1) as exe: cloudpickle_register(ind=1) future = exe.submit( submit_shell_command, @@ -96,7 +96,7 @@ def test_shell_static_executor_args(self): self.assertTrue(future.done()) def test_shell_static_executor_binary(self): - with Executor(max_workers=1) as exe: + with LocalExecutor(max_workers=1) as exe: cloudpickle_register(ind=1) future = exe.submit( submit_shell_command, @@ -109,7 +109,7 @@ def test_shell_static_executor_binary(self): self.assertTrue(future.done()) def test_shell_static_executor_shell(self): - with Executor(max_workers=1) as exe: + with LocalExecutor(max_workers=1) as exe: cloudpickle_register(ind=1) future = exe.submit( submit_shell_command, "echo test", universal_newlines=True, shell=True @@ -119,7 +119,7 @@ def test_shell_static_executor_shell(self): self.assertTrue(future.done()) def test_shell_executor(self): - with Executor(max_workers=2) as exe: + with LocalExecutor(max_workers=2) as exe: cloudpickle_register(ind=1) f_1 = exe.submit( submit_shell_command, ["echo", "test_1"], universal_newlines=True diff --git a/tests/test_shell_interactive.py b/tests/test_shell_interactive.py index e211b559..efccf2b7 100644 --- a/tests/test_shell_interactive.py +++ b/tests/test_shell_interactive.py @@ -4,7 +4,7 @@ import queue import unittest -from executorlib import Executor +from executorlib import LocalExecutor from executorlib.standalone.serialize import cloudpickle_register from executorlib.interactive.shared import execute_parallel_tasks from executorlib.standalone.interactive.spawner import MpiExecSpawner @@ -104,7 +104,7 @@ def test_execute_single_task(self): def test_shell_interactive_executor(self): cloudpickle_register(ind=1) - with Executor( + with LocalExecutor( max_workers=1, init_function=init_process, block_allocation=True, From 07c27a707d6d4b2d6f8cd786e5fb22049863a878 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 1 Feb 2025 11:12:58 +0000 Subject: [PATCH 02/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- executorlib/__init__.py | 9 +++++++-- notebooks/1-local.ipynb | 12 +++++------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/executorlib/__init__.py b/executorlib/__init__.py index 2c99197e..3940d986 100644 --- a/executorlib/__init__.py +++ b/executorlib/__init__.py @@ -1,6 +1,11 @@ -from executorlib.interfaces import LocalExecutor, FluxAllocationExecutor, FluxSubmissionExecutor, SlurmAllocationExecutor, SlurmSubmissionExecutor from executorlib._version import get_versions as _get_versions - +from executorlib.interfaces import ( + FluxAllocationExecutor, + FluxSubmissionExecutor, + LocalExecutor, + SlurmAllocationExecutor, + SlurmSubmissionExecutor, +) __version__ = _get_versions()["version"] __all__: list = [] diff --git a/notebooks/1-local.ipynb b/notebooks/1-local.ipynb index 9e75169f..9c89b494 100644 --- a/notebooks/1-local.ipynb +++ b/notebooks/1-local.ipynb @@ -26,7 +26,9 @@ "id": "b1907f12-7378-423b-9b83-1b65fc0a20f5", "metadata": {}, "outputs": [], - "source": "from executorlib import LocalExecutor" + "source": [ + "from executorlib import LocalExecutor" + ] }, { "cell_type": "markdown", @@ -455,9 +457,7 @@ } ], "source": [ - "with LocalExecutor(\n", - " resource_dict={\"cores\": 2}, block_allocation=True\n", - ") as exe:\n", + "with LocalExecutor(resource_dict={\"cores\": 2}, block_allocation=True) as exe:\n", " fs = exe.submit(calc_mpi, 3)\n", " print(fs.result())" ] @@ -515,9 +515,7 @@ } ], "source": [ - "with LocalExecutor(\n", - " init_function=init_function, block_allocation=True\n", - ") as exe:\n", + "with LocalExecutor(init_function=init_function, block_allocation=True) as exe:\n", " fs = exe.submit(calc_with_preload, 2, j=5)\n", " print(fs.result())" ] From fc0298ca691c440b86b16b13b67e0752305f43fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 1 Feb 2025 12:22:00 +0100 Subject: [PATCH 03/28] fix test --- tests/test_executor_backend_mpi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_executor_backend_mpi.py b/tests/test_executor_backend_mpi.py index 2ccf4770..9d9ba7e2 100644 --- a/tests/test_executor_backend_mpi.py +++ b/tests/test_executor_backend_mpi.py @@ -4,7 +4,7 @@ import time import unittest -from executorlib import LocalExecutor +from executorlib import LocalExecutor, SlurmAllocationExecutor from executorlib.standalone.serialize import cloudpickle_register @@ -131,7 +131,7 @@ def test_validate_max_workers(self): os.environ["SLURM_NTASKS"] = "6" os.environ["SLURM_CPUS_PER_TASK"] = "4" with self.assertRaises(ValueError): - LocalExecutor( + SlurmAllocationExecutor( max_workers=10, resource_dict={"cores": 10, "threads_per_core": 10}, block_allocation=True, From f3b861619c952c4a49c77349d5a811296d58387f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 1 Feb 2025 12:26:49 +0100 Subject: [PATCH 04/28] update benchmark --- tests/benchmark/llh.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/benchmark/llh.py b/tests/benchmark/llh.py index 3fae9ad2..3c6e6a17 100644 --- a/tests/benchmark/llh.py +++ b/tests/benchmark/llh.py @@ -42,39 +42,36 @@ def run_static(mean=0.1, sigma=1.1, runs=32): executor=ThreadPoolExecutor, mean=0.1, sigma=1.1, runs=32, max_workers=4 ) elif run_mode == "block_allocation": - from executorlib import Executor + from executorlib import LocalExecutor run_with_executor( - executor=Executor, + executor=LocalExecutor, mean=0.1, sigma=1.1, runs=32, max_cores=4, - backend="local", block_allocation=True, ) elif run_mode == "executorlib": - from executorlib import Executor + from executorlib import LocalExecutor run_with_executor( - executor=Executor, + executor=LocalExecutor, mean=0.1, sigma=1.1, runs=32, max_cores=4, - backend="local", block_allocation=False, ) elif run_mode == "flux": - from executorlib import Executor + from executorlib import FluxAllocationExecutor run_with_executor( - executor=Executor, + executor=FluxAllocationExecutor, mean=0.1, sigma=1.1, runs=32, max_cores=4, - backend="flux", block_allocation=True, ) elif run_mode == "mpi4py": From 0b38e5fd76bd9e0b06aacbba34982d5ea3f1904f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 1 Feb 2025 12:44:55 +0100 Subject: [PATCH 05/28] move interfaces to individual modules --- executorlib/__init__.py | 8 +- executorlib/interfaces.py | 1005 ---------------------------- executorlib/interfaces/__init__.py | 0 executorlib/interfaces/flux.py | 424 ++++++++++++ executorlib/interfaces/local.py | 196 ++++++ executorlib/interfaces/slurm.py | 408 +++++++++++ 6 files changed, 1033 insertions(+), 1008 deletions(-) delete mode 100644 executorlib/interfaces.py create mode 100644 executorlib/interfaces/__init__.py create mode 100644 executorlib/interfaces/flux.py create mode 100644 executorlib/interfaces/local.py create mode 100644 executorlib/interfaces/slurm.py diff --git a/executorlib/__init__.py b/executorlib/__init__.py index 3940d986..1cf1ea86 100644 --- a/executorlib/__init__.py +++ b/executorlib/__init__.py @@ -1,11 +1,13 @@ from executorlib._version import get_versions as _get_versions -from executorlib.interfaces import ( +from executorlib.interfaces.flux import ( FluxAllocationExecutor, FluxSubmissionExecutor, - LocalExecutor, +) +from executorlib.interfaces.local import LocalExecutor +from executorlib.interfaces.slurm import ( SlurmAllocationExecutor, SlurmSubmissionExecutor, ) __version__ = _get_versions()["version"] -__all__: list = [] +__all__: list = ["FluxAllocationExecutor", "FluxSubmissionExecutor", "LocalExecutor", "SlurmAllocationExecutor", "SlurmSubmissionExecutor"] diff --git a/executorlib/interfaces.py b/executorlib/interfaces.py deleted file mode 100644 index 0c03ac70..00000000 --- a/executorlib/interfaces.py +++ /dev/null @@ -1,1005 +0,0 @@ -from typing import Callable, Optional - -from executorlib._version import get_versions as _get_versions -from executorlib.interactive.executor import ( - ExecutorWithDependencies as _ExecutorWithDependencies, -) -from executorlib.interactive.executor import create_executor as _create_executor -from executorlib.standalone.inputcheck import ( - check_plot_dependency_graph as _check_plot_dependency_graph, -) -from executorlib.standalone.inputcheck import ( - check_pysqa_config_directory as _check_pysqa_config_directory, -) -from executorlib.standalone.inputcheck import ( - check_refresh_rate as _check_refresh_rate, -) - -__version__ = _get_versions()["version"] -__all__: list = [] - - -class LocalExecutor: - """ - The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or - preferable the flux framework for distributing python functions within a given resource allocation. In contrast to - the mpi4py.futures.MPIPoolExecutor the executorlib.Executor can be executed in a serial python process and does not - require the python script to be executed with MPI. It is even possible to execute the executorlib.Executor directly - in an interactive Jupyter notebook. - - Args: - max_workers (int): for backwards compatibility with the standard library, max_workers also defines the number of - cores which can be used in parallel - just like the max_cores parameter. Using max_cores is - recommended, as computers have a limited number of compute cores. - cache_directory (str, optional): The directory to store cache files. Defaults to "cache". - max_cores (int): defines the number cores which can be used in parallel - resource_dict (dict): A dictionary of resources required by the task. With the following keys: - - cores (int): number of MPI cores to be used for each function call - - threads_per_core (int): number of OpenMP threads to be used for each function call - - gpus_per_core (int): number of GPUs per worker - defaults to 0 - - cwd (str/None): current working directory where the parallel python task is executed - - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and - SLURM only) - default False - - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only) - hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the - context of an HPC cluster this essential to be able to communicate to an - Executor running on a different compute node within the same allocation. And - in principle any computer should be able to resolve that their own hostname - points to the same address as localhost. Still MacOS >= 12 seems to disable - this look up for security reasons. So on MacOS it is required to set this - option to true - block_allocation (boolean): To accelerate the submission of a series of python functions with the same resource - requirements, executorlib supports block allocation. In this case all resources have - to be defined on the executor, rather than during the submission of the individual - function. - init_function (None): optional function to preset arguments for functions which are submitted later - disable_dependencies (boolean): Disable resolving future objects during the submission. - refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. - plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For - debugging purposes and to get an overview of the specified dependencies. - plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. - - Examples: - ``` - >>> import numpy as np - >>> from executorlib.interfaces import LocalExecutor - >>> - >>> def calc(i, j, k): - >>> from mpi4py import MPI - >>> size = MPI.COMM_WORLD.Get_size() - >>> rank = MPI.COMM_WORLD.Get_rank() - >>> return np.array([i, j, k]), size, rank - >>> - >>> def init_k(): - >>> return {"k": 3} - >>> - >>> with LocalExecutor(cores=2, init_function=init_k) as p: - >>> fs = p.submit(calc, 2, j=4) - >>> print(fs.result()) - [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] - ``` - """ - - def __init__( - self, - max_workers: Optional[int] = None, - cache_directory: Optional[str] = None, - max_cores: Optional[int] = None, - resource_dict: Optional[dict] = None, - hostname_localhost: Optional[bool] = None, - block_allocation: bool = False, - init_function: Optional[Callable] = None, - disable_dependencies: bool = False, - refresh_rate: float = 0.01, - plot_dependency_graph: bool = False, - plot_dependency_graph_filename: Optional[str] = None, - ): - # Use __new__() instead of __init__(). This function is only implemented to enable auto-completion. - pass - - def __new__( - cls, - max_workers: Optional[int] = None, - cache_directory: Optional[str] = None, - max_cores: Optional[int] = None, - resource_dict: Optional[dict] = None, - hostname_localhost: Optional[bool] = None, - block_allocation: bool = False, - init_function: Optional[Callable] = None, - disable_dependencies: bool = False, - refresh_rate: float = 0.01, - plot_dependency_graph: bool = False, - plot_dependency_graph_filename: Optional[str] = None, - ): - """ - Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor, - executorlib.slurm.PySlurmExecutor or executorlib.flux.PyFluxExecutor depending on which backend is available. The - executorlib.flux.PyFluxExecutor is the preferred choice while the executorlib.mpi.PyMPIExecutor is primarily used - for development and testing. The executorlib.flux.PyFluxExecutor requires flux-core from the flux-framework to be - installed and in addition flux-sched to enable GPU scheduling. Finally, the executorlib.slurm.PySlurmExecutor - requires the SLURM workload manager to be installed on the system. - - Args: - max_workers (int): for backwards compatibility with the standard library, max_workers also defines the - number of cores which can be used in parallel - just like the max_cores parameter. Using - max_cores is recommended, as computers have a limited number of compute cores. - cache_directory (str, optional): The directory to store cache files. Defaults to "cache". - max_cores (int): defines the number cores which can be used in parallel - resource_dict (dict): A dictionary of resources required by the task. With the following keys: - - cores (int): number of MPI cores to be used for each function call - - threads_per_core (int): number of OpenMP threads to be used for each function call - - gpus_per_core (int): number of GPUs per worker - defaults to 0 - - cwd (str/None): current working directory where the parallel python task is executed - - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI - and SLURM only) - default False - - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM - only) - hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the - context of an HPC cluster this essential to be able to communicate to an - Executor running on a different compute node within the same allocation. And - in principle any computer should be able to resolve that their own hostname - points to the same address as localhost. Still MacOS >= 12 seems to disable - this look up for security reasons. So on MacOS it is required to set this - option to true - block_allocation (boolean): To accelerate the submission of a series of python functions with the same - resource requirements, executorlib supports block allocation. In this case all - resources have to be defined on the executor, rather than during the submission - of the individual function. - init_function (None): optional function to preset arguments for functions which are submitted later - disable_dependencies (boolean): Disable resolving future objects during the submission. - refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. - plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For - debugging purposes and to get an overview of the specified dependencies. - plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. - - """ - default_resource_dict: dict = { - "cores": 1, - "threads_per_core": 1, - "gpus_per_core": 0, - "cwd": None, - "openmpi_oversubscribe": False, - "slurm_cmd_args": [], - } - if resource_dict is None: - resource_dict = {} - resource_dict.update( - {k: v for k, v in default_resource_dict.items() if k not in resource_dict} - ) - if not disable_dependencies: - return _ExecutorWithDependencies( - max_workers=max_workers, - backend="local", - cache_directory=cache_directory, - max_cores=max_cores, - resource_dict=resource_dict, - flux_executor=None, - flux_executor_pmi_mode=None, - flux_executor_nesting=False, - flux_log_files=False, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, - refresh_rate=refresh_rate, - plot_dependency_graph=plot_dependency_graph, - plot_dependency_graph_filename=plot_dependency_graph_filename, - ) - else: - _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) - _check_refresh_rate(refresh_rate=refresh_rate) - return _create_executor( - max_workers=max_workers, - backend="local", - cache_directory=cache_directory, - max_cores=max_cores, - resource_dict=resource_dict, - flux_executor=None, - flux_executor_pmi_mode=None, - flux_executor_nesting=False, - flux_log_files=False, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, - ) - - -class SlurmSubmissionExecutor: - """ - The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or - preferable the flux framework for distributing python functions within a given resource allocation. In contrast to - the mpi4py.futures.MPIPoolExecutor the executorlib.Executor can be executed in a serial python process and does not - require the python script to be executed with MPI. It is even possible to execute the executorlib.Executor directly - in an interactive Jupyter notebook. - - Args: - max_workers (int): for backwards compatibility with the standard library, max_workers also defines the number of - cores which can be used in parallel - just like the max_cores parameter. Using max_cores is - recommended, as computers have a limited number of compute cores. - cache_directory (str, optional): The directory to store cache files. Defaults to "cache". - max_cores (int): defines the number cores which can be used in parallel - resource_dict (dict): A dictionary of resources required by the task. With the following keys: - - cores (int): number of MPI cores to be used for each function call - - threads_per_core (int): number of OpenMP threads to be used for each function call - - gpus_per_core (int): number of GPUs per worker - defaults to 0 - - cwd (str/None): current working directory where the parallel python task is executed - - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and - SLURM only) - default False - - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only) - pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend). - hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the - context of an HPC cluster this essential to be able to communicate to an - Executor running on a different compute node within the same allocation. And - in principle any computer should be able to resolve that their own hostname - points to the same address as localhost. Still MacOS >= 12 seems to disable - this look up for security reasons. So on MacOS it is required to set this - option to true - block_allocation (boolean): To accelerate the submission of a series of python functions with the same resource - requirements, executorlib supports block allocation. In this case all resources have - to be defined on the executor, rather than during the submission of the individual - function. - init_function (None): optional function to preset arguments for functions which are submitted later - disable_dependencies (boolean): Disable resolving future objects during the submission. - refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. - plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For - debugging purposes and to get an overview of the specified dependencies. - plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. - - Examples: - ``` - >>> import numpy as np - >>> from executorlib.interfaces import SlurmSubmissionExecutor - >>> - >>> def calc(i, j, k): - >>> from mpi4py import MPI - >>> size = MPI.COMM_WORLD.Get_size() - >>> rank = MPI.COMM_WORLD.Get_rank() - >>> return np.array([i, j, k]), size, rank - >>> - >>> def init_k(): - >>> return {"k": 3} - >>> - >>> with SlurmSubmissionExecutor(cores=2, init_function=init_k) as p: - >>> fs = p.submit(calc, 2, j=4) - >>> print(fs.result()) - [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] - ``` - """ - - def __init__( - self, - max_workers: Optional[int] = None, - cache_directory: Optional[str] = None, - max_cores: Optional[int] = None, - resource_dict: Optional[dict] = None, - pysqa_config_directory: Optional[str] = None, - hostname_localhost: Optional[bool] = None, - block_allocation: bool = False, - init_function: Optional[Callable] = None, - disable_dependencies: bool = False, - refresh_rate: float = 0.01, - plot_dependency_graph: bool = False, - plot_dependency_graph_filename: Optional[str] = None, - ): - # Use __new__() instead of __init__(). This function is only implemented to enable auto-completion. - pass - - def __new__( - cls, - max_workers: Optional[int] = None, - cache_directory: Optional[str] = None, - max_cores: Optional[int] = None, - resource_dict: Optional[dict] = None, - pysqa_config_directory: Optional[str] = None, - hostname_localhost: Optional[bool] = None, - block_allocation: bool = False, - init_function: Optional[Callable] = None, - disable_dependencies: bool = False, - refresh_rate: float = 0.01, - plot_dependency_graph: bool = False, - plot_dependency_graph_filename: Optional[str] = None, - ): - """ - Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor, - executorlib.slurm.PySlurmExecutor or executorlib.flux.PyFluxExecutor depending on which backend is available. The - executorlib.flux.PyFluxExecutor is the preferred choice while the executorlib.mpi.PyMPIExecutor is primarily used - for development and testing. The executorlib.flux.PyFluxExecutor requires flux-core from the flux-framework to be - installed and in addition flux-sched to enable GPU scheduling. Finally, the executorlib.slurm.PySlurmExecutor - requires the SLURM workload manager to be installed on the system. - - Args: - max_workers (int): for backwards compatibility with the standard library, max_workers also defines the - number of cores which can be used in parallel - just like the max_cores parameter. Using - max_cores is recommended, as computers have a limited number of compute cores. - cache_directory (str, optional): The directory to store cache files. Defaults to "cache". - max_cores (int): defines the number cores which can be used in parallel - resource_dict (dict): A dictionary of resources required by the task. With the following keys: - - cores (int): number of MPI cores to be used for each function call - - threads_per_core (int): number of OpenMP threads to be used for each function call - - gpus_per_core (int): number of GPUs per worker - defaults to 0 - - cwd (str/None): current working directory where the parallel python task is executed - - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI - and SLURM only) - default False - - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM - only) - pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend). - hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the - context of an HPC cluster this essential to be able to communicate to an - Executor running on a different compute node within the same allocation. And - in principle any computer should be able to resolve that their own hostname - points to the same address as localhost. Still MacOS >= 12 seems to disable - this look up for security reasons. So on MacOS it is required to set this - option to true - block_allocation (boolean): To accelerate the submission of a series of python functions with the same - resource requirements, executorlib supports block allocation. In this case all - resources have to be defined on the executor, rather than during the submission - of the individual function. - init_function (None): optional function to preset arguments for functions which are submitted later - disable_dependencies (boolean): Disable resolving future objects during the submission. - refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. - plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For - debugging purposes and to get an overview of the specified dependencies. - plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. - - """ - default_resource_dict: dict = { - "cores": 1, - "threads_per_core": 1, - "gpus_per_core": 0, - "cwd": None, - "openmpi_oversubscribe": False, - "slurm_cmd_args": [], - } - if resource_dict is None: - resource_dict = {} - resource_dict.update( - {k: v for k, v in default_resource_dict.items() if k not in resource_dict} - ) - if not plot_dependency_graph: - from executorlib.cache.executor import create_file_executor - - return create_file_executor( - max_workers=max_workers, - backend="slurm_submission", - max_cores=max_cores, - cache_directory=cache_directory, - resource_dict=resource_dict, - flux_executor=None, - flux_executor_pmi_mode=None, - flux_executor_nesting=False, - flux_log_files=False, - pysqa_config_directory=pysqa_config_directory, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, - disable_dependencies=disable_dependencies, - ) - elif not disable_dependencies: - _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) - return _ExecutorWithDependencies( - max_workers=max_workers, - backend="slurm_submission", - cache_directory=cache_directory, - max_cores=max_cores, - resource_dict=resource_dict, - flux_executor=None, - flux_executor_pmi_mode=None, - flux_executor_nesting=False, - flux_log_files=False, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, - refresh_rate=refresh_rate, - plot_dependency_graph=plot_dependency_graph, - plot_dependency_graph_filename=plot_dependency_graph_filename, - ) - else: - _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) - _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) - _check_refresh_rate(refresh_rate=refresh_rate) - return _create_executor( - max_workers=max_workers, - backend="slurm_submission", - cache_directory=cache_directory, - max_cores=max_cores, - resource_dict=resource_dict, - flux_executor=None, - flux_executor_pmi_mode=None, - flux_executor_nesting=False, - flux_log_files=False, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, - ) - - -class SlurmAllocationExecutor: - """ - The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or - preferable the flux framework for distributing python functions within a given resource allocation. In contrast to - the mpi4py.futures.MPIPoolExecutor the executorlib.Executor can be executed in a serial python process and does not - require the python script to be executed with MPI. It is even possible to execute the executorlib.Executor directly - in an interactive Jupyter notebook. - - Args: - max_workers (int): for backwards compatibility with the standard library, max_workers also defines the number of - cores which can be used in parallel - just like the max_cores parameter. Using max_cores is - recommended, as computers have a limited number of compute cores. - cache_directory (str, optional): The directory to store cache files. Defaults to "cache". - max_cores (int): defines the number cores which can be used in parallel - resource_dict (dict): A dictionary of resources required by the task. With the following keys: - - cores (int): number of MPI cores to be used for each function call - - threads_per_core (int): number of OpenMP threads to be used for each function call - - gpus_per_core (int): number of GPUs per worker - defaults to 0 - - cwd (str/None): current working directory where the parallel python task is executed - - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and - SLURM only) - default False - - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only) - hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the - context of an HPC cluster this essential to be able to communicate to an - Executor running on a different compute node within the same allocation. And - in principle any computer should be able to resolve that their own hostname - points to the same address as localhost. Still MacOS >= 12 seems to disable - this look up for security reasons. So on MacOS it is required to set this - option to true - block_allocation (boolean): To accelerate the submission of a series of python functions with the same resource - requirements, executorlib supports block allocation. In this case all resources have - to be defined on the executor, rather than during the submission of the individual - function. - init_function (None): optional function to preset arguments for functions which are submitted later - disable_dependencies (boolean): Disable resolving future objects during the submission. - refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. - plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For - debugging purposes and to get an overview of the specified dependencies. - plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. - - Examples: - ``` - >>> import numpy as np - >>> from executorlib import Executor - >>> - >>> def calc(i, j, k): - >>> from mpi4py import MPI - >>> size = MPI.COMM_WORLD.Get_size() - >>> rank = MPI.COMM_WORLD.Get_rank() - >>> return np.array([i, j, k]), size, rank - >>> - >>> def init_k(): - >>> return {"k": 3} - >>> - >>> with Executor(cores=2, init_function=init_k) as p: - >>> fs = p.submit(calc, 2, j=4) - >>> print(fs.result()) - [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] - ``` - """ - - def __init__( - self, - max_workers: Optional[int] = None, - cache_directory: Optional[str] = None, - max_cores: Optional[int] = None, - resource_dict: Optional[dict] = None, - hostname_localhost: Optional[bool] = None, - block_allocation: bool = False, - init_function: Optional[Callable] = None, - disable_dependencies: bool = False, - refresh_rate: float = 0.01, - plot_dependency_graph: bool = False, - plot_dependency_graph_filename: Optional[str] = None, - ): - # Use __new__() instead of __init__(). This function is only implemented to enable auto-completion. - pass - - def __new__( - cls, - max_workers: Optional[int] = None, - cache_directory: Optional[str] = None, - max_cores: Optional[int] = None, - resource_dict: Optional[dict] = None, - hostname_localhost: Optional[bool] = None, - block_allocation: bool = False, - init_function: Optional[Callable] = None, - disable_dependencies: bool = False, - refresh_rate: float = 0.01, - plot_dependency_graph: bool = False, - plot_dependency_graph_filename: Optional[str] = None, - ): - """ - Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor, - executorlib.slurm.PySlurmExecutor or executorlib.flux.PyFluxExecutor depending on which backend is available. The - executorlib.flux.PyFluxExecutor is the preferred choice while the executorlib.mpi.PyMPIExecutor is primarily used - for development and testing. The executorlib.flux.PyFluxExecutor requires flux-core from the flux-framework to be - installed and in addition flux-sched to enable GPU scheduling. Finally, the executorlib.slurm.PySlurmExecutor - requires the SLURM workload manager to be installed on the system. - - Args: - max_workers (int): for backwards compatibility with the standard library, max_workers also defines the - number of cores which can be used in parallel - just like the max_cores parameter. Using - max_cores is recommended, as computers have a limited number of compute cores. - cache_directory (str, optional): The directory to store cache files. Defaults to "cache". - max_cores (int): defines the number cores which can be used in parallel - resource_dict (dict): A dictionary of resources required by the task. With the following keys: - - cores (int): number of MPI cores to be used for each function call - - threads_per_core (int): number of OpenMP threads to be used for each function call - - gpus_per_core (int): number of GPUs per worker - defaults to 0 - - cwd (str/None): current working directory where the parallel python task is executed - - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI - and SLURM only) - default False - - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM - only) - hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the - context of an HPC cluster this essential to be able to communicate to an - Executor running on a different compute node within the same allocation. And - in principle any computer should be able to resolve that their own hostname - points to the same address as localhost. Still MacOS >= 12 seems to disable - this look up for security reasons. So on MacOS it is required to set this - option to true - block_allocation (boolean): To accelerate the submission of a series of python functions with the same - resource requirements, executorlib supports block allocation. In this case all - resources have to be defined on the executor, rather than during the submission - of the individual function. - init_function (None): optional function to preset arguments for functions which are submitted later - disable_dependencies (boolean): Disable resolving future objects during the submission. - refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. - plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For - debugging purposes and to get an overview of the specified dependencies. - plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. - - """ - default_resource_dict: dict = { - "cores": 1, - "threads_per_core": 1, - "gpus_per_core": 0, - "cwd": None, - "openmpi_oversubscribe": False, - "slurm_cmd_args": [], - } - if resource_dict is None: - resource_dict = {} - resource_dict.update( - {k: v for k, v in default_resource_dict.items() if k not in resource_dict} - ) - if not disable_dependencies: - return _ExecutorWithDependencies( - max_workers=max_workers, - backend="slurm_allocation", - cache_directory=cache_directory, - max_cores=max_cores, - resource_dict=resource_dict, - flux_executor=None, - flux_executor_pmi_mode=None, - flux_executor_nesting=False, - flux_log_files=False, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, - refresh_rate=refresh_rate, - plot_dependency_graph=plot_dependency_graph, - plot_dependency_graph_filename=plot_dependency_graph_filename, - ) - else: - _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) - _check_refresh_rate(refresh_rate=refresh_rate) - return _create_executor( - max_workers=max_workers, - backend="slurm_allocation", - cache_directory=cache_directory, - max_cores=max_cores, - resource_dict=resource_dict, - flux_executor=None, - flux_executor_pmi_mode=None, - flux_executor_nesting=False, - flux_log_files=False, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, - ) - - -class FluxAllocationExecutor: - """ - The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or - preferable the flux framework for distributing python functions within a given resource allocation. In contrast to - the mpi4py.futures.MPIPoolExecutor the executorlib.Executor can be executed in a serial python process and does not - require the python script to be executed with MPI. It is even possible to execute the executorlib.Executor directly - in an interactive Jupyter notebook. - - Args: - max_workers (int): for backwards compatibility with the standard library, max_workers also defines the number of - cores which can be used in parallel - just like the max_cores parameter. Using max_cores is - recommended, as computers have a limited number of compute cores. - cache_directory (str, optional): The directory to store cache files. Defaults to "cache". - max_cores (int): defines the number cores which can be used in parallel - resource_dict (dict): A dictionary of resources required by the task. With the following keys: - - cores (int): number of MPI cores to be used for each function call - - threads_per_core (int): number of OpenMP threads to be used for each function call - - gpus_per_core (int): number of GPUs per worker - defaults to 0 - - cwd (str/None): current working directory where the parallel python task is executed - - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and - SLURM only) - default False - - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only) - flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux - flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only) - flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function. - flux_log_files (bool, optional): Write flux stdout and stderr files. Defaults to False. - hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the - context of an HPC cluster this essential to be able to communicate to an - Executor running on a different compute node within the same allocation. And - in principle any computer should be able to resolve that their own hostname - points to the same address as localhost. Still MacOS >= 12 seems to disable - this look up for security reasons. So on MacOS it is required to set this - option to true - block_allocation (boolean): To accelerate the submission of a series of python functions with the same resource - requirements, executorlib supports block allocation. In this case all resources have - to be defined on the executor, rather than during the submission of the individual - function. - init_function (None): optional function to preset arguments for functions which are submitted later - disable_dependencies (boolean): Disable resolving future objects during the submission. - refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. - plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For - debugging purposes and to get an overview of the specified dependencies. - plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. - - Examples: - ``` - >>> import numpy as np - >>> from executorlib.interfaces import FluxAllocationExecutor - >>> - >>> def calc(i, j, k): - >>> from mpi4py import MPI - >>> size = MPI.COMM_WORLD.Get_size() - >>> rank = MPI.COMM_WORLD.Get_rank() - >>> return np.array([i, j, k]), size, rank - >>> - >>> def init_k(): - >>> return {"k": 3} - >>> - >>> with Executor(cores=2, init_function=init_k) as p: - >>> fs = p.submit(calc, 2, j=4) - >>> print(fs.result()) - [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] - ``` - """ - - def __init__( - self, - max_workers: Optional[int] = None, - cache_directory: Optional[str] = None, - max_cores: Optional[int] = None, - resource_dict: Optional[dict] = None, - flux_executor=None, - flux_executor_pmi_mode: Optional[str] = None, - flux_executor_nesting: bool = False, - flux_log_files: bool = False, - hostname_localhost: Optional[bool] = None, - block_allocation: bool = False, - init_function: Optional[Callable] = None, - disable_dependencies: bool = False, - refresh_rate: float = 0.01, - plot_dependency_graph: bool = False, - plot_dependency_graph_filename: Optional[str] = None, - ): - # Use __new__() instead of __init__(). This function is only implemented to enable auto-completion. - pass - - def __new__( - cls, - max_workers: Optional[int] = None, - cache_directory: Optional[str] = None, - max_cores: Optional[int] = None, - resource_dict: Optional[dict] = None, - flux_executor=None, - flux_executor_pmi_mode: Optional[str] = None, - flux_executor_nesting: bool = False, - flux_log_files: bool = False, - hostname_localhost: Optional[bool] = None, - block_allocation: bool = False, - init_function: Optional[Callable] = None, - disable_dependencies: bool = False, - refresh_rate: float = 0.01, - plot_dependency_graph: bool = False, - plot_dependency_graph_filename: Optional[str] = None, - ): - """ - Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor, - executorlib.slurm.PySlurmExecutor or executorlib.flux.PyFluxExecutor depending on which backend is available. The - executorlib.flux.PyFluxExecutor is the preferred choice while the executorlib.mpi.PyMPIExecutor is primarily used - for development and testing. The executorlib.flux.PyFluxExecutor requires flux-core from the flux-framework to be - installed and in addition flux-sched to enable GPU scheduling. Finally, the executorlib.slurm.PySlurmExecutor - requires the SLURM workload manager to be installed on the system. - - Args: - max_workers (int): for backwards compatibility with the standard library, max_workers also defines the - number of cores which can be used in parallel - just like the max_cores parameter. Using - max_cores is recommended, as computers have a limited number of compute cores. - cache_directory (str, optional): The directory to store cache files. Defaults to "cache". - max_cores (int): defines the number cores which can be used in parallel - resource_dict (dict): A dictionary of resources required by the task. With the following keys: - - cores (int): number of MPI cores to be used for each function call - - threads_per_core (int): number of OpenMP threads to be used for each function call - - gpus_per_core (int): number of GPUs per worker - defaults to 0 - - cwd (str/None): current working directory where the parallel python task is executed - - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI - and SLURM only) - default False - - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM - only) - flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux - flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only) - flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function. - flux_log_files (bool, optional): Write flux stdout and stderr files. Defaults to False. - hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the - context of an HPC cluster this essential to be able to communicate to an - Executor running on a different compute node within the same allocation. And - in principle any computer should be able to resolve that their own hostname - points to the same address as localhost. Still MacOS >= 12 seems to disable - this look up for security reasons. So on MacOS it is required to set this - option to true - block_allocation (boolean): To accelerate the submission of a series of python functions with the same - resource requirements, executorlib supports block allocation. In this case all - resources have to be defined on the executor, rather than during the submission - of the individual function. - init_function (None): optional function to preset arguments for functions which are submitted later - disable_dependencies (boolean): Disable resolving future objects during the submission. - refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. - plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For - debugging purposes and to get an overview of the specified dependencies. - plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. - - """ - default_resource_dict: dict = { - "cores": 1, - "threads_per_core": 1, - "gpus_per_core": 0, - "cwd": None, - "openmpi_oversubscribe": False, - "slurm_cmd_args": [], - } - if resource_dict is None: - resource_dict = {} - resource_dict.update( - {k: v for k, v in default_resource_dict.items() if k not in resource_dict} - ) - if not disable_dependencies: - return _ExecutorWithDependencies( - max_workers=max_workers, - backend="flux_allocation", - cache_directory=cache_directory, - max_cores=max_cores, - resource_dict=resource_dict, - flux_executor=flux_executor, - flux_executor_pmi_mode=flux_executor_pmi_mode, - flux_executor_nesting=flux_executor_nesting, - flux_log_files=flux_log_files, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, - refresh_rate=refresh_rate, - plot_dependency_graph=plot_dependency_graph, - plot_dependency_graph_filename=plot_dependency_graph_filename, - ) - else: - _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) - _check_refresh_rate(refresh_rate=refresh_rate) - return _create_executor( - max_workers=max_workers, - backend="flux_allocation", - cache_directory=cache_directory, - max_cores=max_cores, - resource_dict=resource_dict, - flux_executor=flux_executor, - flux_executor_pmi_mode=flux_executor_pmi_mode, - flux_executor_nesting=flux_executor_nesting, - flux_log_files=flux_log_files, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, - ) - - -class FluxSubmissionExecutor: - """ - The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or - preferable the flux framework for distributing python functions within a given resource allocation. In contrast to - the mpi4py.futures.MPIPoolExecutor the executorlib.Executor can be executed in a serial python process and does not - require the python script to be executed with MPI. It is even possible to execute the executorlib.Executor directly - in an interactive Jupyter notebook. - - Args: - max_workers (int): for backwards compatibility with the standard library, max_workers also defines the number of - cores which can be used in parallel - just like the max_cores parameter. Using max_cores is - recommended, as computers have a limited number of compute cores. - cache_directory (str, optional): The directory to store cache files. Defaults to "cache". - max_cores (int): defines the number cores which can be used in parallel - resource_dict (dict): A dictionary of resources required by the task. With the following keys: - - cores (int): number of MPI cores to be used for each function call - - threads_per_core (int): number of OpenMP threads to be used for each function call - - gpus_per_core (int): number of GPUs per worker - defaults to 0 - - cwd (str/None): current working directory where the parallel python task is executed - - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and - SLURM only) - default False - - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only) - pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend). - hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the - context of an HPC cluster this essential to be able to communicate to an - Executor running on a different compute node within the same allocation. And - in principle any computer should be able to resolve that their own hostname - points to the same address as localhost. Still MacOS >= 12 seems to disable - this look up for security reasons. So on MacOS it is required to set this - option to true - block_allocation (boolean): To accelerate the submission of a series of python functions with the same resource - requirements, executorlib supports block allocation. In this case all resources have - to be defined on the executor, rather than during the submission of the individual - function. - init_function (None): optional function to preset arguments for functions which are submitted later - disable_dependencies (boolean): Disable resolving future objects during the submission. - refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. - plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For - debugging purposes and to get an overview of the specified dependencies. - plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. - - Examples: - ``` - >>> import numpy as np - >>> from executorlib.interfaces import FluxSubmissionExecutor - >>> - >>> def calc(i, j, k): - >>> from mpi4py import MPI - >>> size = MPI.COMM_WORLD.Get_size() - >>> rank = MPI.COMM_WORLD.Get_rank() - >>> return np.array([i, j, k]), size, rank - >>> - >>> def init_k(): - >>> return {"k": 3} - >>> - >>> with Executor(cores=2, init_function=init_k) as p: - >>> fs = p.submit(calc, 2, j=4) - >>> print(fs.result()) - [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] - ``` - """ - - def __init__( - self, - max_workers: Optional[int] = None, - cache_directory: Optional[str] = None, - max_cores: Optional[int] = None, - resource_dict: Optional[dict] = None, - pysqa_config_directory: Optional[str] = None, - hostname_localhost: Optional[bool] = None, - block_allocation: bool = False, - init_function: Optional[Callable] = None, - disable_dependencies: bool = False, - refresh_rate: float = 0.01, - plot_dependency_graph: bool = False, - plot_dependency_graph_filename: Optional[str] = None, - ): - # Use __new__() instead of __init__(). This function is only implemented to enable auto-completion. - pass - - def __new__( - cls, - max_workers: Optional[int] = None, - cache_directory: Optional[str] = None, - max_cores: Optional[int] = None, - resource_dict: Optional[dict] = None, - pysqa_config_directory: Optional[str] = None, - hostname_localhost: Optional[bool] = None, - block_allocation: bool = False, - init_function: Optional[Callable] = None, - disable_dependencies: bool = False, - refresh_rate: float = 0.01, - plot_dependency_graph: bool = False, - plot_dependency_graph_filename: Optional[str] = None, - ): - """ - Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor, - executorlib.slurm.PySlurmExecutor or executorlib.flux.PyFluxExecutor depending on which backend is available. The - executorlib.flux.PyFluxExecutor is the preferred choice while the executorlib.mpi.PyMPIExecutor is primarily used - for development and testing. The executorlib.flux.PyFluxExecutor requires flux-core from the flux-framework to be - installed and in addition flux-sched to enable GPU scheduling. Finally, the executorlib.slurm.PySlurmExecutor - requires the SLURM workload manager to be installed on the system. - - Args: - max_workers (int): for backwards compatibility with the standard library, max_workers also defines the - number of cores which can be used in parallel - just like the max_cores parameter. Using - max_cores is recommended, as computers have a limited number of compute cores. - cache_directory (str, optional): The directory to store cache files. Defaults to "cache". - max_cores (int): defines the number cores which can be used in parallel - resource_dict (dict): A dictionary of resources required by the task. With the following keys: - - cores (int): number of MPI cores to be used for each function call - - threads_per_core (int): number of OpenMP threads to be used for each function call - - gpus_per_core (int): number of GPUs per worker - defaults to 0 - - cwd (str/None): current working directory where the parallel python task is executed - - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI - and SLURM only) - default False - - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM - only) - pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend). - hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the - context of an HPC cluster this essential to be able to communicate to an - Executor running on a different compute node within the same allocation. And - in principle any computer should be able to resolve that their own hostname - points to the same address as localhost. Still MacOS >= 12 seems to disable - this look up for security reasons. So on MacOS it is required to set this - option to true - block_allocation (boolean): To accelerate the submission of a series of python functions with the same - resource requirements, executorlib supports block allocation. In this case all - resources have to be defined on the executor, rather than during the submission - of the individual function. - init_function (None): optional function to preset arguments for functions which are submitted later - disable_dependencies (boolean): Disable resolving future objects during the submission. - refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. - plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For - debugging purposes and to get an overview of the specified dependencies. - plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. - - """ - default_resource_dict: dict = { - "cores": 1, - "threads_per_core": 1, - "gpus_per_core": 0, - "cwd": None, - "openmpi_oversubscribe": False, - "slurm_cmd_args": [], - } - if resource_dict is None: - resource_dict = {} - resource_dict.update( - {k: v for k, v in default_resource_dict.items() if k not in resource_dict} - ) - if not plot_dependency_graph: - from executorlib.cache.executor import create_file_executor - - return create_file_executor( - max_workers=max_workers, - backend="flux_submission", - max_cores=max_cores, - cache_directory=cache_directory, - resource_dict=resource_dict, - flux_executor=None, - flux_executor_pmi_mode=None, - flux_executor_nesting=False, - flux_log_files=False, - pysqa_config_directory=pysqa_config_directory, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, - disable_dependencies=disable_dependencies, - ) - elif not disable_dependencies: - _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) - return _ExecutorWithDependencies( - max_workers=max_workers, - backend="flux_submission", - cache_directory=cache_directory, - max_cores=max_cores, - resource_dict=resource_dict, - flux_executor=None, - flux_executor_pmi_mode=None, - flux_executor_nesting=False, - flux_log_files=False, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, - refresh_rate=refresh_rate, - plot_dependency_graph=plot_dependency_graph, - plot_dependency_graph_filename=plot_dependency_graph_filename, - ) - else: - _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) - _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) - _check_refresh_rate(refresh_rate=refresh_rate) - return _create_executor( - max_workers=max_workers, - backend="flux_submission", - cache_directory=cache_directory, - max_cores=max_cores, - resource_dict=resource_dict, - flux_executor=None, - flux_executor_pmi_mode=None, - flux_executor_nesting=False, - flux_log_files=False, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, - ) diff --git a/executorlib/interfaces/__init__.py b/executorlib/interfaces/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/executorlib/interfaces/flux.py b/executorlib/interfaces/flux.py new file mode 100644 index 00000000..c24db488 --- /dev/null +++ b/executorlib/interfaces/flux.py @@ -0,0 +1,424 @@ +from typing import Callable, Optional + +from executorlib.interactive.executor import ( + ExecutorWithDependencies as _ExecutorWithDependencies, +) +from executorlib.interactive.executor import create_executor as _create_executor +from executorlib.standalone.inputcheck import ( + check_plot_dependency_graph as _check_plot_dependency_graph, +) +from executorlib.standalone.inputcheck import ( + check_pysqa_config_directory as _check_pysqa_config_directory, +) +from executorlib.standalone.inputcheck import ( + check_refresh_rate as _check_refresh_rate, +) + + +class FluxAllocationExecutor: + """ + The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or + preferable the flux framework for distributing python functions within a given resource allocation. In contrast to + the mpi4py.futures.MPIPoolExecutor the executorlib.Executor can be executed in a serial python process and does not + require the python script to be executed with MPI. It is even possible to execute the executorlib.Executor directly + in an interactive Jupyter notebook. + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the number of + cores which can be used in parallel - just like the max_cores parameter. Using max_cores is + recommended, as computers have a limited number of compute cores. + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + max_cores (int): defines the number cores which can be used in parallel + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and + SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only) + flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux + flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only) + flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function. + flux_log_files (bool, optional): Write flux stdout and stderr files. Defaults to False. + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same resource + requirements, executorlib supports block allocation. In this case all resources have + to be defined on the executor, rather than during the submission of the individual + function. + init_function (None): optional function to preset arguments for functions which are submitted later + disable_dependencies (boolean): Disable resolving future objects during the submission. + refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. + plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For + debugging purposes and to get an overview of the specified dependencies. + plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + + Examples: + ``` + >>> import numpy as np + >>> from executorlib.interfaces.flux import FluxAllocationExecutor + >>> + >>> def calc(i, j, k): + >>> from mpi4py import MPI + >>> size = MPI.COMM_WORLD.Get_size() + >>> rank = MPI.COMM_WORLD.Get_rank() + >>> return np.array([i, j, k]), size, rank + >>> + >>> def init_k(): + >>> return {"k": 3} + >>> + >>> with Executor(cores=2, init_function=init_k) as p: + >>> fs = p.submit(calc, 2, j=4) + >>> print(fs.result()) + [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] + ``` + """ + + def __init__( + self, + max_workers: Optional[int] = None, + cache_directory: Optional[str] = None, + max_cores: Optional[int] = None, + resource_dict: Optional[dict] = None, + flux_executor=None, + flux_executor_pmi_mode: Optional[str] = None, + flux_executor_nesting: bool = False, + flux_log_files: bool = False, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, + disable_dependencies: bool = False, + refresh_rate: float = 0.01, + plot_dependency_graph: bool = False, + plot_dependency_graph_filename: Optional[str] = None, + ): + # Use __new__() instead of __init__(). This function is only implemented to enable auto-completion. + pass + + def __new__( + cls, + max_workers: Optional[int] = None, + cache_directory: Optional[str] = None, + max_cores: Optional[int] = None, + resource_dict: Optional[dict] = None, + flux_executor=None, + flux_executor_pmi_mode: Optional[str] = None, + flux_executor_nesting: bool = False, + flux_log_files: bool = False, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, + disable_dependencies: bool = False, + refresh_rate: float = 0.01, + plot_dependency_graph: bool = False, + plot_dependency_graph_filename: Optional[str] = None, + ): + """ + Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor, + executorlib.slurm.PySlurmExecutor or executorlib.flux.PyFluxExecutor depending on which backend is available. The + executorlib.flux.PyFluxExecutor is the preferred choice while the executorlib.mpi.PyMPIExecutor is primarily used + for development and testing. The executorlib.flux.PyFluxExecutor requires flux-core from the flux-framework to be + installed and in addition flux-sched to enable GPU scheduling. Finally, the executorlib.slurm.PySlurmExecutor + requires the SLURM workload manager to be installed on the system. + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the + number of cores which can be used in parallel - just like the max_cores parameter. Using + max_cores is recommended, as computers have a limited number of compute cores. + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + max_cores (int): defines the number cores which can be used in parallel + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI + and SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM + only) + flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux + flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only) + flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function. + flux_log_files (bool, optional): Write flux stdout and stderr files. Defaults to False. + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same + resource requirements, executorlib supports block allocation. In this case all + resources have to be defined on the executor, rather than during the submission + of the individual function. + init_function (None): optional function to preset arguments for functions which are submitted later + disable_dependencies (boolean): Disable resolving future objects during the submission. + refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. + plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For + debugging purposes and to get an overview of the specified dependencies. + plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + + """ + default_resource_dict: dict = { + "cores": 1, + "threads_per_core": 1, + "gpus_per_core": 0, + "cwd": None, + "openmpi_oversubscribe": False, + "slurm_cmd_args": [], + } + if resource_dict is None: + resource_dict = {} + resource_dict.update( + {k: v for k, v in default_resource_dict.items() if k not in resource_dict} + ) + if not disable_dependencies: + return _ExecutorWithDependencies( + max_workers=max_workers, + backend="flux_allocation", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=flux_executor, + flux_executor_pmi_mode=flux_executor_pmi_mode, + flux_executor_nesting=flux_executor_nesting, + flux_log_files=flux_log_files, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + refresh_rate=refresh_rate, + plot_dependency_graph=plot_dependency_graph, + plot_dependency_graph_filename=plot_dependency_graph_filename, + ) + else: + _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) + _check_refresh_rate(refresh_rate=refresh_rate) + return _create_executor( + max_workers=max_workers, + backend="flux_allocation", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=flux_executor, + flux_executor_pmi_mode=flux_executor_pmi_mode, + flux_executor_nesting=flux_executor_nesting, + flux_log_files=flux_log_files, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + ) + + +class FluxSubmissionExecutor: + """ + The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or + preferable the flux framework for distributing python functions within a given resource allocation. In contrast to + the mpi4py.futures.MPIPoolExecutor the executorlib.Executor can be executed in a serial python process and does not + require the python script to be executed with MPI. It is even possible to execute the executorlib.Executor directly + in an interactive Jupyter notebook. + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the number of + cores which can be used in parallel - just like the max_cores parameter. Using max_cores is + recommended, as computers have a limited number of compute cores. + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + max_cores (int): defines the number cores which can be used in parallel + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and + SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only) + pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend). + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same resource + requirements, executorlib supports block allocation. In this case all resources have + to be defined on the executor, rather than during the submission of the individual + function. + init_function (None): optional function to preset arguments for functions which are submitted later + disable_dependencies (boolean): Disable resolving future objects during the submission. + refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. + plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For + debugging purposes and to get an overview of the specified dependencies. + plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + + Examples: + ``` + >>> import numpy as np + >>> from executorlib.interfaces.flux import FluxSubmissionExecutor + >>> + >>> def calc(i, j, k): + >>> from mpi4py import MPI + >>> size = MPI.COMM_WORLD.Get_size() + >>> rank = MPI.COMM_WORLD.Get_rank() + >>> return np.array([i, j, k]), size, rank + >>> + >>> def init_k(): + >>> return {"k": 3} + >>> + >>> with Executor(cores=2, init_function=init_k) as p: + >>> fs = p.submit(calc, 2, j=4) + >>> print(fs.result()) + [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] + ``` + """ + + def __init__( + self, + max_workers: Optional[int] = None, + cache_directory: Optional[str] = None, + max_cores: Optional[int] = None, + resource_dict: Optional[dict] = None, + pysqa_config_directory: Optional[str] = None, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, + disable_dependencies: bool = False, + refresh_rate: float = 0.01, + plot_dependency_graph: bool = False, + plot_dependency_graph_filename: Optional[str] = None, + ): + # Use __new__() instead of __init__(). This function is only implemented to enable auto-completion. + pass + + def __new__( + cls, + max_workers: Optional[int] = None, + cache_directory: Optional[str] = None, + max_cores: Optional[int] = None, + resource_dict: Optional[dict] = None, + pysqa_config_directory: Optional[str] = None, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, + disable_dependencies: bool = False, + refresh_rate: float = 0.01, + plot_dependency_graph: bool = False, + plot_dependency_graph_filename: Optional[str] = None, + ): + """ + Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor, + executorlib.slurm.PySlurmExecutor or executorlib.flux.PyFluxExecutor depending on which backend is available. The + executorlib.flux.PyFluxExecutor is the preferred choice while the executorlib.mpi.PyMPIExecutor is primarily used + for development and testing. The executorlib.flux.PyFluxExecutor requires flux-core from the flux-framework to be + installed and in addition flux-sched to enable GPU scheduling. Finally, the executorlib.slurm.PySlurmExecutor + requires the SLURM workload manager to be installed on the system. + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the + number of cores which can be used in parallel - just like the max_cores parameter. Using + max_cores is recommended, as computers have a limited number of compute cores. + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + max_cores (int): defines the number cores which can be used in parallel + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI + and SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM + only) + pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend). + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same + resource requirements, executorlib supports block allocation. In this case all + resources have to be defined on the executor, rather than during the submission + of the individual function. + init_function (None): optional function to preset arguments for functions which are submitted later + disable_dependencies (boolean): Disable resolving future objects during the submission. + refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. + plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For + debugging purposes and to get an overview of the specified dependencies. + plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + + """ + default_resource_dict: dict = { + "cores": 1, + "threads_per_core": 1, + "gpus_per_core": 0, + "cwd": None, + "openmpi_oversubscribe": False, + "slurm_cmd_args": [], + } + if resource_dict is None: + resource_dict = {} + resource_dict.update( + {k: v for k, v in default_resource_dict.items() if k not in resource_dict} + ) + if not plot_dependency_graph: + from executorlib.cache.executor import create_file_executor + + return create_file_executor( + max_workers=max_workers, + backend="flux_submission", + max_cores=max_cores, + cache_directory=cache_directory, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + pysqa_config_directory=pysqa_config_directory, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + disable_dependencies=disable_dependencies, + ) + elif not disable_dependencies: + _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) + return _ExecutorWithDependencies( + max_workers=max_workers, + backend="flux_submission", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + refresh_rate=refresh_rate, + plot_dependency_graph=plot_dependency_graph, + plot_dependency_graph_filename=plot_dependency_graph_filename, + ) + else: + _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) + _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) + _check_refresh_rate(refresh_rate=refresh_rate) + return _create_executor( + max_workers=max_workers, + backend="flux_submission", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + ) diff --git a/executorlib/interfaces/local.py b/executorlib/interfaces/local.py new file mode 100644 index 00000000..237d6b96 --- /dev/null +++ b/executorlib/interfaces/local.py @@ -0,0 +1,196 @@ +from typing import Callable, Optional + +from executorlib.interactive.executor import ( + ExecutorWithDependencies as _ExecutorWithDependencies, +) +from executorlib.interactive.executor import create_executor as _create_executor +from executorlib.standalone.inputcheck import ( + check_plot_dependency_graph as _check_plot_dependency_graph, +) +from executorlib.standalone.inputcheck import ( + check_refresh_rate as _check_refresh_rate, +) + + +class LocalExecutor: + """ + The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or + preferable the flux framework for distributing python functions within a given resource allocation. In contrast to + the mpi4py.futures.MPIPoolExecutor the executorlib.Executor can be executed in a serial python process and does not + require the python script to be executed with MPI. It is even possible to execute the executorlib.Executor directly + in an interactive Jupyter notebook. + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the number of + cores which can be used in parallel - just like the max_cores parameter. Using max_cores is + recommended, as computers have a limited number of compute cores. + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + max_cores (int): defines the number cores which can be used in parallel + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and + SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only) + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same resource + requirements, executorlib supports block allocation. In this case all resources have + to be defined on the executor, rather than during the submission of the individual + function. + init_function (None): optional function to preset arguments for functions which are submitted later + disable_dependencies (boolean): Disable resolving future objects during the submission. + refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. + plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For + debugging purposes and to get an overview of the specified dependencies. + plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + + Examples: + ``` + >>> import numpy as np + >>> from executorlib.interfaces.local import LocalExecutor + >>> + >>> def calc(i, j, k): + >>> from mpi4py import MPI + >>> size = MPI.COMM_WORLD.Get_size() + >>> rank = MPI.COMM_WORLD.Get_rank() + >>> return np.array([i, j, k]), size, rank + >>> + >>> def init_k(): + >>> return {"k": 3} + >>> + >>> with LocalExecutor(cores=2, init_function=init_k) as p: + >>> fs = p.submit(calc, 2, j=4) + >>> print(fs.result()) + [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] + ``` + """ + + def __init__( + self, + max_workers: Optional[int] = None, + cache_directory: Optional[str] = None, + max_cores: Optional[int] = None, + resource_dict: Optional[dict] = None, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, + disable_dependencies: bool = False, + refresh_rate: float = 0.01, + plot_dependency_graph: bool = False, + plot_dependency_graph_filename: Optional[str] = None, + ): + # Use __new__() instead of __init__(). This function is only implemented to enable auto-completion. + pass + + def __new__( + cls, + max_workers: Optional[int] = None, + cache_directory: Optional[str] = None, + max_cores: Optional[int] = None, + resource_dict: Optional[dict] = None, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, + disable_dependencies: bool = False, + refresh_rate: float = 0.01, + plot_dependency_graph: bool = False, + plot_dependency_graph_filename: Optional[str] = None, + ): + """ + Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor, + executorlib.slurm.PySlurmExecutor or executorlib.flux.PyFluxExecutor depending on which backend is available. The + executorlib.flux.PyFluxExecutor is the preferred choice while the executorlib.mpi.PyMPIExecutor is primarily used + for development and testing. The executorlib.flux.PyFluxExecutor requires flux-core from the flux-framework to be + installed and in addition flux-sched to enable GPU scheduling. Finally, the executorlib.slurm.PySlurmExecutor + requires the SLURM workload manager to be installed on the system. + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the + number of cores which can be used in parallel - just like the max_cores parameter. Using + max_cores is recommended, as computers have a limited number of compute cores. + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + max_cores (int): defines the number cores which can be used in parallel + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI + and SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM + only) + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same + resource requirements, executorlib supports block allocation. In this case all + resources have to be defined on the executor, rather than during the submission + of the individual function. + init_function (None): optional function to preset arguments for functions which are submitted later + disable_dependencies (boolean): Disable resolving future objects during the submission. + refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. + plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For + debugging purposes and to get an overview of the specified dependencies. + plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + + """ + default_resource_dict: dict = { + "cores": 1, + "threads_per_core": 1, + "gpus_per_core": 0, + "cwd": None, + "openmpi_oversubscribe": False, + "slurm_cmd_args": [], + } + if resource_dict is None: + resource_dict = {} + resource_dict.update( + {k: v for k, v in default_resource_dict.items() if k not in resource_dict} + ) + if not disable_dependencies: + return _ExecutorWithDependencies( + max_workers=max_workers, + backend="local", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + refresh_rate=refresh_rate, + plot_dependency_graph=plot_dependency_graph, + plot_dependency_graph_filename=plot_dependency_graph_filename, + ) + else: + _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) + _check_refresh_rate(refresh_rate=refresh_rate) + return _create_executor( + max_workers=max_workers, + backend="local", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + ) diff --git a/executorlib/interfaces/slurm.py b/executorlib/interfaces/slurm.py new file mode 100644 index 00000000..23d516b3 --- /dev/null +++ b/executorlib/interfaces/slurm.py @@ -0,0 +1,408 @@ +from typing import Callable, Optional + +from executorlib.interactive.executor import ( + ExecutorWithDependencies as _ExecutorWithDependencies, +) +from executorlib.interactive.executor import create_executor as _create_executor +from executorlib.standalone.inputcheck import ( + check_plot_dependency_graph as _check_plot_dependency_graph, +) +from executorlib.standalone.inputcheck import ( + check_pysqa_config_directory as _check_pysqa_config_directory, +) +from executorlib.standalone.inputcheck import ( + check_refresh_rate as _check_refresh_rate, +) + + +class SlurmSubmissionExecutor: + """ + The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or + preferable the flux framework for distributing python functions within a given resource allocation. In contrast to + the mpi4py.futures.MPIPoolExecutor the executorlib.Executor can be executed in a serial python process and does not + require the python script to be executed with MPI. It is even possible to execute the executorlib.Executor directly + in an interactive Jupyter notebook. + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the number of + cores which can be used in parallel - just like the max_cores parameter. Using max_cores is + recommended, as computers have a limited number of compute cores. + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + max_cores (int): defines the number cores which can be used in parallel + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and + SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only) + pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend). + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same resource + requirements, executorlib supports block allocation. In this case all resources have + to be defined on the executor, rather than during the submission of the individual + function. + init_function (None): optional function to preset arguments for functions which are submitted later + disable_dependencies (boolean): Disable resolving future objects during the submission. + refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. + plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For + debugging purposes and to get an overview of the specified dependencies. + plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + + Examples: + ``` + >>> import numpy as np + >>> from executorlib.interfaces.slurm import SlurmSubmissionExecutor + >>> + >>> def calc(i, j, k): + >>> from mpi4py import MPI + >>> size = MPI.COMM_WORLD.Get_size() + >>> rank = MPI.COMM_WORLD.Get_rank() + >>> return np.array([i, j, k]), size, rank + >>> + >>> def init_k(): + >>> return {"k": 3} + >>> + >>> with SlurmSubmissionExecutor(cores=2, init_function=init_k) as p: + >>> fs = p.submit(calc, 2, j=4) + >>> print(fs.result()) + [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] + ``` + """ + + def __init__( + self, + max_workers: Optional[int] = None, + cache_directory: Optional[str] = None, + max_cores: Optional[int] = None, + resource_dict: Optional[dict] = None, + pysqa_config_directory: Optional[str] = None, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, + disable_dependencies: bool = False, + refresh_rate: float = 0.01, + plot_dependency_graph: bool = False, + plot_dependency_graph_filename: Optional[str] = None, + ): + # Use __new__() instead of __init__(). This function is only implemented to enable auto-completion. + pass + + def __new__( + cls, + max_workers: Optional[int] = None, + cache_directory: Optional[str] = None, + max_cores: Optional[int] = None, + resource_dict: Optional[dict] = None, + pysqa_config_directory: Optional[str] = None, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, + disable_dependencies: bool = False, + refresh_rate: float = 0.01, + plot_dependency_graph: bool = False, + plot_dependency_graph_filename: Optional[str] = None, + ): + """ + Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor, + executorlib.slurm.PySlurmExecutor or executorlib.flux.PyFluxExecutor depending on which backend is available. The + executorlib.flux.PyFluxExecutor is the preferred choice while the executorlib.mpi.PyMPIExecutor is primarily used + for development and testing. The executorlib.flux.PyFluxExecutor requires flux-core from the flux-framework to be + installed and in addition flux-sched to enable GPU scheduling. Finally, the executorlib.slurm.PySlurmExecutor + requires the SLURM workload manager to be installed on the system. + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the + number of cores which can be used in parallel - just like the max_cores parameter. Using + max_cores is recommended, as computers have a limited number of compute cores. + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + max_cores (int): defines the number cores which can be used in parallel + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI + and SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM + only) + pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend). + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same + resource requirements, executorlib supports block allocation. In this case all + resources have to be defined on the executor, rather than during the submission + of the individual function. + init_function (None): optional function to preset arguments for functions which are submitted later + disable_dependencies (boolean): Disable resolving future objects during the submission. + refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. + plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For + debugging purposes and to get an overview of the specified dependencies. + plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + + """ + default_resource_dict: dict = { + "cores": 1, + "threads_per_core": 1, + "gpus_per_core": 0, + "cwd": None, + "openmpi_oversubscribe": False, + "slurm_cmd_args": [], + } + if resource_dict is None: + resource_dict = {} + resource_dict.update( + {k: v for k, v in default_resource_dict.items() if k not in resource_dict} + ) + if not plot_dependency_graph: + from executorlib.cache.executor import create_file_executor + + return create_file_executor( + max_workers=max_workers, + backend="slurm_submission", + max_cores=max_cores, + cache_directory=cache_directory, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + pysqa_config_directory=pysqa_config_directory, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + disable_dependencies=disable_dependencies, + ) + elif not disable_dependencies: + _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) + return _ExecutorWithDependencies( + max_workers=max_workers, + backend="slurm_submission", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + refresh_rate=refresh_rate, + plot_dependency_graph=plot_dependency_graph, + plot_dependency_graph_filename=plot_dependency_graph_filename, + ) + else: + _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) + _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) + _check_refresh_rate(refresh_rate=refresh_rate) + return _create_executor( + max_workers=max_workers, + backend="slurm_submission", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + ) + + +class SlurmAllocationExecutor: + """ + The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or + preferable the flux framework for distributing python functions within a given resource allocation. In contrast to + the mpi4py.futures.MPIPoolExecutor the executorlib.Executor can be executed in a serial python process and does not + require the python script to be executed with MPI. It is even possible to execute the executorlib.Executor directly + in an interactive Jupyter notebook. + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the number of + cores which can be used in parallel - just like the max_cores parameter. Using max_cores is + recommended, as computers have a limited number of compute cores. + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + max_cores (int): defines the number cores which can be used in parallel + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and + SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only) + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same resource + requirements, executorlib supports block allocation. In this case all resources have + to be defined on the executor, rather than during the submission of the individual + function. + init_function (None): optional function to preset arguments for functions which are submitted later + disable_dependencies (boolean): Disable resolving future objects during the submission. + refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. + plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For + debugging purposes and to get an overview of the specified dependencies. + plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + + Examples: + ``` + >>> import numpy as np + >>> from executorlib.interfaces.slurm import SlurmAllocationExecutor + >>> + >>> def calc(i, j, k): + >>> from mpi4py import MPI + >>> size = MPI.COMM_WORLD.Get_size() + >>> rank = MPI.COMM_WORLD.Get_rank() + >>> return np.array([i, j, k]), size, rank + >>> + >>> def init_k(): + >>> return {"k": 3} + >>> + >>> with SlurmAllocationExecutor(cores=2, init_function=init_k) as p: + >>> fs = p.submit(calc, 2, j=4) + >>> print(fs.result()) + [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] + ``` + """ + + def __init__( + self, + max_workers: Optional[int] = None, + cache_directory: Optional[str] = None, + max_cores: Optional[int] = None, + resource_dict: Optional[dict] = None, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, + disable_dependencies: bool = False, + refresh_rate: float = 0.01, + plot_dependency_graph: bool = False, + plot_dependency_graph_filename: Optional[str] = None, + ): + # Use __new__() instead of __init__(). This function is only implemented to enable auto-completion. + pass + + def __new__( + cls, + max_workers: Optional[int] = None, + cache_directory: Optional[str] = None, + max_cores: Optional[int] = None, + resource_dict: Optional[dict] = None, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, + disable_dependencies: bool = False, + refresh_rate: float = 0.01, + plot_dependency_graph: bool = False, + plot_dependency_graph_filename: Optional[str] = None, + ): + """ + Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor, + executorlib.slurm.PySlurmExecutor or executorlib.flux.PyFluxExecutor depending on which backend is available. The + executorlib.flux.PyFluxExecutor is the preferred choice while the executorlib.mpi.PyMPIExecutor is primarily used + for development and testing. The executorlib.flux.PyFluxExecutor requires flux-core from the flux-framework to be + installed and in addition flux-sched to enable GPU scheduling. Finally, the executorlib.slurm.PySlurmExecutor + requires the SLURM workload manager to be installed on the system. + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the + number of cores which can be used in parallel - just like the max_cores parameter. Using + max_cores is recommended, as computers have a limited number of compute cores. + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + max_cores (int): defines the number cores which can be used in parallel + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI + and SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM + only) + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same + resource requirements, executorlib supports block allocation. In this case all + resources have to be defined on the executor, rather than during the submission + of the individual function. + init_function (None): optional function to preset arguments for functions which are submitted later + disable_dependencies (boolean): Disable resolving future objects during the submission. + refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked. + plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For + debugging purposes and to get an overview of the specified dependencies. + plot_dependency_graph_filename (str): Name of the file to store the plotted graph in. + + """ + default_resource_dict: dict = { + "cores": 1, + "threads_per_core": 1, + "gpus_per_core": 0, + "cwd": None, + "openmpi_oversubscribe": False, + "slurm_cmd_args": [], + } + if resource_dict is None: + resource_dict = {} + resource_dict.update( + {k: v for k, v in default_resource_dict.items() if k not in resource_dict} + ) + if not disable_dependencies: + return _ExecutorWithDependencies( + max_workers=max_workers, + backend="slurm_allocation", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + refresh_rate=refresh_rate, + plot_dependency_graph=plot_dependency_graph, + plot_dependency_graph_filename=plot_dependency_graph_filename, + ) + else: + _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) + _check_refresh_rate(refresh_rate=refresh_rate) + return _create_executor( + max_workers=max_workers, + backend="slurm_allocation", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + ) From 41ca2ad5df09451a9456dc94951d58dd18fdedb1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 1 Feb 2025 11:45:05 +0000 Subject: [PATCH 06/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- executorlib/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/executorlib/__init__.py b/executorlib/__init__.py index 1cf1ea86..502b7958 100644 --- a/executorlib/__init__.py +++ b/executorlib/__init__.py @@ -10,4 +10,10 @@ ) __version__ = _get_versions()["version"] -__all__: list = ["FluxAllocationExecutor", "FluxSubmissionExecutor", "LocalExecutor", "SlurmAllocationExecutor", "SlurmSubmissionExecutor"] +__all__: list = [ + "FluxAllocationExecutor", + "FluxSubmissionExecutor", + "LocalExecutor", + "SlurmAllocationExecutor", + "SlurmSubmissionExecutor", +] From 4e903bc48857d65b882a29f933f9c2cc375d4b94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 1 Feb 2025 14:04:51 +0100 Subject: [PATCH 07/28] fixes --- executorlib/interfaces/flux.py | 52 ++++++++++++++++++--------------- executorlib/interfaces/local.py | 27 +++++++++-------- executorlib/interfaces/slurm.py | 52 ++++++++++++++++++--------------- 3 files changed, 73 insertions(+), 58 deletions(-) diff --git a/executorlib/interfaces/flux.py b/executorlib/interfaces/flux.py index c24db488..53fa8eb5 100644 --- a/executorlib/interfaces/flux.py +++ b/executorlib/interfaces/flux.py @@ -3,7 +3,7 @@ from executorlib.interactive.executor import ( ExecutorWithDependencies as _ExecutorWithDependencies, ) -from executorlib.interactive.executor import create_executor as _create_executor +from executorlib.interactive.create import create_executor as _create_executor from executorlib.standalone.inputcheck import ( check_plot_dependency_graph as _check_plot_dependency_graph, ) @@ -180,18 +180,21 @@ def __new__( ) if not disable_dependencies: return _ExecutorWithDependencies( - max_workers=max_workers, - backend="flux_allocation", - cache_directory=cache_directory, + executor=_create_executor( + max_workers=max_workers, + backend="flux_allocation", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=flux_executor, + flux_executor_pmi_mode=flux_executor_pmi_mode, + flux_executor_nesting=flux_executor_nesting, + flux_log_files=flux_log_files, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + ), max_cores=max_cores, - resource_dict=resource_dict, - flux_executor=flux_executor, - flux_executor_pmi_mode=flux_executor_pmi_mode, - flux_executor_nesting=flux_executor_nesting, - flux_log_files=flux_log_files, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, refresh_rate=refresh_rate, plot_dependency_graph=plot_dependency_graph, plot_dependency_graph_filename=plot_dependency_graph_filename, @@ -388,18 +391,21 @@ def __new__( elif not disable_dependencies: _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) return _ExecutorWithDependencies( - max_workers=max_workers, - backend="flux_submission", - cache_directory=cache_directory, + executor=_create_executor( + max_workers=max_workers, + backend="flux_submission", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + ), max_cores=max_cores, - resource_dict=resource_dict, - flux_executor=None, - flux_executor_pmi_mode=None, - flux_executor_nesting=False, - flux_log_files=False, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, refresh_rate=refresh_rate, plot_dependency_graph=plot_dependency_graph, plot_dependency_graph_filename=plot_dependency_graph_filename, diff --git a/executorlib/interfaces/local.py b/executorlib/interfaces/local.py index 237d6b96..fa5509e2 100644 --- a/executorlib/interfaces/local.py +++ b/executorlib/interfaces/local.py @@ -3,7 +3,7 @@ from executorlib.interactive.executor import ( ExecutorWithDependencies as _ExecutorWithDependencies, ) -from executorlib.interactive.executor import create_executor as _create_executor +from executorlib.interactive.create import create_executor as _create_executor from executorlib.standalone.inputcheck import ( check_plot_dependency_graph as _check_plot_dependency_graph, ) @@ -161,18 +161,21 @@ def __new__( ) if not disable_dependencies: return _ExecutorWithDependencies( - max_workers=max_workers, - backend="local", - cache_directory=cache_directory, + executor=_create_executor( + max_workers=max_workers, + backend="local", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + ), max_cores=max_cores, - resource_dict=resource_dict, - flux_executor=None, - flux_executor_pmi_mode=None, - flux_executor_nesting=False, - flux_log_files=False, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, refresh_rate=refresh_rate, plot_dependency_graph=plot_dependency_graph, plot_dependency_graph_filename=plot_dependency_graph_filename, diff --git a/executorlib/interfaces/slurm.py b/executorlib/interfaces/slurm.py index 23d516b3..6ce74909 100644 --- a/executorlib/interfaces/slurm.py +++ b/executorlib/interfaces/slurm.py @@ -3,7 +3,7 @@ from executorlib.interactive.executor import ( ExecutorWithDependencies as _ExecutorWithDependencies, ) -from executorlib.interactive.executor import create_executor as _create_executor +from executorlib.interactive.create import create_executor as _create_executor from executorlib.standalone.inputcheck import ( check_plot_dependency_graph as _check_plot_dependency_graph, ) @@ -188,18 +188,21 @@ def __new__( elif not disable_dependencies: _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) return _ExecutorWithDependencies( - max_workers=max_workers, - backend="slurm_submission", - cache_directory=cache_directory, + executor=_create_executor( + max_workers=max_workers, + backend="slurm_submission", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + ), max_cores=max_cores, - resource_dict=resource_dict, - flux_executor=None, - flux_executor_pmi_mode=None, - flux_executor_nesting=False, - flux_log_files=False, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, refresh_rate=refresh_rate, plot_dependency_graph=plot_dependency_graph, plot_dependency_graph_filename=plot_dependency_graph_filename, @@ -373,18 +376,21 @@ def __new__( ) if not disable_dependencies: return _ExecutorWithDependencies( - max_workers=max_workers, - backend="slurm_allocation", - cache_directory=cache_directory, + executor=_create_executor( + max_workers=max_workers, + backend="slurm_allocation", + cache_directory=cache_directory, + max_cores=max_cores, + resource_dict=resource_dict, + flux_executor=None, + flux_executor_pmi_mode=None, + flux_executor_nesting=False, + flux_log_files=False, + hostname_localhost=hostname_localhost, + block_allocation=block_allocation, + init_function=init_function, + ), max_cores=max_cores, - resource_dict=resource_dict, - flux_executor=None, - flux_executor_pmi_mode=None, - flux_executor_nesting=False, - flux_log_files=False, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, refresh_rate=refresh_rate, plot_dependency_graph=plot_dependency_graph, plot_dependency_graph_filename=plot_dependency_graph_filename, From 7c199640d4c6c29f16a09ad034eb3f38acfbd88d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 1 Feb 2025 13:05:02 +0000 Subject: [PATCH 08/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- executorlib/interfaces/flux.py | 2 +- executorlib/interfaces/local.py | 2 +- executorlib/interfaces/slurm.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/executorlib/interfaces/flux.py b/executorlib/interfaces/flux.py index 53fa8eb5..013b62e0 100644 --- a/executorlib/interfaces/flux.py +++ b/executorlib/interfaces/flux.py @@ -1,9 +1,9 @@ from typing import Callable, Optional +from executorlib.interactive.create import create_executor as _create_executor from executorlib.interactive.executor import ( ExecutorWithDependencies as _ExecutorWithDependencies, ) -from executorlib.interactive.create import create_executor as _create_executor from executorlib.standalone.inputcheck import ( check_plot_dependency_graph as _check_plot_dependency_graph, ) diff --git a/executorlib/interfaces/local.py b/executorlib/interfaces/local.py index fa5509e2..790e3796 100644 --- a/executorlib/interfaces/local.py +++ b/executorlib/interfaces/local.py @@ -1,9 +1,9 @@ from typing import Callable, Optional +from executorlib.interactive.create import create_executor as _create_executor from executorlib.interactive.executor import ( ExecutorWithDependencies as _ExecutorWithDependencies, ) -from executorlib.interactive.create import create_executor as _create_executor from executorlib.standalone.inputcheck import ( check_plot_dependency_graph as _check_plot_dependency_graph, ) diff --git a/executorlib/interfaces/slurm.py b/executorlib/interfaces/slurm.py index 6ce74909..ecfe50ff 100644 --- a/executorlib/interfaces/slurm.py +++ b/executorlib/interfaces/slurm.py @@ -1,9 +1,9 @@ from typing import Callable, Optional +from executorlib.interactive.create import create_executor as _create_executor from executorlib.interactive.executor import ( ExecutorWithDependencies as _ExecutorWithDependencies, ) -from executorlib.interactive.create import create_executor as _create_executor from executorlib.standalone.inputcheck import ( check_plot_dependency_graph as _check_plot_dependency_graph, ) From 1fb1859022f712ea1889bddb8e9aa94c367aee4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 1 Feb 2025 15:05:34 +0100 Subject: [PATCH 09/28] remove create --- executorlib/interactive/create.py | 287 ---------------------------- executorlib/interfaces/flux.py | 119 +++++++++--- executorlib/interfaces/local.py | 85 +++++--- executorlib/interfaces/slurm.py | 107 +++++++---- tests/test_dependencies_executor.py | 9 +- 5 files changed, 220 insertions(+), 387 deletions(-) delete mode 100644 executorlib/interactive/create.py diff --git a/executorlib/interactive/create.py b/executorlib/interactive/create.py deleted file mode 100644 index 016174a8..00000000 --- a/executorlib/interactive/create.py +++ /dev/null @@ -1,287 +0,0 @@ -from typing import Callable, Optional, Union - -from executorlib.interactive.shared import ( - InteractiveExecutor, - InteractiveStepExecutor, -) -from executorlib.interactive.slurm import SrunSpawner -from executorlib.interactive.slurm import ( - validate_max_workers as validate_max_workers_slurm, -) -from executorlib.standalone.inputcheck import ( - check_command_line_argument_lst, - check_executor, - check_flux_log_files, - check_gpus_per_worker, - check_init_function, - check_nested_flux_executor, - check_oversubscribe, - check_pmi, - validate_number_of_cores, -) -from executorlib.standalone.interactive.spawner import MpiExecSpawner - -try: # The PyFluxExecutor requires flux-base to be installed. - from executorlib.interactive.flux import FluxPythonSpawner - from executorlib.interactive.flux import ( - validate_max_workers as validate_max_workers_flux, - ) -except ImportError: - pass - - -def create_executor( - max_workers: Optional[int] = None, - backend: str = "local", - max_cores: Optional[int] = None, - cache_directory: Optional[str] = None, - resource_dict: dict = {}, - flux_executor=None, - flux_executor_pmi_mode: Optional[str] = None, - flux_executor_nesting: bool = False, - flux_log_files: bool = False, - hostname_localhost: Optional[bool] = None, - block_allocation: bool = False, - init_function: Optional[Callable] = None, -) -> Union[InteractiveStepExecutor, InteractiveExecutor]: - """ - Instead of returning a executorlib.Executor object this function returns either a executorlib.mpi.PyMPIExecutor, - executorlib.slurm.PySlurmExecutor or executorlib.flux.PyFluxExecutor depending on which backend is available. The - executorlib.flux.PyFluxExecutor is the preferred choice while the executorlib.mpi.PyMPIExecutor is primarily used - for development and testing. The executorlib.flux.PyFluxExecutor requires flux-base from the flux-framework to be - installed and in addition flux-sched to enable GPU scheduling. Finally, the executorlib.slurm.PySlurmExecutor - requires the SLURM workload manager to be installed on the system. - - Args: - max_workers (int): for backwards compatibility with the standard library, max_workers also defines the number of - cores which can be used in parallel - just like the max_cores parameter. Using max_cores is - recommended, as computers have a limited number of compute cores. - backend (str): Switch between the different backends "flux", "local" or "slurm". The default is "local". - max_cores (int): defines the number cores which can be used in parallel - cache_directory (str, optional): The directory to store cache files. Defaults to "cache". - resource_dict (dict): A dictionary of resources required by the task. With the following keys: - - cores (int): number of MPI cores to be used for each function call - - threads_per_core (int): number of OpenMP threads to be used for each function call - - gpus_per_core (int): number of GPUs per worker - defaults to 0 - - cwd (str/None): current working directory where the parallel python task is executed - - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and - SLURM only) - default False - - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only) - flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux - flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only) - flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function. - flux_log_files (bool, optional): Write flux stdout and stderr files. Defaults to False. - hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the - context of an HPC cluster this essential to be able to communicate to an Executor - running on a different compute node within the same allocation. And in principle - any computer should be able to resolve that their own hostname points to the same - address as localhost. Still MacOS >= 12 seems to disable this look up for security - reasons. So on MacOS it is required to set this option to true - block_allocation (boolean): To accelerate the submission of a series of python functions with the same - resource requirements, executorlib supports block allocation. In this case all - resources have to be defined on the executor, rather than during the submission - of the individual function. - init_function (None): optional function to preset arguments for functions which are submitted later - """ - if flux_executor is not None and backend != "flux_allocation": - backend = "flux_allocation" - if backend == "flux_allocation": - check_init_function( - block_allocation=block_allocation, init_function=init_function - ) - check_pmi(backend=backend, pmi=flux_executor_pmi_mode) - resource_dict["cache_directory"] = cache_directory - resource_dict["hostname_localhost"] = hostname_localhost - check_oversubscribe( - oversubscribe=resource_dict.get("openmpi_oversubscribe", False) - ) - check_command_line_argument_lst( - command_line_argument_lst=resource_dict.get("slurm_cmd_args", []) - ) - return create_flux_allocation_executor( - max_workers=max_workers, - max_cores=max_cores, - cache_directory=cache_directory, - resource_dict=resource_dict, - flux_executor=flux_executor, - flux_executor_pmi_mode=flux_executor_pmi_mode, - flux_executor_nesting=flux_executor_nesting, - flux_log_files=flux_log_files, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, - ) - elif backend == "slurm_allocation": - check_pmi(backend=backend, pmi=flux_executor_pmi_mode) - check_executor(executor=flux_executor) - check_nested_flux_executor(nested_flux_executor=flux_executor_nesting) - check_flux_log_files(flux_log_files=flux_log_files) - return create_slurm_allocation_executor( - max_workers=max_workers, - max_cores=max_cores, - cache_directory=cache_directory, - resource_dict=resource_dict, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, - ) - elif backend == "local": - check_pmi(backend=backend, pmi=flux_executor_pmi_mode) - check_executor(executor=flux_executor) - check_nested_flux_executor(nested_flux_executor=flux_executor_nesting) - check_flux_log_files(flux_log_files=flux_log_files) - return create_local_executor( - max_workers=max_workers, - max_cores=max_cores, - cache_directory=cache_directory, - resource_dict=resource_dict, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, - ) - else: - raise ValueError( - "The supported backends are slurm_allocation, slurm_submission, flux_allocation, flux_submission and local." - ) - - -def create_flux_allocation_executor( - max_workers: Optional[int] = None, - max_cores: Optional[int] = None, - cache_directory: Optional[str] = None, - resource_dict: dict = {}, - flux_executor=None, - flux_executor_pmi_mode: Optional[str] = None, - flux_executor_nesting: bool = False, - flux_log_files: bool = False, - hostname_localhost: Optional[bool] = None, - block_allocation: bool = False, - init_function: Optional[Callable] = None, -) -> Union[InteractiveStepExecutor, InteractiveExecutor]: - check_init_function(block_allocation=block_allocation, init_function=init_function) - check_pmi(backend="flux_allocation", pmi=flux_executor_pmi_mode) - cores_per_worker = resource_dict.get("cores", 1) - resource_dict["cache_directory"] = cache_directory - resource_dict["hostname_localhost"] = hostname_localhost - check_oversubscribe(oversubscribe=resource_dict.get("openmpi_oversubscribe", False)) - check_command_line_argument_lst( - command_line_argument_lst=resource_dict.get("slurm_cmd_args", []) - ) - if "openmpi_oversubscribe" in resource_dict.keys(): - del resource_dict["openmpi_oversubscribe"] - if "slurm_cmd_args" in resource_dict.keys(): - del resource_dict["slurm_cmd_args"] - resource_dict["flux_executor"] = flux_executor - resource_dict["flux_executor_pmi_mode"] = flux_executor_pmi_mode - resource_dict["flux_executor_nesting"] = flux_executor_nesting - resource_dict["flux_log_files"] = flux_log_files - if block_allocation: - resource_dict["init_function"] = init_function - max_workers = validate_number_of_cores( - max_cores=max_cores, - max_workers=max_workers, - cores_per_worker=cores_per_worker, - set_local_cores=False, - ) - validate_max_workers_flux( - max_workers=max_workers, - cores=cores_per_worker, - threads_per_core=resource_dict.get("threads_per_core", 1), - ) - return InteractiveExecutor( - max_workers=max_workers, - executor_kwargs=resource_dict, - spawner=FluxPythonSpawner, - ) - else: - return InteractiveStepExecutor( - max_cores=max_cores, - max_workers=max_workers, - executor_kwargs=resource_dict, - spawner=FluxPythonSpawner, - ) - - -def create_slurm_allocation_executor( - max_workers: Optional[int] = None, - max_cores: Optional[int] = None, - cache_directory: Optional[str] = None, - resource_dict: dict = {}, - hostname_localhost: Optional[bool] = None, - block_allocation: bool = False, - init_function: Optional[Callable] = None, -) -> Union[InteractiveStepExecutor, InteractiveExecutor]: - check_init_function(block_allocation=block_allocation, init_function=init_function) - cores_per_worker = resource_dict.get("cores", 1) - resource_dict["cache_directory"] = cache_directory - resource_dict["hostname_localhost"] = hostname_localhost - if block_allocation: - resource_dict["init_function"] = init_function - max_workers = validate_number_of_cores( - max_cores=max_cores, - max_workers=max_workers, - cores_per_worker=cores_per_worker, - set_local_cores=False, - ) - validate_max_workers_slurm( - max_workers=max_workers, - cores=cores_per_worker, - threads_per_core=resource_dict.get("threads_per_core", 1), - ) - return InteractiveExecutor( - max_workers=max_workers, - executor_kwargs=resource_dict, - spawner=SrunSpawner, - ) - else: - return InteractiveStepExecutor( - max_cores=max_cores, - max_workers=max_workers, - executor_kwargs=resource_dict, - spawner=SrunSpawner, - ) - - -def create_local_executor( - max_workers: Optional[int] = None, - max_cores: Optional[int] = None, - cache_directory: Optional[str] = None, - resource_dict: dict = {}, - hostname_localhost: Optional[bool] = None, - block_allocation: bool = False, - init_function: Optional[Callable] = None, -) -> Union[InteractiveStepExecutor, InteractiveExecutor]: - check_init_function(block_allocation=block_allocation, init_function=init_function) - cores_per_worker = resource_dict.get("cores", 1) - resource_dict["cache_directory"] = cache_directory - resource_dict["hostname_localhost"] = hostname_localhost - - check_gpus_per_worker(gpus_per_worker=resource_dict.get("gpus_per_core", 0)) - check_command_line_argument_lst( - command_line_argument_lst=resource_dict.get("slurm_cmd_args", []) - ) - if "threads_per_core" in resource_dict.keys(): - del resource_dict["threads_per_core"] - if "gpus_per_core" in resource_dict.keys(): - del resource_dict["gpus_per_core"] - if "slurm_cmd_args" in resource_dict.keys(): - del resource_dict["slurm_cmd_args"] - if block_allocation: - resource_dict["init_function"] = init_function - return InteractiveExecutor( - max_workers=validate_number_of_cores( - max_cores=max_cores, - max_workers=max_workers, - cores_per_worker=cores_per_worker, - set_local_cores=True, - ), - executor_kwargs=resource_dict, - spawner=MpiExecSpawner, - ) - else: - return InteractiveStepExecutor( - max_cores=max_cores, - max_workers=max_workers, - executor_kwargs=resource_dict, - spawner=MpiExecSpawner, - ) diff --git a/executorlib/interfaces/flux.py b/executorlib/interfaces/flux.py index 53fa8eb5..13fe7a47 100644 --- a/executorlib/interfaces/flux.py +++ b/executorlib/interfaces/flux.py @@ -1,19 +1,29 @@ -from typing import Callable, Optional +from typing import Callable, Optional, Union -from executorlib.interactive.executor import ( - ExecutorWithDependencies as _ExecutorWithDependencies, +from executorlib.interactive.shared import ( + InteractiveExecutor, + InteractiveStepExecutor, ) -from executorlib.interactive.create import create_executor as _create_executor +from executorlib.interactive.executor import ExecutorWithDependencies from executorlib.standalone.inputcheck import ( - check_plot_dependency_graph as _check_plot_dependency_graph, -) -from executorlib.standalone.inputcheck import ( - check_pysqa_config_directory as _check_pysqa_config_directory, -) -from executorlib.standalone.inputcheck import ( - check_refresh_rate as _check_refresh_rate, + check_plot_dependency_graph, + check_pysqa_config_directory, + check_refresh_rate, + check_command_line_argument_lst, + check_init_function, + check_oversubscribe, + check_pmi, + validate_number_of_cores, ) +try: # The PyFluxExecutor requires flux-base to be installed. + from executorlib.interactive.flux import FluxPythonSpawner + from executorlib.interactive.flux import ( + validate_max_workers as validate_max_workers_flux, + ) +except ImportError: + pass + class FluxAllocationExecutor: """ @@ -73,7 +83,7 @@ class FluxAllocationExecutor: >>> def init_k(): >>> return {"k": 3} >>> - >>> with Executor(cores=2, init_function=init_k) as p: + >>> with FluxAllocationExecutor(cores=2, init_function=init_k) as p: >>> fs = p.submit(calc, 2, j=4) >>> print(fs.result()) [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] @@ -179,10 +189,9 @@ def __new__( {k: v for k, v in default_resource_dict.items() if k not in resource_dict} ) if not disable_dependencies: - return _ExecutorWithDependencies( - executor=_create_executor( + return ExecutorWithDependencies( + executor=create_flux_executor( max_workers=max_workers, - backend="flux_allocation", cache_directory=cache_directory, max_cores=max_cores, resource_dict=resource_dict, @@ -200,11 +209,10 @@ def __new__( plot_dependency_graph_filename=plot_dependency_graph_filename, ) else: - _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) - _check_refresh_rate(refresh_rate=refresh_rate) - return _create_executor( + check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) + check_refresh_rate(refresh_rate=refresh_rate) + return create_flux_executor( max_workers=max_workers, - backend="flux_allocation", cache_directory=cache_directory, max_cores=max_cores, resource_dict=resource_dict, @@ -389,11 +397,10 @@ def __new__( disable_dependencies=disable_dependencies, ) elif not disable_dependencies: - _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) - return _ExecutorWithDependencies( - executor=_create_executor( + check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) + return ExecutorWithDependencies( + executor=create_flux_executor( max_workers=max_workers, - backend="flux_submission", cache_directory=cache_directory, max_cores=max_cores, resource_dict=resource_dict, @@ -411,12 +418,11 @@ def __new__( plot_dependency_graph_filename=plot_dependency_graph_filename, ) else: - _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) - _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) - _check_refresh_rate(refresh_rate=refresh_rate) - return _create_executor( + check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) + check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) + check_refresh_rate(refresh_rate=refresh_rate) + return create_flux_executor( max_workers=max_workers, - backend="flux_submission", cache_directory=cache_directory, max_cores=max_cores, resource_dict=resource_dict, @@ -428,3 +434,60 @@ def __new__( block_allocation=block_allocation, init_function=init_function, ) + + +def create_flux_executor( + max_workers: Optional[int] = None, + max_cores: Optional[int] = None, + cache_directory: Optional[str] = None, + resource_dict: dict = {}, + flux_executor=None, + flux_executor_pmi_mode: Optional[str] = None, + flux_executor_nesting: bool = False, + flux_log_files: bool = False, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, +) -> Union[InteractiveStepExecutor, InteractiveExecutor]: + check_init_function(block_allocation=block_allocation, init_function=init_function) + check_pmi(backend="flux_allocation", pmi=flux_executor_pmi_mode) + cores_per_worker = resource_dict.get("cores", 1) + resource_dict["cache_directory"] = cache_directory + resource_dict["hostname_localhost"] = hostname_localhost + check_oversubscribe(oversubscribe=resource_dict.get("openmpi_oversubscribe", False)) + check_command_line_argument_lst( + command_line_argument_lst=resource_dict.get("slurm_cmd_args", []) + ) + if "openmpi_oversubscribe" in resource_dict.keys(): + del resource_dict["openmpi_oversubscribe"] + if "slurm_cmd_args" in resource_dict.keys(): + del resource_dict["slurm_cmd_args"] + resource_dict["flux_executor"] = flux_executor + resource_dict["flux_executor_pmi_mode"] = flux_executor_pmi_mode + resource_dict["flux_executor_nesting"] = flux_executor_nesting + resource_dict["flux_log_files"] = flux_log_files + if block_allocation: + resource_dict["init_function"] = init_function + max_workers = validate_number_of_cores( + max_cores=max_cores, + max_workers=max_workers, + cores_per_worker=cores_per_worker, + set_local_cores=False, + ) + validate_max_workers_flux( + max_workers=max_workers, + cores=cores_per_worker, + threads_per_core=resource_dict.get("threads_per_core", 1), + ) + return InteractiveExecutor( + max_workers=max_workers, + executor_kwargs=resource_dict, + spawner=FluxPythonSpawner, + ) + else: + return InteractiveStepExecutor( + max_cores=max_cores, + max_workers=max_workers, + executor_kwargs=resource_dict, + spawner=FluxPythonSpawner, + ) \ No newline at end of file diff --git a/executorlib/interfaces/local.py b/executorlib/interfaces/local.py index fa5509e2..9e28ce11 100644 --- a/executorlib/interfaces/local.py +++ b/executorlib/interfaces/local.py @@ -1,15 +1,19 @@ -from typing import Callable, Optional +from typing import Callable, Optional, Union -from executorlib.interactive.executor import ( - ExecutorWithDependencies as _ExecutorWithDependencies, +from executorlib.interactive.executor import ExecutorWithDependencies +from executorlib.interactive.shared import ( + InteractiveExecutor, + InteractiveStepExecutor, ) -from executorlib.interactive.create import create_executor as _create_executor from executorlib.standalone.inputcheck import ( - check_plot_dependency_graph as _check_plot_dependency_graph, -) -from executorlib.standalone.inputcheck import ( - check_refresh_rate as _check_refresh_rate, + check_plot_dependency_graph, + check_refresh_rate, + check_command_line_argument_lst, + check_gpus_per_worker, + check_init_function, + validate_number_of_cores, ) +from executorlib.standalone.interactive.spawner import MpiExecSpawner class LocalExecutor: @@ -160,17 +164,12 @@ def __new__( {k: v for k, v in default_resource_dict.items() if k not in resource_dict} ) if not disable_dependencies: - return _ExecutorWithDependencies( - executor=_create_executor( + return ExecutorWithDependencies( + executor=create_local_executor( max_workers=max_workers, - backend="local", cache_directory=cache_directory, max_cores=max_cores, resource_dict=resource_dict, - flux_executor=None, - flux_executor_pmi_mode=None, - flux_executor_nesting=False, - flux_log_files=False, hostname_localhost=hostname_localhost, block_allocation=block_allocation, init_function=init_function, @@ -181,19 +180,59 @@ def __new__( plot_dependency_graph_filename=plot_dependency_graph_filename, ) else: - _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) - _check_refresh_rate(refresh_rate=refresh_rate) - return _create_executor( + check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) + check_refresh_rate(refresh_rate=refresh_rate) + return create_local_executor( max_workers=max_workers, - backend="local", cache_directory=cache_directory, max_cores=max_cores, resource_dict=resource_dict, - flux_executor=None, - flux_executor_pmi_mode=None, - flux_executor_nesting=False, - flux_log_files=False, hostname_localhost=hostname_localhost, block_allocation=block_allocation, init_function=init_function, ) + + +def create_local_executor( + max_workers: Optional[int] = None, + max_cores: Optional[int] = None, + cache_directory: Optional[str] = None, + resource_dict: dict = {}, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, +) -> Union[InteractiveStepExecutor, InteractiveExecutor]: + check_init_function(block_allocation=block_allocation, init_function=init_function) + cores_per_worker = resource_dict.get("cores", 1) + resource_dict["cache_directory"] = cache_directory + resource_dict["hostname_localhost"] = hostname_localhost + + check_gpus_per_worker(gpus_per_worker=resource_dict.get("gpus_per_core", 0)) + check_command_line_argument_lst( + command_line_argument_lst=resource_dict.get("slurm_cmd_args", []) + ) + if "threads_per_core" in resource_dict.keys(): + del resource_dict["threads_per_core"] + if "gpus_per_core" in resource_dict.keys(): + del resource_dict["gpus_per_core"] + if "slurm_cmd_args" in resource_dict.keys(): + del resource_dict["slurm_cmd_args"] + if block_allocation: + resource_dict["init_function"] = init_function + return InteractiveExecutor( + max_workers=validate_number_of_cores( + max_cores=max_cores, + max_workers=max_workers, + cores_per_worker=cores_per_worker, + set_local_cores=True, + ), + executor_kwargs=resource_dict, + spawner=MpiExecSpawner, + ) + else: + return InteractiveStepExecutor( + max_cores=max_cores, + max_workers=max_workers, + executor_kwargs=resource_dict, + spawner=MpiExecSpawner, + ) diff --git a/executorlib/interfaces/slurm.py b/executorlib/interfaces/slurm.py index 6ce74909..34ecfdf2 100644 --- a/executorlib/interfaces/slurm.py +++ b/executorlib/interfaces/slurm.py @@ -1,17 +1,20 @@ -from typing import Callable, Optional +from typing import Callable, Optional, Union -from executorlib.interactive.executor import ( - ExecutorWithDependencies as _ExecutorWithDependencies, +from executorlib.interactive.shared import ( + InteractiveExecutor, + InteractiveStepExecutor, ) -from executorlib.interactive.create import create_executor as _create_executor -from executorlib.standalone.inputcheck import ( - check_plot_dependency_graph as _check_plot_dependency_graph, -) -from executorlib.standalone.inputcheck import ( - check_pysqa_config_directory as _check_pysqa_config_directory, +from executorlib.interactive.executor import ExecutorWithDependencies +from executorlib.interactive.slurm import SrunSpawner +from executorlib.interactive.slurm import ( + validate_max_workers as validate_max_workers_slurm, ) from executorlib.standalone.inputcheck import ( - check_refresh_rate as _check_refresh_rate, + check_init_function, + validate_number_of_cores, + check_plot_dependency_graph, + check_pysqa_config_directory, + check_refresh_rate, ) @@ -186,18 +189,13 @@ def __new__( disable_dependencies=disable_dependencies, ) elif not disable_dependencies: - _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) - return _ExecutorWithDependencies( - executor=_create_executor( + check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) + return ExecutorWithDependencies( + executor=create_slurm_executor( max_workers=max_workers, - backend="slurm_submission", cache_directory=cache_directory, max_cores=max_cores, resource_dict=resource_dict, - flux_executor=None, - flux_executor_pmi_mode=None, - flux_executor_nesting=False, - flux_log_files=False, hostname_localhost=hostname_localhost, block_allocation=block_allocation, init_function=init_function, @@ -208,19 +206,14 @@ def __new__( plot_dependency_graph_filename=plot_dependency_graph_filename, ) else: - _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) - _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) - _check_refresh_rate(refresh_rate=refresh_rate) - return _create_executor( + check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) + check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) + check_refresh_rate(refresh_rate=refresh_rate) + return create_slurm_executor( max_workers=max_workers, - backend="slurm_submission", cache_directory=cache_directory, max_cores=max_cores, resource_dict=resource_dict, - flux_executor=None, - flux_executor_pmi_mode=None, - flux_executor_nesting=False, - flux_log_files=False, hostname_localhost=hostname_localhost, block_allocation=block_allocation, init_function=init_function, @@ -375,17 +368,12 @@ def __new__( {k: v for k, v in default_resource_dict.items() if k not in resource_dict} ) if not disable_dependencies: - return _ExecutorWithDependencies( - executor=_create_executor( + return ExecutorWithDependencies( + executor=create_slurm_executor( max_workers=max_workers, - backend="slurm_allocation", cache_directory=cache_directory, max_cores=max_cores, resource_dict=resource_dict, - flux_executor=None, - flux_executor_pmi_mode=None, - flux_executor_nesting=False, - flux_log_files=False, hostname_localhost=hostname_localhost, block_allocation=block_allocation, init_function=init_function, @@ -396,19 +384,54 @@ def __new__( plot_dependency_graph_filename=plot_dependency_graph_filename, ) else: - _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) - _check_refresh_rate(refresh_rate=refresh_rate) - return _create_executor( + check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) + check_refresh_rate(refresh_rate=refresh_rate) + return create_slurm_executor( max_workers=max_workers, - backend="slurm_allocation", cache_directory=cache_directory, max_cores=max_cores, resource_dict=resource_dict, - flux_executor=None, - flux_executor_pmi_mode=None, - flux_executor_nesting=False, - flux_log_files=False, hostname_localhost=hostname_localhost, block_allocation=block_allocation, init_function=init_function, ) + + +def create_slurm_executor( + max_workers: Optional[int] = None, + max_cores: Optional[int] = None, + cache_directory: Optional[str] = None, + resource_dict: dict = {}, + hostname_localhost: Optional[bool] = None, + block_allocation: bool = False, + init_function: Optional[Callable] = None, +) -> Union[InteractiveStepExecutor, InteractiveExecutor]: + check_init_function(block_allocation=block_allocation, init_function=init_function) + cores_per_worker = resource_dict.get("cores", 1) + resource_dict["cache_directory"] = cache_directory + resource_dict["hostname_localhost"] = hostname_localhost + if block_allocation: + resource_dict["init_function"] = init_function + max_workers = validate_number_of_cores( + max_cores=max_cores, + max_workers=max_workers, + cores_per_worker=cores_per_worker, + set_local_cores=False, + ) + validate_max_workers_slurm( + max_workers=max_workers, + cores=cores_per_worker, + threads_per_core=resource_dict.get("threads_per_core", 1), + ) + return InteractiveExecutor( + max_workers=max_workers, + executor_kwargs=resource_dict, + spawner=SrunSpawner, + ) + else: + return InteractiveStepExecutor( + max_cores=max_cores, + max_workers=max_workers, + executor_kwargs=resource_dict, + spawner=SrunSpawner, + ) \ No newline at end of file diff --git a/tests/test_dependencies_executor.py b/tests/test_dependencies_executor.py index 98089c0f..6758fef7 100644 --- a/tests/test_dependencies_executor.py +++ b/tests/test_dependencies_executor.py @@ -5,7 +5,7 @@ from queue import Queue from executorlib import LocalExecutor -from executorlib.interactive.create import create_executor +from executorlib.interfaces.local import create_local_executor from executorlib.interactive.shared import execute_tasks_with_dependencies from executorlib.standalone.plot import generate_nodes_and_edges from executorlib.standalone.serialize import cloudpickle_register @@ -96,10 +96,6 @@ def test_executor_dependency_plot_filename(self): self.assertTrue(os.path.exists(graph_file)) os.remove(graph_file) - def test_create_executor_error(self): - with self.assertRaises(ValueError): - create_executor(backend="toast", resource_dict={"cores": 1}) - def test_dependency_steps(self): cloudpickle_register(ind=1) fs1 = Future() @@ -123,7 +119,7 @@ def test_dependency_steps(self): "resource_dict": {"cores": 1}, } ) - executor = create_executor( + executor = create_local_executor( max_workers=1, max_cores=2, resource_dict={ @@ -134,7 +130,6 @@ def test_dependency_steps(self): "openmpi_oversubscribe": False, "slurm_cmd_args": [], }, - backend="local", ) process = RaisingThread( target=execute_tasks_with_dependencies, From 4f4516b4e6088b572c6168675973e207e088000e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 1 Feb 2025 14:06:22 +0000 Subject: [PATCH 10/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- executorlib/interfaces/flux.py | 10 +++++----- executorlib/interfaces/local.py | 4 ++-- executorlib/interfaces/slurm.py | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/executorlib/interfaces/flux.py b/executorlib/interfaces/flux.py index 13fe7a47..aef96410 100644 --- a/executorlib/interfaces/flux.py +++ b/executorlib/interfaces/flux.py @@ -1,18 +1,18 @@ from typing import Callable, Optional, Union +from executorlib.interactive.executor import ExecutorWithDependencies from executorlib.interactive.shared import ( InteractiveExecutor, InteractiveStepExecutor, ) -from executorlib.interactive.executor import ExecutorWithDependencies from executorlib.standalone.inputcheck import ( - check_plot_dependency_graph, - check_pysqa_config_directory, - check_refresh_rate, check_command_line_argument_lst, check_init_function, check_oversubscribe, + check_plot_dependency_graph, check_pmi, + check_pysqa_config_directory, + check_refresh_rate, validate_number_of_cores, ) @@ -490,4 +490,4 @@ def create_flux_executor( max_workers=max_workers, executor_kwargs=resource_dict, spawner=FluxPythonSpawner, - ) \ No newline at end of file + ) diff --git a/executorlib/interfaces/local.py b/executorlib/interfaces/local.py index 9e28ce11..73ed4b18 100644 --- a/executorlib/interfaces/local.py +++ b/executorlib/interfaces/local.py @@ -6,11 +6,11 @@ InteractiveStepExecutor, ) from executorlib.standalone.inputcheck import ( - check_plot_dependency_graph, - check_refresh_rate, check_command_line_argument_lst, check_gpus_per_worker, check_init_function, + check_plot_dependency_graph, + check_refresh_rate, validate_number_of_cores, ) from executorlib.standalone.interactive.spawner import MpiExecSpawner diff --git a/executorlib/interfaces/slurm.py b/executorlib/interfaces/slurm.py index 34ecfdf2..afde3b2f 100644 --- a/executorlib/interfaces/slurm.py +++ b/executorlib/interfaces/slurm.py @@ -1,20 +1,20 @@ from typing import Callable, Optional, Union +from executorlib.interactive.executor import ExecutorWithDependencies from executorlib.interactive.shared import ( InteractiveExecutor, InteractiveStepExecutor, ) -from executorlib.interactive.executor import ExecutorWithDependencies from executorlib.interactive.slurm import SrunSpawner from executorlib.interactive.slurm import ( validate_max_workers as validate_max_workers_slurm, ) from executorlib.standalone.inputcheck import ( check_init_function, - validate_number_of_cores, check_plot_dependency_graph, check_pysqa_config_directory, check_refresh_rate, + validate_number_of_cores, ) @@ -434,4 +434,4 @@ def create_slurm_executor( max_workers=max_workers, executor_kwargs=resource_dict, spawner=SrunSpawner, - ) \ No newline at end of file + ) From 454b336691a0da26ff09fab9f5dfc68c97ae2833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 1 Feb 2025 15:16:28 +0100 Subject: [PATCH 11/28] remove redundant functionality --- executorlib/interfaces/flux.py | 21 +-------------------- executorlib/interfaces/slurm.py | 17 +---------------- 2 files changed, 2 insertions(+), 36 deletions(-) diff --git a/executorlib/interfaces/flux.py b/executorlib/interfaces/flux.py index aef96410..2aadc3e2 100644 --- a/executorlib/interfaces/flux.py +++ b/executorlib/interfaces/flux.py @@ -11,7 +11,6 @@ check_oversubscribe, check_plot_dependency_graph, check_pmi, - check_pysqa_config_directory, check_refresh_rate, validate_number_of_cores, ) @@ -396,8 +395,7 @@ def __new__( init_function=init_function, disable_dependencies=disable_dependencies, ) - elif not disable_dependencies: - check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) + else: return ExecutorWithDependencies( executor=create_flux_executor( max_workers=max_workers, @@ -417,23 +415,6 @@ def __new__( plot_dependency_graph=plot_dependency_graph, plot_dependency_graph_filename=plot_dependency_graph_filename, ) - else: - check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) - check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) - check_refresh_rate(refresh_rate=refresh_rate) - return create_flux_executor( - max_workers=max_workers, - cache_directory=cache_directory, - max_cores=max_cores, - resource_dict=resource_dict, - flux_executor=None, - flux_executor_pmi_mode=None, - flux_executor_nesting=False, - flux_log_files=False, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, - ) def create_flux_executor( diff --git a/executorlib/interfaces/slurm.py b/executorlib/interfaces/slurm.py index afde3b2f..c1132866 100644 --- a/executorlib/interfaces/slurm.py +++ b/executorlib/interfaces/slurm.py @@ -12,7 +12,6 @@ from executorlib.standalone.inputcheck import ( check_init_function, check_plot_dependency_graph, - check_pysqa_config_directory, check_refresh_rate, validate_number_of_cores, ) @@ -188,8 +187,7 @@ def __new__( init_function=init_function, disable_dependencies=disable_dependencies, ) - elif not disable_dependencies: - check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) + else: return ExecutorWithDependencies( executor=create_slurm_executor( max_workers=max_workers, @@ -205,19 +203,6 @@ def __new__( plot_dependency_graph=plot_dependency_graph, plot_dependency_graph_filename=plot_dependency_graph_filename, ) - else: - check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) - check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) - check_refresh_rate(refresh_rate=refresh_rate) - return create_slurm_executor( - max_workers=max_workers, - cache_directory=cache_directory, - max_cores=max_cores, - resource_dict=resource_dict, - hostname_localhost=hostname_localhost, - block_allocation=block_allocation, - init_function=init_function, - ) class SlurmAllocationExecutor: From b89bca7aca9338d67e4f7a0d18aa81d36f0cdbe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 1 Feb 2025 15:41:26 +0100 Subject: [PATCH 12/28] Move graph tests --- tests/test_dependencies_executor.py | 94 --------- tests/test_plot_dependency.py | 293 ++++++++++++++++++++++++++++ 2 files changed, 293 insertions(+), 94 deletions(-) create mode 100644 tests/test_plot_dependency.py diff --git a/tests/test_dependencies_executor.py b/tests/test_dependencies_executor.py index 6758fef7..9fc06789 100644 --- a/tests/test_dependencies_executor.py +++ b/tests/test_dependencies_executor.py @@ -1,5 +1,4 @@ from concurrent.futures import Future -import os import unittest from time import sleep from queue import Queue @@ -7,7 +6,6 @@ from executorlib import LocalExecutor from executorlib.interfaces.local import create_local_executor from executorlib.interactive.shared import execute_tasks_with_dependencies -from executorlib.standalone.plot import generate_nodes_and_edges from executorlib.standalone.serialize import cloudpickle_register from executorlib.standalone.thread import RaisingThread @@ -52,50 +50,6 @@ def test_executor(self): future_2 = exe.submit(add_function, 1, parameter_2=future_1) self.assertEqual(future_2.result(), 4) - @unittest.skipIf( - skip_graphviz_test, - "graphviz is not installed, so the plot_dependency_graph tests are skipped.", - ) - def test_executor_dependency_plot(self): - with LocalExecutor( - max_cores=1, - plot_dependency_graph=True, - ) as exe: - cloudpickle_register(ind=1) - future_1 = exe.submit(add_function, 1, parameter_2=2) - future_2 = exe.submit(add_function, 1, parameter_2=future_1) - self.assertTrue(future_1.done()) - self.assertTrue(future_2.done()) - self.assertEqual(len(exe._future_hash_dict), 2) - self.assertEqual(len(exe._task_hash_dict), 2) - nodes, edges = generate_nodes_and_edges( - task_hash_dict=exe._task_hash_dict, - future_hash_inverse_dict={ - v: k for k, v in exe._future_hash_dict.items() - }, - ) - self.assertEqual(len(nodes), 5) - self.assertEqual(len(edges), 4) - - @unittest.skipIf( - skip_graphviz_test, - "graphviz is not installed, so the plot_dependency_graph tests are skipped.", - ) - def test_executor_dependency_plot_filename(self): - graph_file = os.path.join(os.path.dirname(__file__), "test.png") - with LocalExecutor( - max_cores=1, - plot_dependency_graph=False, - plot_dependency_graph_filename=graph_file, - ) as exe: - cloudpickle_register(ind=1) - future_1 = exe.submit(add_function, 1, parameter_2=2) - future_2 = exe.submit(add_function, 1, parameter_2=future_1) - self.assertTrue(future_1.done()) - self.assertTrue(future_2.done()) - self.assertTrue(os.path.exists(graph_file)) - os.remove(graph_file) - def test_dependency_steps(self): cloudpickle_register(ind=1) fs1 = Future() @@ -176,54 +130,6 @@ def test_many_to_one(self): ) self.assertEqual(future_sum.result(), 15) - @unittest.skipIf( - skip_graphviz_test, - "graphviz is not installed, so the plot_dependency_graph tests are skipped.", - ) - def test_many_to_one_plot(self): - length = 5 - parameter = 1 - with LocalExecutor( - max_cores=2, - plot_dependency_graph=True, - ) as exe: - cloudpickle_register(ind=1) - future_lst = exe.submit( - generate_tasks, - length=length, - resource_dict={"cores": 1}, - ) - lst = [] - for i in range(length): - lst.append( - exe.submit( - calc_from_lst, - lst=future_lst, - ind=i, - parameter=parameter, - resource_dict={"cores": 1}, - ) - ) - future_sum = exe.submit( - merge, - lst=lst, - resource_dict={"cores": 1}, - ) - self.assertTrue(future_lst.done()) - for l in lst: - self.assertTrue(l.done()) - self.assertTrue(future_sum.done()) - self.assertEqual(len(exe._future_hash_dict), 7) - self.assertEqual(len(exe._task_hash_dict), 7) - nodes, edges = generate_nodes_and_edges( - task_hash_dict=exe._task_hash_dict, - future_hash_inverse_dict={ - v: k for k, v in exe._future_hash_dict.items() - }, - ) - self.assertEqual(len(nodes), 18) - self.assertEqual(len(edges), 21) - class TestExecutorErrors(unittest.TestCase): def test_block_allocation_false_one_worker(self): diff --git a/tests/test_plot_dependency.py b/tests/test_plot_dependency.py new file mode 100644 index 00000000..465ddd3f --- /dev/null +++ b/tests/test_plot_dependency.py @@ -0,0 +1,293 @@ +import os +import unittest +from time import sleep + +from executorlib import LocalExecutor, SlurmAllocationExecutor, SlurmSubmissionExecutor +from executorlib.standalone.plot import generate_nodes_and_edges +from executorlib.standalone.serialize import cloudpickle_register + + +try: + import pygraphviz + + skip_graphviz_test = False +except ImportError: + skip_graphviz_test = True + + +def add_function(parameter_1, parameter_2): + sleep(0.2) + return parameter_1 + parameter_2 + + +def generate_tasks(length): + sleep(0.2) + return range(length) + + +def calc_from_lst(lst, ind, parameter): + sleep(0.2) + return lst[ind] + parameter + + +def merge(lst): + sleep(0.2) + return sum(lst) + + +@unittest.skipIf( + skip_graphviz_test, + "graphviz is not installed, so the plot_dependency_graph tests are skipped.", +) +class TestLocalExecutorWithDependencies(unittest.TestCase): + def test_executor_dependency_plot(self): + with LocalExecutor( + max_cores=1, + plot_dependency_graph=True, + ) as exe: + cloudpickle_register(ind=1) + future_1 = exe.submit(add_function, 1, parameter_2=2) + future_2 = exe.submit(add_function, 1, parameter_2=future_1) + self.assertTrue(future_1.done()) + self.assertTrue(future_2.done()) + self.assertEqual(len(exe._future_hash_dict), 2) + self.assertEqual(len(exe._task_hash_dict), 2) + nodes, edges = generate_nodes_and_edges( + task_hash_dict=exe._task_hash_dict, + future_hash_inverse_dict={ + v: k for k, v in exe._future_hash_dict.items() + }, + ) + self.assertEqual(len(nodes), 5) + self.assertEqual(len(edges), 4) + + def test_executor_dependency_plot_filename(self): + graph_file = os.path.join(os.path.dirname(__file__), "test.png") + with LocalExecutor( + max_cores=1, + plot_dependency_graph=False, + plot_dependency_graph_filename=graph_file, + ) as exe: + cloudpickle_register(ind=1) + future_1 = exe.submit(add_function, 1, parameter_2=2) + future_2 = exe.submit(add_function, 1, parameter_2=future_1) + self.assertTrue(future_1.done()) + self.assertTrue(future_2.done()) + self.assertTrue(os.path.exists(graph_file)) + os.remove(graph_file) + + def test_many_to_one_plot(self): + length = 5 + parameter = 1 + with LocalExecutor( + max_cores=2, + plot_dependency_graph=True, + ) as exe: + cloudpickle_register(ind=1) + future_lst = exe.submit( + generate_tasks, + length=length, + resource_dict={"cores": 1}, + ) + lst = [] + for i in range(length): + lst.append( + exe.submit( + calc_from_lst, + lst=future_lst, + ind=i, + parameter=parameter, + resource_dict={"cores": 1}, + ) + ) + future_sum = exe.submit( + merge, + lst=lst, + resource_dict={"cores": 1}, + ) + self.assertTrue(future_lst.done()) + for l in lst: + self.assertTrue(l.done()) + self.assertTrue(future_sum.done()) + self.assertEqual(len(exe._future_hash_dict), 7) + self.assertEqual(len(exe._task_hash_dict), 7) + nodes, edges = generate_nodes_and_edges( + task_hash_dict=exe._task_hash_dict, + future_hash_inverse_dict={ + v: k for k, v in exe._future_hash_dict.items() + }, + ) + self.assertEqual(len(nodes), 18) + self.assertEqual(len(edges), 21) + + +@unittest.skipIf( + skip_graphviz_test, + "graphviz is not installed, so the plot_dependency_graph tests are skipped.", +) +class TestSlurmAllocationExecutorWithDependencies(unittest.TestCase): + def test_executor_dependency_plot(self): + with SlurmAllocationExecutor( + max_cores=1, + plot_dependency_graph=True, + ) as exe: + cloudpickle_register(ind=1) + future_1 = exe.submit(add_function, 1, parameter_2=2) + future_2 = exe.submit(add_function, 1, parameter_2=future_1) + self.assertTrue(future_1.done()) + self.assertTrue(future_2.done()) + self.assertEqual(len(exe._future_hash_dict), 2) + self.assertEqual(len(exe._task_hash_dict), 2) + nodes, edges = generate_nodes_and_edges( + task_hash_dict=exe._task_hash_dict, + future_hash_inverse_dict={ + v: k for k, v in exe._future_hash_dict.items() + }, + ) + self.assertEqual(len(nodes), 5) + self.assertEqual(len(edges), 4) + + def test_executor_dependency_plot_filename(self): + graph_file = os.path.join(os.path.dirname(__file__), "test.png") + with SlurmAllocationExecutor( + max_cores=1, + plot_dependency_graph=False, + plot_dependency_graph_filename=graph_file, + ) as exe: + cloudpickle_register(ind=1) + future_1 = exe.submit(add_function, 1, parameter_2=2) + future_2 = exe.submit(add_function, 1, parameter_2=future_1) + self.assertTrue(future_1.done()) + self.assertTrue(future_2.done()) + self.assertTrue(os.path.exists(graph_file)) + os.remove(graph_file) + + def test_many_to_one_plot(self): + length = 5 + parameter = 1 + with SlurmAllocationExecutor( + max_cores=2, + plot_dependency_graph=True, + ) as exe: + cloudpickle_register(ind=1) + future_lst = exe.submit( + generate_tasks, + length=length, + resource_dict={"cores": 1}, + ) + lst = [] + for i in range(length): + lst.append( + exe.submit( + calc_from_lst, + lst=future_lst, + ind=i, + parameter=parameter, + resource_dict={"cores": 1}, + ) + ) + future_sum = exe.submit( + merge, + lst=lst, + resource_dict={"cores": 1}, + ) + self.assertTrue(future_lst.done()) + for l in lst: + self.assertTrue(l.done()) + self.assertTrue(future_sum.done()) + self.assertEqual(len(exe._future_hash_dict), 7) + self.assertEqual(len(exe._task_hash_dict), 7) + nodes, edges = generate_nodes_and_edges( + task_hash_dict=exe._task_hash_dict, + future_hash_inverse_dict={ + v: k for k, v in exe._future_hash_dict.items() + }, + ) + self.assertEqual(len(nodes), 18) + self.assertEqual(len(edges), 21) + + +@unittest.skipIf( + skip_graphviz_test, + "graphviz is not installed, so the plot_dependency_graph tests are skipped.", +) +class TestSlurmSubmissionExecutorWithDependencies(unittest.TestCase): + def test_executor_dependency_plot(self): + with SlurmSubmissionExecutor( + max_cores=1, + plot_dependency_graph=True, + ) as exe: + cloudpickle_register(ind=1) + future_1 = exe.submit(add_function, 1, parameter_2=2) + future_2 = exe.submit(add_function, 1, parameter_2=future_1) + self.assertTrue(future_1.done()) + self.assertTrue(future_2.done()) + self.assertEqual(len(exe._future_hash_dict), 2) + self.assertEqual(len(exe._task_hash_dict), 2) + nodes, edges = generate_nodes_and_edges( + task_hash_dict=exe._task_hash_dict, + future_hash_inverse_dict={ + v: k for k, v in exe._future_hash_dict.items() + }, + ) + self.assertEqual(len(nodes), 5) + self.assertEqual(len(edges), 4) + + def test_executor_dependency_plot_filename(self): + graph_file = os.path.join(os.path.dirname(__file__), "test.png") + with SlurmSubmissionExecutor( + max_cores=1, + plot_dependency_graph=False, + plot_dependency_graph_filename=graph_file, + ) as exe: + cloudpickle_register(ind=1) + future_1 = exe.submit(add_function, 1, parameter_2=2) + future_2 = exe.submit(add_function, 1, parameter_2=future_1) + self.assertTrue(future_1.done()) + self.assertTrue(future_2.done()) + self.assertTrue(os.path.exists(graph_file)) + os.remove(graph_file) + + def test_many_to_one_plot(self): + length = 5 + parameter = 1 + with SlurmSubmissionExecutor( + max_cores=2, + plot_dependency_graph=True, + ) as exe: + cloudpickle_register(ind=1) + future_lst = exe.submit( + generate_tasks, + length=length, + resource_dict={"cores": 1}, + ) + lst = [] + for i in range(length): + lst.append( + exe.submit( + calc_from_lst, + lst=future_lst, + ind=i, + parameter=parameter, + resource_dict={"cores": 1}, + ) + ) + future_sum = exe.submit( + merge, + lst=lst, + resource_dict={"cores": 1}, + ) + self.assertTrue(future_lst.done()) + for l in lst: + self.assertTrue(l.done()) + self.assertTrue(future_sum.done()) + self.assertEqual(len(exe._future_hash_dict), 7) + self.assertEqual(len(exe._task_hash_dict), 7) + nodes, edges = generate_nodes_and_edges( + task_hash_dict=exe._task_hash_dict, + future_hash_inverse_dict={ + v: k for k, v in exe._future_hash_dict.items() + }, + ) + self.assertEqual(len(nodes), 18) + self.assertEqual(len(edges), 21) From 89240d2df7f6926b38424642b0d8c5fc8a528b1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 1 Feb 2025 15:49:44 +0100 Subject: [PATCH 13/28] remove core check --- tests/test_plot_dependency.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_plot_dependency.py b/tests/test_plot_dependency.py index 465ddd3f..8a2ec093 100644 --- a/tests/test_plot_dependency.py +++ b/tests/test_plot_dependency.py @@ -214,7 +214,6 @@ def test_many_to_one_plot(self): class TestSlurmSubmissionExecutorWithDependencies(unittest.TestCase): def test_executor_dependency_plot(self): with SlurmSubmissionExecutor( - max_cores=1, plot_dependency_graph=True, ) as exe: cloudpickle_register(ind=1) @@ -236,7 +235,6 @@ def test_executor_dependency_plot(self): def test_executor_dependency_plot_filename(self): graph_file = os.path.join(os.path.dirname(__file__), "test.png") with SlurmSubmissionExecutor( - max_cores=1, plot_dependency_graph=False, plot_dependency_graph_filename=graph_file, ) as exe: @@ -252,7 +250,6 @@ def test_many_to_one_plot(self): length = 5 parameter = 1 with SlurmSubmissionExecutor( - max_cores=2, plot_dependency_graph=True, ) as exe: cloudpickle_register(ind=1) From 8f01dfac5527c85f1e469df6d7783351be17cde5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 1 Feb 2025 16:00:17 +0100 Subject: [PATCH 14/28] clean up --- tests/test_plot_dependency.py | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/tests/test_plot_dependency.py b/tests/test_plot_dependency.py index 8a2ec093..3da92da4 100644 --- a/tests/test_plot_dependency.py +++ b/tests/test_plot_dependency.py @@ -147,21 +147,6 @@ def test_executor_dependency_plot(self): self.assertEqual(len(nodes), 5) self.assertEqual(len(edges), 4) - def test_executor_dependency_plot_filename(self): - graph_file = os.path.join(os.path.dirname(__file__), "test.png") - with SlurmAllocationExecutor( - max_cores=1, - plot_dependency_graph=False, - plot_dependency_graph_filename=graph_file, - ) as exe: - cloudpickle_register(ind=1) - future_1 = exe.submit(add_function, 1, parameter_2=2) - future_2 = exe.submit(add_function, 1, parameter_2=future_1) - self.assertTrue(future_1.done()) - self.assertTrue(future_2.done()) - self.assertTrue(os.path.exists(graph_file)) - os.remove(graph_file) - def test_many_to_one_plot(self): length = 5 parameter = 1 @@ -232,20 +217,6 @@ def test_executor_dependency_plot(self): self.assertEqual(len(nodes), 5) self.assertEqual(len(edges), 4) - def test_executor_dependency_plot_filename(self): - graph_file = os.path.join(os.path.dirname(__file__), "test.png") - with SlurmSubmissionExecutor( - plot_dependency_graph=False, - plot_dependency_graph_filename=graph_file, - ) as exe: - cloudpickle_register(ind=1) - future_1 = exe.submit(add_function, 1, parameter_2=2) - future_2 = exe.submit(add_function, 1, parameter_2=future_1) - self.assertTrue(future_1.done()) - self.assertTrue(future_2.done()) - self.assertTrue(os.path.exists(graph_file)) - os.remove(graph_file) - def test_many_to_one_plot(self): length = 5 parameter = 1 From d9fcc6160cb925198a02c27f28794c0c1bff616c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 1 Feb 2025 17:25:59 +0100 Subject: [PATCH 15/28] more tests --- tests/test_executor_backend_flux.py | 14 +++ tests/test_plot_dependency_flux.py | 179 ++++++++++++++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 tests/test_plot_dependency_flux.py diff --git a/tests/test_executor_backend_flux.py b/tests/test_executor_backend_flux.py index 8ce476f0..dd7fab44 100644 --- a/tests/test_executor_backend_flux.py +++ b/tests/test_executor_backend_flux.py @@ -56,6 +56,20 @@ def test_flux_executor_serial(self): self.assertTrue(fs_1.done()) self.assertTrue(fs_2.done()) + def test_flux_executor_serial_no_depencies(self): + with FluxAllocationExecutor( + max_cores=2, + flux_executor=self.executor, + block_allocation=True, + disable_dependencies=True, + ) as exe: + fs_1 = exe.submit(calc, 1) + fs_2 = exe.submit(calc, 2) + self.assertEqual(fs_1.result(), 1) + self.assertEqual(fs_2.result(), 2) + self.assertTrue(fs_1.done()) + self.assertTrue(fs_2.done()) + def test_flux_executor_threads(self): with FluxAllocationExecutor( max_cores=1, diff --git a/tests/test_plot_dependency_flux.py b/tests/test_plot_dependency_flux.py new file mode 100644 index 00000000..e16bfaa0 --- /dev/null +++ b/tests/test_plot_dependency_flux.py @@ -0,0 +1,179 @@ +import unittest +from time import sleep + +from executorlib import FluxAllocationExecutor, FluxSubmissionExecutor +from executorlib.standalone.plot import generate_nodes_and_edges +from executorlib.standalone.serialize import cloudpickle_register + + +try: + import pygraphviz + import flux.job + from executorlib.interactive.flux import FluxPythonSpawner + + skip_flux_test = "FLUX_URI" not in os.environ + pmi = os.environ.get("PYMPIPOOL_PMIX", None) + + skip_graphviz_flux_test = False +except ImportError: + skip_graphviz_flux_test = True + + +def add_function(parameter_1, parameter_2): + sleep(0.2) + return parameter_1 + parameter_2 + + +def generate_tasks(length): + sleep(0.2) + return range(length) + + +def calc_from_lst(lst, ind, parameter): + sleep(0.2) + return lst[ind] + parameter + + +def merge(lst): + sleep(0.2) + return sum(lst) + + +@unittest.skipIf( + skip_graphviz_flux_test, + "either graphviz or flux are not installed, so the plot_dependency_graph tests are skipped.", +) +class TestFluxAllocationExecutorWithDependencies(unittest.TestCase): + def test_executor_dependency_plot(self): + with FluxAllocationExecutor( + max_cores=1, + plot_dependency_graph=True, + ) as exe: + cloudpickle_register(ind=1) + future_1 = exe.submit(add_function, 1, parameter_2=2) + future_2 = exe.submit(add_function, 1, parameter_2=future_1) + self.assertTrue(future_1.done()) + self.assertTrue(future_2.done()) + self.assertEqual(len(exe._future_hash_dict), 2) + self.assertEqual(len(exe._task_hash_dict), 2) + nodes, edges = generate_nodes_and_edges( + task_hash_dict=exe._task_hash_dict, + future_hash_inverse_dict={ + v: k for k, v in exe._future_hash_dict.items() + }, + ) + self.assertEqual(len(nodes), 5) + self.assertEqual(len(edges), 4) + + def test_many_to_one_plot(self): + length = 5 + parameter = 1 + with FluxAllocationExecutor( + max_cores=2, + plot_dependency_graph=True, + ) as exe: + cloudpickle_register(ind=1) + future_lst = exe.submit( + generate_tasks, + length=length, + resource_dict={"cores": 1}, + ) + lst = [] + for i in range(length): + lst.append( + exe.submit( + calc_from_lst, + lst=future_lst, + ind=i, + parameter=parameter, + resource_dict={"cores": 1}, + ) + ) + future_sum = exe.submit( + merge, + lst=lst, + resource_dict={"cores": 1}, + ) + self.assertTrue(future_lst.done()) + for l in lst: + self.assertTrue(l.done()) + self.assertTrue(future_sum.done()) + self.assertEqual(len(exe._future_hash_dict), 7) + self.assertEqual(len(exe._task_hash_dict), 7) + nodes, edges = generate_nodes_and_edges( + task_hash_dict=exe._task_hash_dict, + future_hash_inverse_dict={ + v: k for k, v in exe._future_hash_dict.items() + }, + ) + self.assertEqual(len(nodes), 18) + self.assertEqual(len(edges), 21) + + +@unittest.skipIf( + skip_graphviz_test, + "graphviz is not installed, so the plot_dependency_graph tests are skipped.", +) +class TestFluxSubmissionExecutorWithDependencies(unittest.TestCase): + def test_executor_dependency_plot(self): + with FluxSubmissionExecutor( + plot_dependency_graph=True, + ) as exe: + cloudpickle_register(ind=1) + future_1 = exe.submit(add_function, 1, parameter_2=2) + future_2 = exe.submit(add_function, 1, parameter_2=future_1) + self.assertTrue(future_1.done()) + self.assertTrue(future_2.done()) + self.assertEqual(len(exe._future_hash_dict), 2) + self.assertEqual(len(exe._task_hash_dict), 2) + nodes, edges = generate_nodes_and_edges( + task_hash_dict=exe._task_hash_dict, + future_hash_inverse_dict={ + v: k for k, v in exe._future_hash_dict.items() + }, + ) + self.assertEqual(len(nodes), 5) + self.assertEqual(len(edges), 4) + + def test_many_to_one_plot(self): + length = 5 + parameter = 1 + with FluxSubmissionExecutor( + plot_dependency_graph=True, + ) as exe: + cloudpickle_register(ind=1) + future_lst = exe.submit( + generate_tasks, + length=length, + resource_dict={"cores": 1}, + ) + lst = [] + for i in range(length): + lst.append( + exe.submit( + calc_from_lst, + lst=future_lst, + ind=i, + parameter=parameter, + resource_dict={"cores": 1}, + ) + ) + future_sum = exe.submit( + merge, + lst=lst, + resource_dict={"cores": 1}, + ) + self.assertTrue(future_lst.done()) + for l in lst: + self.assertTrue(l.done()) + self.assertTrue(future_sum.done()) + self.assertEqual(len(exe._future_hash_dict), 7) + self.assertEqual(len(exe._task_hash_dict), 7) + nodes, edges = generate_nodes_and_edges( + task_hash_dict=exe._task_hash_dict, + future_hash_inverse_dict={ + v: k for k, v in exe._future_hash_dict.items() + }, + ) + self.assertEqual(len(nodes), 18) + self.assertEqual(len(edges), 21) From 28185b2c0274f23e66806ec30ac1d3630ebc9a96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 1 Feb 2025 17:27:29 +0100 Subject: [PATCH 16/28] fix import --- tests/test_plot_dependency_flux.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_plot_dependency_flux.py b/tests/test_plot_dependency_flux.py index e16bfaa0..6c3bd536 100644 --- a/tests/test_plot_dependency_flux.py +++ b/tests/test_plot_dependency_flux.py @@ -1,3 +1,4 @@ +import os import unittest from time import sleep From 685c5dc826fe14fee5cab7d17c9682aaa19dd680 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 1 Feb 2025 17:29:37 +0100 Subject: [PATCH 17/28] more fixes --- tests/test_plot_dependency_flux.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_plot_dependency_flux.py b/tests/test_plot_dependency_flux.py index 6c3bd536..96249c9e 100644 --- a/tests/test_plot_dependency_flux.py +++ b/tests/test_plot_dependency_flux.py @@ -42,7 +42,7 @@ def merge(lst): @unittest.skipIf( skip_graphviz_flux_test, - "either graphviz or flux are not installed, so the plot_dependency_graph tests are skipped.", + "Either graphviz or flux are not installed, so the plot_dependency_graph tests are skipped.", ) class TestFluxAllocationExecutorWithDependencies(unittest.TestCase): def test_executor_dependency_plot(self): @@ -112,8 +112,8 @@ def test_many_to_one_plot(self): @unittest.skipIf( - skip_graphviz_test, - "graphviz is not installed, so the plot_dependency_graph tests are skipped.", + skip_graphviz_flux_test, + "Either graphviz or flux are not installed, so the plot_dependency_graph tests are skipped.", ) class TestFluxSubmissionExecutorWithDependencies(unittest.TestCase): def test_executor_dependency_plot(self): From 672196e9dd82093de69df8b27996548637fa81a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 1 Feb 2025 19:25:17 +0100 Subject: [PATCH 18/28] Rename LocalExecutor to SingleNodeExecutor --- executorlib/__init__.py | 4 +-- .../interfaces/{local.py => single.py} | 6 ++-- notebooks/1-local.ipynb | 32 +++++++++---------- notebooks/4-developer.ipynb | 6 ++-- tests/benchmark/llh.py | 8 ++--- tests/test_cache_executor_interactive.py | 4 +-- tests/test_dependencies_executor.py | 16 +++++----- tests/test_executor_backend_mpi.py | 16 +++++----- tests/test_executor_backend_mpi_noblock.py | 14 ++++---- tests/test_integration_pyiron_workflow.py | 14 ++++---- tests/test_plot_dependency.py | 8 ++--- tests/test_shell_executor.py | 10 +++--- tests/test_shell_interactive.py | 4 +-- 13 files changed, 70 insertions(+), 72 deletions(-) rename executorlib/interfaces/{local.py => single.py} (98%) diff --git a/executorlib/__init__.py b/executorlib/__init__.py index 502b7958..20b6cfa2 100644 --- a/executorlib/__init__.py +++ b/executorlib/__init__.py @@ -3,7 +3,7 @@ FluxAllocationExecutor, FluxSubmissionExecutor, ) -from executorlib.interfaces.local import LocalExecutor +from executorlib.interfaces.single import SingleNodeExecutor from executorlib.interfaces.slurm import ( SlurmAllocationExecutor, SlurmSubmissionExecutor, @@ -13,7 +13,7 @@ __all__: list = [ "FluxAllocationExecutor", "FluxSubmissionExecutor", - "LocalExecutor", + "SingleNodeExecutor", "SlurmAllocationExecutor", "SlurmSubmissionExecutor", ] diff --git a/executorlib/interfaces/local.py b/executorlib/interfaces/single.py similarity index 98% rename from executorlib/interfaces/local.py rename to executorlib/interfaces/single.py index 73ed4b18..14392451 100644 --- a/executorlib/interfaces/local.py +++ b/executorlib/interfaces/single.py @@ -16,7 +16,7 @@ from executorlib.standalone.interactive.spawner import MpiExecSpawner -class LocalExecutor: +class SingleNodeExecutor: """ The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or preferable the flux framework for distributing python functions within a given resource allocation. In contrast to @@ -59,7 +59,7 @@ class LocalExecutor: Examples: ``` >>> import numpy as np - >>> from executorlib.interfaces.local import LocalExecutor + >>> from executorlib.interfaces.local import SingleNodeExecutor >>> >>> def calc(i, j, k): >>> from mpi4py import MPI @@ -70,7 +70,7 @@ class LocalExecutor: >>> def init_k(): >>> return {"k": 3} >>> - >>> with LocalExecutor(cores=2, init_function=init_k) as p: + >>> with SingleNodeExecutor(cores=2, init_function=init_k) as p: >>> fs = p.submit(calc, 2, j=4) >>> print(fs.result()) [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] diff --git a/notebooks/1-local.ipynb b/notebooks/1-local.ipynb index 9c89b494..d8a70d91 100644 --- a/notebooks/1-local.ipynb +++ b/notebooks/1-local.ipynb @@ -26,9 +26,7 @@ "id": "b1907f12-7378-423b-9b83-1b65fc0a20f5", "metadata": {}, "outputs": [], - "source": [ - "from executorlib import LocalExecutor" - ] + "source": "from executorlib import SingleNodeExecutor" }, { "cell_type": "markdown", @@ -56,7 +54,7 @@ ], "source": [ "%%time\n", - "with LocalExecutor() as exe:\n", + "with SingleNodeExecutor() as exe:\n", " future = exe.submit(sum, [1, 1])\n", " print(future.result())" ] @@ -87,7 +85,7 @@ ], "source": [ "%%time\n", - "with LocalExecutor() as exe:\n", + "with SingleNodeExecutor() as exe:\n", " future_lst = [exe.submit(sum, [i, i]) for i in range(2, 5)]\n", " print([f.result() for f in future_lst])" ] @@ -118,7 +116,7 @@ ], "source": [ "%%time\n", - "with LocalExecutor() as exe:\n", + "with SingleNodeExecutor() as exe:\n", " results = exe.map(sum, [[5, 5], [6, 6], [7, 7]])\n", " print(list(results))" ] @@ -191,7 +189,7 @@ } ], "source": [ - "with LocalExecutor() as exe:\n", + "with SingleNodeExecutor() as exe:\n", " fs = exe.submit(calc_mpi, 3, resource_dict={\"cores\": 2})\n", " print(fs.result())" ] @@ -219,7 +217,7 @@ } ], "source": [ - "with LocalExecutor(resource_dict={\"cores\": 2}) as exe:\n", + "with SingleNodeExecutor(resource_dict={\"cores\": 2}) as exe:\n", " fs = exe.submit(calc_mpi, 3)\n", " print(fs.result())" ] @@ -285,7 +283,7 @@ } ], "source": [ - "with LocalExecutor() as exe:\n", + "with SingleNodeExecutor() as exe:\n", " fs = exe.submit(calc_with_threads, 3, resource_dict={\"threads_per_core\": 2})\n", " print(fs.result())" ] @@ -313,7 +311,7 @@ } ], "source": [ - "with LocalExecutor(resource_dict={\"threads_per_core\": 2}) as exe:\n", + "with SingleNodeExecutor(resource_dict={\"threads_per_core\": 2}) as exe:\n", " fs = exe.submit(calc_with_threads, 3)\n", " print(fs.result())" ] @@ -364,7 +362,7 @@ ], "source": [ "%%time\n", - "with LocalExecutor(max_workers=2, block_allocation=True) as exe:\n", + "with SingleNodeExecutor(max_workers=2, block_allocation=True) as exe:\n", " future = exe.submit(sum, [1, 1])\n", " print(future.result())" ] @@ -457,7 +455,7 @@ } ], "source": [ - "with LocalExecutor(resource_dict={\"cores\": 2}, block_allocation=True) as exe:\n", + "with SingleNodeExecutor(resource_dict={\"cores\": 2}, block_allocation=True) as exe:\n", " fs = exe.submit(calc_mpi, 3)\n", " print(fs.result())" ] @@ -515,7 +513,7 @@ } ], "source": [ - "with LocalExecutor(init_function=init_function, block_allocation=True) as exe:\n", + "with SingleNodeExecutor(init_function=init_function, block_allocation=True) as exe:\n", " fs = exe.submit(calc_with_preload, 2, j=5)\n", " print(fs.result())" ] @@ -557,7 +555,7 @@ ], "source": [ "%%time\n", - "with LocalExecutor(cache_directory=\"./cache\") as exe:\n", + "with SingleNodeExecutor(cache_directory=\"./cache\") as exe:\n", " future_lst = [exe.submit(sum, [i, i]) for i in range(1, 4)]\n", " print([f.result() for f in future_lst])" ] @@ -590,7 +588,7 @@ ], "source": [ "%%time\n", - "with LocalExecutor(cache_directory=\"./cache\") as exe:\n", + "with SingleNodeExecutor(cache_directory=\"./cache\") as exe:\n", " future_lst = [exe.submit(sum, [i, i]) for i in range(1, 4)]\n", " print([f.result() for f in future_lst])" ] @@ -683,7 +681,7 @@ } ], "source": [ - "with LocalExecutor() as exe:\n", + "with SingleNodeExecutor() as exe:\n", " future = 0\n", " for i in range(1, 4):\n", " future = exe.submit(calc_add, i, future)\n", @@ -811,7 +809,7 @@ } ], "source": [ - "with LocalExecutor(plot_dependency_graph=True) as exe:\n", + "with SingleNodeExecutor(plot_dependency_graph=True) as exe:\n", " future = 0\n", " for i in range(1, 4):\n", " future = exe.submit(calc_add, i, future)\n", diff --git a/notebooks/4-developer.ipynb b/notebooks/4-developer.ipynb index b86a34ed..e76c3f88 100644 --- a/notebooks/4-developer.ipynb +++ b/notebooks/4-developer.ipynb @@ -56,7 +56,7 @@ "id": "83515b16-c4d5-4b02-acd7-9e1eb57fd335", "metadata": {}, "outputs": [], - "source": "from executorlib import LocalExecutor" + "source": "from executorlib import SingleNodeExecutor" }, { "cell_type": "code", @@ -91,7 +91,7 @@ } ], "source": [ - "with LocalExecutor() as exe:\n", + "with SingleNodeExecutor() as exe:\n", " future = exe.submit(\n", " execute_shell_command,\n", " [\"echo\", \"test\"],\n", @@ -248,7 +248,7 @@ } ], "source": [ - "with LocalExecutor(\n", + "with SingleNodeExecutor(\n", " max_workers=1,\n", " init_function=init_process,\n", " block_allocation=True,\n", diff --git a/tests/benchmark/llh.py b/tests/benchmark/llh.py index 3c6e6a17..e275aba8 100644 --- a/tests/benchmark/llh.py +++ b/tests/benchmark/llh.py @@ -42,10 +42,10 @@ def run_static(mean=0.1, sigma=1.1, runs=32): executor=ThreadPoolExecutor, mean=0.1, sigma=1.1, runs=32, max_workers=4 ) elif run_mode == "block_allocation": - from executorlib import LocalExecutor + from executorlib import SingleNodeExecutor run_with_executor( - executor=LocalExecutor, + executor=SingleNodeExecutor, mean=0.1, sigma=1.1, runs=32, @@ -53,10 +53,10 @@ def run_static(mean=0.1, sigma=1.1, runs=32): block_allocation=True, ) elif run_mode == "executorlib": - from executorlib import LocalExecutor + from executorlib import SingleNodeExecutor run_with_executor( - executor=LocalExecutor, + executor=SingleNodeExecutor, mean=0.1, sigma=1.1, runs=32, diff --git a/tests/test_cache_executor_interactive.py b/tests/test_cache_executor_interactive.py index efd46780..cc5752da 100644 --- a/tests/test_cache_executor_interactive.py +++ b/tests/test_cache_executor_interactive.py @@ -2,7 +2,7 @@ import shutil import unittest -from executorlib import LocalExecutor +from executorlib import SingleNodeExecutor try: from executorlib.standalone.hdf import get_cache_data @@ -18,7 +18,7 @@ class TestCacheFunctions(unittest.TestCase): def test_cache_data(self): cache_directory = "./cache" - with LocalExecutor(cache_directory=cache_directory) as exe: + with SingleNodeExecutor(cache_directory=cache_directory) as exe: future_lst = [exe.submit(sum, [i, i]) for i in range(1, 4)] result_lst = [f.result() for f in future_lst] diff --git a/tests/test_dependencies_executor.py b/tests/test_dependencies_executor.py index 9fc06789..690bf397 100644 --- a/tests/test_dependencies_executor.py +++ b/tests/test_dependencies_executor.py @@ -3,8 +3,8 @@ from time import sleep from queue import Queue -from executorlib import LocalExecutor -from executorlib.interfaces.local import create_local_executor +from executorlib import SingleNodeExecutor +from executorlib.interfaces.single import create_local_executor from executorlib.interactive.shared import execute_tasks_with_dependencies from executorlib.standalone.serialize import cloudpickle_register from executorlib.standalone.thread import RaisingThread @@ -44,7 +44,7 @@ def raise_error(): class TestExecutorWithDependencies(unittest.TestCase): def test_executor(self): - with LocalExecutor(max_cores=1) as exe: + with SingleNodeExecutor(max_cores=1) as exe: cloudpickle_register(ind=1) future_1 = exe.submit(add_function, 1, parameter_2=2) future_2 = exe.submit(add_function, 1, parameter_2=future_1) @@ -105,7 +105,7 @@ def test_dependency_steps(self): def test_many_to_one(self): length = 5 parameter = 1 - with LocalExecutor(max_cores=2) as exe: + with SingleNodeExecutor(max_cores=2) as exe: cloudpickle_register(ind=1) future_lst = exe.submit( generate_tasks, @@ -134,24 +134,24 @@ def test_many_to_one(self): class TestExecutorErrors(unittest.TestCase): def test_block_allocation_false_one_worker(self): with self.assertRaises(RuntimeError): - with LocalExecutor(max_cores=1, block_allocation=False) as exe: + with SingleNodeExecutor(max_cores=1, block_allocation=False) as exe: cloudpickle_register(ind=1) _ = exe.submit(raise_error) def test_block_allocation_true_one_worker(self): with self.assertRaises(RuntimeError): - with LocalExecutor(max_cores=1, block_allocation=True) as exe: + with SingleNodeExecutor(max_cores=1, block_allocation=True) as exe: cloudpickle_register(ind=1) _ = exe.submit(raise_error) def test_block_allocation_false_two_workers(self): with self.assertRaises(RuntimeError): - with LocalExecutor(max_cores=2, block_allocation=False) as exe: + with SingleNodeExecutor(max_cores=2, block_allocation=False) as exe: cloudpickle_register(ind=1) _ = exe.submit(raise_error) def test_block_allocation_true_two_workers(self): with self.assertRaises(RuntimeError): - with LocalExecutor(max_cores=2, block_allocation=True) as exe: + with SingleNodeExecutor(max_cores=2, block_allocation=True) as exe: cloudpickle_register(ind=1) _ = exe.submit(raise_error) diff --git a/tests/test_executor_backend_mpi.py b/tests/test_executor_backend_mpi.py index 9d9ba7e2..fa39d619 100644 --- a/tests/test_executor_backend_mpi.py +++ b/tests/test_executor_backend_mpi.py @@ -4,7 +4,7 @@ import time import unittest -from executorlib import LocalExecutor, SlurmAllocationExecutor +from executorlib import SingleNodeExecutor, SlurmAllocationExecutor from executorlib.standalone.serialize import cloudpickle_register @@ -34,7 +34,7 @@ def mpi_funct_sleep(i): class TestExecutorBackend(unittest.TestCase): def test_meta_executor_serial(self): - with LocalExecutor(max_cores=2, block_allocation=True) as exe: + with SingleNodeExecutor(max_cores=2, block_allocation=True) as exe: cloudpickle_register(ind=1) fs_1 = exe.submit(calc, 1) fs_2 = exe.submit(calc, 2) @@ -44,7 +44,7 @@ def test_meta_executor_serial(self): self.assertTrue(fs_2.done()) def test_meta_executor_single(self): - with LocalExecutor(max_cores=1, block_allocation=True) as exe: + with SingleNodeExecutor(max_cores=1, block_allocation=True) as exe: cloudpickle_register(ind=1) fs_1 = exe.submit(calc, 1) fs_2 = exe.submit(calc, 2) @@ -55,7 +55,7 @@ def test_meta_executor_single(self): def test_oversubscribe(self): with self.assertRaises(ValueError): - with LocalExecutor(max_cores=1, block_allocation=True) as exe: + with SingleNodeExecutor(max_cores=1, block_allocation=True) as exe: cloudpickle_register(ind=1) fs_1 = exe.submit(calc, 1, resource_dict={"cores": 2}) @@ -63,7 +63,7 @@ def test_oversubscribe(self): skip_mpi4py_test, "mpi4py is not installed, so the mpi4py tests are skipped." ) def test_meta_executor_parallel(self): - with LocalExecutor( + with SingleNodeExecutor( max_workers=2, resource_dict={"cores": 2}, block_allocation=True, @@ -75,7 +75,7 @@ def test_meta_executor_parallel(self): def test_errors(self): with self.assertRaises(TypeError): - LocalExecutor( + SingleNodeExecutor( max_cores=1, resource_dict={"cores": 1, "gpus_per_core": 1}, ) @@ -89,7 +89,7 @@ def tearDown(self): skip_mpi4py_test, "mpi4py is not installed, so the mpi4py tests are skipped." ) def test_meta_executor_parallel_cache(self): - with LocalExecutor( + with SingleNodeExecutor( max_workers=2, resource_dict={"cores": 2}, block_allocation=True, @@ -114,7 +114,7 @@ class TestWorkingDirectory(unittest.TestCase): def test_output_files_cwd(self): dirname = os.path.abspath(os.path.dirname(__file__)) os.makedirs(dirname, exist_ok=True) - with LocalExecutor( + with SingleNodeExecutor( max_cores=1, resource_dict={"cores": 1, "cwd": dirname}, block_allocation=True, diff --git a/tests/test_executor_backend_mpi_noblock.py b/tests/test_executor_backend_mpi_noblock.py index e9c27091..03f21ef6 100644 --- a/tests/test_executor_backend_mpi_noblock.py +++ b/tests/test_executor_backend_mpi_noblock.py @@ -1,6 +1,6 @@ import unittest -from executorlib import LocalExecutor +from executorlib import SingleNodeExecutor from executorlib.standalone.serialize import cloudpickle_register @@ -14,7 +14,7 @@ def resource_dict(resource_dict): class TestExecutorBackend(unittest.TestCase): def test_meta_executor_serial_with_dependencies(self): - with LocalExecutor( + with SingleNodeExecutor( max_cores=2, block_allocation=False, disable_dependencies=True, @@ -28,7 +28,7 @@ def test_meta_executor_serial_with_dependencies(self): self.assertTrue(fs_2.done()) def test_meta_executor_serial_without_dependencies(self): - with LocalExecutor( + with SingleNodeExecutor( max_cores=2, block_allocation=False, disable_dependencies=False, @@ -42,7 +42,7 @@ def test_meta_executor_serial_without_dependencies(self): self.assertTrue(fs_2.done()) def test_meta_executor_single(self): - with LocalExecutor( + with SingleNodeExecutor( max_cores=1, block_allocation=False, ) as exe: @@ -56,7 +56,7 @@ def test_meta_executor_single(self): def test_errors(self): with self.assertRaises(TypeError): - LocalExecutor( + SingleNodeExecutor( max_cores=1, resource_dict={ "cores": 1, @@ -64,13 +64,13 @@ def test_errors(self): }, ) with self.assertRaises(ValueError): - with LocalExecutor( + with SingleNodeExecutor( max_cores=1, block_allocation=False, ) as exe: exe.submit(resource_dict, resource_dict={}) with self.assertRaises(ValueError): - with LocalExecutor( + with SingleNodeExecutor( max_cores=1, block_allocation=True, ) as exe: diff --git a/tests/test_integration_pyiron_workflow.py b/tests/test_integration_pyiron_workflow.py index dad81894..f1f8505f 100644 --- a/tests/test_integration_pyiron_workflow.py +++ b/tests/test_integration_pyiron_workflow.py @@ -12,7 +12,7 @@ from typing import Callable import unittest -from executorlib import LocalExecutor +from executorlib import SingleNodeExecutor from executorlib.standalone.serialize import cloudpickle_register @@ -75,7 +75,7 @@ def slowly_returns_dynamic(dynamic_arg): return dynamic_arg dynamic_dynamic = slowly_returns_dynamic() - executor = LocalExecutor(block_allocation=True, max_workers=1) + executor = SingleNodeExecutor(block_allocation=True, max_workers=1) cloudpickle_register(ind=1) dynamic_object = does_nothing() fs = executor.submit(dynamic_dynamic.run, dynamic_object) @@ -105,7 +105,7 @@ def slowly_returns_42(): self.assertIsNone( dynamic_42.result, msg="Just a sanity check that the test is set up right" ) - executor = LocalExecutor(block_allocation=True, max_workers=1) + executor = SingleNodeExecutor(block_allocation=True, max_workers=1) cloudpickle_register(ind=1) fs = executor.submit(dynamic_42.run) fs.add_done_callback(dynamic_42.process_result) @@ -136,7 +136,7 @@ def returns_42(): dynamic_42.running, msg="Sanity check that the test starts in the expected condition", ) - executor = LocalExecutor(block_allocation=True, max_workers=1) + executor = SingleNodeExecutor(block_allocation=True, max_workers=1) cloudpickle_register(ind=1) fs = executor.submit(dynamic_42.run) fs.add_done_callback(dynamic_42.process_result) @@ -160,7 +160,7 @@ def raise_error(): raise RuntimeError re = raise_error() - executor = LocalExecutor(block_allocation=True, max_workers=1) + executor = SingleNodeExecutor(block_allocation=True, max_workers=1) cloudpickle_register(ind=1) fs = executor.submit(re.run) with self.assertRaises( @@ -190,7 +190,7 @@ def slowly_returns_dynamic(): return inside_variable dynamic_dynamic = slowly_returns_dynamic() - executor = LocalExecutor(block_allocation=True, max_workers=1) + executor = SingleNodeExecutor(block_allocation=True, max_workers=1) cloudpickle_register(ind=1) fs = executor.submit(dynamic_dynamic.run) self.assertIsInstance( @@ -219,7 +219,7 @@ def slow(): return fortytwo f = slow() - executor = LocalExecutor(block_allocation=True, max_workers=1) + executor = SingleNodeExecutor(block_allocation=True, max_workers=1) cloudpickle_register(ind=1) fs = executor.submit(f.run) self.assertEqual( diff --git a/tests/test_plot_dependency.py b/tests/test_plot_dependency.py index 3da92da4..6da6a82e 100644 --- a/tests/test_plot_dependency.py +++ b/tests/test_plot_dependency.py @@ -2,7 +2,7 @@ import unittest from time import sleep -from executorlib import LocalExecutor, SlurmAllocationExecutor, SlurmSubmissionExecutor +from executorlib import SingleNodeExecutor, SlurmAllocationExecutor, SlurmSubmissionExecutor from executorlib.standalone.plot import generate_nodes_and_edges from executorlib.standalone.serialize import cloudpickle_register @@ -41,7 +41,7 @@ def merge(lst): ) class TestLocalExecutorWithDependencies(unittest.TestCase): def test_executor_dependency_plot(self): - with LocalExecutor( + with SingleNodeExecutor( max_cores=1, plot_dependency_graph=True, ) as exe: @@ -63,7 +63,7 @@ def test_executor_dependency_plot(self): def test_executor_dependency_plot_filename(self): graph_file = os.path.join(os.path.dirname(__file__), "test.png") - with LocalExecutor( + with SingleNodeExecutor( max_cores=1, plot_dependency_graph=False, plot_dependency_graph_filename=graph_file, @@ -79,7 +79,7 @@ def test_executor_dependency_plot_filename(self): def test_many_to_one_plot(self): length = 5 parameter = 1 - with LocalExecutor( + with SingleNodeExecutor( max_cores=2, plot_dependency_graph=True, ) as exe: diff --git a/tests/test_shell_executor.py b/tests/test_shell_executor.py index 00f1ca53..971befd0 100644 --- a/tests/test_shell_executor.py +++ b/tests/test_shell_executor.py @@ -3,7 +3,7 @@ import queue import unittest -from executorlib import LocalExecutor +from executorlib import SingleNodeExecutor from executorlib.standalone.serialize import cloudpickle_register from executorlib.interactive.shared import execute_parallel_tasks from executorlib.standalone.interactive.spawner import MpiExecSpawner @@ -83,7 +83,7 @@ def test_broken_executable(self): ) def test_shell_static_executor_args(self): - with LocalExecutor(max_workers=1) as exe: + with SingleNodeExecutor(max_workers=1) as exe: cloudpickle_register(ind=1) future = exe.submit( submit_shell_command, @@ -96,7 +96,7 @@ def test_shell_static_executor_args(self): self.assertTrue(future.done()) def test_shell_static_executor_binary(self): - with LocalExecutor(max_workers=1) as exe: + with SingleNodeExecutor(max_workers=1) as exe: cloudpickle_register(ind=1) future = exe.submit( submit_shell_command, @@ -109,7 +109,7 @@ def test_shell_static_executor_binary(self): self.assertTrue(future.done()) def test_shell_static_executor_shell(self): - with LocalExecutor(max_workers=1) as exe: + with SingleNodeExecutor(max_workers=1) as exe: cloudpickle_register(ind=1) future = exe.submit( submit_shell_command, "echo test", universal_newlines=True, shell=True @@ -119,7 +119,7 @@ def test_shell_static_executor_shell(self): self.assertTrue(future.done()) def test_shell_executor(self): - with LocalExecutor(max_workers=2) as exe: + with SingleNodeExecutor(max_workers=2) as exe: cloudpickle_register(ind=1) f_1 = exe.submit( submit_shell_command, ["echo", "test_1"], universal_newlines=True diff --git a/tests/test_shell_interactive.py b/tests/test_shell_interactive.py index efccf2b7..553639af 100644 --- a/tests/test_shell_interactive.py +++ b/tests/test_shell_interactive.py @@ -4,7 +4,7 @@ import queue import unittest -from executorlib import LocalExecutor +from executorlib import SingleNodeExecutor from executorlib.standalone.serialize import cloudpickle_register from executorlib.interactive.shared import execute_parallel_tasks from executorlib.standalone.interactive.spawner import MpiExecSpawner @@ -104,7 +104,7 @@ def test_execute_single_task(self): def test_shell_interactive_executor(self): cloudpickle_register(ind=1) - with LocalExecutor( + with SingleNodeExecutor( max_workers=1, init_function=init_process, block_allocation=True, From 5072318f89c8c4447dba79f72d0bf75068ed97c0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 1 Feb 2025 18:25:27 +0000 Subject: [PATCH 19/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_plot_dependency.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_plot_dependency.py b/tests/test_plot_dependency.py index 6da6a82e..03165b6b 100644 --- a/tests/test_plot_dependency.py +++ b/tests/test_plot_dependency.py @@ -2,7 +2,11 @@ import unittest from time import sleep -from executorlib import SingleNodeExecutor, SlurmAllocationExecutor, SlurmSubmissionExecutor +from executorlib import ( + SingleNodeExecutor, + SlurmAllocationExecutor, + SlurmSubmissionExecutor, +) from executorlib.standalone.plot import generate_nodes_and_edges from executorlib.standalone.serialize import cloudpickle_register From 8a37e60d1f3c74cc80247a43ce3377bf53009ebf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 1 Feb 2025 19:48:45 +0100 Subject: [PATCH 20/28] test block allocation --- tests/test_plot_dependency.py | 5 +++++ tests/test_plot_dependency_flux.py | 2 ++ 2 files changed, 7 insertions(+) diff --git a/tests/test_plot_dependency.py b/tests/test_plot_dependency.py index 6da6a82e..2da6de6f 100644 --- a/tests/test_plot_dependency.py +++ b/tests/test_plot_dependency.py @@ -44,6 +44,7 @@ def test_executor_dependency_plot(self): with SingleNodeExecutor( max_cores=1, plot_dependency_graph=True, + block_allocation=True, ) as exe: cloudpickle_register(ind=1) future_1 = exe.submit(add_function, 1, parameter_2=2) @@ -65,6 +66,7 @@ def test_executor_dependency_plot_filename(self): graph_file = os.path.join(os.path.dirname(__file__), "test.png") with SingleNodeExecutor( max_cores=1, + block_allocation=False, plot_dependency_graph=False, plot_dependency_graph_filename=graph_file, ) as exe: @@ -81,6 +83,7 @@ def test_many_to_one_plot(self): parameter = 1 with SingleNodeExecutor( max_cores=2, + block_allocation=False, plot_dependency_graph=True, ) as exe: cloudpickle_register(ind=1) @@ -129,6 +132,7 @@ class TestSlurmAllocationExecutorWithDependencies(unittest.TestCase): def test_executor_dependency_plot(self): with SlurmAllocationExecutor( max_cores=1, + block_allocation=True, plot_dependency_graph=True, ) as exe: cloudpickle_register(ind=1) @@ -152,6 +156,7 @@ def test_many_to_one_plot(self): parameter = 1 with SlurmAllocationExecutor( max_cores=2, + block_allocation=False, plot_dependency_graph=True, ) as exe: cloudpickle_register(ind=1) diff --git a/tests/test_plot_dependency_flux.py b/tests/test_plot_dependency_flux.py index 96249c9e..5b4586e9 100644 --- a/tests/test_plot_dependency_flux.py +++ b/tests/test_plot_dependency_flux.py @@ -49,6 +49,7 @@ def test_executor_dependency_plot(self): with FluxAllocationExecutor( max_cores=1, plot_dependency_graph=True, + block_allocation=False, ) as exe: cloudpickle_register(ind=1) future_1 = exe.submit(add_function, 1, parameter_2=2) @@ -72,6 +73,7 @@ def test_many_to_one_plot(self): with FluxAllocationExecutor( max_cores=2, plot_dependency_graph=True, + block_allocation=True, ) as exe: cloudpickle_register(ind=1) future_lst = exe.submit( From b7d5a8c203553b357ddd9c025d63227721e1740f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 1 Feb 2025 19:52:29 +0100 Subject: [PATCH 21/28] fix --- tests/test_plot_dependency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_plot_dependency.py b/tests/test_plot_dependency.py index aaf64344..0a62628b 100644 --- a/tests/test_plot_dependency.py +++ b/tests/test_plot_dependency.py @@ -136,7 +136,7 @@ class TestSlurmAllocationExecutorWithDependencies(unittest.TestCase): def test_executor_dependency_plot(self): with SlurmAllocationExecutor( max_cores=1, - block_allocation=True, + block_allocation=False, plot_dependency_graph=True, ) as exe: cloudpickle_register(ind=1) From a0bfc9f6a637e7b4c9031220fa3b2556abbd4a23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 1 Feb 2025 19:59:37 +0100 Subject: [PATCH 22/28] flux fix --- .github/workflows/unittest-flux-mpich.yml | 2 +- .github/workflows/unittest-flux-openmpi.yml | 2 +- tests/test_plot_dependency_flux.py | 5 +---- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/unittest-flux-mpich.yml b/.github/workflows/unittest-flux-mpich.yml index 7d2b5b76..79ff685a 100644 --- a/.github/workflows/unittest-flux-mpich.yml +++ b/.github/workflows/unittest-flux-mpich.yml @@ -34,4 +34,4 @@ jobs: timeout-minutes: 5 run: > flux start - python -m unittest tests/test_flux_executor.py tests/test_executor_backend_flux.py tests/test_cache_executor_pysqa_flux.py; + python -m unittest tests/test_flux_executor.py tests/test_executor_backend_flux.py tests/test_cache_executor_pysqa_flux.py tests/test_plot_dependency_flux.py; diff --git a/.github/workflows/unittest-flux-openmpi.yml b/.github/workflows/unittest-flux-openmpi.yml index c954553e..1cb61314 100644 --- a/.github/workflows/unittest-flux-openmpi.yml +++ b/.github/workflows/unittest-flux-openmpi.yml @@ -34,7 +34,7 @@ jobs: timeout-minutes: 5 run: > flux start - coverage run -a --omit="executorlib/_version.py,tests/*" -m unittest tests/test_flux_executor.py tests/test_executor_backend_flux.py tests/test_cache_executor_pysqa_flux.py; + coverage run -a --omit="executorlib/_version.py,tests/*" -m unittest tests/test_flux_executor.py tests/test_executor_backend_flux.py tests/test_cache_executor_pysqa_flux.py tests/test_plot_dependency_flux.py; coverage xml env: PYMPIPOOL_PMIX: "pmix" diff --git a/tests/test_plot_dependency_flux.py b/tests/test_plot_dependency_flux.py index 5b4586e9..4bbe1593 100644 --- a/tests/test_plot_dependency_flux.py +++ b/tests/test_plot_dependency_flux.py @@ -12,10 +12,7 @@ import flux.job from executorlib.interactive.flux import FluxPythonSpawner - skip_flux_test = "FLUX_URI" not in os.environ - pmi = os.environ.get("PYMPIPOOL_PMIX", None) - - skip_graphviz_flux_test = False + skip_graphviz_flux_test = "FLUX_URI" not in os.environ except ImportError: skip_graphviz_flux_test = True From abeac75eb0c2e0a50e2fd33e70716c3e901bfbec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 2 Feb 2025 14:14:07 +0100 Subject: [PATCH 23/28] Rename interfaces --- executorlib/__init__.py | 16 ++++++++-------- executorlib/interfaces/flux.py | 10 +++++----- executorlib/interfaces/slurm.py | 12 ++++++------ notebooks/3-hpc-allocation.ipynb | 18 +++++++++--------- tests/benchmark/llh.py | 4 ++-- tests/test_cache_executor_pysqa_flux.py | 4 ++-- tests/test_executor_backend_flux.py | 20 ++++++++++---------- tests/test_executor_backend_mpi.py | 4 ++-- tests/test_plot_dependency.py | 12 ++++++------ tests/test_plot_dependency_flux.py | 10 +++++----- 10 files changed, 55 insertions(+), 55 deletions(-) diff --git a/executorlib/__init__.py b/executorlib/__init__.py index 20b6cfa2..a95649e8 100644 --- a/executorlib/__init__.py +++ b/executorlib/__init__.py @@ -1,19 +1,19 @@ from executorlib._version import get_versions as _get_versions from executorlib.interfaces.flux import ( - FluxAllocationExecutor, - FluxSubmissionExecutor, + FluxJobExecutor, + FluxClusterExecutor, ) from executorlib.interfaces.single import SingleNodeExecutor from executorlib.interfaces.slurm import ( - SlurmAllocationExecutor, - SlurmSubmissionExecutor, + SlurmJobExecutor, + SlurmClusterExecutor, ) __version__ = _get_versions()["version"] __all__: list = [ - "FluxAllocationExecutor", - "FluxSubmissionExecutor", + "FluxJobExecutor", + "FluxClusterExecutor", "SingleNodeExecutor", - "SlurmAllocationExecutor", - "SlurmSubmissionExecutor", + "SlurmJobExecutor", + "SlurmClusterExecutor", ] diff --git a/executorlib/interfaces/flux.py b/executorlib/interfaces/flux.py index 2aadc3e2..187a4921 100644 --- a/executorlib/interfaces/flux.py +++ b/executorlib/interfaces/flux.py @@ -24,7 +24,7 @@ pass -class FluxAllocationExecutor: +class FluxJobExecutor: """ The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or preferable the flux framework for distributing python functions within a given resource allocation. In contrast to @@ -71,7 +71,7 @@ class FluxAllocationExecutor: Examples: ``` >>> import numpy as np - >>> from executorlib.interfaces.flux import FluxAllocationExecutor + >>> from executorlib.interfaces.flux import FluxJobExecutor >>> >>> def calc(i, j, k): >>> from mpi4py import MPI @@ -82,7 +82,7 @@ class FluxAllocationExecutor: >>> def init_k(): >>> return {"k": 3} >>> - >>> with FluxAllocationExecutor(cores=2, init_function=init_k) as p: + >>> with FluxJobExecutor(cores=2, init_function=init_k) as p: >>> fs = p.submit(calc, 2, j=4) >>> print(fs.result()) [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] @@ -225,7 +225,7 @@ def __new__( ) -class FluxSubmissionExecutor: +class FluxClusterExecutor: """ The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or preferable the flux framework for distributing python functions within a given resource allocation. In contrast to @@ -269,7 +269,7 @@ class FluxSubmissionExecutor: Examples: ``` >>> import numpy as np - >>> from executorlib.interfaces.flux import FluxSubmissionExecutor + >>> from executorlib.interfaces.flux import FluxClusterExecutor >>> >>> def calc(i, j, k): >>> from mpi4py import MPI diff --git a/executorlib/interfaces/slurm.py b/executorlib/interfaces/slurm.py index c1132866..426fba36 100644 --- a/executorlib/interfaces/slurm.py +++ b/executorlib/interfaces/slurm.py @@ -17,7 +17,7 @@ ) -class SlurmSubmissionExecutor: +class SlurmClusterExecutor: """ The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or preferable the flux framework for distributing python functions within a given resource allocation. In contrast to @@ -61,7 +61,7 @@ class SlurmSubmissionExecutor: Examples: ``` >>> import numpy as np - >>> from executorlib.interfaces.slurm import SlurmSubmissionExecutor + >>> from executorlib.interfaces.slurm import SlurmClusterExecutor >>> >>> def calc(i, j, k): >>> from mpi4py import MPI @@ -72,7 +72,7 @@ class SlurmSubmissionExecutor: >>> def init_k(): >>> return {"k": 3} >>> - >>> with SlurmSubmissionExecutor(cores=2, init_function=init_k) as p: + >>> with SlurmClusterExecutor(cores=2, init_function=init_k) as p: >>> fs = p.submit(calc, 2, j=4) >>> print(fs.result()) [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] @@ -205,7 +205,7 @@ def __new__( ) -class SlurmAllocationExecutor: +class SlurmJobExecutor: """ The executorlib.Executor leverages either the message passing interface (MPI), the SLURM workload manager or preferable the flux framework for distributing python functions within a given resource allocation. In contrast to @@ -248,7 +248,7 @@ class SlurmAllocationExecutor: Examples: ``` >>> import numpy as np - >>> from executorlib.interfaces.slurm import SlurmAllocationExecutor + >>> from executorlib.interfaces.slurm import SlurmJobExecutor >>> >>> def calc(i, j, k): >>> from mpi4py import MPI @@ -259,7 +259,7 @@ class SlurmAllocationExecutor: >>> def init_k(): >>> return {"k": 3} >>> - >>> with SlurmAllocationExecutor(cores=2, init_function=init_k) as p: + >>> with SlurmJobExecutor(cores=2, init_function=init_k) as p: >>> fs = p.submit(calc, 2, j=4) >>> print(fs.result()) [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] diff --git a/notebooks/3-hpc-allocation.ipynb b/notebooks/3-hpc-allocation.ipynb index 8375d2fd..26cb60fe 100644 --- a/notebooks/3-hpc-allocation.ipynb +++ b/notebooks/3-hpc-allocation.ipynb @@ -31,7 +31,7 @@ "id": "133b751f-0925-4d11-99f0-3f8dd9360b54", "metadata": {}, "outputs": [], - "source": "from executorlib import SlurmAllocationExecutor" + "source": "from executorlib import SlurmJobExecutor" }, { "cell_type": "markdown", @@ -109,9 +109,9 @@ } ], "source": [ - "from executorlib import FluxAllocationExecutor\n", + "from executorlib import FluxJobExecutor\n", "\n", - "with FluxAllocationExecutor(flux_executor_pmi_mode=\"pmix\") as exe:\n", + "with FluxJobExecutor(flux_executor_pmi_mode=\"pmix\") as exe:\n", " fs = exe.submit(calc_mpi, 3, resource_dict={\"cores\": 2})\n", " print(fs.result())" ] @@ -162,7 +162,7 @@ } ], "source": [ - "with FluxAllocationExecutor(\n", + "with FluxJobExecutor(\n", " flux_executor_pmi_mode=\"pmix\",\n", " max_workers=2,\n", " init_function=init_function,\n", @@ -217,7 +217,7 @@ } ], "source": [ - "with FluxAllocationExecutor(flux_executor_pmi_mode=\"pmix\") as exe:\n", + "with FluxJobExecutor(flux_executor_pmi_mode=\"pmix\") as exe:\n", " future = 0\n", " for i in range(1, 4):\n", " future = exe.submit(add_funct, i, future)\n", @@ -248,7 +248,7 @@ } ], "source": [ - "with FluxAllocationExecutor(\n", + "with FluxJobExecutor(\n", " flux_executor_pmi_mode=\"pmix\", cache_directory=\"./cache\"\n", ") as exe:\n", " future_lst = [exe.submit(sum, [i, i]) for i in range(1, 4)]\n", @@ -299,9 +299,9 @@ "outputs": [], "source": [ "def calc_nested():\n", - " from executorlib import FluxAllocationExecutor\n", + " from executorlib import FluxJobExecutor\n", "\n", - " with FluxAllocationExecutor(flux_executor_pmi_mode=\"pmix\") as exe:\n", + " with FluxJobExecutor(flux_executor_pmi_mode=\"pmix\") as exe:\n", " fs = exe.submit(sum, [1, 1])\n", " return fs.result()" ] @@ -321,7 +321,7 @@ } ], "source": [ - "with FluxAllocationExecutor(\n", + "with FluxJobExecutor(\n", " flux_executor_pmi_mode=\"pmix\", flux_executor_nesting=True\n", ") as exe:\n", " fs = exe.submit(calc_nested)\n", diff --git a/tests/benchmark/llh.py b/tests/benchmark/llh.py index e275aba8..d1488d21 100644 --- a/tests/benchmark/llh.py +++ b/tests/benchmark/llh.py @@ -64,10 +64,10 @@ def run_static(mean=0.1, sigma=1.1, runs=32): block_allocation=False, ) elif run_mode == "flux": - from executorlib import FluxAllocationExecutor + from executorlib import FluxJobExecutor run_with_executor( - executor=FluxAllocationExecutor, + executor=FluxJobExecutor, mean=0.1, sigma=1.1, runs=32, diff --git a/tests/test_cache_executor_pysqa_flux.py b/tests/test_cache_executor_pysqa_flux.py index 3e8f7c87..4a35c488 100644 --- a/tests/test_cache_executor_pysqa_flux.py +++ b/tests/test_cache_executor_pysqa_flux.py @@ -3,7 +3,7 @@ import unittest import shutil -from executorlib import FluxSubmissionExecutor +from executorlib import FluxClusterExecutor from executorlib.standalone.serialize import cloudpickle_register try: @@ -32,7 +32,7 @@ def mpi_funct(i): ) class TestCacheExecutorPysqa(unittest.TestCase): def test_executor(self): - with FluxSubmissionExecutor( + with FluxClusterExecutor( resource_dict={"cores": 2, "cwd": "cache"}, block_allocation=False, cache_directory="cache", diff --git a/tests/test_executor_backend_flux.py b/tests/test_executor_backend_flux.py index dd7fab44..c11fcb3e 100644 --- a/tests/test_executor_backend_flux.py +++ b/tests/test_executor_backend_flux.py @@ -3,7 +3,7 @@ import numpy as np -from executorlib import FluxAllocationExecutor +from executorlib import FluxJobExecutor try: @@ -44,7 +44,7 @@ def setUp(self): self.executor = flux.job.FluxExecutor() def test_flux_executor_serial(self): - with FluxAllocationExecutor( + with FluxJobExecutor( max_cores=2, flux_executor=self.executor, block_allocation=True, @@ -57,7 +57,7 @@ def test_flux_executor_serial(self): self.assertTrue(fs_2.done()) def test_flux_executor_serial_no_depencies(self): - with FluxAllocationExecutor( + with FluxJobExecutor( max_cores=2, flux_executor=self.executor, block_allocation=True, @@ -71,7 +71,7 @@ def test_flux_executor_serial_no_depencies(self): self.assertTrue(fs_2.done()) def test_flux_executor_threads(self): - with FluxAllocationExecutor( + with FluxJobExecutor( max_cores=1, resource_dict={"threads_per_core": 2}, flux_executor=self.executor, @@ -85,7 +85,7 @@ def test_flux_executor_threads(self): self.assertTrue(fs_2.done()) def test_flux_executor_parallel(self): - with FluxAllocationExecutor( + with FluxJobExecutor( max_cores=2, resource_dict={"cores": 2}, flux_executor=self.executor, @@ -97,7 +97,7 @@ def test_flux_executor_parallel(self): self.assertTrue(fs_1.done()) def test_single_task(self): - with FluxAllocationExecutor( + with FluxJobExecutor( max_cores=2, resource_dict={"cores": 2}, flux_executor=self.executor, @@ -115,7 +115,7 @@ def test_output_files_cwd(self): os.makedirs(dirname, exist_ok=True) file_stdout = os.path.join(dirname, "flux.out") file_stderr = os.path.join(dirname, "flux.err") - with FluxAllocationExecutor( + with FluxJobExecutor( max_cores=1, resource_dict={"cores": 1, "cwd": dirname}, flux_executor=self.executor, @@ -135,7 +135,7 @@ def test_output_files_cwd(self): def test_output_files_abs(self): file_stdout = os.path.abspath("flux.out") file_stderr = os.path.abspath("flux.err") - with FluxAllocationExecutor( + with FluxJobExecutor( max_cores=1, resource_dict={"cores": 1}, flux_executor=self.executor, @@ -153,7 +153,7 @@ def test_output_files_abs(self): os.remove(file_stderr) def test_internal_memory(self): - with FluxAllocationExecutor( + with FluxJobExecutor( max_cores=1, resource_dict={"cores": 1}, init_function=set_global, @@ -167,7 +167,7 @@ def test_internal_memory(self): def test_validate_max_workers(self): with self.assertRaises(ValueError): - FluxAllocationExecutor( + FluxJobExecutor( max_workers=10, resource_dict={"cores": 10, "threads_per_core": 10}, flux_executor=self.executor, diff --git a/tests/test_executor_backend_mpi.py b/tests/test_executor_backend_mpi.py index fa39d619..ca7abe39 100644 --- a/tests/test_executor_backend_mpi.py +++ b/tests/test_executor_backend_mpi.py @@ -4,7 +4,7 @@ import time import unittest -from executorlib import SingleNodeExecutor, SlurmAllocationExecutor +from executorlib import SingleNodeExecutor, SlurmJobExecutor from executorlib.standalone.serialize import cloudpickle_register @@ -131,7 +131,7 @@ def test_validate_max_workers(self): os.environ["SLURM_NTASKS"] = "6" os.environ["SLURM_CPUS_PER_TASK"] = "4" with self.assertRaises(ValueError): - SlurmAllocationExecutor( + SlurmJobExecutor( max_workers=10, resource_dict={"cores": 10, "threads_per_core": 10}, block_allocation=True, diff --git a/tests/test_plot_dependency.py b/tests/test_plot_dependency.py index 0a62628b..3b79f960 100644 --- a/tests/test_plot_dependency.py +++ b/tests/test_plot_dependency.py @@ -4,8 +4,8 @@ from executorlib import ( SingleNodeExecutor, - SlurmAllocationExecutor, - SlurmSubmissionExecutor, + SlurmJobExecutor, + SlurmClusterExecutor, ) from executorlib.standalone.plot import generate_nodes_and_edges from executorlib.standalone.serialize import cloudpickle_register @@ -134,7 +134,7 @@ def test_many_to_one_plot(self): ) class TestSlurmAllocationExecutorWithDependencies(unittest.TestCase): def test_executor_dependency_plot(self): - with SlurmAllocationExecutor( + with SlurmJobExecutor( max_cores=1, block_allocation=False, plot_dependency_graph=True, @@ -158,7 +158,7 @@ def test_executor_dependency_plot(self): def test_many_to_one_plot(self): length = 5 parameter = 1 - with SlurmAllocationExecutor( + with SlurmJobExecutor( max_cores=2, block_allocation=False, plot_dependency_graph=True, @@ -207,7 +207,7 @@ def test_many_to_one_plot(self): ) class TestSlurmSubmissionExecutorWithDependencies(unittest.TestCase): def test_executor_dependency_plot(self): - with SlurmSubmissionExecutor( + with SlurmClusterExecutor( plot_dependency_graph=True, ) as exe: cloudpickle_register(ind=1) @@ -229,7 +229,7 @@ def test_executor_dependency_plot(self): def test_many_to_one_plot(self): length = 5 parameter = 1 - with SlurmSubmissionExecutor( + with SlurmClusterExecutor( plot_dependency_graph=True, ) as exe: cloudpickle_register(ind=1) diff --git a/tests/test_plot_dependency_flux.py b/tests/test_plot_dependency_flux.py index 4bbe1593..75c06f2b 100644 --- a/tests/test_plot_dependency_flux.py +++ b/tests/test_plot_dependency_flux.py @@ -2,7 +2,7 @@ import unittest from time import sleep -from executorlib import FluxAllocationExecutor, FluxSubmissionExecutor +from executorlib import FluxJobExecutor, FluxClusterExecutor from executorlib.standalone.plot import generate_nodes_and_edges from executorlib.standalone.serialize import cloudpickle_register @@ -43,7 +43,7 @@ def merge(lst): ) class TestFluxAllocationExecutorWithDependencies(unittest.TestCase): def test_executor_dependency_plot(self): - with FluxAllocationExecutor( + with FluxJobExecutor( max_cores=1, plot_dependency_graph=True, block_allocation=False, @@ -67,7 +67,7 @@ def test_executor_dependency_plot(self): def test_many_to_one_plot(self): length = 5 parameter = 1 - with FluxAllocationExecutor( + with FluxJobExecutor( max_cores=2, plot_dependency_graph=True, block_allocation=True, @@ -116,7 +116,7 @@ def test_many_to_one_plot(self): ) class TestFluxSubmissionExecutorWithDependencies(unittest.TestCase): def test_executor_dependency_plot(self): - with FluxSubmissionExecutor( + with FluxClusterExecutor( plot_dependency_graph=True, ) as exe: cloudpickle_register(ind=1) @@ -138,7 +138,7 @@ def test_executor_dependency_plot(self): def test_many_to_one_plot(self): length = 5 parameter = 1 - with FluxSubmissionExecutor( + with FluxClusterExecutor( plot_dependency_graph=True, ) as exe: cloudpickle_register(ind=1) From 814c0d2e81ba495dca7c79d7ed7d00b399267ab1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 2 Feb 2025 13:14:18 +0000 Subject: [PATCH 24/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- executorlib/__init__.py | 4 ++-- notebooks/3-hpc-allocation.ipynb | 36 +++++++++++++++----------------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/executorlib/__init__.py b/executorlib/__init__.py index a95649e8..16180430 100644 --- a/executorlib/__init__.py +++ b/executorlib/__init__.py @@ -1,12 +1,12 @@ from executorlib._version import get_versions as _get_versions from executorlib.interfaces.flux import ( - FluxJobExecutor, FluxClusterExecutor, + FluxJobExecutor, ) from executorlib.interfaces.single import SingleNodeExecutor from executorlib.interfaces.slurm import ( - SlurmJobExecutor, SlurmClusterExecutor, + SlurmJobExecutor, ) __version__ = _get_versions()["version"] diff --git a/notebooks/3-hpc-allocation.ipynb b/notebooks/3-hpc-allocation.ipynb index 26cb60fe..fc0f84c6 100644 --- a/notebooks/3-hpc-allocation.ipynb +++ b/notebooks/3-hpc-allocation.ipynb @@ -31,7 +31,9 @@ "id": "133b751f-0925-4d11-99f0-3f8dd9360b54", "metadata": {}, "outputs": [], - "source": "from executorlib import SlurmJobExecutor" + "source": [ + "from executorlib import SlurmJobExecutor" + ] }, { "cell_type": "markdown", @@ -248,9 +250,7 @@ } ], "source": [ - "with FluxJobExecutor(\n", - " flux_executor_pmi_mode=\"pmix\", cache_directory=\"./cache\"\n", - ") as exe:\n", + "with FluxJobExecutor(flux_executor_pmi_mode=\"pmix\", cache_directory=\"./cache\") as exe:\n", " future_lst = [exe.submit(sum, [i, i]) for i in range(1, 4)]\n", " print([f.result() for f in future_lst])" ] @@ -321,9 +321,7 @@ } ], "source": [ - "with FluxJobExecutor(\n", - " flux_executor_pmi_mode=\"pmix\", flux_executor_nesting=True\n", - ") as exe:\n", + "with FluxJobExecutor(flux_executor_pmi_mode=\"pmix\", flux_executor_nesting=True) as exe:\n", " fs = exe.submit(calc_nested)\n", " print(fs.result())" ] @@ -377,18 +375,18 @@ "output_type": "stream", "text": [ " JOBID USER NAME ST NTASKS NNODES TIME INFO\n", - "\u001B[01;32m ƒDqBpVYK jan python CD 1 1 0.695s fedora\n", - "\u001B[0;0m\u001B[01;32m ƒDxdEtYf jan python CD 1 1 0.225s fedora\n", - "\u001B[0;0m\u001B[01;32m ƒDVahzPq jan python CD 1 1 0.254s fedora\n", - "\u001B[0;0m\u001B[01;32m ƒDSsZJXH jan python CD 1 1 0.316s fedora\n", - "\u001B[0;0m\u001B[01;32m ƒDSu3Hod jan python CD 1 1 0.277s fedora\n", - "\u001B[0;0m\u001B[01;32m ƒDFbkmFD jan python CD 1 1 0.247s fedora\n", - "\u001B[0;0m\u001B[01;32m ƒD9eKeas jan python CD 1 1 0.227s fedora\n", - "\u001B[0;0m\u001B[01;32m ƒD3iNXCs jan python CD 1 1 0.224s fedora\n", - "\u001B[0;0m\u001B[01;32m ƒCoZ3P5q jan python CD 1 1 0.261s fedora\n", - "\u001B[0;0m\u001B[01;32m ƒCoXZPoV jan python CD 1 1 0.261s fedora\n", - "\u001B[0;0m\u001B[01;32m ƒCZ1URjd jan python CD 2 1 0.360s fedora\n", - "\u001B[0;0m" + "\u001b[01;32m ƒDqBpVYK jan python CD 1 1 0.695s fedora\n", + "\u001b[0;0m\u001b[01;32m ƒDxdEtYf jan python CD 1 1 0.225s fedora\n", + "\u001b[0;0m\u001b[01;32m ƒDVahzPq jan python CD 1 1 0.254s fedora\n", + "\u001b[0;0m\u001b[01;32m ƒDSsZJXH jan python CD 1 1 0.316s fedora\n", + "\u001b[0;0m\u001b[01;32m ƒDSu3Hod jan python CD 1 1 0.277s fedora\n", + "\u001b[0;0m\u001b[01;32m ƒDFbkmFD jan python CD 1 1 0.247s fedora\n", + "\u001b[0;0m\u001b[01;32m ƒD9eKeas jan python CD 1 1 0.227s fedora\n", + "\u001b[0;0m\u001b[01;32m ƒD3iNXCs jan python CD 1 1 0.224s fedora\n", + "\u001b[0;0m\u001b[01;32m ƒCoZ3P5q jan python CD 1 1 0.261s fedora\n", + "\u001b[0;0m\u001b[01;32m ƒCoXZPoV jan python CD 1 1 0.261s fedora\n", + "\u001b[0;0m\u001b[01;32m ƒCZ1URjd jan python CD 2 1 0.360s fedora\n", + "\u001b[0;0m" ] } ], From acf46939091d244acfb747fd3426e0798995792d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 2 Feb 2025 14:27:08 +0100 Subject: [PATCH 25/28] fix docstrings --- executorlib/interfaces/flux.py | 4 ++-- executorlib/interfaces/single.py | 4 ++-- executorlib/interfaces/slurm.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/executorlib/interfaces/flux.py b/executorlib/interfaces/flux.py index 187a4921..4b1c57d9 100644 --- a/executorlib/interfaces/flux.py +++ b/executorlib/interfaces/flux.py @@ -82,7 +82,7 @@ class FluxJobExecutor: >>> def init_k(): >>> return {"k": 3} >>> - >>> with FluxJobExecutor(cores=2, init_function=init_k) as p: + >>> with FluxJobExecutor(max_workers=2, init_function=init_k) as p: >>> fs = p.submit(calc, 2, j=4) >>> print(fs.result()) [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] @@ -280,7 +280,7 @@ class FluxClusterExecutor: >>> def init_k(): >>> return {"k": 3} >>> - >>> with Executor(cores=2, init_function=init_k) as p: + >>> with FluxClusterExecutor(max_workers=2, init_function=init_k) as p: >>> fs = p.submit(calc, 2, j=4) >>> print(fs.result()) [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] diff --git a/executorlib/interfaces/single.py b/executorlib/interfaces/single.py index 14392451..9f5e1ba7 100644 --- a/executorlib/interfaces/single.py +++ b/executorlib/interfaces/single.py @@ -59,7 +59,7 @@ class SingleNodeExecutor: Examples: ``` >>> import numpy as np - >>> from executorlib.interfaces.local import SingleNodeExecutor + >>> from executorlib.interfaces.single import SingleNodeExecutor >>> >>> def calc(i, j, k): >>> from mpi4py import MPI @@ -70,7 +70,7 @@ class SingleNodeExecutor: >>> def init_k(): >>> return {"k": 3} >>> - >>> with SingleNodeExecutor(cores=2, init_function=init_k) as p: + >>> with SingleNodeExecutor(max_workers=2, init_function=init_k) as p: >>> fs = p.submit(calc, 2, j=4) >>> print(fs.result()) [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] diff --git a/executorlib/interfaces/slurm.py b/executorlib/interfaces/slurm.py index 426fba36..e55174a2 100644 --- a/executorlib/interfaces/slurm.py +++ b/executorlib/interfaces/slurm.py @@ -72,7 +72,7 @@ class SlurmClusterExecutor: >>> def init_k(): >>> return {"k": 3} >>> - >>> with SlurmClusterExecutor(cores=2, init_function=init_k) as p: + >>> with SlurmClusterExecutor(max_workers=2, init_function=init_k) as p: >>> fs = p.submit(calc, 2, j=4) >>> print(fs.result()) [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] @@ -259,7 +259,7 @@ class SlurmJobExecutor: >>> def init_k(): >>> return {"k": 3} >>> - >>> with SlurmJobExecutor(cores=2, init_function=init_k) as p: + >>> with SlurmJobExecutor(max_workers=2, init_function=init_k) as p: >>> fs = p.submit(calc, 2, j=4) >>> print(fs.result()) [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] From 0fe7dfe4a7574c0923562f3d254dfe86e0ef3f87 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Mon, 3 Feb 2025 15:34:56 +0100 Subject: [PATCH 26/28] add docstrings to create_*_executor() function --- executorlib/interfaces/flux.py | 38 ++++++++++++++++++++++++++++++ executorlib/interfaces/single.py | 40 +++++++++++++++++++++++++++++--- executorlib/interfaces/slurm.py | 34 +++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 3 deletions(-) diff --git a/executorlib/interfaces/flux.py b/executorlib/interfaces/flux.py index 4b1c57d9..1aefb5f9 100644 --- a/executorlib/interfaces/flux.py +++ b/executorlib/interfaces/flux.py @@ -430,6 +430,44 @@ def create_flux_executor( block_allocation: bool = False, init_function: Optional[Callable] = None, ) -> Union[InteractiveStepExecutor, InteractiveExecutor]: + """ + Create a flux executor + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the + number of cores which can be used in parallel - just like the max_cores parameter. Using + max_cores is recommended, as computers have a limited number of compute cores. + max_cores (int): defines the number cores which can be used in parallel + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI + and SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM + only) + flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux + flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only) + flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function. + flux_log_files (bool, optional): Write flux stdout and stderr files. Defaults to False. + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same + resource requirements, executorlib supports block allocation. In this case all + resources have to be defined on the executor, rather than during the submission + of the individual function. + init_function (None): optional function to preset arguments for functions which are submitted later + + Returns: + InteractiveStepExecutor/ InteractiveExecutor + """ check_init_function(block_allocation=block_allocation, init_function=init_function) check_pmi(backend="flux_allocation", pmi=flux_executor_pmi_mode) cores_per_worker = resource_dict.get("cores", 1) diff --git a/executorlib/interfaces/single.py b/executorlib/interfaces/single.py index 9f5e1ba7..e8a0019a 100644 --- a/executorlib/interfaces/single.py +++ b/executorlib/interfaces/single.py @@ -165,7 +165,7 @@ def __new__( ) if not disable_dependencies: return ExecutorWithDependencies( - executor=create_local_executor( + executor=create_single_node_executor( max_workers=max_workers, cache_directory=cache_directory, max_cores=max_cores, @@ -182,7 +182,7 @@ def __new__( else: check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) check_refresh_rate(refresh_rate=refresh_rate) - return create_local_executor( + return create_single_node_executor( max_workers=max_workers, cache_directory=cache_directory, max_cores=max_cores, @@ -193,7 +193,7 @@ def __new__( ) -def create_local_executor( +def create_single_node_executor( max_workers: Optional[int] = None, max_cores: Optional[int] = None, cache_directory: Optional[str] = None, @@ -202,6 +202,40 @@ def create_local_executor( block_allocation: bool = False, init_function: Optional[Callable] = None, ) -> Union[InteractiveStepExecutor, InteractiveExecutor]: + """ + Create a single node executor + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the + number of cores which can be used in parallel - just like the max_cores parameter. Using + max_cores is recommended, as computers have a limited number of compute cores. + max_cores (int): defines the number cores which can be used in parallel + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI + and SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM + only) + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same + resource requirements, executorlib supports block allocation. In this case all + resources have to be defined on the executor, rather than during the submission + of the individual function. + init_function (None): optional function to preset arguments for functions which are submitted later + + Returns: + InteractiveStepExecutor/ InteractiveExecutor + """ check_init_function(block_allocation=block_allocation, init_function=init_function) cores_per_worker = resource_dict.get("cores", 1) resource_dict["cache_directory"] = cache_directory diff --git a/executorlib/interfaces/slurm.py b/executorlib/interfaces/slurm.py index e55174a2..26c2a95c 100644 --- a/executorlib/interfaces/slurm.py +++ b/executorlib/interfaces/slurm.py @@ -391,6 +391,40 @@ def create_slurm_executor( block_allocation: bool = False, init_function: Optional[Callable] = None, ) -> Union[InteractiveStepExecutor, InteractiveExecutor]: + """ + Create a SLURM executor + + Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the + number of cores which can be used in parallel - just like the max_cores parameter. Using + max_cores is recommended, as computers have a limited number of compute cores. + max_cores (int): defines the number cores which can be used in parallel + cache_directory (str, optional): The directory to store cache files. Defaults to "cache". + resource_dict (dict): A dictionary of resources required by the task. With the following keys: + - cores (int): number of MPI cores to be used for each function call + - threads_per_core (int): number of OpenMP threads to be used for each function call + - gpus_per_core (int): number of GPUs per worker - defaults to 0 + - cwd (str/None): current working directory where the parallel python task is executed + - openmpi_oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI + and SLURM only) - default False + - slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM + only) + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + block_allocation (boolean): To accelerate the submission of a series of python functions with the same + resource requirements, executorlib supports block allocation. In this case all + resources have to be defined on the executor, rather than during the submission + of the individual function. + init_function (None): optional function to preset arguments for functions which are submitted later + + Returns: + InteractiveStepExecutor/ InteractiveExecutor + """ check_init_function(block_allocation=block_allocation, init_function=init_function) cores_per_worker = resource_dict.get("cores", 1) resource_dict["cache_directory"] = cache_directory From 87b3380f7d9df6697f3141b26b9d550f360bf6b0 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Mon, 3 Feb 2025 16:05:39 +0100 Subject: [PATCH 27/28] fix test --- tests/test_dependencies_executor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_dependencies_executor.py b/tests/test_dependencies_executor.py index 690bf397..aaf21a7d 100644 --- a/tests/test_dependencies_executor.py +++ b/tests/test_dependencies_executor.py @@ -4,7 +4,7 @@ from queue import Queue from executorlib import SingleNodeExecutor -from executorlib.interfaces.single import create_local_executor +from executorlib.interfaces.single import create_single_node_executor from executorlib.interactive.shared import execute_tasks_with_dependencies from executorlib.standalone.serialize import cloudpickle_register from executorlib.standalone.thread import RaisingThread @@ -73,7 +73,7 @@ def test_dependency_steps(self): "resource_dict": {"cores": 1}, } ) - executor = create_local_executor( + executor = create_single_node_executor( max_workers=1, max_cores=2, resource_dict={ From 0cf4b6a7500723834c616723f36396535baa86c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Mon, 3 Feb 2025 20:24:45 +0100 Subject: [PATCH 28/28] merge --- executorlib/interfaces/flux.py | 12 +++++++----- executorlib/interfaces/single.py | 12 +++++++----- executorlib/interfaces/slurm.py | 6 ++++-- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/executorlib/interfaces/flux.py b/executorlib/interfaces/flux.py index 1aefb5f9..30fb839d 100644 --- a/executorlib/interfaces/flux.py +++ b/executorlib/interfaces/flux.py @@ -421,7 +421,7 @@ def create_flux_executor( max_workers: Optional[int] = None, max_cores: Optional[int] = None, cache_directory: Optional[str] = None, - resource_dict: dict = {}, + resource_dict: Optional[dict] = None, flux_executor=None, flux_executor_pmi_mode: Optional[str] = None, flux_executor_nesting: bool = False, @@ -468,18 +468,20 @@ def create_flux_executor( Returns: InteractiveStepExecutor/ InteractiveExecutor """ - check_init_function(block_allocation=block_allocation, init_function=init_function) - check_pmi(backend="flux_allocation", pmi=flux_executor_pmi_mode) + if resource_dict is None: + resource_dict = {} cores_per_worker = resource_dict.get("cores", 1) resource_dict["cache_directory"] = cache_directory resource_dict["hostname_localhost"] = hostname_localhost + check_init_function(block_allocation=block_allocation, init_function=init_function) + check_pmi(backend="flux_allocation", pmi=flux_executor_pmi_mode) check_oversubscribe(oversubscribe=resource_dict.get("openmpi_oversubscribe", False)) check_command_line_argument_lst( command_line_argument_lst=resource_dict.get("slurm_cmd_args", []) ) - if "openmpi_oversubscribe" in resource_dict.keys(): + if "openmpi_oversubscribe" in resource_dict: del resource_dict["openmpi_oversubscribe"] - if "slurm_cmd_args" in resource_dict.keys(): + if "slurm_cmd_args" in resource_dict: del resource_dict["slurm_cmd_args"] resource_dict["flux_executor"] = flux_executor resource_dict["flux_executor_pmi_mode"] = flux_executor_pmi_mode diff --git a/executorlib/interfaces/single.py b/executorlib/interfaces/single.py index e8a0019a..ec594c44 100644 --- a/executorlib/interfaces/single.py +++ b/executorlib/interfaces/single.py @@ -197,7 +197,7 @@ def create_single_node_executor( max_workers: Optional[int] = None, max_cores: Optional[int] = None, cache_directory: Optional[str] = None, - resource_dict: dict = {}, + resource_dict: Optional[dict] = None, hostname_localhost: Optional[bool] = None, block_allocation: bool = False, init_function: Optional[Callable] = None, @@ -236,20 +236,22 @@ def create_single_node_executor( Returns: InteractiveStepExecutor/ InteractiveExecutor """ - check_init_function(block_allocation=block_allocation, init_function=init_function) + if resource_dict is None: + resource_dict = {} cores_per_worker = resource_dict.get("cores", 1) resource_dict["cache_directory"] = cache_directory resource_dict["hostname_localhost"] = hostname_localhost + check_init_function(block_allocation=block_allocation, init_function=init_function) check_gpus_per_worker(gpus_per_worker=resource_dict.get("gpus_per_core", 0)) check_command_line_argument_lst( command_line_argument_lst=resource_dict.get("slurm_cmd_args", []) ) - if "threads_per_core" in resource_dict.keys(): + if "threads_per_core" in resource_dict: del resource_dict["threads_per_core"] - if "gpus_per_core" in resource_dict.keys(): + if "gpus_per_core" in resource_dict: del resource_dict["gpus_per_core"] - if "slurm_cmd_args" in resource_dict.keys(): + if "slurm_cmd_args" in resource_dict: del resource_dict["slurm_cmd_args"] if block_allocation: resource_dict["init_function"] = init_function diff --git a/executorlib/interfaces/slurm.py b/executorlib/interfaces/slurm.py index 26c2a95c..308c1f31 100644 --- a/executorlib/interfaces/slurm.py +++ b/executorlib/interfaces/slurm.py @@ -386,7 +386,7 @@ def create_slurm_executor( max_workers: Optional[int] = None, max_cores: Optional[int] = None, cache_directory: Optional[str] = None, - resource_dict: dict = {}, + resource_dict: Optional[dict] = None, hostname_localhost: Optional[bool] = None, block_allocation: bool = False, init_function: Optional[Callable] = None, @@ -425,10 +425,12 @@ def create_slurm_executor( Returns: InteractiveStepExecutor/ InteractiveExecutor """ - check_init_function(block_allocation=block_allocation, init_function=init_function) + if resource_dict is None: + resource_dict = {} cores_per_worker = resource_dict.get("cores", 1) resource_dict["cache_directory"] = cache_directory resource_dict["hostname_localhost"] = hostname_localhost + check_init_function(block_allocation=block_allocation, init_function=init_function) if block_allocation: resource_dict["init_function"] = init_function max_workers = validate_number_of_cores(