Remove conda support in executorlib

jan-janssen · jan-janssen · commit 76291be87d3a · 2024-10-01T09:15:13.000+02:00
Users who want conda support have to first decorate the function and then submit it with executorlib.
diff --git a/.ci_support/environment-mpich.yml b/.ci_support/environment-mpich.yml
@@ -5,7 +5,6 @@ dependencies:
 - numpy
 - mpich
 - cloudpickle =3.0.0
-- conda_subprocess =0.0.4
 - mpi4py =4.0.0
 - pyzmq =26.2.0
 - h5py =3.11.0
diff --git a/.ci_support/environment-old.yml b/.ci_support/environment-old.yml
@@ -5,7 +5,6 @@ dependencies:
 - numpy
 - openmpi =4.1.4
 - cloudpickle =2.0.0
-- conda_subprocess =0.0.3
 - mpi4py =3.1.4
 - pyzmq =25.0.0
 - h5py =3.6.0
diff --git a/.ci_support/environment-openmpi.yml b/.ci_support/environment-openmpi.yml
@@ -5,7 +5,6 @@ dependencies:
 - numpy
 - openmpi
 - cloudpickle =3.0.0
-- conda_subprocess =0.0.4
 - mpi4py =4.0.0
 - pyzmq =26.2.0
 - h5py =3.11.0
diff --git a/.ci_support/environment-win.yml b/.ci_support/environment-win.yml
@@ -5,7 +5,6 @@ dependencies:
 - numpy
 - msmpi
 - cloudpickle =3.0.0
-- conda_subprocess =0.0.4
 - mpi4py =4.0.0
 - pyzmq =26.2.0
 - h5py =3.11.0
diff --git a/.github/workflows/unittest-flux-mpich.yml b/.github/workflows/unittest-flux-mpich.yml
@@ -27,7 +27,6 @@ jobs:
       shell: bash -l {0}
       timeout-minutes: 5
       run: |
-        conda create -y -n py312 python=3.12.1 executorlib
         pip install . --no-deps --no-build-isolation
         python -m unittest discover tests
     - name: Test Flux
diff --git a/.github/workflows/unittest-flux-openmpi.yml b/.github/workflows/unittest-flux-openmpi.yml
@@ -27,7 +27,6 @@ jobs:
       shell: bash -l {0}
       timeout-minutes: 5
       run: |
-        conda create -y -n py312 python=3.12.1 executorlib
         pip install . --no-deps --no-build-isolation
         coverage run -a --omit="executorlib/_version.py,tests/*" -m unittest discover tests
     - name: Test Flux with OpenMPI
diff --git a/.github/workflows/unittest-mpich.yml b/.github/workflows/unittest-mpich.yml
@@ -42,7 +42,6 @@ jobs:
       timeout-minutes: 5
       run: |
         pip install versioneer[toml]==0.29
-        conda create -y -n py312 python=3.12.1 executorlib
         pip install . --no-deps --no-build-isolation
         cd tests
         python -m unittest discover .
diff --git a/.github/workflows/unittest-openmpi.yml b/.github/workflows/unittest-openmpi.yml
@@ -42,7 +42,6 @@ jobs:
       timeout-minutes: 5
       run: |
         pip install versioneer[toml]==0.29
-        conda create -y -n py312 python=3.12.1 executorlib
         pip install . --no-deps --no-build-isolation
         cd tests
         python -m unittest discover .
diff --git a/.github/workflows/unittest-win.yml b/.github/workflows/unittest-win.yml
@@ -32,7 +32,6 @@ jobs:
       timeout-minutes: 5
       run: |
         pip install versioneer[toml]==0.29
-        conda create -y -n py312 python=3.12.1 executorlib
         pip install . --no-deps --no-build-isolation
         cd tests
         python -m unittest discover .
diff --git a/.github/workflows/unittests-old.yml b/.github/workflows/unittests-old.yml
@@ -26,7 +26,6 @@ jobs:
       timeout-minutes: 5
       run: |
         pip install versioneer[toml]==0.29
-        conda create -y -n py312 python=3.12.1 executorlib
         pip install . --no-deps --no-build-isolation
         cd tests
         python -m unittest discover .
diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -25,6 +25,6 @@ sphinx:
 # Optionally build your docs in additional formats such as PDF and ePub
 formats: []
 
-# Install pyiron from conda
+# Install executorlib from conda
 conda:
   environment: .ci_support/environment-docs.yml
diff --git a/executorlib/__init__.py b/executorlib/__init__.py
@@ -51,8 +51,6 @@ class Executor:
         flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux
         flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only)
         flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function.
-        conda_environment_name (str): name of the conda environment to initialize
-        conda_environment_path (str): path of the conda environment to initialize
         hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
                                       context of an HPC cluster this essential to be able to communicate to an
                                       Executor running on a different compute node within the same allocation. And
@@ -105,8 +103,6 @@ def __init__(
         flux_executor=None,
         flux_executor_pmi_mode: Optional[str] = None,
         flux_executor_nesting: bool = False,
-        conda_environment_name: Optional[str] = None,
-        conda_environment_path: Optional[str] = None,
         hostname_localhost: bool = False,
         block_allocation: bool = True,
         init_function: Optional[callable] = None,
@@ -131,8 +127,6 @@ def __new__(
         flux_executor=None,
         flux_executor_pmi_mode: Optional[str] = None,
         flux_executor_nesting: bool = False,
-        conda_environment_name: Optional[str] = None,
-        conda_environment_path: Optional[str] = None,
         hostname_localhost: bool = False,
         block_allocation: bool = True,
         init_function: Optional[callable] = None,
@@ -163,8 +157,6 @@ def __new__(
             flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux
             flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only)
             flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function.
-            conda_environment_name (str): name of the conda environment to initialize
-            conda_environment_path (str): path of the conda environment to initialize
             hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
                                       context of an HPC cluster this essential to be able to communicate to an
                                       Executor running on a different compute node within the same allocation. And
@@ -197,8 +189,6 @@ def __new__(
                 flux_executor=flux_executor,
                 flux_executor_pmi_mode=flux_executor_pmi_mode,
                 flux_executor_nesting=flux_executor_nesting,
-                conda_environment_name=conda_environment_name,
-                conda_environment_path=conda_environment_path,
                 hostname_localhost=hostname_localhost,
                 block_allocation=block_allocation,
                 init_function=init_function,
@@ -221,8 +211,6 @@ def __new__(
                 flux_executor=flux_executor,
                 flux_executor_pmi_mode=flux_executor_pmi_mode,
                 flux_executor_nesting=flux_executor_nesting,
-                conda_environment_name=conda_environment_name,
-                conda_environment_path=conda_environment_path,
                 hostname_localhost=hostname_localhost,
                 block_allocation=block_allocation,
                 init_function=init_function,
diff --git a/executorlib/interactive/__init__.py b/executorlib/interactive/__init__.py
@@ -40,8 +40,6 @@ def create_executor(
     flux_executor=None,
     flux_executor_pmi_mode: Optional[str] = None,
     flux_executor_nesting: bool = False,
-    conda_environment_name: Optional[str] = None,
-    conda_environment_path: Optional[str] = None,
     hostname_localhost: bool = False,
     block_allocation: bool = False,
     init_function: Optional[callable] = None,
@@ -69,8 +67,6 @@ def create_executor(
         flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux
         flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only)
         flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function.
-        conda_environment_name (str): name of the conda environment to initialize
-        conda_environment_path (str): path of the conda environment to initialize
         hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
                                       context of an HPC cluster this essential to be able to communicate to an Executor
                                       running on a different compute node within the same allocation. And in principle
@@ -92,8 +88,6 @@ def create_executor(
         "cores": cores_per_worker,
         "hostname_localhost": hostname_localhost,
         "cwd": cwd,
-        "conda_environment_name": conda_environment_name,
-        "conda_environment_path": conda_environment_path,
     }
     if backend == "flux":
         check_oversubscribe(oversubscribe=openmpi_oversubscribe)
diff --git a/executorlib/interactive/flux.py b/executorlib/interactive/flux.py
@@ -47,27 +47,19 @@ def __init__(
     def bootup(
         self,
         command_lst: list[str],
-        conda_environment_name: Optional[str] = None,
-        conda_environment_path: Optional[str] = None,
     ):
         """
         Boot up the client process to connect to the SocketInterface.
 
         Args:
             command_lst (list[str]): List of strings to start the client process.
-            conda_environment_name (str, optional): Name of the conda environment to initialize. Defaults to None.
-            conda_environment_path (str, optional): Path of the conda environment to initialize. Defaults to None.
         Raises:
             ValueError: If oversubscribing is not supported for the Flux adapter or if conda environments are not supported.
         """
         if self._openmpi_oversubscribe:
             raise ValueError(
                 "Oversubscribing is currently not supported for the Flux adapter."
             )
-        if conda_environment_name is not None or conda_environment_path is not None:
-            raise ValueError(
-                "Conda environments are currently not supported for the Flux adapter."
-            )
         if self._flux_executor is None:
             self._flux_executor = flux.job.FluxExecutor()
         if not self._flux_executor_nesting:
diff --git a/executorlib/shared/communication.py b/executorlib/shared/communication.py
@@ -1,5 +1,4 @@
 from socket import gethostname
-from typing import Optional
 
 import cloudpickle
 import zmq
@@ -76,21 +75,15 @@ def bind_to_random_port(self):
     def bootup(
         self,
         command_lst: list[str],
-        conda_environment_name: Optional[str] = None,
-        conda_environment_path: Optional[str] = None,
     ):
         """
         Boot up the client process to connect to the SocketInterface.
 
         Args:
             command_lst (list): list of strings to start the client process
-            conda_environment_name (str): name of the conda environment to initialize
-            conda_environment_path (str): path of the conda environment to initialize
         """
         self._spawner.bootup(
             command_lst=command_lst,
-            conda_environment_name=conda_environment_name,
-            conda_environment_path=conda_environment_path,
         )
 
     def shutdown(self, wait: bool = True):
@@ -127,8 +120,6 @@ def interface_bootup(
     command_lst: list[str],
     connections,
     hostname_localhost: bool = False,
-    conda_environment_name: Optional[str] = None,
-    conda_environment_path: Optional[str] = None,
 ):
     """
     Start interface for ZMQ communication
@@ -144,8 +135,6 @@ def interface_bootup(
                                       points to the same address as localhost. Still MacOS >= 12 seems to disable
                                       this look up for security reasons. So on MacOS it is required to set this
                                       option to true
-        conda_environment_name (str): name of the conda environment to initialize
-        conda_environment_path (str): path of the conda environment to initialize
 
     Returns:
          executorlib.shared.communication.SocketInterface: socket interface for zmq communication
@@ -162,8 +151,6 @@ def interface_bootup(
     ]
     interface.bootup(
         command_lst=command_lst,
-        conda_environment_name=conda_environment_name,
-        conda_environment_path=conda_environment_path,
     )
     return interface
 
diff --git a/executorlib/shared/executor.py b/executorlib/shared/executor.py
@@ -304,8 +304,6 @@ def execute_parallel_tasks(
     spawner: BaseSpawner = MpiExecSpawner,
     hostname_localhost: bool = False,
     init_function: Optional[Callable] = None,
-    conda_environment_name: Optional[str] = None,
-    conda_environment_path: Optional[str] = None,
     **kwargs,
 ) -> None:
     """
@@ -323,17 +321,13 @@ def execute_parallel_tasks(
                                      this look up for security reasons. So on MacOS it is required to set this
                                      option to true
        init_function (callable): optional function to preset arguments for functions which are submitted later
-       conda_environment_name (str): name of the conda environment to initialize
-       conda_environment_path (str): path of the conda environment to initialize
     """
     interface = interface_bootup(
         command_lst=_get_backend_path(
             cores=cores,
         ),
         connections=spawner(cores=cores, **kwargs),
         hostname_localhost=hostname_localhost,
-        conda_environment_path=conda_environment_path,
-        conda_environment_name=conda_environment_name,
     )
     if init_function is not None:
         interface.send_dict(
diff --git a/executorlib/shared/spawner.py b/executorlib/shared/spawner.py
@@ -23,16 +23,12 @@ def __init__(self, cwd: str, cores: int = 1, openmpi_oversubscribe: bool = False
     def bootup(
         self,
         command_lst: list[str],
-        conda_environment_name: Optional[str] = None,
-        conda_environment_path: Optional[str] = None,
     ):
         """
         Method to start the interface.
 
         Args:
             command_lst (list[str]): The command list to execute.
-            conda_environment_name (str, optional): The prefix name. Defaults to None.
-            conda_environment_path (str, optional): The prefix path. Defaults to None.
         """
         raise NotImplementedError
 
@@ -80,33 +76,18 @@ def __init__(
     def bootup(
         self,
         command_lst: list[str],
-        conda_environment_name: Optional[str] = None,
-        conda_environment_path: Optional[str] = None,
     ):
         """
         Method to start the subprocess interface.
 
         Args:
             command_lst (list[str]): The command list to execute.
-            conda_environment_name (str, optional): The prefix name. Defaults to None.
-            conda_environment_path (str, optional): The prefix path. Defaults to None.
         """
-        if conda_environment_name is None and conda_environment_path is None:
-            self._process = subprocess.Popen(
-                args=self.generate_command(command_lst=command_lst),
-                cwd=self._cwd,
-                stdin=subprocess.DEVNULL,
-            )
-        else:
-            import conda_subprocess
-
-            self._process = conda_subprocess.Popen(
-                args=self.generate_command(command_lst=command_lst),
-                cwd=self._cwd,
-                stdin=subprocess.DEVNULL,
-                prefix_path=conda_environment_path,
-                prefix_name=conda_environment_name,
-            )
+        self._process = subprocess.Popen(
+            args=self.generate_command(command_lst=command_lst),
+            cwd=self._cwd,
+            stdin=subprocess.DEVNULL,
+        )
 
     def generate_command(self, command_lst: list[str]) -> list[str]:
         """
diff --git a/executorlib/shell/executor.py b/executorlib/shell/executor.py
diff --git a/notebooks/examples.ipynb b/notebooks/examples.ipynb
diff --git a/pyproject.toml b/pyproject.toml
diff --git a/tests/test_executor_conda.py b/tests/test_executor_conda.py
diff --git a/tests/test_flux_executor.py b/tests/test_flux_executor.py
