fix(ordered_pip): re-create venv if it has been garbage collected (#285)

feat(ordered_pip): modify_context() auto-builds the underlying pip venv if it's been garbage collected

Signed-off-by: Vassilis Vassiliadis <[email protected]>

Author: VassilisVassiliadis

orchestrator/utilities/ray_env/ordered_pip.py

@@ -333,6 +333,20 @@ def modify_context(
         if not len(phases):
             return
 
+        env_dir = self.get_path_to_pip_venv(uris[0])
+
+        if not os.path.isdir(env_dir):
+            logger.warning(
+                f"The pip environment at {env_dir} has been garbage collected - recreating it"
+            )
+            import asyncio
+
+            asyncio.run(
+                self.create(
+                    uris[0], runtime_env=runtime_env, context=context, logger=logger
+                )
+            )
+
         self._pip_plugin.modify_context(
             uris=uris,
             runtime_env=RuntimeEnv(pip=phases[0]),