Use subprocesses to avoid import conflicts

Author: NicolasHug

test_run_python_code.py

@@ -1,4 +1,5 @@
-import importlib
+from subprocess import check_output, STDOUT, CalledProcessError
+import sys
 import pytest
 import glob
 
@@ -23,5 +24,18 @@ def test_run_file(file_path):
     if 'pytorch_fairseq_roberta' in file_path:
         pytest.skip("temporarily disabled")
 
-    module = file_path[:-3].replace('/', '.')  # :-3 to remove the .py extension
-    importlib.import_module(module)
+    # We just run the python files in a separate sub-process. We really want a
+    # subprocess here because otherwise we might run into package versions
+    # issues: imagine script A that needs torchvivion 0.9 and script B that
+    # needs torchvision 0.10. If script A is run prior to script B in the same
+    # process, script B will still be run with torchvision 0.9 because the only
+    # "import torchvision" statement that counts is the first one, and even
+    # torchub sys.path shenanigans can do nothing about this. By creating
+    # subprocesses we're sure that all file executions are fully independent.
+    try:
+        # This is inspired (and heavily simplified) from
+        # https://github.com/cloudpipe/cloudpickle/blob/343da119685f622da2d1658ef7b3e2516a01817f/tests/testutils.py#L177
+        out = check_output([sys.executable, file_path], stderr=STDOUT)
+        print(out.decode())
+    except CalledProcessError as e:
+        raise RuntimeError(f"Script {file_path} errored with output:\n{e.output.decode()}")