Make pep561 tests parrallel and sandboxed

mypy/sitepkgs.py

@@ -7,7 +7,8 @@
 possible.
 """
 
-
+import sys
+sys.path = sys.path[1:]  # we don't want to pick up mypy.types
 from distutils.sysconfig import get_python_lib
 import site
 MYPY = False

mypy/test/testpep561.py

@@ -1,8 +1,11 @@
 from contextlib import contextmanager
 import os
+import random
 import shutil
+import string
 import sys
-from typing import Iterator, List
+import tempfile
+from typing import Iterator, List, Generator
 from unittest import TestCase, main
 
 import mypy.api
@@ -19,112 +22,112 @@
 
 
 def check_mypy_run(cmd_line: List[str],
-                   expected_out: str,
+                   python_executable: str = sys.executable,
+                   expected_out: str = '',
                    expected_err: str = '',
                    expected_returncode: int = 1) -> None:
     """Helper to run mypy and check the output."""
+    if python_executable != sys.executable:
+        cmd_line.append('--python-executable={}'.format(python_executable))
     out, err, returncode = mypy.api.run(cmd_line)
     assert out == expected_out, err
     assert err == expected_err, out
     assert returncode == expected_returncode, returncode
 
 
-def is_in_venv() -> bool:
-    """Returns whether we are running inside a venv.
-
-    Based on https://stackoverflow.com/a/42580137.
-
-    """
-    if hasattr(sys, 'real_prefix'):
-        return True
-    else:
-        return hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix
+class TestPEP561(TestCase):
 
+    @contextmanager
+    def virtualenv(self, python_executable: str = sys.executable) -> Generator[str, None, None]:
+        """Context manager that creates a virtualenv in a temporary directory
+
+        returns the path to the created Python executable"""
+        with tempfile.TemporaryDirectory() as venv_dir:
+            run_command([sys.executable, '-m', 'virtualenv', '-p{}'.format(python_executable),
+                        venv_dir], cwd=os.getcwd())
+            if sys.platform == 'win32':
+                yield os.path.abspath(os.path.join(venv_dir, 'Scripts', 'python'))
+            else:
+                yield os.path.abspath(os.path.join(venv_dir, 'bin', 'python'))
 
-class TestPEP561(TestCase):
     @contextmanager
     def install_package(self, pkg: str,
                         python_executable: str = sys.executable) -> Iterator[None]:
         """Context manager to temporarily install a package from test-data/packages/pkg/"""
         working_dir = os.path.join(package_path, pkg)
         install_cmd = [python_executable, '-m', 'pip', 'install', '.']
-        # if we aren't in a virtualenv, install in the
-        # user package directory so we don't need sudo
-        if not is_in_venv() or python_executable != sys.executable:
-            install_cmd.append('--user')
         returncode, lines = run_command(install_cmd, cwd=working_dir)
         if returncode != 0:
             self.fail('\n'.join(lines))
-        try:
-            yield
-        finally:
-            returncode, lines = run_command([python_executable, '-m', 'pip', 'uninstall',
-                                             '-y', pkg], cwd=package_path)
-            if returncode != 0:
-                self.fail('\n'.join(lines))
+        yield
+
+    def setUp(self) -> None:
+        self.temp_file_dir = tempfile.TemporaryDirectory()
+        self.tempfile = os.path.join(self.temp_file_dir.name, 'simple.py')
+        with open(self.tempfile, 'w+') as file:
+            file.write(SIMPLE_PROGRAM)
+        self.msg_list = \
+            "{}:4: error: Revealed type is 'builtins.list[builtins.str]'\n".format(self.tempfile)
+        self.msg_tuple = \
+            "{}:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n".format(self.tempfile)
+
+    def tearDown(self) -> None:
+        self.temp_file_dir.cleanup()
 
     def test_get_pkg_dirs(self) -> None:
         """Check that get_package_dirs works."""
         dirs = _get_site_packages_dirs(sys.executable)
         assert dirs
 
-    def test_typed_pkg(self) -> None:
-        """Tests type checking based on installed packages.
-
-        This test CANNOT be split up, concurrency means that simultaneously
-        installing/uninstalling will break tests.
-        """
-        test_file = 'simple.py'
-        if not os.path.isdir('test-packages-data'):
-            os.mkdir('test-packages-data')
-        old_cwd = os.getcwd()
-        os.chdir('test-packages-data')
-        with open(test_file, 'w') as f:
-            f.write(SIMPLE_PROGRAM)
-        try:
-            with self.install_package('typedpkg-stubs'):
+    def test_typedpkg_stub_package(self) -> None:
+        with self.virtualenv() as python_executable:
+            with self.install_package('typedpkg-stubs', python_executable):
                 check_mypy_run(
-                    [test_file],
-                    "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n"
+                    [self.tempfile],
+                    python_executable,
+                    self.msg_list,
                 )
 
-            # The Python 2 tests are intentionally placed after a Python 3 test to check
-            # the package_dir_cache is behaving correctly.
-            python2 = try_find_python2_interpreter()
-            if python2:
-                with self.install_package('typedpkg-stubs', python2):
+    def test_typedpkg(self) -> None:
+        with self.virtualenv() as python_executable:
+            with self.install_package('typedpkg', python_executable):
+                check_mypy_run(
+                    [self.tempfile],
+                    python_executable,
+                    self.msg_tuple,
+                )
+
+    def test_stub_and_typed_pkg(self) -> None:
+        with self.virtualenv() as python_executable:
+            with self.install_package('typedpkg', python_executable):
+                with self.install_package('typedpkg-stubs', python_executable):
                     check_mypy_run(
-                        ['--python-executable={}'.format(python2), test_file],
-                        "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n"
+                        [self.tempfile],
+                        python_executable,
+                        self.msg_list,
                     )
-                with self.install_package('typedpkg', python2):
+
+    def test_typedpkg_stubs_python2(self) -> None:
+        python2 = try_find_python2_interpreter()
+        if python2:
+            with self.virtualenv(python2) as py2:
+                with self.install_package('typedpkg-stubs', py2):
                     check_mypy_run(
-                        ['--python-executable={}'.format(python2), 'simple.py'],
-                        "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n"
+                        [self.tempfile],
+                        py2,
+                        self.msg_list,
                     )
 
-                with self.install_package('typedpkg', python2):
-                    with self.install_package('typedpkg-stubs', python2):
-                        check_mypy_run(
-                            ['--python-executable={}'.format(python2), test_file],
-                            "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n"
-                        )
-
-            with self.install_package('typedpkg'):
-                check_mypy_run(
-                    [test_file],
-                    "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n"
-                )
-
-            with self.install_package('typedpkg'):
-                with self.install_package('typedpkg-stubs'):
+    def test_typedpkg_python2(self) -> None:
+        python2 = try_find_python2_interpreter()
+        if python2:
+            with self.virtualenv(python2) as py2:
+                with self.install_package('typedpkg', py2):
                     check_mypy_run(
-                        [test_file],
-                        "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n"
+                        [self.tempfile],
+                        py2,
+                        self.msg_tuple,
                     )
-        finally:
-            os.chdir(old_cwd)
-            shutil.rmtree('test-packages-data')
 
 
 if __name__ == '__main__':

test-requirements.txt

@@ -9,3 +9,4 @@ pytest-cov>=2.4.0
 typed-ast>=1.1.0,<1.2.0
 typing>=3.5.2; python_version < '3.5'
 py>=1.5.2
+virtualenv