move test_stubs and test_samples to pytest (#5142)

Using the api, similarly to #5087.

stdlibsamples is not included in this PR since it requires special handling of working directory and paths.

This is another item for #1673.

Author: elazarg

mypy/test/helpers.py

@@ -4,11 +4,13 @@
 import sys
 import time
 import shutil
+from os.path import dirname
 
 from typing import List, Iterable, Dict, Tuple, Callable, Any, Optional
 
 from mypy import defaults
 from mypy.test.config import test_temp_dir
+import mypy.api as api
 
 import pytest  # type: ignore  # no pytest in typeshed
 
@@ -27,6 +29,21 @@
 MIN_LINE_LENGTH_FOR_ALIGNMENT = 5
 
 
+def run_mypy(args: List[str]) -> None:
+    __tracebackhide__ = True
+    outval, errval, status = api.run(args + ['--show-traceback',
+                                             '--no-site-packages'])
+    if status != 0:
+        sys.stdout.write(outval)
+        sys.stderr.write(errval)
+        pytest.fail(msg="Sample check failed", pytrace=False)
+
+
+def use_builtins_fixtures(options):
+    root_dir = dirname(dirname(dirname(__file__)))
+    options.path_prefix = os.path.join(root_dir, 'test-data', 'unit', 'lib-stub')
+
+
 def assert_string_arrays_equal(expected: List[str], actual: List[str],
                                msg: str) -> None:
     """Assert that two string arrays are equal.

mypy/test/testsamples.py

@@ -0,0 +1,49 @@
+"""Self check mypy package"""
+import sys
+import os.path
+from typing import List, Set
+
+from mypy.test.helpers import Suite, run_mypy
+
+
+class SamplesSuite(Suite):
+    def test_stubs(self) -> None:
+        # We only test each module in the one version mypy prefers to find.
+        # TODO: test stubs for other versions, especially Python 2 stubs.
+        seen = set()  # type: Set[str]
+        modules = []
+        # TODO: This should also test Python 2, and pass pyversion accordingly.
+        for version in ["2and3", "3", "3.5"]:
+            # FIX: remove 'builtins', this directory does not exist
+            for stub_type in ['builtins', 'stdlib', 'third_party']:
+                stubdir = os.path.join('typeshed', stub_type, version)
+                for f in find_files(stubdir, suffix='.pyi'):
+                    module = file_to_module(f[len(stubdir) + 1:])
+                    if module not in seen:
+                        seen.add(module)
+                        modules.extend(['-m', module])
+        if modules:
+            # these require at least 3.5 otherwise it will fail trying to import zipapp
+            run_mypy(['--python-version=3.5'] + modules)
+
+    def test_samples(self) -> None:
+        for f in find_files(os.path.join('test-data', 'samples'), suffix='.py'):
+            mypy_args = ['--no-strict-optional']
+            if f == os.path.join('test-data', 'samples', 'crawl2.py'):
+                # This test requires 3.5 for async functions
+                mypy_args.append('--python-version=3.5')
+            run_mypy(mypy_args + [f])
+
+
+def find_files(base: str, prefix: str = '', suffix: str = '') -> List[str]:
+    return [os.path.join(root, f)
+            for root, dirs, files in os.walk(base)
+            for f in files
+            if f.startswith(prefix) and f.endswith(suffix)]
+
+
+def file_to_module(file: str) -> str:
+    rv = os.path.splitext(file)[0].replace(os.sep, '.')
+    if rv.endswith('.__init__'):
+        rv = rv[:-len('.__init__')]
+    return rv

mypy/test/testselfcheck.py

@@ -1,11 +1,6 @@
 """Self check mypy package"""
-import sys
-from typing import List
 
-import pytest  # type: ignore
-
-from mypy.test.helpers import Suite
-from mypy.api import run
+from mypy.test.helpers import Suite, run_mypy
 
 
 class SelfCheckSuite(Suite):
@@ -14,16 +9,3 @@ def test_mypy_package(self) -> None:
 
     def test_testrunner(self) -> None:
         run_mypy(['runtests.py', 'waiter.py'])
-
-
-def run_mypy(args: List[str]) -> None:
-    __tracebackhide__ = True
-    outval, errval, status = run(args + ['--config-file', 'mypy_self_check.ini',
-                                         '--show-traceback',
-                                         '--no-site-packages'])
-    if status != 0:
-        sys.stdout.write(outval)
-        errval = '\n'.join(line for line in errval.split('\n')
-                           if 'mypy_self_check.ini' not in line)
-        sys.stderr.write(errval)
-        pytest.fail(msg="Self check failed", pytrace=False)

runtests.py

@@ -9,6 +9,7 @@
 import sys
 
 from waiter import Waiter, LazySubprocess
+from mypy.test.testsamples import find_files, file_to_module
 
 
 def get_versions() -> List[str]:
@@ -106,20 +107,6 @@ def list_tasks(self) -> None:
             print('{id}:{task}'.format(id=id, task=task.name))
 
 
-def find_files(base: str, prefix: str = '', suffix: str = '') -> List[str]:
-    return [join(root, f)
-            for root, dirs, files in os.walk(base)
-            for f in files
-            if f.startswith(prefix) and f.endswith(suffix)]
-
-
-def file_to_module(file: str) -> str:
-    rv = os.path.splitext(file)[0].replace(os.sep, '.')
-    if rv.endswith('.__init__'):
-        rv = rv[:-len('.__init__')]
-    return rv
-
-
 def test_path(*names: str) -> List[str]:
     return [os.path.join('mypy', 'test', '{}.py'.format(name))
             for name in names]
@@ -155,6 +142,7 @@ def test_path(*names: str) -> List[str]:
     'testpythoneval',
     'testcmdline',
     'teststubgen',
+    'testsamples',
 )
 
 SELFCHECK_FILES = test_path(
@@ -170,23 +158,6 @@ def add_pytest(driver: Driver) -> None:
                       [('self-check', name) for name in SELFCHECK_FILES])
 
 
-def add_stubs(driver: Driver) -> None:
-    # We only test each module in the one version mypy prefers to find.
-    # TODO: test stubs for other versions, especially Python 2 stubs.
-
-    modules = {'typing'}
-    # TODO: This should also test Python 2, and pass pyversion accordingly.
-    for version in ["2and3", "3", "3.5"]:
-        for stub_type in ['builtins', 'stdlib', 'third_party']:
-            stubdir = join('typeshed', stub_type, version)
-            for f in find_files(stubdir, suffix='.pyi'):
-                module = file_to_module(f[len(stubdir) + 1:])
-                modules.add(module)
-
-    # these require at least 3.5 otherwise it will fail trying to import zipapp
-    driver.add_mypy_modules('stubs', sorted(modules), extra_args=['--python-version=3.5'])
-
-
 def add_stdlibsamples(driver: Driver) -> None:
     seen = set()  # type: Set[str]
     stdlibsamples_dir = join(driver.cwd, 'test-data', 'stdlib-samples', '3.2', 'test')
@@ -202,15 +173,6 @@ def add_stdlibsamples(driver: Driver) -> None:
                                 cwd=stdlibsamples_dir, extra_args=['--no-strict-optional'])
 
 
-def add_samples(driver: Driver) -> None:
-    for f in find_files(os.path.join('test-data', 'samples'), suffix='.py'):
-        mypy_args = ['--no-strict-optional']
-        if f == os.path.join('test-data', 'samples', 'crawl2.py'):
-            # This test requires 3.5 for async functions
-            mypy_args.append('--python-version=3.5')
-        driver.add_mypy_cmd('file {}'.format(f), mypy_args + [f])
-
-
 def usage(status: int) -> None:
     print('Usage: %s [-h | -v | -q | --lf | --ff | [-x] FILTER | -a ARG | -p ARG]'
           '... [-- FILTER ...]'
@@ -348,9 +310,7 @@ def main() -> None:
 
     driver.add_flake8()
     add_pytest(driver)
-    add_stubs(driver)
     add_stdlibsamples(driver)
-    add_samples(driver)
 
     if list_only:
         driver.list_tasks()