refactor test discovery

Author: elazarg

mypy/myunit/__init__.py

@@ -105,30 +105,19 @@ def fail() -> None:
     raise AssertionFailure()
 
 
-class TestCase:
-    def __init__(self, name: str, suite: 'Optional[Suite]' = None,
-                 func: Optional[Callable[[], None]] = None) -> None:
-        self.func = func
+class ProtoTestCase:
+    def __init__(self, name: str) -> None:
         self.name = name
-        self.suite = suite
         self.old_cwd = None  # type: Optional[str]
         self.tmpdir = None  # type: Optional[tempfile.TemporaryDirectory[str]]
 
-    def run(self) -> None:
-        if self.func:
-            self.func()
-
     def set_up(self) -> None:
         self.old_cwd = os.getcwd()
         self.tmpdir = tempfile.TemporaryDirectory(prefix='mypy-test-')
         os.chdir(self.tmpdir.name)
         os.mkdir('tmp')
-        if self.suite:
-            self.suite.set_up()
 
     def tear_down(self) -> None:
-        if self.suite:
-            self.suite.tear_down()
         assert self.old_cwd is not None and self.tmpdir is not None, \
             "test was not properly set up"
         os.chdir(self.old_cwd)
@@ -140,6 +129,28 @@ def tear_down(self) -> None:
         self.tmpdir = None
 
 
+class TestCase(ProtoTestCase):
+    def __init__(self, name: str, suite: 'Optional[Suite]' = None,
+                 func: Optional[Callable[[], None]] = None) -> None:
+        super().__init__(name)
+        self.func = func
+        self.suite = suite
+
+    def run(self) -> None:
+        if self.func:
+            self.func()
+
+    def set_up(self) -> None:
+        super().set_up()
+        if self.suite:
+            self.suite.set_up()
+
+    def tear_down(self) -> None:
+        if self.suite:
+            self.suite.tear_down()
+        super().tear_down()
+
+
 class Suite:
     def __init__(self) -> None:
         self.prefix = typename(type(self)) + '.'

mypy/test/data.py

@@ -8,35 +8,33 @@
 import shutil
 
 import pytest  # type: ignore  # no pytest in typeshed
-from typing import Callable, List, Tuple, Set, Optional, Iterator, Any, Dict
-
-from mypy.myunit import TestCase, SkipTestCaseException
+from typing import List, Tuple, Set, Optional, Iterator, Any, Dict
+import typing
 
+from mypy.myunit import ProtoTestCase
+from mypy.test.config import test_data_prefix
 
 root_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..'))
 
 
 def parse_test_cases(
         path: str,
-        perform: Optional[Callable[['DataDrivenTestCase'], None]],
         base_path: str = '.',
         optional_out: bool = False,
-        include_path: Optional[str] = None,
         native_sep: bool = False) -> List['DataDrivenTestCase']:
     """Parse a file with test case descriptions.
 
     Return an array of test cases.
 
-    NB this function and DataDrivenTestCase are shared between the
+    NB: this function and DataDrivenTestCase were shared between the
     myunit and pytest codepaths -- if something looks redundant,
     that's likely the reason.
     """
     if native_sep:
         join = os.path.join
     else:
         join = posixpath.join  # type: ignore
-    if not include_path:
-        include_path = os.path.dirname(path)
+    include_path = os.path.dirname(path)
     with open(path, encoding='utf-8') as f:
         lst = f.readlines()
     for i in range(len(lst)):
@@ -167,7 +165,7 @@ def parse_test_cases(
                 arg0 = p[i0].arg
                 assert arg0 is not None
                 tc = DataDrivenTestCase(arg0, input, tcout, tcout2, path,
-                                        p[i0].line, lastline, perform,
+                                        p[i0].line, lastline,
                                         files, output_files, stale_modules,
                                         rechecked_modules, deleted_paths, native_sep)
                 out.append(tc)
@@ -179,7 +177,7 @@ def parse_test_cases(
     return out
 
 
-class DataDrivenTestCase(TestCase):
+class DataDrivenTestCase(ProtoTestCase):
     input = None  # type: List[str]
     output = None  # type: List[str]  # Output for the first pass
     output2 = None  # type: Dict[int, List[str]]  # Output for runs 2+, indexed by run number
@@ -202,7 +200,6 @@ def __init__(self,
                  file: str,
                  line: int,
                  lastline: int,
-                 perform: Optional[Callable[['DataDrivenTestCase'], None]],
                  files: List[Tuple[str, str]],
                  output_files: List[Tuple[str, str]],
                  expected_stale_modules: Dict[int, Set[str]],
@@ -217,7 +214,6 @@ def __init__(self,
         self.lastline = lastline
         self.file = file
         self.line = line
-        self.perform = perform
         self.files = files
         self.output_files = output_files
         self.expected_stale_modules = expected_stale_modules
@@ -268,13 +264,6 @@ def add_dirs(self, dir: str) -> List[str]:
             os.mkdir(dir)
             return dirs
 
-    def run(self) -> None:
-        if self.name.endswith('-skip'):
-            raise SkipTestCaseException()
-        else:
-            assert self.perform is not None, 'Tests without `perform` should not be `run`'
-            self.perform(self)
-
     def tear_down(self) -> None:
         # First remove files.
         for is_dir, path in reversed(self.clean_up):
@@ -507,13 +496,37 @@ def pytest_addoption(parser: Any) -> None:
 def pytest_pycollect_makeitem(collector: Any, name: str, obj: Any) -> Any:
     if not isinstance(obj, type) or not issubclass(obj, DataSuite):
         return None
+    if obj is DataSuite:
+        return None
     return MypyDataSuite(name, parent=collector)
 
 
 class MypyDataSuite(pytest.Class):  # type: ignore  # inheriting from Any
     def collect(self) -> Iterator['MypyDataCase']:
-        for case in self.obj.cases():
-            yield MypyDataCase(case.name, self, case)
+        cls = self.obj
+        for f in cls.files:
+            for case in parse_test_cases(os.path.join(test_data_prefix, f),
+                                         base_path=cls.base_path,
+                                         optional_out=cls.optional_out,
+                                         native_sep=cls.native_sep):
+                if cls.require_stable and not has_stable_flags(case):
+                    continue
+                if cls.require_incremental and not is_incremental(case):
+                    continue
+                yield MypyDataCase(case.name, self, case)
+
+
+def is_incremental(testcase: DataDrivenTestCase) -> bool:
+    return 'incremental' in testcase.name.lower() or 'incremental' in testcase.file
+
+
+def has_stable_flags(testcase: DataDrivenTestCase) -> bool:
+    if any(re.match(r'# flags[2-9]:', line) for line in testcase.input):
+        return False
+    for filename, contents in testcase.files:
+        if os.path.basename(filename).startswith('mypy.ini.'):
+            return False
+    return True
 
 
 class MypyDataCase(pytest.Item):  # type: ignore  # inheriting from Any
@@ -556,12 +569,15 @@ def repr_failure(self, excinfo: Any) -> str:
 
 
 class DataSuite:
+    files = None  # type: typing.ClassVar[List[str]]
+    base_path = '.'  # type: typing.ClassVar[str]
+    optional_out = False  # type: typing.ClassVar[bool]
+    native_sep = False  # type: typing.ClassVar[bool]
+    require_stable = False  # type: typing.ClassVar[bool]
+    require_incremental = False  # type: typing.ClassVar[bool]
+
     def __init__(self, *, update_data: bool) -> None:
         self.update_data = update_data
 
-    @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        return []
-
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         raise NotImplementedError

mypy/test/testcheck.py

@@ -3,18 +3,16 @@
 import os
 import re
 import shutil
-import sys
-import time
-import typed_ast
 
 from typing import Dict, List, Optional, Set, Tuple
+import typing
 
 from mypy import build, defaults
 from mypy.main import process_options
 from mypy.build import BuildSource, find_module_clear_caches
 from mypy.myunit import AssertionFailure
-from mypy.test.config import test_temp_dir, test_data_prefix
-from mypy.test.data import parse_test_cases, DataDrivenTestCase, DataSuite
+from mypy.test.config import test_temp_dir
+from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import (
     assert_string_arrays_equal, normalize_error_messages,
     retry_on_error, testcase_pyversion, update_testcase_output,
@@ -25,7 +23,7 @@
 from mypy import experiments
 
 # List of files that contain test case descriptions.
-files = [
+typecheck_files = [
     'check-basic.test',
     'check-callable.test',
     'check-classes.test',
@@ -83,14 +81,9 @@
 
 
 class TypeCheckSuite(DataSuite):
-
-    @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        c = []  # type: List[DataDrivenTestCase]
-        for f in files:
-            c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  None, test_temp_dir, True)
-        return c
+    files = typecheck_files  # type: typing.ClassVar[List[str]]
+    base_path = test_temp_dir  # type: typing.ClassVar[str]
+    optional_out = True  # type: typing.ClassVar[bool]
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         incremental = ('incremental' in testcase.name.lower()

mypy/test/testcmdline.py

@@ -9,12 +9,13 @@
 import subprocess
 import sys
 
-from typing import Tuple, List, Dict, Set
+from typing import List
+import typing
 
-from mypy.myunit import Suite, SkipTestCaseException, AssertionFailure
-from mypy.test.config import test_data_prefix, test_temp_dir
+from mypy.myunit import AssertionFailure
+from mypy.test.config import test_temp_dir
 from mypy.test.data import fix_cobertura_filename
-from mypy.test.data import parse_test_cases, DataDrivenTestCase, DataSuite
+from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import assert_string_arrays_equal, normalize_error_messages
 from mypy.version import __version__, base_version
 
@@ -28,24 +29,17 @@
 ]
 
 
-class PythonEvaluationSuite(DataSuite):
-
-    @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        c = []  # type: List[DataDrivenTestCase]
-        for f in cmdline_files:
-            c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  test_python_evaluation,
-                                  base_path=test_temp_dir,
-                                  optional_out=True,
-                                  native_sep=True)
-        return c
+class PythonCmdlineSuite(DataSuite):
+    files = cmdline_files  # type: typing.ClassVar[List[str]]
+    base_path = test_temp_dir  # type: typing.ClassVar[str]
+    optional_out = True  # type: typing.ClassVar[bool]
+    native_sep = True  # type: typing.ClassVar[bool]
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
-        test_python_evaluation(testcase)
+        test_python_cmdline(testcase)
 
 
-def test_python_evaluation(testcase: DataDrivenTestCase) -> None:
+def test_python_cmdline(testcase: DataDrivenTestCase) -> None:
     assert testcase.old_cwd is not None, "test was not properly set up"
     # Write the program to a file.
     program = '_program.py'

mypy/test/testdeps.py

@@ -2,32 +2,24 @@
 
 import os
 from typing import List, Tuple, Dict, Optional
+import typing
 
 from mypy import build
 from mypy.build import BuildSource
 from mypy.errors import CompileError
 from mypy.nodes import MypyFile, Expression
 from mypy.options import Options
 from mypy.server.deps import get_dependencies
-from mypy.test.config import test_temp_dir, test_data_prefix
-from mypy.test.data import parse_test_cases, DataDrivenTestCase, DataSuite
+from mypy.test.config import test_temp_dir
+from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import assert_string_arrays_equal
 from mypy.types import Type
 
-files = [
-    'deps.test'
-]
-
 
 class GetDependenciesSuite(DataSuite):
-
-    @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        c = []  # type: List[DataDrivenTestCase]
-        for f in files:
-            c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  None, test_temp_dir, True)
-        return c
+    files = ['deps.test']  # type: typing.ClassVar[List[str]]
+    base_path = test_temp_dir  # type: typing.ClassVar[str]
+    optional_out = True  # type: typing.ClassVar[bool]
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         src = '\n'.join(testcase.input)

mypy/test/testdiff.py

@@ -2,32 +2,23 @@
 
 import os
 from typing import List, Tuple, Dict, Optional
+import typing
 
 from mypy import build
 from mypy.build import BuildSource
 from mypy.errors import CompileError
 from mypy.nodes import MypyFile
 from mypy.options import Options
 from mypy.server.astdiff import compare_symbol_tables
-from mypy.test.config import test_temp_dir, test_data_prefix
-from mypy.test.data import parse_test_cases, DataDrivenTestCase, DataSuite
+from mypy.test.config import test_temp_dir
+from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import assert_string_arrays_equal
 
 
-files = [
-    'diff.test'
-]
-
-
 class ASTDiffSuite(DataSuite):
-
-    @classmethod
-    def cases(cls) -> List[DataDrivenTestCase]:
-        c = []  # type: List[DataDrivenTestCase]
-        for f in files:
-            c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  None, test_temp_dir, True)
-        return c
+    files = ['diff.test']  # type: typing.ClassVar[List[str]]
+    base_path = test_temp_dir  # type: typing.ClassVar[str]
+    optional_out = True  # type: typing.ClassVar[bool]
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         first_src = '\n'.join(testcase.input)