Start moving to pytest

.travis.yml

@@ -9,7 +9,7 @@ python:
   # - "pypy3"
 
 install:
-  - pip install -r test-requirements.txt
+  - pip install -r requirements-testing.txt
   - python setup.py install
 
 script:

README.md

@@ -193,6 +193,14 @@ pass inferior arguments via `-a`:
     $ PYTHONPATH=$PWD scripts/myunit -m mypy.test.testlex -v '*backslash*'
     $ ./runtests.py mypy.test.testlex -a -v -a '*backslash*'
 
+Mypy is currently in the process of converting its tests from myunit to pytest.
+Some of the tests, such as `test_check`, have already been converted. To run
+these individually, you can either use `./runtests.py` but pass options with
+`-t` instead of `-a`, or you can just run `py.test` itself:
+
+    $ ./runtests.py pytest -t -k -t NestedListAssignment
+    $ py.test -k NestedListAssignment
+
 You can also run the type checker for manual testing without
 installing anything by setting up the Python module search path
 suitably (the lib-typing/3.2 path entry is not needed for Python 3.5

conftest.py

@@ -0,0 +1,13 @@
+from mypy.test.helpers import PytestSuite
+import inspect
+import pytest
+
+def pytest_addoption(parser):
+    parser.addoption('--update-testcases', action='store_true',
+                     dest='UPDATE_TESTCASES')
+
+def pytest_pycollect_makeitem(collector, name, obj):
+     if (inspect.isclass(obj) and issubclass(obj, PytestSuite) and
+            obj is not PytestSuite):
+        print(name)
+        obj.collect_tests()

mypy/test/data.py

@@ -6,14 +6,14 @@
 from os import remove, rmdir
 import shutil
 
-from typing import Callable, List, Tuple
+from typing import Callable, List, Tuple, Any
 
 from mypy.myunit import TestCase, SkipTestCaseException
 
 
 def parse_test_cases(
         path: str,
-        perform: Callable[['DataDrivenTestCase'], None],
+        perform: Callable[..., None],
         base_path: str = '.',
         optional_out: bool = False,
         include_path: str = None,
@@ -89,29 +89,35 @@ def parse_test_cases(
 
 
 class DataDrivenTestCase(TestCase):
+    name = None  # type: str
     input = None  # type: List[str]
     output = None  # type: List[str]
 
     file = ''
     line = 0
 
-    perform = None  # type: Callable[['DataDrivenTestCase'], None]
+    # NOTE: For info on the ..., see `run`.
+    perform = None  # type: Callable[..., None]
 
     # (file path, file content) tuples
     files = None  # type: List[Tuple[str, str]]
 
     clean_up = None  # type: List[Tuple[bool, str]]
 
+    marked_skip = False
+
     def __init__(self, name, input, output, file, line, lastline,
                  perform, files):
         super().__init__(name)
+        self.name = name
         self.input = input
         self.output = output
         self.lastline = lastline
         self.file = file
         self.line = line
         self.perform = perform
         self.files = files
+        self.marked_skip = self.name.endswith('-skip')
 
     def set_up(self) -> None:
         super().set_up()
@@ -137,19 +143,34 @@ def add_dirs(self, dir: str) -> List[str]:
             os.mkdir(dir)
             return dirs
 
-    def run(self):
-        if self.name.endswith('-skip'):
-            raise SkipTestCaseException()
+    def run(self, obj: Any = None):
+        if obj is None:
+            # XXX: The unit tests are being converted over to pytest. Due to
+            # modifications requires to make BOTH run at the moment, this branch
+            # is necessary. It should be removed once all the tests relying on
+            # DataDrivenTestCase are converted to pytest.
+            if self.marked_skip:
+                raise SkipTestCaseException()
+            else:
+                self.perform(self)
         else:
-            self.perform(self)
+            assert not self.marked_skip
+            # Because perform is an unbound method, it needs to be passed it
+            # own self, which is obj. In the future, after all tests are moved
+            # over to pytest, this whole class should probably be generic, to
+            # allow annotating obj. In the mean time, there isn't a cleaner way
+            # to handle this...
+            self.perform(obj, self)
 
     def tear_down(self) -> None:
         # First remove files.
         for is_dir, path in reversed(self.clean_up):
-            if not is_dir:
+            if not is_dir and os.path.exists(path):
                 remove(path)
         # Then remove directories.
         for is_dir, path in reversed(self.clean_up):
+            if not os.path.exists(path):
+                continue
             if is_dir:
                 pycache = os.path.join(path, '__pycache__')
                 if os.path.isdir(pycache):

mypy/test/helpers.py

@@ -1,6 +1,7 @@
 import sys
 import re
 import os
+import pytest  # type: ignore
 
 from typing import List, Dict, Tuple
 
@@ -263,18 +264,18 @@ def num_skipped_suffix_lines(a1: List[str], a2: List[str]) -> int:
     return max(0, num_eq - 4)
 
 
-def testfile_pyversion(path: str) -> Tuple[int, int]:
+def pyversion_testfile(path: str) -> Tuple[int, int]:
     if path.endswith('python2.test'):
         return defaults.PYTHON2_VERSION
     else:
         return defaults.PYTHON3_VERSION
 
 
-def testcase_pyversion(path: str, testcase_name: str) -> Tuple[int, int]:
+def pyversion_testcase(path: str, testcase_name: str) -> Tuple[int, int]:
     if testcase_name.endswith('python2'):
         return defaults.PYTHON2_VERSION
     else:
-        return testfile_pyversion(path)
+        return pyversion_testfile(path)
 
 
 def normalize_error_messages(messages: List[str]) -> List[str]:
@@ -284,3 +285,42 @@ def normalize_error_messages(messages: List[str]) -> List[str]:
     for m in messages:
         a.append(m.replace(os.sep, '/'))
     return a
+
+
+@pytest.fixture(scope='function')
+def test(request):
+    test = request.function.test
+    test.set_up()
+    request.addfinalizer(test.tear_down)
+    return test
+
+
+class PytestSuite:
+    """Assists in setting up data-driven test cases for pytest."""
+    @classmethod
+    def collect_tests(cls):
+        """
+        Sets up the child class's test case. The child must have a method
+        `cases` that returns a list of `DataDrivenTestCase`s. This method will
+        load the data-driven test cases and use setattr to assign it to the
+        class, which will allow pytest to recognize the test.
+
+        This will be called during test collection (see conftest.py in the root
+        of the repository).
+        """
+        c = cls.cases()  # type: List[DataDrivenTestCase]
+        for test in c:
+            def func(self, test):
+                test.run(self)
+            if test.marked_skip:
+                func = pytest.mark.skip(reason='Test ends with -skip')(func)
+            if 'FastParse' in test.name and not test.marked_skip:
+                try:
+                    import mypy.fastparse
+                except SystemExit:
+                    func = pytest.mark.skip(
+                        reason='You must install the typed_ast package in '
+                               'order to run this test')(func)
+            func.test = test
+            setattr(cls, test.name, func)
+            # setattr(cls, test.name.replace('test', 'test_', 1), func)

mypy/test/testcheck.py → mypy/test/test_check.py

@@ -4,18 +4,18 @@
 import re
 import shutil
 import sys
+import mypy
+import pytest  # type: ignore
 
 from typing import Tuple, List, Dict, Set
 
 from mypy import build
-import mypy.myunit  # for mutable globals (ick!)
 from mypy.build import BuildSource, find_module_clear_caches
-from mypy.myunit import Suite, AssertionFailure
 from mypy.test.config import test_temp_dir, test_data_prefix
 from mypy.test.data import parse_test_cases, DataDrivenTestCase
 from mypy.test.helpers import (
-    assert_string_arrays_equal, normalize_error_messages,
-    testcase_pyversion, update_testcase_output,
+    normalize_error_messages, pyversion_testcase, update_testcase_output,
+    PytestSuite, test
 )
 from mypy.errors import CompileError
 
@@ -61,13 +61,13 @@
 ]
 
 
-class TypeCheckSuite(Suite):
-
-    def cases(self) -> List[DataDrivenTestCase]:
+class TestTypeCheck(PytestSuite):
+    @classmethod
+    def cases(cls):
         c = []  # type: List[DataDrivenTestCase]
         for f in files:
             c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  self.run_test, test_temp_dir, True)
+                                  cls.run_test, test_temp_dir, True)
         return c
 
     def run_test(self, testcase: DataDrivenTestCase) -> None:
@@ -88,7 +88,7 @@ def clear_cache(self) -> None:
 
     def run_test_once(self, testcase: DataDrivenTestCase, incremental=0) -> None:
         find_module_clear_caches()
-        pyversion = testcase_pyversion(testcase.file, testcase.name)
+        pyversion = pyversion_testcase(testcase.file, testcase.name)
         program_text = '\n'.join(testcase.input)
         module_name, program_name, program_text = self.parse_options(program_text)
         flags = self.parse_flags(program_text)
@@ -122,13 +122,12 @@ def run_test_once(self, testcase: DataDrivenTestCase, incremental=0) -> None:
             a = e.messages
         a = normalize_error_messages(a)
 
-        if output != a and mypy.myunit.UPDATE_TESTCASES:
+        if output != a and pytest.config.getoption('UPDATE_TESTCASES'):
             update_testcase_output(testcase, a, mypy.myunit.APPEND_TESTCASES)
 
-        assert_string_arrays_equal(
-            output, a,
+        assert output == a, \
             'Invalid type checker output ({}, line {})'.format(
-                testcase.file, testcase.line))
+                testcase.file, testcase.line)
 
         if incremental and res:
             self.verify_cache(module_name, program_name, a, res.manager)
@@ -145,9 +144,7 @@ def verify_cache(self, module_name: str, program_name: str, a: List[str],
         modules = self.find_module_files()
         modules.update({module_name: program_name})
         missing_paths = self.find_missing_cache_files(modules, manager)
-        if missing_paths != error_paths:
-            raise AssertionFailure("cache data discrepancy %s != %s" %
-                                   (missing_paths, error_paths))
+        assert missing_paths == error_paths, 'cache data discrepancy'
 
     def find_error_paths(self, a: List[str]) -> Set[str]:
         hits = set()

mypy/test/testcmdline.py → mypy/test/test_cmdline.py

@@ -14,7 +14,7 @@
 from mypy.myunit import Suite, SkipTestCaseException
 from mypy.test.config import test_data_prefix, test_temp_dir
 from mypy.test.data import parse_test_cases, DataDrivenTestCase
-from mypy.test.helpers import assert_string_arrays_equal
+from mypy.test.helpers import PytestSuite, test
 
 # Path to Python 3 interpreter
 python3_path = sys.executable
@@ -23,20 +23,20 @@
 cmdline_files = ['cmdline.test']
 
 
-class PythonEvaluationSuite(Suite):
-
-    def cases(self) -> List[DataDrivenTestCase]:
+class TestPythonEvaluation(PytestSuite):
+    @classmethod
+    def cases(cls) -> List[DataDrivenTestCase]:
         c = []  # type: List[DataDrivenTestCase]
         for f in cmdline_files:
             c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  test_python_evaluation,
+                                  python_evaluation_test,
                                   base_path=test_temp_dir,
                                   optional_out=True,
                                   native_sep=True)
         return c
 
 
-def test_python_evaluation(testcase: DataDrivenTestCase) -> None:
+def python_evaluation_test(obj: None, testcase: DataDrivenTestCase) -> None:
     # Write the program to a file.
     program = '_program.py'
     program_path = os.path.join(test_temp_dir, program)
@@ -58,9 +58,8 @@ def test_python_evaluation(testcase: DataDrivenTestCase) -> None:
     # Remove temp file.
     os.remove(program_path)
     # Compare actual output to expected.
-    assert_string_arrays_equal(testcase.output, out,
-                               'Invalid output ({}, line {})'.format(
-                                   testcase.file, testcase.line))
+    assert testcase.output == out, 'Invalid output ({}, line {})'.format(
+        testcase.file, testcase.line)
 
 
 def parse_args(line: str) -> List[str]: