Start moving to pytest

.travis.yml

@@ -9,7 +9,7 @@ python:
   # - "pypy3"
 
 install:
-  - pip install -r test-requirements.txt
+  - pip install -r requirements-testing.txt
   - python setup.py install
 
 script:

README.md

@@ -193,6 +193,13 @@ pass inferior arguments via `-a`:
     $ PYTHONPATH=$PWD scripts/myunit -m mypy.test.testlex -v '*backslash*'
     $ ./runtests.py mypy.test.testlex -a -v -a '*backslash*'
 
+Mypy is currently in the process of converting its tests from myunit to pytest.
+Some of the tests, such as `test_check`, have already been converted. To run
+these, you must use pytest instead, and use `-k` instead of `-a`:
+
+    $ ./runtests.py pytest -t -k -t NestedListAssignment
+    $ py.test -k NestedListAssignment
+
 You can also run the type checker for manual testing without
 installing anything by setting up the Python module search path
 suitably (the lib-typing/3.2 path entry is not needed for Python 3.5

conftest.py

@@ -0,0 +1,22 @@
+import pytest
+
+def pytest_addoption(parser):
+    parser.addoption('--update-testcases', action='store_true',
+                     dest='UPDATE_TESTCASES')
+
+# mypy.test.helpers defines several top-level utility functions that
+# pytest will pick up as tests. This removes them.
+def pytest_collection_modifyitems(items):
+    to_remove = []
+
+    for i, item in enumerate(items):
+        if (isinstance(item, pytest.Function) and
+                item.function.__module__ == 'mypy.test.helpers' and
+                # This is to prevent removing data-driven tests, which are
+                # defined by mypy.test.helpers.PytestSuite.
+                not getattr(item.function, 'is_test_attr', False)):
+            to_remove.append(i)
+
+    # reversed is to prevent changing indexes that haven't been removed yet.
+    for index in reversed(to_remove):
+        items.pop(index)

mypy/test/data.py

@@ -6,14 +6,14 @@
 from os import remove, rmdir
 import shutil
 
-from typing import Callable, List, Tuple
+from typing import Callable, List, Tuple, Any
 
 from mypy.myunit import TestCase, SkipTestCaseException
 
 
 def parse_test_cases(
         path: str,
-        perform: Callable[['DataDrivenTestCase'], None],
+        perform: Callable[[Any, 'DataDrivenTestCase'], None],
         base_path: str = '.',
         optional_out: bool = False,
         include_path: str = None,
@@ -89,29 +89,34 @@ def parse_test_cases(
 
 
 class DataDrivenTestCase(TestCase):
+    name = None  # type: str
     input = None  # type: List[str]
     output = None  # type: List[str]
 
     file = ''
     line = 0
 
-    perform = None  # type: Callable[['DataDrivenTestCase'], None]
+    perform = None  # type: Callable[..., None]
 
     # (file path, file content) tuples
     files = None  # type: List[Tuple[str, str]]
 
     clean_up = None  # type: List[Tuple[bool, str]]
 
+    is_skip = False
+
     def __init__(self, name, input, output, file, line, lastline,
                  perform, files):
         super().__init__(name)
+        self.name = name
         self.input = input
         self.output = output
         self.lastline = lastline
         self.file = file
         self.line = line
         self.perform = perform
         self.files = files
+        self.is_skip = self.name.endswith('-skip')
 
     def set_up(self) -> None:
         super().set_up()
@@ -137,19 +142,29 @@ def add_dirs(self, dir: str) -> List[str]:
             os.mkdir(dir)
             return dirs
 
-    def run(self):
-        if self.name.endswith('-skip'):
-            raise SkipTestCaseException()
+    def run(self, obj: Any = None):
+        if obj is None:
+            # XXX: The unit tests are being converted over to pytest. Due to
+            # modifications requires to make BOTH run at the moment, this branch
+            # is necessary. It should be removed once all the tests relying on
+            # DataDrivenTestCase are converted to pytest.
+            if self.is_skip:
+                raise SkipTestCaseException()
+            else:
+                self.perform(self)
         else:
-            self.perform(self)
+            assert not self.is_skip
+            self.perform(obj, self)
 
     def tear_down(self) -> None:
         # First remove files.
         for is_dir, path in reversed(self.clean_up):
-            if not is_dir:
+            if not is_dir and os.path.exists(path):
                 remove(path)
         # Then remove directories.
         for is_dir, path in reversed(self.clean_up):
+            if not os.path.exists(path):
+                continue
             if is_dir:
                 pycache = os.path.join(path, '__pycache__')
                 if os.path.isdir(pycache):

mypy/test/helpers.py

@@ -1,6 +1,7 @@
 import sys
 import re
 import os
+import pytest  # type: ignore
 
 from typing import List, Dict, Tuple
 
@@ -284,3 +285,43 @@ def normalize_error_messages(messages: List[str]) -> List[str]:
     for m in messages:
         a.append(m.replace(os.sep, '/'))
     return a
+
+
+@pytest.fixture(scope='function')
+def test(request):
+    test = request.function.test
+    test.set_up()
+    request.addfinalizer(test.tear_down)
+    return test
+
+
+class PytestSuite:
+    """Assists in setting up data-driven test cases for pytest."""
+    @classmethod
+    def setup_tests(cls):
+        """
+        Sets up the child class's test case. The child must have a method
+        `cases` that returns a list of `DataDrivenTestCase`s. This method will
+        load the data-driven test cases and use setattr to assign it to the
+        class, which will allow pytest to recognize the test.
+
+        Note that this method **must** be run after the definition of any
+        functions used by PytestSuite.cases.
+        """
+        c = cls.cases()  # type: List[DataDrivenTestCase]
+        for test in c:
+            def func(self, test):
+                test.run(self)
+            func.is_test_attr = True
+            if test.is_skip:
+                func = pytest.mark.skip(reason='Test ends with -skip')(func)
+            if 'FastParse' in test.name and not test.is_skip:
+                try:
+                    import mypy.fastparse
+                except SystemExit:
+                    func = pytest.mark.skip(
+                        reason='You must install the typed_ast package in '
+                               'order to run this test')(func)
+            func.test = test
+            setattr(cls, test.name, func)
+            # setattr(cls, test.name.replace('test', 'test_', 1), func)

mypy/test/testcheck.py → mypy/test/test_check.py

@@ -4,18 +4,18 @@
 import re
 import shutil
 import sys
+import mypy
+import pytest  # type: ignore
 
 from typing import Tuple, List, Dict, Set
 
 from mypy import build
-import mypy.myunit  # for mutable globals (ick!)
 from mypy.build import BuildSource, find_module_clear_caches
-from mypy.myunit import Suite, AssertionFailure
 from mypy.test.config import test_temp_dir, test_data_prefix
 from mypy.test.data import parse_test_cases, DataDrivenTestCase
 from mypy.test.helpers import (
-    assert_string_arrays_equal, normalize_error_messages,
-    testcase_pyversion, update_testcase_output,
+    normalize_error_messages, testcase_pyversion, update_testcase_output,
+    PytestSuite, test
 )
 from mypy.errors import CompileError
 
@@ -61,13 +61,13 @@
 ]
 
 
-class TypeCheckSuite(Suite):
-
-    def cases(self) -> List[DataDrivenTestCase]:
+class TestTypeCheck(PytestSuite):
+    @classmethod
+    def cases(cls):
         c = []  # type: List[DataDrivenTestCase]
         for f in files:
             c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  self.run_test, test_temp_dir, True)
+                                  cls.run_test, test_temp_dir, True)
         return c
 
     def run_test(self, testcase: DataDrivenTestCase) -> None:
@@ -122,13 +122,12 @@ def run_test_once(self, testcase: DataDrivenTestCase, incremental=0) -> None:
             a = e.messages
         a = normalize_error_messages(a)
 
-        if output != a and mypy.myunit.UPDATE_TESTCASES:
+        if output != a and pytest.config.getoption('UPDATE_TESTCASES'):
             update_testcase_output(testcase, a, mypy.myunit.APPEND_TESTCASES)
 
-        assert_string_arrays_equal(
-            output, a,
+        assert output == a, \
             'Invalid type checker output ({}, line {})'.format(
-                testcase.file, testcase.line))
+                testcase.file, testcase.line)
 
         if incremental and res:
             self.verify_cache(module_name, program_name, a, res.manager)
@@ -145,9 +144,7 @@ def verify_cache(self, module_name: str, program_name: str, a: List[str],
         modules = self.find_module_files()
         modules.update({module_name: program_name})
         missing_paths = self.find_missing_cache_files(modules, manager)
-        if missing_paths != error_paths:
-            raise AssertionFailure("cache data discrepancy %s != %s" %
-                                   (missing_paths, error_paths))
+        assert missing_paths == error_paths, 'cache data discrepancy'
 
     def find_error_paths(self, a: List[str]) -> Set[str]:
         hits = set()
@@ -209,3 +206,5 @@ def parse_flags(self, program_text: str) -> List[str]:
             return m.group(1).split()
         else:
             return []
+
+TestTypeCheck.setup_tests()

mypy/test/testcmdline.py → mypy/test/test_cmdline.py

@@ -14,7 +14,7 @@
 from mypy.myunit import Suite, SkipTestCaseException
 from mypy.test.config import test_data_prefix, test_temp_dir
 from mypy.test.data import parse_test_cases, DataDrivenTestCase
-from mypy.test.helpers import assert_string_arrays_equal
+from mypy.test.helpers import PytestSuite, test
 
 # Path to Python 3 interpreter
 python3_path = sys.executable
@@ -23,20 +23,20 @@
 cmdline_files = ['cmdline.test']
 
 
-class PythonEvaluationSuite(Suite):
-
-    def cases(self) -> List[DataDrivenTestCase]:
+class TestPythonEvaluation(PytestSuite):
+    @classmethod
+    def cases(cls) -> List[DataDrivenTestCase]:
         c = []  # type: List[DataDrivenTestCase]
         for f in cmdline_files:
             c += parse_test_cases(os.path.join(test_data_prefix, f),
-                                  test_python_evaluation,
+                                  python_evaluation_test,
                                   base_path=test_temp_dir,
                                   optional_out=True,
                                   native_sep=True)
         return c
 
 
-def test_python_evaluation(testcase: DataDrivenTestCase) -> None:
+def python_evaluation_test(obj: None, testcase: DataDrivenTestCase) -> None:
     # Write the program to a file.
     program = '_program.py'
     program_path = os.path.join(test_temp_dir, program)
@@ -58,9 +58,8 @@ def test_python_evaluation(testcase: DataDrivenTestCase) -> None:
     # Remove temp file.
     os.remove(program_path)
     # Compare actual output to expected.
-    assert_string_arrays_equal(testcase.output, out,
-                               'Invalid output ({}, line {})'.format(
-                                   testcase.file, testcase.line))
+    assert testcase.output == out, 'Invalid output ({}, line {})'.format(
+        testcase.file, testcase.line)
 
 
 def parse_args(line: str) -> List[str]:
@@ -78,3 +77,5 @@ def parse_args(line: str) -> List[str]:
     if not m:
         return []  # No args; mypy will spit out an error.
     return m.group(1).split()
+
+TestPythonEvaluation.setup_tests()