Move pythoneval to use mypy.api.run

This replaces the old subprocess based method. In addition, the test cases
are no longer run, in order to reduce test time. This has led to a 10-20%
speedup in pytest. This might help with python#3895, as it removes the
subprocesses.

Fixes python#1671

Author: emmatyping

mypy/test/testpythoneval.py

@@ -10,12 +10,9 @@
       this suite would slow down the main suite too much.
 """
 
-from contextlib import contextmanager
-import errno
 import os
 import os.path
 import re
-import subprocess
 import sys
 
 import pytest  # type: ignore  # no pytest in typeshed
@@ -25,6 +22,7 @@
 from mypy.test.data import DataDrivenTestCase, parse_test_cases, DataSuite
 from mypy.test.helpers import assert_string_arrays_equal
 from mypy.util import try_find_python2_interpreter
+from mypy.api import run
 
 # Files which contain test case descriptions.
 python_eval_files = ['pythoneval.test',
@@ -61,73 +59,38 @@ def test_python_evaluation(testcase: DataDrivenTestCase) -> None:
     version.
     """
     assert testcase.old_cwd is not None, "test was not properly set up"
-    mypy_cmdline = [
-        python3_path,
-        os.path.join(testcase.old_cwd, 'scripts', 'mypy'),
-        '--show-traceback',
-    ]
+    mypy_cmdline = ['--show-traceback']
     py2 = testcase.name.lower().endswith('python2')
     if py2:
         mypy_cmdline.append('--py2')
-        interpreter = try_find_python2_interpreter()
-        if interpreter is None:
-            # Skip, can't find a Python 2 interpreter.
-            pytest.skip()
-            # placate the type checker
-            return
-    else:
-        interpreter = python3_path
-
     # Write the program to a file.
     program = '_' + testcase.name + '.py'
-    mypy_cmdline.append(program)
     program_path = os.path.join(test_temp_dir, program)
+    mypy_cmdline.append(program_path)
     with open(program_path, 'w') as file:
         for s in testcase.input:
             file.write('{}\n'.format(s))
     # Type check the program.
     # This uses the same PYTHONPATH as the current process.
-    returncode, out = run(mypy_cmdline)
-    if returncode == 0:
-        # Execute the program.
-        returncode, interp_out = run([interpreter, program])
-        out += interp_out
+    out, err, returncode = run(mypy_cmdline)
+    output = split_lines(out, err)
     # Remove temp file.
     os.remove(program_path)
-    assert_string_arrays_equal(adapt_output(testcase), out,
+    assert_string_arrays_equal(adapt_output(testcase), output,
                                'Invalid output ({}, line {})'.format(
                                    testcase.file, testcase.line))
 
 
-def split_lines(*streams: bytes) -> List[str]:
+def split_lines(*streams: str) -> List[str]:
     """Returns a single list of string lines from the byte streams in args."""
     return [
         s.rstrip('\n\r')
         for stream in streams
-        for s in str(stream, 'utf8').splitlines()
+        for s in stream.splitlines()
     ]
 
 
 def adapt_output(testcase: DataDrivenTestCase) -> List[str]:
     """Translates the generic _program.py into the actual filename."""
     program = '_' + testcase.name + '.py'
-    return [program_re.sub(program, line) for line in testcase.output]
-
-
-def run(
-    cmdline: List[str], *, env: Optional[Dict[str, str]] = None, timeout: int = 30
-) -> Tuple[int, List[str]]:
-    """A poor man's subprocess.run() for 3.3 and 3.4 compatibility."""
-    process = subprocess.Popen(
-        cmdline,
-        env=env,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        cwd=test_temp_dir,
-    )
-    try:
-        out, err = process.communicate(timeout=timeout)
-    except subprocess.TimeoutExpired:
-        out = err = b''
-        process.kill()
-    return process.returncode, split_lines(out, err)
+    return [test_temp_dir + os.sep + program_re.sub(program, line) for line in testcase.output]

test-data/unit/python2eval.test

@@ -1,5 +1,4 @@
--- Test cases for type checking mypy programs using full stubs and running
--- using CPython (Python 2 mode).
+-- Test cases for type checking mypy programs using full stubs
 --
 -- These are mostly regression tests -- no attempt is made to make these
 -- complete.
@@ -22,46 +21,32 @@ print x
 x = u'foo'
 print repr(x)
 [out]
-xyz
-u'foo'
 
 [case testXrangeAndRange_python2]
 for i in xrange(2):
     print i
 for i in range(3):
     print i
 [out]
-0
-1
-0
-1
-2
 
 [case testIterator_python2]
 import typing, sys
 x = iter('bar')
 print x.next(), x.next()
 [out]
-b a
 
 [case testEncodeAndDecode_python2]
 print 'a'.encode('latin1')
 print 'b'.decode('latin1')
 print u'c'.encode('latin1')
 print u'd'.decode('latin1')
 [out]
-a
-b
-c
-d
 
 [case testHasKey_python2]
 d = {1: 'x'}
 print d.has_key(1)
 print d.has_key(2)
 [out]
-True
-False
 
 [case testIntegerDivision_python2]
 x = 1 / 2
@@ -86,8 +71,6 @@ def f(x): # type: (AnyStr) -> AnyStr
 print f('')
 print f(u'')
 [out]
-foo
-zar
 
 [case testGenericPatterns_python2]
 from typing import Pattern
@@ -98,7 +81,6 @@ b = None  # type: Pattern[str]
 b = re.compile('foo*')
 print(p.match(u'fooo').group(0))
 [out]
-fooo
 
 [case testGenericMatch_python2]
 from typing import Match
@@ -107,26 +89,22 @@ def f(m): # type: (Match[str]) -> None
     print(m.group(0))
 f(re.match('x*', 'xxy'))
 [out]
-xx
 
 [case testVariableLengthTuple_python2]
 from typing import Tuple, cast
 x = cast(Tuple[int, ...], ())
 print(x)
 [out]
-()
 
 [case testFromFuturePrintFunction_python2]
 from __future__ import print_function
 print('a', 'b')
 [out]
-a b
 
 [case testFromFutureImportUnicodeLiterals_python2]
 from __future__ import unicode_literals
 print '>', ['a', b'b', u'c']
 [out]
-> [u'a', 'b', u'c']
 
 [case testUnicodeLiteralsKwargs_python2]
 from __future__ import unicode_literals
@@ -182,7 +160,6 @@ def f(a): # type: (Sequence[T]) -> None
     print a
 f(tuple())
 [out]
-()
 
 [case testReadOnlyProperty_python2]
 import typing
@@ -192,7 +169,6 @@ class A:
         return 1
 print(A().foo + 2)
 [out]
-3
 
 [case testIOTypes_python2]
 from typing import IO, TextIO, BinaryIO, Any
@@ -219,7 +195,6 @@ if 1 == 2: # Don't want to run the code below, since it would create a file.
     f.close()
 print('ok')
 [out]
-ok
 
 [case testStringIO_python2]
 import typing
@@ -228,7 +203,6 @@ c = io.StringIO()
 c.write(u'\x89')
 print(repr(c.getvalue()))
 [out]
-u'\x89'
 
 [case testBytesIO_python2]
 import typing
@@ -237,7 +211,6 @@ c = io.BytesIO()
 c.write('\x89')
 print(repr(c.getvalue()))
 [out]
-'\x89'
 
 [case testTextIOWrapper_python2]
 import typing
@@ -246,7 +219,6 @@ b = io.BytesIO(u'\xab'.encode('utf8'))
 w = io.TextIOWrapper(b, encoding='utf8')
 print(repr(w.read()))
 [out]
-u'\xab'
 
 [case testIoOpen_python2]
 import typing
@@ -257,7 +229,6 @@ if 1 == 2: # Only type check, do not execute
     f.close()
 print 'ok'
 [out]
-ok
 
 [case testUnionType_python2]
 from typing import Union
@@ -269,8 +240,6 @@ def f(x): # type: (Union[int, str]) -> str
 print f(12)
 print f('ab')
 [out]
-12
-ab
 
 [case testStrAdd_python2]
 import typing
@@ -301,7 +270,6 @@ X = namedtuple('X', ['a', 'b'])
 x = X(a=1, b='s')
 print x.a, x.b
 [out]
-1 s
 
 [case testNamedTupleError_python2]
 import typing
@@ -328,9 +296,6 @@ print 5 + 8j
 print 3j * 2.0
 print 4j / 2.0
 [out]
-(5+8j)
-6j
-2j
 
 [case testNamedTupleWithTypes_python2]
 from typing import NamedTuple
@@ -341,9 +306,6 @@ a, b = n
 print a, b
 print n[0]
 [out]
-N(a=1, b='x')
-1 x
-1
 
 [case testUnionTypeAlias_python2]
 from typing import Union
@@ -363,7 +325,6 @@ class A(object):
     __metaclass__ = MyType
 print(type(A()).__name__)
 [out]
-Ax
 
 [case testSequenceIndexAndCount_python2]
 from typing import Sequence
@@ -372,8 +333,6 @@ def f(x): # type: (Sequence[int]) -> None
     print(x.count(1))
 f([0, 0, 1, 1, 1])
 [out]
-2
-3
 
 [case testOptional_python2]
 from typing import Optional
@@ -419,7 +378,6 @@ class B(A):
 b = B()
 print b.x + 1
 [out]
-4
 
 [case testReModuleBytesPython2]
 # Regression tests for various overloads in the re module -- bytes version