Fix bugs in test driver

Author: o11c

mypy/build.py

@@ -41,6 +41,7 @@
 DO_NOT_RUN = 'do-not-run'        # Only type check, don't run the program
 VERBOSE = 'verbose'              # More verbose messages (for troubleshooting)
 MODULE = 'module'                # Build/run module as a script
+PROGRAM_TEXT = 'program-text'    # Build/run command-line argument as a script
 TEST_BUILTINS = 'test-builtins'  # Use stub builtins to speed up tests
 DUMP_TYPE_STATS = 'dump-type-stats'
 DUMP_INFER_STATS = 'dump-infer-stats'
@@ -87,6 +88,7 @@ def __init__(self, files: Dict[str, MypyFile],
 def build(program_path: str,
           target: int,
           module: str = None,
+          argument: str = None,
           program_text: Union[str, bytes] = None,
           alt_lib_path: str = None,
           bin_dir: str = None,
@@ -145,9 +147,11 @@ def build(program_path: str,
     if alt_lib_path:
         lib_path.insert(0, alt_lib_path)
 
-    program_path = program_path or lookup_program(module, lib_path)
     if program_text is None:
+        program_path = program_path or lookup_program(module, lib_path)
         program_text = read_program(program_path)
+    else:
+        program_path = program_path or '<string>'
 
     reports = Reports(program_path, data_dir, report_dirs)
 
@@ -173,6 +177,7 @@ def build(program_path: str,
 
 
 def default_data_dir(bin_dir: str) -> str:
+    # TODO fix this logic
     if not bin_dir:
         # Default to current directory.
         return ''
@@ -220,6 +225,9 @@ def default_lib_path(data_dir: str, target: int, pyversion: int,
         for v in versions:
             path.append(os.path.join(data_dir, 'stubs', v))
 
+    # Add stubs for third-party libraries needed by mypy.
+    path.append(os.path.join(data_dir, 'stubs', 'third-party'))
+
     # Add fallback path that can be used if we have a broken installation.
     if sys.platform != 'win32':
         path.append('/usr/local/lib/mypy')
@@ -446,7 +454,7 @@ def lookup_state(self, module: str) -> 'State':
         for state in self.states:
             if state.id == module:
                 return state
-        raise RuntimeError('%s not found' % str)
+        raise RuntimeError('%s not found' % module)
 
     def all_imported_modules_in_file(self,
                                      file: MypyFile) -> List[Tuple[str, int]]:
@@ -461,7 +469,7 @@ def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str:
             rel = imp.relative
             if rel == 0:
                 return imp.id
-            if os.path.basename(file.path) == '__init__.py':
+            if os.path.basename(file.path).startswith('__init__.'):
                 rel -= 1
             if rel != 0:
                 file_id = ".".join(file_id.split(".")[:-rel])
@@ -504,7 +512,7 @@ def get_python_out_path(self, f: MypyFile) -> str:
             return os.path.join(self.output_dir, basename(f.path))
         else:
             components = f.fullname().split('.')
-            if os.path.basename(f.path) == '__init__.py':
+            if os.path.basename(f.path).startswith('__init__.'):
                 components.append('__init__.py')
             else:
                 components[-1] += '.py'

mypy/codec/pytokenize.py

@@ -92,7 +92,7 @@
 from token import *
 
 import token
-x = None
+x = None # type: str
 __all__ = [x for x in dir(token) if not x.startswith("_")]
 __all__ += ["COMMENT", "tokenize", "generate_tokens", "NL", "untokenize"]
 del x
@@ -187,7 +187,7 @@ def maybe(*choices): return group(*choices) + '?'
             'r': None, 'R': None, 'u': None, 'U': None,
             'b': None, 'B': None}
 
-triple_quoted = {}
+triple_quoted = {} # type: Dict[str, str]
 for t in ("'''", '"""',
           "r'''", 'r"""', "R'''", 'R"""',
           "u'''", 'u"""', "U'''", 'U"""',
@@ -197,7 +197,7 @@ def maybe(*choices): return group(*choices) + '?'
           "br'''", 'br"""', "Br'''", 'Br"""',
           "bR'''", 'bR"""', "BR'''", 'BR"""'):
     triple_quoted[t] = t
-single_quoted = {}
+single_quoted = {} # type: Dict[str, str]
 for t in ("'", '"',
           "r'", 'r"', "R'", 'R"',
           "u'", 'u"', "U'", 'U"',
@@ -333,6 +333,10 @@ def generate_tokens(readline):
     contline = None
     indents = [0]
 
+    if 0:
+        # type hints for mypy
+        strstart = (0, 0)
+        endprog = re.compile('')
     while 1:                                   # loop over lines in stream
         try:
             line = readline()

mypy/main.py

@@ -29,10 +29,10 @@ def __init__(self) -> None:
 
 def main() -> None:
     bin_dir = find_bin_directory()
-    path, module, options = process_options(sys.argv[1:])
+    path, module, program_text, options = process_options(sys.argv[1:])
     try:
         if options.target == build.TYPE_CHECK:
-            type_check_only(path, module, bin_dir, options)
+            type_check_only(path, module, program_text, bin_dir, options)
         else:
             raise RuntimeError('unsupported target %d' % options.target)
     except CompileError as e:
@@ -66,10 +66,11 @@ def readlinkabs(link: str) -> str:
     return os.path.join(os.path.dirname(link), path)
 
 
-def type_check_only(path: str, module: str, bin_dir: str, options: Options) -> None:
+def type_check_only(path: str, module: str, program_text: str, bin_dir: str, options: Options) -> None:
     # Type check the program and dependencies and translate to Python.
     build.build(path,
                 module=module,
+                program_text=program_text,
                 bin_dir=bin_dir,
                 target=build.TYPE_CHECK,
                 pyversion=options.pyversion,
@@ -79,7 +80,7 @@ def type_check_only(path: str, module: str, bin_dir: str, options: Options) -> N
                 python_path=options.python_path)
 
 
-def process_options(args: List[str]) -> Tuple[str, str, Options]:
+def process_options(args: List[str]) -> Tuple[str, str, str, Options]:
     """Process command line arguments.
 
     Return (mypy program path (or None),
@@ -99,7 +100,10 @@ def process_options(args: List[str]) -> Tuple[str, str, Options]:
             args = args[1:]
         elif args[0] == '-m' and args[1:]:
             options.build_flags.append(build.MODULE)
-            return None, args[1], options
+            return None, args[1], None, options
+        elif args[0] == '-c' and args[1:]:
+            options.build_flags.append(build.PROGRAM_TEXT)
+            return None, None, args[1], options
         elif args[0] in ('-h', '--help'):
             help = True
             args = args[1:]
@@ -142,7 +146,7 @@ def process_options(args: List[str]) -> Tuple[str, str, Options]:
         usage('--py2 specified, '
                 'but --use-python-path will search in sys.path of Python 3')
 
-    return args[0], None, options
+    return args[0], None, None, options
 
 
 # Don't generate this from mypy.reports, not all are meant to be public.
@@ -177,6 +181,7 @@ def usage(msg: str = None) -> None:
   --<fmt>-report dir generate a <fmt> report of type precision under dir/
                      <fmt> may be one of: %s
   -m mod             type check module
+  -c string          type check string
   --verbose          more verbose messages
   --use-python-path  search for modules in sys.path of running Python
   --version          show the current version information

mypy/nodes.py

@@ -159,7 +159,7 @@ def accept(self, visitor: NodeVisitor[T]) -> T:
 
     def is_package_init_file(self) -> bool:
         return not (self.path is None) and len(self.path) != 0 \
-            and os.path.basename(self.path) == '__init__.py'
+            and os.path.basename(self.path).startswith('__init__.')
 
 class ImportBase(Node):
     """Base class for all import statements."""

mypy/waiter.py

@@ -0,0 +1,107 @@
+from typing import Dict, List, Optional, Tuple
+
+import os
+from subprocess import Popen
+import sys
+
+
+class WaiterError(Exception):
+    pass
+
+class LazySubprocess:
+
+    def __init__(self, name: str, args: List[str], *, cwd: Optional[str] = None, env: Optional[Dict[str, str]] = None) -> None:
+        self.name = name
+        self.args = args
+        self.cwd = cwd
+        self.env = env
+
+    def __call__(self) -> Popen:
+        return Popen(self.args, cwd=self.cwd, env=self.env)
+
+
+class Waiter:
+    """Run subprocesses in parallel and wait for them.
+
+    Usage:
+
+    waiter = Waiter()
+    waiter.add('sleep 9')
+    waiter.add('sleep 10')
+    if not waiter.run():
+        print('error')
+    """
+    def __init__(self, limit: int = 0) -> None:
+        self.queue = [] # type: List[LazySubprocess]
+        self.next = 0
+        self.current = {} # type: Dict[int, Tuple[int, Popen]]
+        if limit == 0:
+            try:
+                sched_getaffinity = os.sched_getaffinity
+            except AttributeError:
+                limit = 2
+            else:
+                # Note: only count CPUs we are allowed to use. It is a
+                # major mistake to count *all* CPUs on the machine.
+                limit = len(sched_getaffinity(0))
+        try:
+            os.waitid
+            os.P_ALL
+            os.WEXITED
+            os.WNOWAIT
+        except AttributeError:
+            limit = 1
+        # Temporarily force until test parallelization bugs are fixed.
+        limit = 1
+        self.limit = limit
+        assert limit > 0
+        print('%-8s %d' % ('PARALLEL', limit))
+
+    def add(self, cmd: LazySubprocess) -> int:
+        rv = len(self.queue)
+        self.queue.append(cmd)
+        return rv
+
+    def _start1(self) -> None:
+        cmd = self.queue[self.next]
+        name = cmd.name
+        proc = cmd()
+        num = self.next
+        self.current[proc.pid] = (num, proc)
+        print('%-8s #%d %s' % ('START', num, name))
+        self.next += 1
+
+    def _wait1(self) -> List[str]:
+        if self.limit > 1:
+            # WNOWAIT is not to be confused with WNOHANG
+            pid = os.waitid(os.P_ALL, -1, os.WEXITED | os.WNOWAIT).si_pid
+            num, proc = self.current.pop(pid)
+        else:
+            pid, (num, proc) = self.current.popitem()
+        name = self.queue[num].name
+        rc = proc.wait()
+        if rc >= 0:
+            msg = 'EXIT %d' % rc
+        else:
+            msg = 'SIG %d' % -rc
+        print('%-8s #%d %s' % (msg, num, name))
+        if rc != 0:
+            return [name]
+        else:
+            return []
+
+    def run(self) -> None:
+        print('SUMMARY  %d tasks selected' % len(self.queue))
+        failures = [] # type: List[str]
+        while self.current or self.next < len(self.queue):
+            while len(self.current) < self.limit and self.next < len(self.queue):
+                self._start1()
+            failures += self._wait1()
+        if failures:
+            print('SUMMARY  %d/%d tasks failed' % (len(failures), len(self.queue)))
+            for f in failures:
+                print(' FAILURE %s' % f)
+            print('SUMMARY  %d/%d tasks failed' % (len(failures), len(self.queue)))
+            sys.exit(1)
+        else:
+            print('SUMMARY  all %d tasks passed' % len(self.queue))

samples/codec/example.py

@@ -2,8 +2,9 @@
 
 # very simple script to test type annotations
 
+from typing import Tuple
 
-def f(x: int, y: str='abc') -> [int,
+def f(x: int, y: str='abc') -> Tuple[int,
                                 str]:
     return x, y
 

setup.py

@@ -49,12 +49,7 @@ def find_data_files(base, globs):
 
 data_files = []
 
-# Is there a reason not to just use:
-#data_files += find_data_files('stubs', ...)
-
-for py_version in ['3.4', '3.3', '3.2', '2.7']:
-    base = os.path.join('stubs', py_version)
-    data_files += find_data_files(base, ['*.py', '*.pyi'])
+data_files += find_data_files('stubs', ['*.py', '*.pyi'])
 
 data_files += find_data_files('xml', ['*.xsd', '*.xslt', '*.css'])