Parser redesign

.gitignore

@@ -1,10 +1,19 @@
-build/
-__pycache__
+# Backup and compiled files
+__pycache__/
 *.pyc
 *~
+
+# Unclassified
 @*
-/build
 /env
 docs/build/
 *.iml
 /out/
+
+# Used by setup.py and tox
+/build/
+/dist/
+/elsewhere/
+/.tox/
+/MANIFEST
+/lib-typing/3.2/mypy_lang.egg-info/

.travis.yml

@@ -1,23 +1,27 @@
 sudo: false
 language: python
-python:
-  - "3.2"
-  - "3.3"
-  - "3.4"
-  - "3.5"
-  # Pypy build is disaabled because it doubles the travis build time, and it rarely fails
-  # unless one one of the other builds fails.
-  # - "pypy3"
 
 install:
-  - pip install -r test-requirements.txt
-  - python setup.py install
+  - pip install tox
 
 script:
-  - python runtests.py -v
+  - tox
 
 notifications:
   irc:
     channels:
       - "chat.freenode.net##mypy"
     use_notice: true
+
+matrix:
+  include:
+    - python: "3.2"
+      env: TOXENV=py32
+    - python: "3.3"
+      env: TOXENV=py33
+    - python: "3.4"
+      env: TOXENV=py34
+    - python: "3.5"
+      env: TOXENV=py35
+    - python: "pypy3"
+      env: TOXENV=pypy3

mypy/build.py

@@ -27,7 +27,7 @@
 from mypy import parse
 from mypy import stats
 from mypy.report import Reports
-from mypy import defaults
+from mypy.syntax.dialect import Dialect, Implementation, default_implementation
 
 
 # We need to know the location of this file to load data, but
@@ -95,8 +95,7 @@ def build(program_path: str,
           argument: str = None,
           program_text: Union[str, bytes] = None,
           alt_lib_path: str = None,
-          bin_dir: str = None,
-          pyversion: Tuple[int, int] = defaults.PYTHON3_VERSION,
+          implementation: Implementation = None,
           custom_typing_module: str = None,
           report_dirs: Dict[str, str] = {},
           flags: List[str] = None,
@@ -116,19 +115,18 @@ def build(program_path: str,
       program_text: the main source file contents; if omitted, read from file
       alt_lib_dir: an additional directory for looking up library modules
         (takes precedence over other directories)
-      bin_dir: directory containing the mypy script, used for finding data
-        directories; if omitted, use '.' as the data directory
-      pyversion: Python version (major, minor)
+      implementation: Python version
       custom_typing_module: if not None, use this module id as an alias for typing
       flags: list of build options (e.g. COMPILE_ONLY)
     """
     flags = flags or []
     module = module or '__main__'
+    implementation = implementation or default_implementation()
 
-    data_dir = default_data_dir(bin_dir)
+    data_dir = default_data_dir()
 
     # Determine the default module search path.
-    lib_path = default_lib_path(data_dir, target, pyversion, python_path)
+    lib_path = default_lib_path(data_dir, target, implementation, python_path)
 
     if TEST_BUILTINS in flags:
         # Use stub builtins (to speed up test cases and to make them easier to
@@ -160,7 +158,7 @@ def build(program_path: str,
     #
     # Ignore current directory prefix in error messages.
     manager = BuildManager(data_dir, lib_path, target,
-                           pyversion=pyversion, flags=flags,
+                           dialect=implementation.base_dialect, flags=flags,
                            ignore_prefix=os.getcwd(),
                            custom_typing_module=custom_typing_module,
                            reports=reports)
@@ -176,32 +174,15 @@ def build(program_path: str,
     return result
 
 
-def default_data_dir(bin_dir: str) -> str:
-    # TODO fix this logic
-    if not bin_dir:
-        # Default to directory containing this file's parent.
-        return os.path.dirname(os.path.dirname(__file__))
-    base = os.path.basename(bin_dir)
-    dir = os.path.dirname(bin_dir)
-    if (sys.platform == 'win32' and base.lower() == 'mypy'
-            and not os.path.isdir(os.path.join(dir, 'stubs'))):
-        # Installed, on Windows.
-        return os.path.join(dir, 'Lib', 'mypy')
-    elif base == 'mypy':
-        # Assume that we have a repo check out or unpacked source tarball.
-        return os.path.dirname(bin_dir)
-    elif base == 'bin':
-        # Installed to somewhere (can be under /usr/local or anywhere).
-        return os.path.join(dir, 'lib', 'mypy')
-    elif base == 'python3':
-        # Assume we installed python3 with brew on os x
-        return os.path.join(os.path.dirname(dir), 'lib', 'mypy')
-    else:
-        # Don't know where to find the data files!
-        raise RuntimeError("Broken installation: can't determine base dir")
+def is_installed() -> bool:
+    return 'site-packages' in __file__ or 'dist-packages' in __file__
+
+
+def default_data_dir() -> str:
+    return os.path.join(os.path.dirname(__file__), 'data')
 
 
-def default_lib_path(data_dir: str, target: int, pyversion: Tuple[int, int],
+def default_lib_path(data_dir: str, target: int, implementation: Implementation,
         python_path: bool) -> List[str]:
     """Return default standard library search paths."""
     # IDEA: Make this more portable.
@@ -216,14 +197,12 @@ def default_lib_path(data_dir: str, target: int, pyversion: Tuple[int, int],
     # stubs/x.y directory of the mypy installation. Additionally, stubs
     # for earlier versions in the same major version will be added, and
     # as a last resort, third-party stubs will be added.
-    if pyversion == 2:
-        major, minor = 2, 7
+    major = implementation.base_dialect.major
+    minor = implementation.base_dialect.minor
+    if major == 3:
+        version_dir = '3.2'
+        third_party_dir = 'third-party-3.2'
     else:
-        # See bug #886
-        major, minor = sys.version_info[0], sys.version_info[1]
-    version_dir = '3.2'
-    third_party_dir = 'third-party-3.2'
-    if pyversion[0] < 3:
         version_dir = '2.7'
         third_party_dir = 'third-party-2.7'
     path.append(os.path.join(data_dir, 'stubs', version_dir))
@@ -244,13 +223,9 @@ def default_lib_path(data_dir: str, target: int, pyversion: Tuple[int, int],
             if os.path.isdir(third_party_stubdir):
                 path.append(third_party_stubdir)
 
-    # Add fallback path that can be used if we have a broken installation.
-    if sys.platform != 'win32':
-        path.append('/usr/local/lib/mypy')
-
     # Contents of Python's sys.path go last, to prefer the stubs
     if python_path:
-        path.extend(sys.path)
+        path.extend(implementation.python_path)
 
     return path
 
@@ -292,7 +267,7 @@ class BuildManager:
                        Semantic analyzer, pass 3
       type_checker:    Type checker
       errors:          Used for reporting all errors
-      pyversion:       Python version (major, minor)
+      dialect:         Python version
       flags:           Build options
       states:          States of all individual files that are being
                        processed. Each file in a build is always represented
@@ -310,7 +285,7 @@ class BuildManager:
     def __init__(self, data_dir: str,
                  lib_path: List[str],
                  target: int,
-                 pyversion: Tuple[int, int],
+                 dialect: Dialect,
                  flags: List[str],
                  ignore_prefix: str,
                  custom_typing_module: str,
@@ -320,16 +295,16 @@ def __init__(self, data_dir: str,
         self.errors.set_ignore_prefix(ignore_prefix)
         self.lib_path = lib_path
         self.target = target
-        self.pyversion = pyversion
+        self.dialect = dialect
         self.flags = flags
         self.custom_typing_module = custom_typing_module
         self.reports = reports
         self.semantic_analyzer = SemanticAnalyzer(lib_path, self.errors,
-                                                  pyversion=pyversion)
+                                                  dialect=dialect)
         self.semantic_analyzer_pass3 = ThirdPass(self.errors)
         self.type_checker = TypeChecker(self.errors,
                                         self.semantic_analyzer.modules,
-                                        self.pyversion)
+                                        dialect)
         self.states = []  # type: List[State]
         self.module_files = {}  # type: Dict[str, str]
         self.module_deps = {}  # type: Dict[Tuple[str, str], bool]
@@ -740,7 +715,7 @@ def parse(self, source_text: Union[str, bytes], fnam: str) -> MypyFile:
         """
         num_errs = self.errors().num_messages()
         tree = parse.parse(source_text, fnam, self.errors(),
-                           pyversion=self.manager.pyversion,
+                           dialect=self.manager.dialect,
                            custom_typing_module=self.manager.custom_typing_module)
         tree._fullname = self.id
         if self.errors().num_messages() != num_errs:

mypy/checker.py

@@ -33,7 +33,7 @@
 from mypy.sametypes import is_same_type
 from mypy.messages import MessageBuilder
 import mypy.checkexpr
-from mypy import defaults
+from mypy.syntax.dialect import Dialect, default_dialect
 from mypy import messages
 from mypy.subtypes import (
     is_subtype, is_equivalent, is_proper_subtype,
@@ -302,7 +302,7 @@ class TypeChecker(NodeVisitor[Type]):
     """
 
     # Target Python version
-    pyversion = defaults.PYTHON3_VERSION
+    dialect = None  # type: Dialect
     # Are we type checking a stub?
     is_stub = False
     # Error message reporter
@@ -337,7 +337,7 @@ class TypeChecker(NodeVisitor[Type]):
     modules = None  # type: Dict[str, MypyFile]
 
     def __init__(self, errors: Errors, modules: Dict[str, MypyFile],
-                 pyversion: Tuple[int, int] = defaults.PYTHON3_VERSION) -> None:
+                 dialect: Dialect = default_dialect()) -> None:
         """Construct a type checker.
 
         Use errors to report type check errors. Assume symtable has been
@@ -346,7 +346,7 @@ def __init__(self, errors: Errors, modules: Dict[str, MypyFile],
         self.expr_checker
         self.errors = errors
         self.modules = modules
-        self.pyversion = pyversion
+        self.dialect = dialect
         self.msg = MessageBuilder(errors)
         self.type_map = {}
         self.binder = ConditionalTypeBinder()
@@ -1548,7 +1548,7 @@ def type_check_raise(self, e: Node, s: RaiseStmt) -> None:
                     # Good!
                     return None
                 # Else fall back to the checks below (which will fail).
-        if isinstance(typ, TupleType) and self.pyversion[0] == 2:
+        if isinstance(typ, TupleType) and self.dialect.major == 2:
             # allow `raise type, value, traceback`
             # https://docs.python.org/2/reference/simple_stmts.html#the-raise-statement
             # TODO: Also check tuple item types.
@@ -1660,7 +1660,7 @@ def analyze_iterable_item_type(self, expr: Node) -> Type:
             method = echk.analyze_external_member_access('__iter__', iterable,
                                                          expr)
             iterator = echk.check_call(method, [], [], expr)[0]
-            if self.pyversion[0] >= 3:
+            if self.dialect.major >= 3:
                 nextmethod = '__next__'
             else:
                 nextmethod = 'next'

mypy/checkexpr.py

@@ -727,7 +727,7 @@ def visit_complex_expr(self, e: ComplexExpr) -> Type:
 
     def visit_ellipsis(self, e: EllipsisExpr) -> Type:
         """Type check '...'."""
-        if self.chk.pyversion[0] >= 3:
+        if self.chk.dialect.major >= 3:
             return self.named_type('builtins.ellipsis')
         else:
             # '...' is not valid in normal Python 2 code, but it can
@@ -821,7 +821,7 @@ def visit_comparison_expr(self, e: ComparisonExpr) -> Type:
         return result
 
     def get_operator_method(self, op: str) -> str:
-        if op == '/' and self.chk.pyversion[0] == 2:
+        if op == '/' and self.chk.dialect.major == 2:
             # TODO also check for "from __future__ import division"
             return '__div__'
         else:
@@ -893,7 +893,7 @@ def check_op(self, method: str, base_type: Type, arg: Node,
                                            self.msg)
 
     def get_reverse_op_method(self, method: str) -> str:
-        if method == '__div__' and self.chk.pyversion[0] == 2:
+        if method == '__div__' and self.chk.dialect.major == 2:
             return '__rdiv__'
         else:
             return nodes.reverse_op_methods[method]

mypy/constraints.py

@@ -273,6 +273,12 @@ def infer_against_any(self, types: List[Type]) -> List[Constraint]:
             res.extend(infer_constraints(t, AnyType(), self.direction))
         return res
 
+    def visit_overloaded(self, type: Overloaded) -> List[Constraint]:
+        res = []  # type: List[Constraint]
+        for t in type.items():
+            res.extend(infer_constraints(t, self.actual, self.direction))
+        return res
+
 
 def negate_constraints(constraints: List[Constraint]) -> List[Constraint]:
     res = []  # type: List[Constraint]

mypy/data/stubs/3.2/platform.pyi

@@ -0,0 +1,35 @@
+# Stubs for platform (Python 3.5)
+
+from typing import Tuple, NamedTuple
+
+from os import devnull as DEV_NULL
+
+def libc_ver(executable: str = ..., lib: str = '', version: str = '', chunksize: int = 16384) -> Tuple[str, str]: ...
+def linux_distribution(distname: str = '', version: str = '', id: str = '', supported_dists: Tuple[str, ...] = ..., full_distribution_name: bool = True) -> Tuple[str, str, str]: ...
+def dist(distname: str = '', version: str = '', id: str = '', supported_dists: Tuple[str, ...] = ...) -> Tuple[str, str, str]: ...
+from os import popen
+def win32_ver(release: str = '', version: str = '', csd: str = '', ptype: str = '') -> Tuple[str, str, str, str]: ...
+def mac_ver(release: str = '', versioninfo: Tuple[str, str, str] = ..., machine: str = '') -> Tuple[str, Tuple[str, str, str], str]: ...
+def java_ver(release: str = '', vendor: str = '', vminfo: Tuple[str, str, str] = ..., osinfo: Tuple[str, str, str] = ...) -> Tuple[str, str, Tuple[str, str, str], Tuple[str, str, str]]: ...
+def system_alias(system: str, release: str, version: str) -> Tuple[str, str, str]: ...
+def architecture(executable: str = ..., bits: str = '', linkage: str = '') -> Tuple[str, str]: ...
+
+uname_result = NamedTuple('uname_result', [('system', str), ('node', str), ('release', str), ('version', str), ('machine', str), ('processor', str)])
+
+def uname() -> uname_result: ...
+def system() -> str: ...
+def node() -> str: ...
+def release() -> str: ...
+def version() -> str: ...
+def machine() -> str: ...
+def processor() -> str: ...
+
+def python_implementation() -> str: ...
+def python_version() -> str: ...
+def python_version_tuple() -> Tuple[str, str, str]: ...
+def python_branch() -> str: ...
+def python_revision() -> str: ...
+def python_build() -> Tuple[str, str]: ...
+def python_compiler() -> str: ...
+
+def platform(aliased: bool = False, terse: bool = False) -> str: ...