Use new Dialect objects to store version everywhere

Author: o11c

mypy/build.py

@@ -27,7 +27,7 @@
 from mypy import parse
 from mypy import stats
 from mypy.report import Reports
-from mypy import defaults
+from mypy.syntax.dialect import Dialect, Implementation, default_implementation
 
 
 # We need to know the location of this file to load data, but
@@ -95,7 +95,7 @@ def build(program_path: str,
           argument: str = None,
           program_text: Union[str, bytes] = None,
           alt_lib_path: str = None,
-          pyversion: Tuple[int, int] = defaults.PYTHON3_VERSION,
+          implementation: Implementation = None,
           custom_typing_module: str = None,
           report_dirs: Dict[str, str] = {},
           flags: List[str] = None,
@@ -115,17 +115,18 @@ def build(program_path: str,
       program_text: the main source file contents; if omitted, read from file
       alt_lib_dir: an additional directory for looking up library modules
         (takes precedence over other directories)
-      pyversion: Python version (major, minor)
+      implementation: Python version
       custom_typing_module: if not None, use this module id as an alias for typing
       flags: list of build options (e.g. COMPILE_ONLY)
     """
     flags = flags or []
     module = module or '__main__'
+    implementation = implementation or default_implementation()
 
     data_dir = default_data_dir()
 
     # Determine the default module search path.
-    lib_path = default_lib_path(data_dir, target, pyversion, python_path)
+    lib_path = default_lib_path(data_dir, target, implementation, python_path)
 
     if TEST_BUILTINS in flags:
         # Use stub builtins (to speed up test cases and to make them easier to
@@ -157,7 +158,7 @@ def build(program_path: str,
     #
     # Ignore current directory prefix in error messages.
     manager = BuildManager(data_dir, lib_path, target,
-                           pyversion=pyversion, flags=flags,
+                           dialect=implementation.base_dialect, flags=flags,
                            ignore_prefix=os.getcwd(),
                            custom_typing_module=custom_typing_module,
                            reports=reports)
@@ -181,7 +182,7 @@ def default_data_dir() -> str:
     return os.path.join(os.path.dirname(__file__), 'data')
 
 
-def default_lib_path(data_dir: str, target: int, pyversion: Tuple[int, int],
+def default_lib_path(data_dir: str, target: int, implementation: Implementation,
         python_path: bool) -> List[str]:
     """Return default standard library search paths."""
     # IDEA: Make this more portable.
@@ -196,14 +197,12 @@ def default_lib_path(data_dir: str, target: int, pyversion: Tuple[int, int],
     # stubs/x.y directory of the mypy installation. Additionally, stubs
     # for earlier versions in the same major version will be added, and
     # as a last resort, third-party stubs will be added.
-    if pyversion == 2:
-        major, minor = 2, 7
+    major = implementation.base_dialect.major
+    minor = implementation.base_dialect.minor
+    if major == 3:
+        version_dir = '3.2'
+        third_party_dir = 'third-party-3.2'
     else:
-        # See bug #886
-        major, minor = sys.version_info[0], sys.version_info[1]
-    version_dir = '3.2'
-    third_party_dir = 'third-party-3.2'
-    if pyversion[0] < 3:
         version_dir = '2.7'
         third_party_dir = 'third-party-2.7'
     path.append(os.path.join(data_dir, 'stubs', version_dir))
@@ -224,13 +223,9 @@ def default_lib_path(data_dir: str, target: int, pyversion: Tuple[int, int],
             if os.path.isdir(third_party_stubdir):
                 path.append(third_party_stubdir)
 
-    # Add fallback path that can be used if we have a broken installation.
-    if sys.platform != 'win32':
-        path.append('/usr/local/lib/mypy')
-
     # Contents of Python's sys.path go last, to prefer the stubs
     if python_path:
-        path.extend(sys.path)
+        path.extend(implementation.python_path)
 
     return path
 
@@ -272,7 +267,7 @@ class BuildManager:
                        Semantic analyzer, pass 3
       type_checker:    Type checker
       errors:          Used for reporting all errors
-      pyversion:       Python version (major, minor)
+      dialect:         Python version
       flags:           Build options
       states:          States of all individual files that are being
                        processed. Each file in a build is always represented
@@ -290,7 +285,7 @@ class BuildManager:
     def __init__(self, data_dir: str,
                  lib_path: List[str],
                  target: int,
-                 pyversion: Tuple[int, int],
+                 dialect: Dialect,
                  flags: List[str],
                  ignore_prefix: str,
                  custom_typing_module: str,
@@ -300,16 +295,16 @@ def __init__(self, data_dir: str,
         self.errors.set_ignore_prefix(ignore_prefix)
         self.lib_path = lib_path
         self.target = target
-        self.pyversion = pyversion
+        self.dialect = dialect
         self.flags = flags
         self.custom_typing_module = custom_typing_module
         self.reports = reports
         self.semantic_analyzer = SemanticAnalyzer(lib_path, self.errors,
-                                                  pyversion=pyversion)
+                                                  dialect=dialect)
         self.semantic_analyzer_pass3 = ThirdPass(self.errors)
         self.type_checker = TypeChecker(self.errors,
                                         self.semantic_analyzer.modules,
-                                        self.pyversion)
+                                        dialect)
         self.states = []  # type: List[State]
         self.module_files = {}  # type: Dict[str, str]
         self.module_deps = {}  # type: Dict[Tuple[str, str], bool]
@@ -720,7 +715,7 @@ def parse(self, source_text: Union[str, bytes], fnam: str) -> MypyFile:
         """
         num_errs = self.errors().num_messages()
         tree = parse.parse(source_text, fnam, self.errors(),
-                           pyversion=self.manager.pyversion,
+                           dialect=self.manager.dialect,
                            custom_typing_module=self.manager.custom_typing_module)
         tree._fullname = self.id
         if self.errors().num_messages() != num_errs:

mypy/checker.py

@@ -33,7 +33,7 @@
 from mypy.sametypes import is_same_type
 from mypy.messages import MessageBuilder
 import mypy.checkexpr
-from mypy import defaults
+from mypy.syntax.dialect import Dialect, default_dialect
 from mypy import messages
 from mypy.subtypes import (
     is_subtype, is_equivalent, is_proper_subtype,
@@ -302,7 +302,7 @@ class TypeChecker(NodeVisitor[Type]):
     """
 
     # Target Python version
-    pyversion = defaults.PYTHON3_VERSION
+    dialect = None  # type: Dialect
     # Are we type checking a stub?
     is_stub = False
     # Error message reporter
@@ -337,7 +337,7 @@ class TypeChecker(NodeVisitor[Type]):
     modules = None  # type: Dict[str, MypyFile]
 
     def __init__(self, errors: Errors, modules: Dict[str, MypyFile],
-                 pyversion: Tuple[int, int] = defaults.PYTHON3_VERSION) -> None:
+                 dialect: Dialect = default_dialect()) -> None:
         """Construct a type checker.
 
         Use errors to report type check errors. Assume symtable has been
@@ -346,7 +346,7 @@ def __init__(self, errors: Errors, modules: Dict[str, MypyFile],
         self.expr_checker
         self.errors = errors
         self.modules = modules
-        self.pyversion = pyversion
+        self.dialect = dialect
         self.msg = MessageBuilder(errors)
         self.type_map = {}
         self.binder = ConditionalTypeBinder()
@@ -1548,7 +1548,7 @@ def type_check_raise(self, e: Node, s: RaiseStmt) -> None:
                     # Good!
                     return None
                 # Else fall back to the checks below (which will fail).
-        if isinstance(typ, TupleType) and self.pyversion[0] == 2:
+        if isinstance(typ, TupleType) and self.dialect.major == 2:
             # allow `raise type, value, traceback`
             # https://docs.python.org/2/reference/simple_stmts.html#the-raise-statement
             # TODO: Also check tuple item types.
@@ -1660,7 +1660,7 @@ def analyze_iterable_item_type(self, expr: Node) -> Type:
             method = echk.analyze_external_member_access('__iter__', iterable,
                                                          expr)
             iterator = echk.check_call(method, [], [], expr)[0]
-            if self.pyversion[0] >= 3:
+            if self.dialect.major >= 3:
                 nextmethod = '__next__'
             else:
                 nextmethod = 'next'

mypy/checkexpr.py

@@ -727,7 +727,7 @@ def visit_complex_expr(self, e: ComplexExpr) -> Type:
 
     def visit_ellipsis(self, e: EllipsisExpr) -> Type:
         """Type check '...'."""
-        if self.chk.pyversion[0] >= 3:
+        if self.chk.dialect.major >= 3:
             return self.named_type('builtins.ellipsis')
         else:
             # '...' is not valid in normal Python 2 code, but it can
@@ -821,7 +821,7 @@ def visit_comparison_expr(self, e: ComparisonExpr) -> Type:
         return result
 
     def get_operator_method(self, op: str) -> str:
-        if op == '/' and self.chk.pyversion[0] == 2:
+        if op == '/' and self.chk.dialect.major == 2:
             # TODO also check for "from __future__ import division"
             return '__div__'
         else:
@@ -893,7 +893,7 @@ def check_op(self, method: str, base_type: Type, arg: Node,
                                            self.msg)
 
     def get_reverse_op_method(self, method: str) -> str:
-        if method == '__div__' and self.chk.pyversion[0] == 2:
+        if method == '__div__' and self.chk.dialect.major == 2:
             return '__rdiv__'
         else:
             return nodes.reverse_op_methods[method]

mypy/defaults.py

@@ -8,8 +8,8 @@
 
 import re
 
+from mypy.syntax.dialect import Dialect, default_dialect
 from mypy.util import short_type
-from mypy import defaults
 from typing import List, Callable, Dict, Any, Match, Pattern, Set, Union, Tuple
 
 
@@ -159,14 +159,13 @@ def __str__(self):
 
 
 def lex(string: Union[str, bytes], first_line: int = 1,
-        pyversion: Tuple[int, int] = defaults.PYTHON3_VERSION,
-        is_stub_file: bool = False) -> Tuple[List[Token], Set[int]]:
+        dialect: Dialect = default_dialect()) -> Tuple[List[Token], Set[int]]:
     """Analyze string, and return an array of token objects and the lines to ignore.
 
     The last token is always Eof. The intention is to ignore any
     semantic and type check errors on the ignored lines.
     """
-    l = Lexer(pyversion, is_stub_file=is_stub_file)
+    l = Lexer(dialect)
     l.lex(string, first_line)
     return l.tok, l.ignored_lines
 
@@ -293,19 +292,17 @@ class Lexer:
     # newlines within parentheses/brackets.
     open_brackets = None  # type: List[str]
 
-    pyversion = defaults.PYTHON3_VERSION
+    dialect = None  # type: Dialect
 
     # Ignore errors on these lines (defined using '# type: ignore').
     ignored_lines = None  # type: Set[int]
 
-    def __init__(self, pyversion: Tuple[int, int] = defaults.PYTHON3_VERSION,
-                 is_stub_file: bool = False) -> None:
+    def __init__(self, dialect: Dialect = default_dialect()) -> None:
         self.map = [self.unknown_character] * 256
         self.tok = []
         self.indents = [0]
         self.open_brackets = []
-        self.pyversion = pyversion
-        self.is_stub_file = is_stub_file
+        self.dialect = dialect
         self.ignored_lines = set()
         # Fill in the map from valid character codes to relevant lexer methods.
         for seq, method in [('ABCDEFGHIJKLMNOPQRSTUVWXYZ', self.lex_name),
@@ -325,12 +322,12 @@ def __init__(self, pyversion: Tuple[int, int] = defaults.PYTHON3_VERSION,
                             ('-+*/<>%&|^~=!,@', self.lex_misc)]:
             for c in seq:
                 self.map[ord(c)] = method
-        if pyversion[0] == 2:
+        if dialect.major == 2:
             self.keywords = keywords_common | keywords2
             # Decimal/hex/octal/binary literal or integer complex literal
             self.number_exp1 = re.compile('(0[xXoObB][0-9a-fA-F]+|[0-9]+)[lL]?')
 
-        if pyversion[0] == 3:
+        if dialect.major == 3:
             self.keywords = keywords_common | keywords3
             self.number_exp1 = re.compile('0[xXoObB][0-9a-fA-F]+|[0-9]+')
 
@@ -397,7 +394,7 @@ def find_encoding(self, text: bytes) -> Tuple[str, int]:
             line = 2 if result.group(1) else 1
             return result.group(3).decode('ascii'), line
         else:
-            default_encoding = 'utf8' if self.pyversion[0] >= 3 else 'ascii'
+            default_encoding = 'utf8' if self.dialect.major >= 3 else 'ascii'
             return default_encoding, -1
 
     def report_unicode_decode_error(self, exc: UnicodeDecodeError, text: bytes) -> None:
@@ -487,7 +484,7 @@ def lex_number(self) -> None:
             self.add_token(LexError(' ' * maxlen, NUMERIC_LITERAL_ERROR))
         elif len(s1) == maxlen:
             # Integer literal.
-            if self.pyversion[0] >= 3 and self.octal_int.match(s1):
+            if self.dialect.major >= 3 and self.octal_int.match(s1):
                 # Python 2 style octal literal such as 0377 not supported in Python 3.
                 self.add_token(LexError(s1, NUMERIC_LITERAL_ERROR))
             else:

mypy/lex.py

@@ -11,7 +11,7 @@
 from typing import Dict, List, Tuple
 
 from mypy import build
-from mypy import defaults
+from mypy.syntax.dialect import Implementation, default_implementation
 from mypy.errors import CompileError
 
 from mypy.version import __version__
@@ -22,7 +22,7 @@ def __init__(self) -> None:
         # Set default options.
         self.target = build.TYPE_CHECK
         self.build_flags = []  # type: List[str]
-        self.pyversion = defaults.PYTHON3_VERSION
+        self.implementation = None  # type: Implementation
         self.custom_typing_module = None  # type: str
         self.report_dirs = {}  # type: Dict[str, str]
         self.python_path = False
@@ -58,7 +58,7 @@ def type_check_only(path: str, module: str, program_text: str,
                 module=module,
                 program_text=program_text,
                 target=build.TYPE_CHECK,
-                pyversion=options.pyversion,
+                implementation=options.implementation,
                 custom_typing_module=options.custom_typing_module,
                 report_dirs=options.report_dirs,
                 flags=options.build_flags,
@@ -75,23 +75,21 @@ def process_options(args: List[str]) -> Tuple[str, str, str, Options]:
     options = Options()
     help = False
     ver = False
+    python_executable = None  # type: str
+    force_py2 = False
+
     while args and args[0].startswith('-'):
         if args[0] == '--verbose':
             options.build_flags.append(build.VERBOSE)
             args = args[1:]
         elif args[0] == '--py2':
             # Use Python 2 mode.
-            options.pyversion = defaults.PYTHON2_VERSION
+            force_py2 = True
             args = args[1:]
-        elif args[0] == '--python-version':
-            version_components = args[1].split(".")[0:2]
-            if len(version_components) != 2:
-                fail("Invalid python version {} (expected format: 'x.y')".format(
-                    repr(args[1])))
-            if not all(item.isdigit() for item in version_components):
-                fail("Found non-digit in python version: {}".format(
-                    args[1]))
-            options.pyversion = (int(version_components[0]), int(version_components[1]))
+        elif args[0] == '--python-executable':
+            if len(args) < 2:
+                fail('argument required')
+            python_executable = args[1]
             args = args[2:]
         elif args[0] == '-m' and args[1:]:
             options.build_flags.append(build.MODULE)
@@ -137,9 +135,12 @@ def process_options(args: List[str]) -> Tuple[str, str, str, Options]:
     if args[1:]:
         usage('Extra argument: {}'.format(args[1]))
 
-    if options.python_path and options.pyversion[0] == 2:
-        usage('Python version 2 (or --py2) specified, '
-              'but --use-python-path will search in sys.path of Python 3')
+    if python_executable is not None:
+        options.implementation = Implementation(python_executable)
+        if force_py2 and options.implementation.base_dialect.major != 2:
+            usage('given --python-executable is not --py2')
+    else:
+        options.implementation = default_implementation(force_py2=force_py2)
 
     return args[0], None, None, options
 
@@ -182,9 +183,12 @@ def usage(msg: str = None) -> None:
   --verbose          more verbose messages
   --use-python-path  search for modules in sys.path of running Python
   --version          show the current version information
+  --python-executable    emulate this python interpreter
+  --py2              deprecated, automatically find a python2 interpreter
 
 Environment variables:
   MYPYPATH     additional module search path
+  MYPY_PYTHON  interpreter to emulate
 """ % ', '.join(REPORTS))
     sys.exit(2)