From 6898db186c8d12c983825e2494a704f8ff828c0f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 3 Oct 2018 14:55:34 +0100 Subject: [PATCH 01/12] Rewrite visitor logic and remove Python base class --- mypy/fastparse2.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index 970d1f0ce000..f5d5e1125304 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -71,7 +71,7 @@ T = TypeVar('T', bound=Union[ast27.expr, ast27.stmt]) U = TypeVar('U', bound=Node) -V = TypeVar('V') +N = TypeVar('N', bound=Node) # There is no way to create reasonable fallbacks at this stage, # they must be patched later. @@ -130,13 +130,6 @@ def wrapper(self: 'ASTConverter', ast: T) -> U: return wrapper -def find(f: Callable[[V], bool], seq: Sequence[V]) -> Optional[V]: - for item in seq: - if f(item): - return item - return None - - def is_no_type_check_decorator(expr: ast27.expr) -> bool: if isinstance(expr, ast27.Name): return expr.id == 'no_type_check' @@ -146,7 +139,7 @@ def is_no_type_check_decorator(expr: ast27.expr) -> bool: return False -class ASTConverter(ast27.NodeTransformer): +class ASTConverter: def __init__(self, options: Options, is_stub: bool, @@ -158,16 +151,27 @@ def __init__(self, self.is_stub = is_stub self.errors = errors + # Cache of visit_X methods keyed by type of visited object + self.visitor_cache = {} # type: Dict[type, Callable[[Optional[AST]], Any]] + def fail(self, msg: str, line: int, column: int) -> None: self.errors.report(line, column, msg, blocker=True) - def generic_visit(self, node: ast27.AST) -> None: - raise RuntimeError('AST node not implemented: ' + str(type(node))) - def visit(self, node: Optional[ast27.AST]) -> Any: # same as in typed_ast stub if node is None: return None - return super().visit(node) + typeobj = type(node) + visitor = self.visitor_cache.get(typeobj) + if visitor is None: + method = 'visit_' + node.__class__.__name__ + visitor = getattr(self, method) + self.visitor_cache[typeobj] = visitor + return visitor(node) + + def set_line(self, node: N, n: Union[ast27.expr, ast27.stmt]) -> N: + node.line = n.lineno + node.column = n.col_offset + return node def translate_expr_list(self, l: Sequence[ast27.AST]) -> List[Expression]: res = [] # type: List[Expression] From a98025d3a71128c7e75ea726b07dfa384c9552d0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 3 Oct 2018 15:02:03 +0100 Subject: [PATCH 02/12] Refactor decorators away --- mypy/fastparse2.py | 271 +++++++++++++++++++++++---------------------- 1 file changed, 137 insertions(+), 134 deletions(-) diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index f5d5e1125304..f6375564733d 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -305,7 +305,6 @@ def visit_Module(self, mod: ast27.Module) -> MypyFile: # stmt* body, expr* decorator_list, expr? returns, string? type_comment) # arguments = (arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults, # arg? kwarg, expr* defaults) - @with_line def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: converter = TypeConverter(self.errors, line=n.lineno) args, decompose_stmts = self.transform_args(n.args, n.lineno) @@ -392,8 +391,11 @@ def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: func_def.is_decorated = True func_def.set_line(n.lineno + len(n.decorator_list)) func_def.body.set_line(func_def.get_line()) - return Decorator(func_def, self.translate_expr_list(n.decorator_list), var) + dec = Decorator(func_def, self.translate_expr_list(n.decorator_list), var) + dec.set_line(n.lineno, n.col_offset) + return dec else: + func_def.set_line(n.lineno, n.col_offset) return func_def def set_type_optional(self, type: Optional[Type], initializer: Optional[Expression]) -> None: @@ -486,7 +488,6 @@ def stringify_name(self, n: ast27.AST) -> str: # keyword* keywords, # stmt* body, # expr* decorator_list) - @with_line def visit_ClassDef(self, n: ast27.ClassDef) -> ClassDef: self.class_nesting += 1 @@ -496,82 +497,82 @@ def visit_ClassDef(self, n: ast27.ClassDef) -> ClassDef: self.translate_expr_list(n.bases), metaclass=None) cdef.decorators = self.translate_expr_list(n.decorator_list) + self.set_line(cdef, n) self.class_nesting -= 1 return cdef # Return(expr? value) - @with_line def visit_Return(self, n: ast27.Return) -> ReturnStmt: - return ReturnStmt(self.visit(n.value)) + s = ReturnStmt(self.visit(n.value)) + return self.set_line(s, n) # Delete(expr* targets) - @with_line def visit_Delete(self, n: ast27.Delete) -> DelStmt: if len(n.targets) > 1: tup = TupleExpr(self.translate_expr_list(n.targets)) tup.set_line(n.lineno) - return DelStmt(tup) + s = DelStmt(tup) else: - return DelStmt(self.visit(n.targets[0])) + s = DelStmt(self.visit(n.targets[0])) + return self.set_line(s, n) # Assign(expr* targets, expr value, string? type_comment) - @with_line def visit_Assign(self, n: ast27.Assign) -> AssignmentStmt: typ = None if n.type_comment: typ = parse_type_comment(n.type_comment, n.lineno, self.errors) - return AssignmentStmt(self.translate_expr_list(n.targets), - self.visit(n.value), - type=typ) + s = AssignmentStmt(self.translate_expr_list(n.targets), + self.visit(n.value), + type=typ) + return self.set_line(s, n) # AugAssign(expr target, operator op, expr value) - @with_line def visit_AugAssign(self, n: ast27.AugAssign) -> OperatorAssignmentStmt: - return OperatorAssignmentStmt(self.from_operator(n.op), - self.visit(n.target), - self.visit(n.value)) + s = OperatorAssignmentStmt(self.from_operator(n.op), + self.visit(n.target), + self.visit(n.value)) + return self.set_line(s, n) # For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) - @with_line def visit_For(self, n: ast27.For) -> ForStmt: if n.type_comment is not None: target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) else: target_type = None - return ForStmt(self.visit(n.target), - self.visit(n.iter), - self.as_required_block(n.body, n.lineno), - self.as_block(n.orelse, n.lineno), - target_type) + s = ForStmt(self.visit(n.target), + self.visit(n.iter), + self.as_required_block(n.body, n.lineno), + self.as_block(n.orelse, n.lineno), + target_type) + return self.set_line(s, n) # While(expr test, stmt* body, stmt* orelse) - @with_line def visit_While(self, n: ast27.While) -> WhileStmt: - return WhileStmt(self.visit(n.test), - self.as_required_block(n.body, n.lineno), - self.as_block(n.orelse, n.lineno)) + s = WhileStmt(self.visit(n.test), + self.as_required_block(n.body, n.lineno), + self.as_block(n.orelse, n.lineno)) + return self.set_line(s, n) # If(expr test, stmt* body, stmt* orelse) - @with_line def visit_If(self, n: ast27.If) -> IfStmt: - return IfStmt([self.visit(n.test)], - [self.as_required_block(n.body, n.lineno)], - self.as_block(n.orelse, n.lineno)) + s = IfStmt([self.visit(n.test)], + [self.as_required_block(n.body, n.lineno)], + self.as_block(n.orelse, n.lineno)) + return self.set_line(s, n) # With(withitem* items, stmt* body, string? type_comment) - @with_line def visit_With(self, n: ast27.With) -> WithStmt: if n.type_comment is not None: target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) else: target_type = None - return WithStmt([self.visit(n.context_expr)], - [self.visit(n.optional_vars)], - self.as_required_block(n.body, n.lineno), - target_type) + s = WithStmt([self.visit(n.context_expr)], + [self.visit(n.optional_vars)], + self.as_required_block(n.body, n.lineno), + target_type) + return self.set_line(s, n) - @with_line def visit_Raise(self, n: ast27.Raise) -> RaiseStmt: if n.type is None: e = None @@ -584,19 +585,20 @@ def visit_Raise(self, n: ast27.Raise) -> RaiseStmt: else: e = TupleExpr([self.visit(n.type), self.visit(n.inst), self.visit(n.tback)]) - return RaiseStmt(e, None) + s = RaiseStmt(e, None) + return self.set_line(s, n) # TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) - @with_line def visit_TryExcept(self, n: ast27.TryExcept) -> TryStmt: - return self.try_handler(n.body, n.handlers, n.orelse, [], n.lineno) + s = self.try_handler(n.body, n.handlers, n.orelse, [], n.lineno) + return self.set_line(s, n) - @with_line def visit_TryFinally(self, n: ast27.TryFinally) -> TryStmt: if len(n.body) == 1 and isinstance(n.body[0], ast27.TryExcept): - return self.try_handler([n.body[0]], [], [], n.finalbody, n.lineno) + s = self.try_handler([n.body[0]], [], [], n.finalbody, n.lineno) else: - return self.try_handler(n.body, [], [], n.finalbody, n.lineno) + s = self.try_handler(n.body, [], [], n.finalbody, n.lineno) + return self.set_line(s, n) def try_handler(self, body: List[ast27.stmt], @@ -624,27 +626,26 @@ def try_handler(self, self.as_block(orelse, lineno), self.as_block(finalbody, lineno)) - @with_line def visit_Print(self, n: ast27.Print) -> PrintStmt: - return PrintStmt(self.translate_expr_list(n.values), n.nl, self.visit(n.dest)) + s = PrintStmt(self.translate_expr_list(n.values), n.nl, self.visit(n.dest)) + return self.set_line(s, n) - @with_line def visit_Exec(self, n: ast27.Exec) -> ExecStmt: - return ExecStmt(self.visit(n.body), - self.visit(n.globals), - self.visit(n.locals)) + s = ExecStmt(self.visit(n.body), + self.visit(n.globals), + self.visit(n.locals)) + return self.set_line(s, n) - @with_line def visit_Repr(self, n: ast27.Repr) -> BackquoteExpr: - return BackquoteExpr(self.visit(n.value)) + s = BackquoteExpr(self.visit(n.value)) + return self.set_line(s, n) # Assert(expr test, expr? msg) - @with_line def visit_Assert(self, n: ast27.Assert) -> AssertStmt: - return AssertStmt(self.visit(n.test), self.visit(n.msg)) + s = AssertStmt(self.visit(n.test), self.visit(n.msg)) + return self.set_line(s, n) # Import(alias* names) - @with_line def visit_Import(self, n: ast27.Import) -> Import: names = [] # type: List[Tuple[str, Optional[str]]] for alias in n.names: @@ -658,10 +659,9 @@ def visit_Import(self, n: ast27.Import) -> Import: names.append((name, asname)) i = Import(names) self.imports.append(i) - return i + return self.set_line(i, n) # ImportFrom(identifier? module, alias* names, int? level) - @with_line def visit_ImportFrom(self, n: ast27.ImportFrom) -> ImportBase: assert n.level is not None if len(n.names) == 1 and n.names[0].name == '*': @@ -672,37 +672,37 @@ def visit_ImportFrom(self, n: ast27.ImportFrom) -> ImportBase: n.level, [(a.name, a.asname) for a in n.names]) self.imports.append(i) - return i + return self.set_line(i, n) # Global(identifier* names) - @with_line def visit_Global(self, n: ast27.Global) -> GlobalDecl: - return GlobalDecl(n.names) + s = GlobalDecl(n.names) + return self.set_line(s, n) # Expr(expr value) - @with_line def visit_Expr(self, n: ast27.Expr) -> ExpressionStmt: value = self.visit(n.value) - return ExpressionStmt(value) + s = ExpressionStmt(value) + return self.set_line(s, n) # Pass - @with_line def visit_Pass(self, n: ast27.Pass) -> PassStmt: - return PassStmt() + s = PassStmt() + return self.set_line(s, n) # Break - @with_line def visit_Break(self, n: ast27.Break) -> BreakStmt: - return BreakStmt() + s = BreakStmt() + return self.set_line(s, n) # Continue - @with_line def visit_Continue(self, n: ast27.Continue) -> ContinueStmt: - return ContinueStmt() + s = ContinueStmt() + return self.set_line(s, n) # --- expr --- + # BoolOp(boolop op, expr* values) - @with_line def visit_BoolOp(self, n: ast27.BoolOp) -> OpExpr: # mypy translates (1 and 2 and 3) as (1 and (2 and 3)) assert len(n.values) >= 2 @@ -720,20 +720,20 @@ def group(vals: List[Expression]) -> OpExpr: else: return OpExpr(op, vals[0], group(vals[1:])) - return group(self.translate_expr_list(n.values)) + e = group(self.translate_expr_list(n.values)) + return self.set_line(e, n) # BinOp(expr left, operator op, expr right) - @with_line def visit_BinOp(self, n: ast27.BinOp) -> OpExpr: op = self.from_operator(n.op) if op is None: raise RuntimeError('cannot translate BinOp ' + str(type(n.op))) - return OpExpr(op, self.visit(n.left), self.visit(n.right)) + e = OpExpr(op, self.visit(n.left), self.visit(n.right)) + return self.set_line(e, n) # UnaryOp(unaryop op, expr operand) - @with_line def visit_UnaryOp(self, n: ast27.UnaryOp) -> UnaryExpr: op = None if isinstance(n.op, ast27.Invert): @@ -748,10 +748,10 @@ def visit_UnaryOp(self, n: ast27.UnaryOp) -> UnaryExpr: if op is None: raise RuntimeError('cannot translate UnaryOp ' + str(type(n.op))) - return UnaryExpr(op, self.visit(n.operand)) + e = UnaryExpr(op, self.visit(n.operand)) + return self.set_line(e, n) # Lambda(arguments args, expr body) - @with_line def visit_Lambda(self, n: ast27.Lambda) -> LambdaExpr: args, decompose_stmts = self.transform_args(n.args, n.lineno) @@ -762,76 +762,76 @@ def visit_Lambda(self, n: ast27.Lambda) -> LambdaExpr: if decompose_stmts: body.body = decompose_stmts + body.body - return LambdaExpr(args, body) + e = LambdaExpr(args, body) + return self.set_line(e, n) # IfExp(expr test, expr body, expr orelse) - @with_line def visit_IfExp(self, n: ast27.IfExp) -> ConditionalExpr: - return ConditionalExpr(self.visit(n.test), - self.visit(n.body), - self.visit(n.orelse)) + e = ConditionalExpr(self.visit(n.test), + self.visit(n.body), + self.visit(n.orelse)) + return self.set_line(e, n) # Dict(expr* keys, expr* values) - @with_line def visit_Dict(self, n: ast27.Dict) -> DictExpr: - return DictExpr(list(zip(self.translate_expr_list(n.keys), - self.translate_expr_list(n.values)))) + e = DictExpr(list(zip(self.translate_expr_list(n.keys), + self.translate_expr_list(n.values)))) + return self.set_line(e, n) # Set(expr* elts) - @with_line def visit_Set(self, n: ast27.Set) -> SetExpr: - return SetExpr(self.translate_expr_list(n.elts)) + e = SetExpr(self.translate_expr_list(n.elts)) + return self.set_line(e, n) # ListComp(expr elt, comprehension* generators) - @with_line def visit_ListComp(self, n: ast27.ListComp) -> ListComprehension: - return ListComprehension(self.visit_GeneratorExp(cast(ast27.GeneratorExp, n))) + e = ListComprehension(self.visit_GeneratorExp(cast(ast27.GeneratorExp, n))) + return self.set_line(e, n) # SetComp(expr elt, comprehension* generators) - @with_line def visit_SetComp(self, n: ast27.SetComp) -> SetComprehension: - return SetComprehension(self.visit_GeneratorExp(cast(ast27.GeneratorExp, n))) + e = SetComprehension(self.visit_GeneratorExp(cast(ast27.GeneratorExp, n))) + return self.set_line(e, n) # DictComp(expr key, expr value, comprehension* generators) - @with_line def visit_DictComp(self, n: ast27.DictComp) -> DictionaryComprehension: targets = [self.visit(c.target) for c in n.generators] iters = [self.visit(c.iter) for c in n.generators] ifs_list = [self.translate_expr_list(c.ifs) for c in n.generators] - return DictionaryComprehension(self.visit(n.key), - self.visit(n.value), - targets, - iters, - ifs_list, - [False for _ in n.generators]) + e = DictionaryComprehension(self.visit(n.key), + self.visit(n.value), + targets, + iters, + ifs_list, + [False for _ in n.generators]) + return self.set_line(e, n) # GeneratorExp(expr elt, comprehension* generators) - @with_line def visit_GeneratorExp(self, n: ast27.GeneratorExp) -> GeneratorExpr: targets = [self.visit(c.target) for c in n.generators] iters = [self.visit(c.iter) for c in n.generators] ifs_list = [self.translate_expr_list(c.ifs) for c in n.generators] - return GeneratorExpr(self.visit(n.elt), - targets, - iters, - ifs_list, - [False for _ in n.generators]) + e = GeneratorExpr(self.visit(n.elt), + targets, + iters, + ifs_list, + [False for _ in n.generators]) + return self.set_line(e, n) # Yield(expr? value) - @with_line def visit_Yield(self, n: ast27.Yield) -> YieldExpr: - return YieldExpr(self.visit(n.value)) + e = YieldExpr(self.visit(n.value)) + return self.set_line(e, n) # Compare(expr left, cmpop* ops, expr* comparators) - @with_line def visit_Compare(self, n: ast27.Compare) -> ComparisonExpr: operators = [self.from_comp_operator(o) for o in n.ops] operands = self.translate_expr_list([n.left] + n.comparators) - return ComparisonExpr(operators, operands) + e = ComparisonExpr(operators, operands) + return self.set_line(e, n) # Call(expr func, expr* args, keyword* keywords) # keyword = (identifier? arg, expr value) - @with_line def visit_Call(self, n: ast27.Call) -> CallExpr: arg_types = [] # type: List[ast27.expr] arg_kinds = [] # type: List[int] @@ -855,18 +855,18 @@ def visit_Call(self, n: ast27.Call) -> CallExpr: arg_kinds.append(ARG_STAR2) signature.append(None) - return CallExpr(self.visit(n.func), - self.translate_expr_list(arg_types), - arg_kinds, - signature) + e = CallExpr(self.visit(n.func), + self.translate_expr_list(arg_types), + arg_kinds, + signature) + return self.set_line(e, n) # Num(object n) -- a number as a PyObject. - @with_line - def visit_Num(self, new: ast27.Num) -> Expression: - value = new.n + def visit_Num(self, n: ast27.Num) -> Expression: + value = n.n is_inverse = False - if str(new.n).startswith('-'): # Hackish because of complex. - value = -new.n + if str(n.n).startswith('-'): # Hackish because of complex. + value = -n.n is_inverse = True if isinstance(value, int): @@ -881,63 +881,66 @@ def visit_Num(self, new: ast27.Num) -> Expression: if is_inverse: expr = UnaryExpr('-', expr) - return expr + return self.set_line(expr, n) # Str(string s) - @with_line - def visit_Str(self, s: ast27.Str) -> Expression: + def visit_Str(self, n: ast27.Str) -> Expression: # Hack: assume all string literals in Python 2 stubs are normal # strs (i.e. not unicode). All stubs are parsed with the Python 3 # parser, which causes unprefixed string literals to be interpreted # as unicode instead of bytes. This hack is generally okay, # because mypy considers str literals to be compatible with # unicode. - if isinstance(s.s, bytes): - n = s.s + if isinstance(n.s, bytes): + s = n.s # The following line is a bit hacky, but is the best way to maintain # compatibility with how mypy currently parses the contents of bytes literals. - contents = str(n)[2:-1] - return StrExpr(contents) + contents = str(s)[2:-1] + e = StrExpr(contents) + return self.set_line(e, n) else: - return UnicodeExpr(s.s) + e = UnicodeExpr(n.s) + return self.set_line(e, n) # Ellipsis def visit_Ellipsis(self, n: ast27.Ellipsis) -> EllipsisExpr: return EllipsisExpr() # Attribute(expr value, identifier attr, expr_context ctx) - @with_line def visit_Attribute(self, n: ast27.Attribute) -> Expression: if (isinstance(n.value, ast27.Call) and isinstance(n.value.func, ast27.Name) and n.value.func.id == 'super'): - return SuperExpr(n.attr, self.visit(n.value)) + e = SuperExpr(n.attr, self.visit(n.value)) + return self.set_line(e, n) - return MemberExpr(self.visit(n.value), n.attr) + e = MemberExpr(self.visit(n.value), n.attr) + return self.set_line(e, n) # Subscript(expr value, slice slice, expr_context ctx) - @with_line def visit_Subscript(self, n: ast27.Subscript) -> IndexExpr: - return IndexExpr(self.visit(n.value), self.visit(n.slice)) + e = IndexExpr(self.visit(n.value), self.visit(n.slice)) + return self.set_line(e, n) # Name(identifier id, expr_context ctx) - @with_line def visit_Name(self, n: ast27.Name) -> NameExpr: - return NameExpr(n.id) + e = NameExpr(n.id) + return self.set_line(e, n) # List(expr* elts, expr_context ctx) - @with_line def visit_List(self, n: ast27.List) -> Union[ListExpr, TupleExpr]: expr_list = [self.visit(e) for e in n.elts] # type: List[Expression] if isinstance(n.ctx, ast27.Store): # [x, y] = z and (x, y) = z means exactly the same thing - return TupleExpr(expr_list) - return ListExpr(expr_list) + e = TupleExpr(expr_list) + return self.set_line(e, n) + e = ListExpr(expr_list) + return self.set_line(e, n) # Tuple(expr* elts, expr_context ctx) - @with_line def visit_Tuple(self, n: ast27.Tuple) -> TupleExpr: - return TupleExpr([self.visit(e) for e in n.elts]) + e = TupleExpr([self.visit(e) for e in n.elts]) + return self.set_line(e, n) # --- slice --- From 940de39daf3b9216c14c5286620242e142c35a9d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 3 Oct 2018 15:55:20 +0100 Subject: [PATCH 03/12] Refactor nested functions away --- mypy/fastparse2.py | 72 +++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index f6375564733d..e838d6a45d8a 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -414,38 +414,11 @@ def transform_args(self, converter = TypeConverter(self.errors, line=line) decompose_stmts = [] # type: List[Statement] - def extract_names(arg: ast27.expr) -> List[str]: - if isinstance(arg, ast27.Name): - return [arg.id] - elif isinstance(arg, ast27.Tuple): - return [name for elt in arg.elts for name in extract_names(elt)] - else: - return [] - - def convert_arg(index: int, arg: ast27.expr) -> Var: - if isinstance(arg, ast27.Name): - v = arg.id - elif isinstance(arg, ast27.Tuple): - v = '__tuple_arg_{}'.format(index + 1) - rvalue = NameExpr(v) - rvalue.set_line(line) - assignment = AssignmentStmt([self.visit(arg)], rvalue) - assignment.set_line(line) - decompose_stmts.append(assignment) - else: - raise RuntimeError("'{}' is not a valid argument.".format(ast27.dump(arg))) - return Var(v) - - def get_type(i: int) -> Optional[Type]: - if i < len(type_comments): - comment = type_comments[i] - if comment is not None: - return converter.visit_raw_str(comment) - return None - - args = [(convert_arg(i, arg), get_type(i)) for i, arg in enumerate(n.args)] + args = [(self.convert_arg(i, arg, line, decompose_stmts), + self.get_type(i, type_comments, converter)) + for i, arg in enumerate(n.args)] defaults = self.translate_expr_list(n.defaults) - names = [name for arg in n.args for name in extract_names(arg)] # type: List[str] + names = [name for arg in n.args for name in self.extract_names(arg)] # type: List[str] new_args = [] # type: List[Argument] num_no_defaults = len(args) - len(defaults) @@ -459,12 +432,15 @@ def get_type(i: int) -> Optional[Type]: # *arg if n.vararg is not None: - new_args.append(Argument(Var(n.vararg), get_type(len(args)), None, ARG_STAR)) + new_args.append(Argument(Var(n.vararg), + self.get_type(len(args), type_comments, converter), + None, + ARG_STAR)) names.append(n.vararg) # **kwarg if n.kwarg is not None: - typ = get_type(len(args) + (0 if n.vararg is None else 1)) + typ = self.get_type(len(args) + (0 if n.vararg is None else 1), type_comments, converter) new_args.append(Argument(Var(n.kwarg), typ, None, ARG_STAR2)) names.append(n.kwarg) @@ -475,6 +451,36 @@ def fail_arg(msg: str, arg: None) -> None: return new_args, decompose_stmts + def extract_names(self, arg: ast27.expr) -> List[str]: + if isinstance(arg, ast27.Name): + return [arg.id] + elif isinstance(arg, ast27.Tuple): + return [name for elt in arg.elts for name in self.extract_names(elt)] + else: + return [] + + def convert_arg(self, index: int, arg: ast27.expr, line: int, decompose_stmts: List[Statement]) -> Var: + if isinstance(arg, ast27.Name): + v = arg.id + elif isinstance(arg, ast27.Tuple): + v = '__tuple_arg_{}'.format(index + 1) + rvalue = NameExpr(v) + rvalue.set_line(line) + assignment = AssignmentStmt([self.visit(arg)], rvalue) + assignment.set_line(line) + decompose_stmts.append(assignment) + else: + raise RuntimeError("'{}' is not a valid argument.".format(ast27.dump(arg))) + return Var(v) + + def get_type(self, i: int, type_comments: Sequence[Optional[str]], + converter: TypeConverter) -> Optional[Type]: + if i < len(type_comments): + comment = type_comments[i] + if comment is not None: + return converter.visit_raw_str(comment) + return None + def stringify_name(self, n: ast27.AST) -> str: if isinstance(n, ast27.Name): return n.id From 3026c6a0917462d30a7b4bea5155d2fbd74453e5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 3 Oct 2018 15:55:36 +0100 Subject: [PATCH 04/12] Remove dead code --- mypy/fastparse2.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index e838d6a45d8a..16da53985fe6 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -69,8 +69,6 @@ 'Python 3.3 and greater.', file=sys.stderr) sys.exit(1) -T = TypeVar('T', bound=Union[ast27.expr, ast27.stmt]) -U = TypeVar('U', bound=Node) N = TypeVar('N', bound=Node) # There is no way to create reasonable fallbacks at this stage, @@ -120,16 +118,6 @@ def parse(source: Union[str, bytes], return tree -def with_line(f: Callable[['ASTConverter', T], U]) -> Callable[['ASTConverter', T], U]: - # mypyc doesn't properly populate all the fields that @wraps expects - # @wraps(f) - def wrapper(self: 'ASTConverter', ast: T) -> U: - node = f(self, ast) - node.set_line(ast.lineno, ast.col_offset) - return node - return wrapper - - def is_no_type_check_decorator(expr: ast27.expr) -> bool: if isinstance(expr, ast27.Name): return expr.id == 'no_type_check' From 5652e975532e9e72769746a3c4b5c84a0bb06fc6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 3 Oct 2018 16:00:24 +0100 Subject: [PATCH 05/12] Use fewer qualified type names --- mypy/fastparse2.py | 51 ++++++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index 16da53985fe6..6aaf6eefcd22 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -50,6 +50,13 @@ try: from typed_ast import ast27 + from typed_ast.ast27 import ( + AST, + Call, + Name, + Attribute, + Tuple as ast27_Tuple, + ) from typed_ast import ast3 except ImportError: if sys.version_info.minor > 2: @@ -119,10 +126,10 @@ def parse(source: Union[str, bytes], def is_no_type_check_decorator(expr: ast27.expr) -> bool: - if isinstance(expr, ast27.Name): + if isinstance(expr, Name): return expr.id == 'no_type_check' - elif isinstance(expr, ast27.Attribute): - if isinstance(expr.value, ast27.Name): + elif isinstance(expr, Attribute): + if isinstance(expr.value, Name): return expr.value.id == 'typing' and expr.attr == 'no_type_check' return False @@ -145,7 +152,7 @@ def __init__(self, def fail(self, msg: str, line: int, column: int) -> None: self.errors.report(line, column, msg, blocker=True) - def visit(self, node: Optional[ast27.AST]) -> Any: # same as in typed_ast stub + def visit(self, node: Optional[AST]) -> Any: # same as in typed_ast stub if node is None: return None typeobj = type(node) @@ -161,7 +168,7 @@ def set_line(self, node: N, n: Union[ast27.expr, ast27.stmt]) -> N: node.column = n.col_offset return node - def translate_expr_list(self, l: Sequence[ast27.AST]) -> List[Expression]: + def translate_expr_list(self, l: Sequence[AST]) -> List[Expression]: res = [] # type: List[Expression] for e in l: exp = self.visit(e) @@ -169,7 +176,7 @@ def translate_expr_list(self, l: Sequence[ast27.AST]) -> List[Expression]: res.append(exp) return res - def translate_stmt_list(self, l: Sequence[ast27.AST]) -> List[Statement]: + def translate_stmt_list(self, l: Sequence[AST]) -> List[Statement]: res = [] # type: List[Statement] for e in l: stmt = self.visit(e) @@ -190,7 +197,7 @@ def translate_stmt_list(self, l: Sequence[ast27.AST]) -> List[Statement]: ast27.BitXor: '^', ast27.BitAnd: '&', ast27.FloorDiv: '//' - } # type: Final[Dict[typing.Type[ast27.AST], str]] + } # type: Final[Dict[typing.Type[AST], str]] def from_operator(self, op: ast27.operator) -> str: op_name = ASTConverter.op_map.get(type(op)) @@ -212,7 +219,7 @@ def from_operator(self, op: ast27.operator) -> str: ast27.IsNot: 'is not', ast27.In: 'in', ast27.NotIn: 'not in' - } # type: Final[Dict[typing.Type[ast27.AST], str]] + } # type: Final[Dict[typing.Type[AST], str]] def from_comp_operator(self, op: ast27.cmpop) -> str: op_name = ASTConverter.comp_op_map.get(type(op)) @@ -440,17 +447,17 @@ def fail_arg(msg: str, arg: None) -> None: return new_args, decompose_stmts def extract_names(self, arg: ast27.expr) -> List[str]: - if isinstance(arg, ast27.Name): + if isinstance(arg, Name): return [arg.id] - elif isinstance(arg, ast27.Tuple): + elif isinstance(arg, ast27_Tuple): return [name for elt in arg.elts for name in self.extract_names(elt)] else: return [] def convert_arg(self, index: int, arg: ast27.expr, line: int, decompose_stmts: List[Statement]) -> Var: - if isinstance(arg, ast27.Name): + if isinstance(arg, Name): v = arg.id - elif isinstance(arg, ast27.Tuple): + elif isinstance(arg, ast27_Tuple): v = '__tuple_arg_{}'.format(index + 1) rvalue = NameExpr(v) rvalue.set_line(line) @@ -469,10 +476,10 @@ def get_type(self, i: int, type_comments: Sequence[Optional[str]], return converter.visit_raw_str(comment) return None - def stringify_name(self, n: ast27.AST) -> str: - if isinstance(n, ast27.Name): + def stringify_name(self, n: AST) -> str: + if isinstance(n, Name): return n.id - elif isinstance(n, ast27.Attribute): + elif isinstance(n, Attribute): return "{}.{}".format(self.stringify_name(n.value), n.attr) else: assert False, "can't stringify " + str(type(n)) @@ -604,7 +611,7 @@ def try_handler(self, for item in handlers: if item.name is None: vs.append(None) - elif isinstance(item.name, ast27.Name): + elif isinstance(item.name, Name): vs.append(NameExpr(item.name.id)) else: self.fail("Sorry, `except , ` is not supported", @@ -826,7 +833,7 @@ def visit_Compare(self, n: ast27.Compare) -> ComparisonExpr: # Call(expr func, expr* args, keyword* keywords) # keyword = (identifier? arg, expr value) - def visit_Call(self, n: ast27.Call) -> CallExpr: + def visit_Call(self, n: Call) -> CallExpr: arg_types = [] # type: List[ast27.expr] arg_kinds = [] # type: List[int] signature = [] # type: List[Optional[str]] @@ -901,9 +908,9 @@ def visit_Ellipsis(self, n: ast27.Ellipsis) -> EllipsisExpr: return EllipsisExpr() # Attribute(expr value, identifier attr, expr_context ctx) - def visit_Attribute(self, n: ast27.Attribute) -> Expression: - if (isinstance(n.value, ast27.Call) and - isinstance(n.value.func, ast27.Name) and + def visit_Attribute(self, n: Attribute) -> Expression: + if (isinstance(n.value, Call) and + isinstance(n.value.func, Name) and n.value.func.id == 'super'): e = SuperExpr(n.attr, self.visit(n.value)) return self.set_line(e, n) @@ -917,7 +924,7 @@ def visit_Subscript(self, n: ast27.Subscript) -> IndexExpr: return self.set_line(e, n) # Name(identifier id, expr_context ctx) - def visit_Name(self, n: ast27.Name) -> NameExpr: + def visit_Name(self, n: Name) -> NameExpr: e = NameExpr(n.id) return self.set_line(e, n) @@ -932,7 +939,7 @@ def visit_List(self, n: ast27.List) -> Union[ListExpr, TupleExpr]: return self.set_line(e, n) # Tuple(expr* elts, expr_context ctx) - def visit_Tuple(self, n: ast27.Tuple) -> TupleExpr: + def visit_Tuple(self, n: ast27_Tuple) -> TupleExpr: e = TupleExpr([self.visit(e) for e in n.elts]) return self.set_line(e, n) From 23f7ad2fd05d8d2f6d7b31ad356363f88d1af7b7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 3 Oct 2018 16:28:09 +0100 Subject: [PATCH 06/12] Various micro-optimizations --- mypy/fastparse2.py | 72 ++++++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index 6aaf6eefcd22..16e24b4743bd 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -58,6 +58,10 @@ Tuple as ast27_Tuple, ) from typed_ast import ast3 + from typed_ast.ast3 import ( + FunctionType, + Ellipsis as ast3_Ellipsis, + ) except ImportError: if sys.version_info.minor > 2: try: @@ -301,8 +305,9 @@ def visit_Module(self, mod: ast27.Module) -> MypyFile: # arguments = (arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults, # arg? kwarg, expr* defaults) def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: - converter = TypeConverter(self.errors, line=n.lineno) - args, decompose_stmts = self.transform_args(n.args, n.lineno) + lineno = n.lineno + converter = TypeConverter(self.errors, line=lineno) + args, decompose_stmts = self.transform_args(n.args, lineno) arg_kinds = [arg.kind for arg in args] arg_names = [arg.variable.name() for arg in args] # type: List[Optional[str]] @@ -311,16 +316,17 @@ def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: arg_names = [None] * len(arg_names) arg_types = [] # type: List[Optional[Type]] + type_comment = n.type_comment if (n.decorator_list and any(is_no_type_check_decorator(d) for d in n.decorator_list)): arg_types = [None] * len(args) return_type = None - elif n.type_comment is not None and len(n.type_comment) > 0: + elif type_comment is not None and len(type_comment) > 0: try: - func_type_ast = ast3.parse(n.type_comment, '', 'func_type') - assert isinstance(func_type_ast, ast3.FunctionType) + func_type_ast = ast3.parse(type_comment, '', 'func_type') + assert isinstance(func_type_ast, FunctionType) # for ellipsis arg if (len(func_type_ast.argtypes) == 1 and - isinstance(func_type_ast.argtypes[0], ast3.Ellipsis)): + isinstance(func_type_ast.argtypes[0], ast3_Ellipsis)): arg_types = [a.type_annotation if a.type_annotation is not None else AnyType(TypeOfAny.unannotated) @@ -328,7 +334,7 @@ def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: else: # PEP 484 disallows both type annotations and type comments if any(a.type_annotation is not None for a in args): - self.fail(messages.DUPLICATE_TYPE_SIGNATURES, n.lineno, n.col_offset) + self.fail(messages.DUPLICATE_TYPE_SIGNATURES, lineno, n.col_offset) arg_types = [a if a is not None else AnyType(TypeOfAny.unannotated) for a in converter.translate_expr_list(func_type_ast.argtypes)] return_type = converter.visit(func_type_ast.returns) @@ -337,7 +343,7 @@ def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: if self.in_class() and len(arg_types) < len(args): arg_types.insert(0, AnyType(TypeOfAny.special_form)) except SyntaxError: - self.fail(TYPE_COMMENT_SYNTAX_ERROR, n.lineno, n.col_offset) + self.fail(TYPE_COMMENT_SYNTAX_ERROR, lineno, n.col_offset) arg_types = [AnyType(TypeOfAny.from_error)] * len(args) return_type = AnyType(TypeOfAny.from_error) else: @@ -351,11 +357,11 @@ def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: if any(arg_types) or return_type: if len(arg_types) != 1 and any(isinstance(t, EllipsisType) for t in arg_types): self.fail("Ellipses cannot accompany other argument types " - "in function type signature.", n.lineno, 0) + "in function type signature.", lineno, 0) elif len(arg_types) > len(arg_kinds): - self.fail('Type signature has too many arguments', n.lineno, 0) + self.fail('Type signature has too many arguments', lineno, 0) elif len(arg_types) < len(arg_kinds): - self.fail('Type signature has too few arguments', n.lineno, 0) + self.fail('Type signature has too few arguments', lineno, 0) else: any_type = AnyType(TypeOfAny.unannotated) func_type = CallableType([a if a is not None else any_type for a in arg_types], @@ -364,7 +370,7 @@ def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: return_type if return_type is not None else any_type, _dummy_fallback) - body = self.as_required_block(n.body, n.lineno) + body = self.as_required_block(n.body, lineno) if decompose_stmts: body.body = decompose_stmts + body.body func_def = FuncDef(n.name, @@ -376,7 +382,7 @@ def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: func_def.unanalyzed_type = func_def.type.copy_modified() if func_type is not None: func_type.definition = func_def - func_type.line = n.lineno + func_type.line = lineno if n.decorator_list: var = Var(func_def.name()) @@ -384,13 +390,13 @@ def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: var.set_line(n.decorator_list[0].lineno) func_def.is_decorated = True - func_def.set_line(n.lineno + len(n.decorator_list)) + func_def.set_line(lineno + len(n.decorator_list)) func_def.body.set_line(func_def.get_line()) dec = Decorator(func_def, self.translate_expr_list(n.decorator_list), var) - dec.set_line(n.lineno, n.col_offset) + dec.set_line(lineno, n.col_offset) return dec else: - func_def.set_line(n.lineno, n.col_offset) + func_def.set_line(lineno, n.col_offset) return func_def def set_type_optional(self, type: Optional[Type], initializer: Optional[Expression]) -> None: @@ -409,11 +415,12 @@ def transform_args(self, converter = TypeConverter(self.errors, line=line) decompose_stmts = [] # type: List[Statement] + n_args = n.args args = [(self.convert_arg(i, arg, line, decompose_stmts), self.get_type(i, type_comments, converter)) - for i, arg in enumerate(n.args)] + for i, arg in enumerate(n_args)] defaults = self.translate_expr_list(n.defaults) - names = [name for arg in n.args for name in self.extract_names(arg)] # type: List[str] + names = [name for arg in n_args for name in self.extract_names(arg)] # type: List[str] new_args = [] # type: List[Argument] num_no_defaults = len(args) - len(defaults) @@ -838,18 +845,20 @@ def visit_Call(self, n: Call) -> CallExpr: arg_kinds = [] # type: List[int] signature = [] # type: List[Optional[str]] - arg_types.extend(n.args) - arg_kinds.extend(ARG_POS for a in n.args) - signature.extend(None for a in n.args) + args = n.args + arg_types.extend(args) + arg_kinds.extend(ARG_POS for a in args) + signature.extend(None for a in args) if n.starargs is not None: arg_types.append(n.starargs) arg_kinds.append(ARG_STAR) signature.append(None) - arg_types.extend(k.value for k in n.keywords) - arg_kinds.extend(ARG_NAMED for k in n.keywords) - signature.extend(k.arg for k in n.keywords) + keywords = n.keywords + arg_types.extend(k.value for k in keywords) + arg_kinds.extend(ARG_NAMED for k in keywords) + signature.extend(k.arg for k in keywords) if n.kwargs is not None: arg_types.append(n.kwargs) @@ -909,13 +918,14 @@ def visit_Ellipsis(self, n: ast27.Ellipsis) -> EllipsisExpr: # Attribute(expr value, identifier attr, expr_context ctx) def visit_Attribute(self, n: Attribute) -> Expression: - if (isinstance(n.value, Call) and - isinstance(n.value.func, Name) and - n.value.func.id == 'super'): - e = SuperExpr(n.attr, self.visit(n.value)) - return self.set_line(e, n) - - e = MemberExpr(self.visit(n.value), n.attr) + member_expr = MemberExpr(self.visit(n.value), n.attr) + obj = member_expr.expr + if (isinstance(obj, CallExpr) and + isinstance(obj.callee, NameExpr) and + obj.callee.name == 'super'): + e = SuperExpr(member_expr.name, obj) # type: Expression + else: + e = member_expr return self.set_line(e, n) # Subscript(expr value, slice slice, expr_context ctx) From 33d4e90cc18e49f3d356883cf1f804ea96ccf8b9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 3 Oct 2018 17:56:35 +0100 Subject: [PATCH 07/12] Minor tweaks and add test --- mypy/fastparse2.py | 5 +++-- mypy/strconv.py | 2 +- test-data/unit/parse-python2.test | 24 +++++++++++++++++++++++- test-data/unit/parse.test | 5 ++++- 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index 16e24b4743bd..080083c624a8 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -396,7 +396,7 @@ def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: dec.set_line(lineno, n.col_offset) return dec else: - func_def.set_line(lineno, n.col_offset) + func_def.set_line(lineno, n.col_offset) # Overrides set_line -- can't use self.set_line return func_def def set_type_optional(self, type: Optional[Type], initializer: Optional[Expression]) -> None: @@ -771,7 +771,8 @@ def visit_Lambda(self, n: ast27.Lambda) -> LambdaExpr: body.body = decompose_stmts + body.body e = LambdaExpr(args, body) - return self.set_line(e, n) + e.set_line(n.lineno, n.col_offset) # Overrides set_line -- can't use self.set_line + return e # IfExp(expr test, expr body, expr orelse) def visit_IfExp(self, n: ast27.IfExp) -> ConditionalExpr: diff --git a/mypy/strconv.py b/mypy/strconv.py index 58771a9f6466..e3fb189f6b45 100644 --- a/mypy/strconv.py +++ b/mypy/strconv.py @@ -440,7 +440,7 @@ def visit_index_expr(self, o: 'mypy.nodes.IndexExpr') -> str: return self.dump([o.base, o.index], o) def visit_super_expr(self, o: 'mypy.nodes.SuperExpr') -> str: - return self.dump([o.name], o) + return self.dump([o.name, o.call], o) def visit_type_application(self, o: 'mypy.nodes.TypeApplication') -> str: return self.dump([o.expr, ('Types', o.types)], o) diff --git a/test-data/unit/parse-python2.test b/test-data/unit/parse-python2.test index 562a21834df0..c3bea460d04f 100644 --- a/test-data/unit/parse-python2.test +++ b/test-data/unit/parse-python2.test @@ -211,7 +211,7 @@ MypyFile:1( IntExpr(1) IntExpr(2)))) -[case testLambdaInListComprehensionInPython2] +[case testListComprehensionInPython2] ([ 0 for x in 1, 2 if 3 ]) [out] MypyFile:1( @@ -395,3 +395,25 @@ MypyFile:1( TupleExpr:1( IntExpr(1) IntExpr(2))))) + +[case testSuperInPython2] +class A: + def f(self): + super(A, self).x +[out] +MypyFile:1( + ClassDef:1( + A + FuncDef:2( + f + Args( + Var(self)) + Block:2( + ExpressionStmt:3( + SuperExpr:3( + x + CallExpr:3( + NameExpr(super) + Args( + NameExpr(A) + NameExpr(self))))))))) diff --git a/test-data/unit/parse.test b/test-data/unit/parse.test index 2ab8dc3c4333..1490a7fdd741 100644 --- a/test-data/unit/parse.test +++ b/test-data/unit/parse.test @@ -2356,7 +2356,10 @@ super().x MypyFile:1( ExpressionStmt:1( SuperExpr:1( - x))) + x + CallExpr:1( + NameExpr(super) + Args())))) [case testKeywordAndDictArgs] f(x = y, **kwargs) From 2d260b076e29f106ba782a45b19dad135ddf7873 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 3 Oct 2018 18:19:37 +0100 Subject: [PATCH 08/12] Refactor nested function --- mypy/fastparse2.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index 080083c624a8..e886ceb8cddd 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -442,7 +442,9 @@ def transform_args(self, # **kwarg if n.kwarg is not None: - typ = self.get_type(len(args) + (0 if n.vararg is None else 1), type_comments, converter) + typ = self.get_type(len(args) + (0 if n.vararg is None else 1), + type_comments, + converter) new_args.append(Argument(Var(n.kwarg), typ, None, ARG_STAR2)) names.append(n.kwarg) @@ -461,7 +463,8 @@ def extract_names(self, arg: ast27.expr) -> List[str]: else: return [] - def convert_arg(self, index: int, arg: ast27.expr, line: int, decompose_stmts: List[Statement]) -> Var: + def convert_arg(self, index: int, arg: ast27.expr, line: int, + decompose_stmts: List[Statement]) -> Var: if isinstance(arg, Name): v = arg.id elif isinstance(arg, ast27_Tuple): @@ -722,15 +725,15 @@ def visit_BoolOp(self, n: ast27.BoolOp) -> OpExpr: raise RuntimeError('unknown BoolOp ' + str(type(n))) # potentially inefficient! - def group(vals: List[Expression]) -> OpExpr: - if len(vals) == 2: - return OpExpr(op, vals[0], vals[1]) - else: - return OpExpr(op, vals[0], group(vals[1:])) - - e = group(self.translate_expr_list(n.values)) + e = self.group(self.translate_expr_list(n.values), op) return self.set_line(e, n) + def group(self, vals: List[Expression], op: str) -> OpExpr: + if len(vals) == 2: + return OpExpr(op, vals[0], vals[1]) + else: + return OpExpr(op, vals[0], self.group(vals[1:], op)) + # BinOp(expr left, operator op, expr right) def visit_BinOp(self, n: ast27.BinOp) -> OpExpr: op = self.from_operator(n.op) From c9806486cf6f23d88f64e9d85f27125e4d367af0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 3 Oct 2018 18:19:51 +0100 Subject: [PATCH 09/12] Add test cases --- test-data/unit/parse-python2.test | 391 ++++++++++++++++++++++++++++++ 1 file changed, 391 insertions(+) diff --git a/test-data/unit/parse-python2.test b/test-data/unit/parse-python2.test index c3bea460d04f..a3148291fe97 100644 --- a/test-data/unit/parse-python2.test +++ b/test-data/unit/parse-python2.test @@ -417,3 +417,394 @@ MypyFile:1( Args( NameExpr(A) NameExpr(self))))))))) + +[case testTypeCommentsInPython2] +x = 1 # type: List[int] + +def f(x, y=0): + # type: (List[int], str) -> None + pass +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + IntExpr(1) + List?[int?]) + FuncDef:3( + f + Args( + Var(x) + default( + Var(y) + IntExpr(0))) + def (x: List?[int?], y: str? =) -> None? + Block:3( + PassStmt:5()))) + +[case testMultiLineTypeCommentInPython2] +def f(x, # type: List[int] + y, + z=1, # type: str + ): + # type: (...) -> None + pass +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var(x) + Var(y) + default( + Var(z) + IntExpr(1))) + def (x: List?[int?], y: Any, z: str? =) -> None? + Block:1( + PassStmt:6()))) + +[case testIfStmtInPython2] +if x: + y +elif z: + a +else: + b +[out] +MypyFile:1( + IfStmt:1( + If( + NameExpr(x)) + Then( + ExpressionStmt:2( + NameExpr(y))) + Else( + IfStmt:3( + If( + NameExpr(z)) + Then( + ExpressionStmt:4( + NameExpr(a))) + Else( + ExpressionStmt:6( + NameExpr(b))))))) + +[case testWhileStmtInPython2] +while x: + y +else: + z +[out] +MypyFile:1( + WhileStmt:1( + NameExpr(x) + Block:1( + ExpressionStmt:2( + NameExpr(y))) + Else( + ExpressionStmt:4( + NameExpr(z))))) + +[case testForStmtInPython2] +for x, y in z: + a +else: + b +[out] +MypyFile:1( + ForStmt:1( + TupleExpr:1( + NameExpr(x) + NameExpr(y)) + NameExpr(z) + Block:1( + ExpressionStmt:2( + NameExpr(a))) + Else( + ExpressionStmt:4( + NameExpr(b))))) + +[case testWithStmtInPython2] +with x as y: + z +[out] +MypyFile:1( + WithStmt:1( + Expr( + NameExpr(x)) + Target( + NameExpr(y)) + Block:1( + ExpressionStmt:2( + NameExpr(z))))) + +[case testExpressionsInPython2] +x[y] +x + y +~z +x.y +([x, y]) +{x, y} +{x: y} +x < y > z +[out] +MypyFile:1( + ExpressionStmt:1( + IndexExpr:1( + NameExpr(x) + NameExpr(y))) + ExpressionStmt:2( + OpExpr:2( + + + NameExpr(x) + NameExpr(y))) + ExpressionStmt:3( + UnaryExpr:3( + ~ + NameExpr(z))) + ExpressionStmt:4( + MemberExpr:4( + NameExpr(x) + y)) + ExpressionStmt:5( + ListExpr:5( + NameExpr(x) + NameExpr(y))) + ExpressionStmt:6( + SetExpr:6( + NameExpr(x) + NameExpr(y))) + ExpressionStmt:7( + DictExpr:7( + NameExpr(x) + NameExpr(y))) + ExpressionStmt:8( + ComparisonExpr:8( + < + > + NameExpr(x) + NameExpr(y) + NameExpr(z)))) + +[case testSlicingInPython2] +x[y:] +x[y:z] +x[::y] +[out] +MypyFile:1( + ExpressionStmt:1( + IndexExpr:1( + NameExpr(x) + SliceExpr:-1( + NameExpr(y) + ))) + ExpressionStmt:2( + IndexExpr:2( + NameExpr(x) + SliceExpr:-1( + NameExpr(y) + NameExpr(z)))) + ExpressionStmt:3( + IndexExpr:3( + NameExpr(x) + SliceExpr:-1( + + + NameExpr(y))))) + +[case testStarArgsInPython2] +def f(*x): # type: (*int) -> None + pass +f(x, *y) +[out] +MypyFile:1( + FuncDef:1( + f + def (*x: int?) -> None? + VarArg( + Var(x)) + Block:1( + PassStmt:2())) + ExpressionStmt:3( + CallExpr:3( + NameExpr(f) + Args( + NameExpr(x) + NameExpr(y)) + VarArg))) + +[case testKwArgsInPython2] +def f(**x): # type: (**int) -> None + pass +f(x, **y) +[out] +MypyFile:1( + FuncDef:1( + f + def (**x: int?) -> None? + DictVarArg( + Var(x)) + Block:1( + PassStmt:2())) + ExpressionStmt:3( + CallExpr:3( + NameExpr(f) + Args( + NameExpr(x)) + DictVarArg( + NameExpr(y))))) + +[case testBoolOpInPython2] +x and y or z +[out] +MypyFile:1( + ExpressionStmt:1( + OpExpr:1( + or + OpExpr:1( + and + NameExpr(x) + NameExpr(y)) + NameExpr(z)))) + +[case testImportsInPython2] +from x import y, z as zz +import m +import n as nn +from aa import * +[out] +MypyFile:1( + ImportFrom:1(x, [y, z : zz]) + Import:2(m) + Import:3(n : nn) + ImportAll:4(aa)) + +[case testTryFinallyInPython2] +try: + x +finally: + y +[out] +MypyFile:1( + TryStmt:1( + Block:1( + ExpressionStmt:2( + NameExpr(x))) + Finally( + ExpressionStmt:4( + NameExpr(y))))) + +[case testRaiseInPython2] +raise +raise x +[out] +MypyFile:1( + RaiseStmt:1() + RaiseStmt:2( + NameExpr(x))) + +[case testAssignmentInPython2] +x = y +x, (y, z) = aa +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + NameExpr(y)) + AssignmentStmt:2( + TupleExpr:2( + NameExpr(x) + TupleExpr:2( + NameExpr(y) + NameExpr(z))) + NameExpr(aa))) + +[case testAugmentedAssignmentInPython2] +x += y +x *= 2 +[out] +MypyFile:1( + OperatorAssignmentStmt:1( + + + NameExpr(x) + NameExpr(y)) + OperatorAssignmentStmt:2( + * + NameExpr(x) + IntExpr(2))) + +[case testDelStatementInPython2] +del x +del x.y, x[y] +[out] +MypyFile:1( + DelStmt:1( + NameExpr(x)) + DelStmt:2( + TupleExpr:2( + MemberExpr:2( + NameExpr(x) + y) + IndexExpr:2( + NameExpr(x) + NameExpr(y))))) + +[case testClassDecoratorInPython2] +@dec() +class C: + pass +[out] +MypyFile:1( + ClassDef:1( + C + Decorators( + CallExpr:1( + NameExpr(dec) + Args())) + PassStmt:3())) + +[case testFunctionDecaratorInPython2] +@dec() +def f(): + pass +[out] +MypyFile:1( + Decorator:1( + Var(f) + CallExpr:1( + NameExpr(dec) + Args()) + FuncDef:2( + f + Block:2( + PassStmt:3())))) + +[case testOverloadedFunctionInPython2] +@overload +def g(): + pass +@overload +def g(): + pass +def g(): + pass +[out] +MypyFile:1( + OverloadedFuncDef:1( + Decorator:1( + Var(g) + NameExpr(overload) + FuncDef:2( + g + Block:2( + PassStmt:3()))) + Decorator:4( + Var(g) + NameExpr(overload) + FuncDef:5( + g + Block:5( + PassStmt:6()))) + FuncDef:7( + g + Block:7( + PassStmt:8())))) From c5e28fc0d1cf0852e1e89960529f9c0b78c8e0ae Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 3 Oct 2018 18:28:37 +0100 Subject: [PATCH 10/12] Fix lint and type checking --- mypy/fastparse2.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index e886ceb8cddd..c8f31588169b 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -396,7 +396,8 @@ def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: dec.set_line(lineno, n.col_offset) return dec else: - func_def.set_line(lineno, n.col_offset) # Overrides set_line -- can't use self.set_line + # Overrides set_line -- can't use self.set_line + func_def.set_line(lineno, n.col_offset) return func_def def set_type_optional(self, type: Optional[Type], initializer: Optional[Expression]) -> None: @@ -792,8 +793,8 @@ def visit_Dict(self, n: ast27.Dict) -> DictExpr: # Set(expr* elts) def visit_Set(self, n: ast27.Set) -> SetExpr: - e = SetExpr(self.translate_expr_list(n.elts)) - return self.set_line(e, n) + e = SetExpr(self.translate_expr_list(n.elts)) + return self.set_line(e, n) # ListComp(expr elt, comprehension* generators) def visit_ListComp(self, n: ast27.ListComp) -> ListComprehension: @@ -890,7 +891,7 @@ def visit_Num(self, n: ast27.Num) -> Expression: elif isinstance(value, complex): expr = ComplexExpr(value) else: - raise RuntimeError('num not implemented for ' + str(type(new.n))) + raise RuntimeError('num not implemented for ' + str(type(n.n))) if is_inverse: expr = UnaryExpr('-', expr) @@ -939,7 +940,7 @@ def visit_Subscript(self, n: ast27.Subscript) -> IndexExpr: # Name(identifier id, expr_context ctx) def visit_Name(self, n: Name) -> NameExpr: - e = NameExpr(n.id) + e = NameExpr(n.id) return self.set_line(e, n) # List(expr* elts, expr_context ctx) @@ -947,9 +948,9 @@ def visit_List(self, n: ast27.List) -> Union[ListExpr, TupleExpr]: expr_list = [self.visit(e) for e in n.elts] # type: List[Expression] if isinstance(n.ctx, ast27.Store): # [x, y] = z and (x, y) = z means exactly the same thing - e = TupleExpr(expr_list) - return self.set_line(e, n) - e = ListExpr(expr_list) + e = TupleExpr(expr_list) # type: Union[ListExpr, TupleExpr] + else: + e = ListExpr(expr_list) return self.set_line(e, n) # Tuple(expr* elts, expr_context ctx) From 2d5d524b0f18ac8b71b4b0cd9ab3cf70717d36f4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 4 Oct 2018 12:52:19 +0100 Subject: [PATCH 11/12] Fix mypyc issues --- mypy/fastparse2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index c8f31588169b..ad885ae9d58f 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -911,7 +911,7 @@ def visit_Str(self, n: ast27.Str) -> Expression: # The following line is a bit hacky, but is the best way to maintain # compatibility with how mypy currently parses the contents of bytes literals. contents = str(s)[2:-1] - e = StrExpr(contents) + e = StrExpr(contents) # type: Union[StrExpr, UnicodeExpr] return self.set_line(e, n) else: e = UnicodeExpr(n.s) From 416000e79e900bac4ecec631c68ff127828b59b9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 5 Oct 2018 12:02:31 +0100 Subject: [PATCH 12/12] Address feedback --- mypy/fastparse2.py | 122 +++++++++++++++++++++++---------------------- 1 file changed, 63 insertions(+), 59 deletions(-) diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index ad885ae9d58f..e33469341140 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -515,18 +515,18 @@ def visit_ClassDef(self, n: ast27.ClassDef) -> ClassDef: # Return(expr? value) def visit_Return(self, n: ast27.Return) -> ReturnStmt: - s = ReturnStmt(self.visit(n.value)) - return self.set_line(s, n) + stmt = ReturnStmt(self.visit(n.value)) + return self.set_line(stmt, n) # Delete(expr* targets) def visit_Delete(self, n: ast27.Delete) -> DelStmt: if len(n.targets) > 1: tup = TupleExpr(self.translate_expr_list(n.targets)) tup.set_line(n.lineno) - s = DelStmt(tup) + stmt = DelStmt(tup) else: - s = DelStmt(self.visit(n.targets[0])) - return self.set_line(s, n) + stmt = DelStmt(self.visit(n.targets[0])) + return self.set_line(stmt, n) # Assign(expr* targets, expr value, string? type_comment) def visit_Assign(self, n: ast27.Assign) -> AssignmentStmt: @@ -534,17 +534,17 @@ def visit_Assign(self, n: ast27.Assign) -> AssignmentStmt: if n.type_comment: typ = parse_type_comment(n.type_comment, n.lineno, self.errors) - s = AssignmentStmt(self.translate_expr_list(n.targets), - self.visit(n.value), - type=typ) - return self.set_line(s, n) + stmt = AssignmentStmt(self.translate_expr_list(n.targets), + self.visit(n.value), + type=typ) + return self.set_line(stmt, n) # AugAssign(expr target, operator op, expr value) def visit_AugAssign(self, n: ast27.AugAssign) -> OperatorAssignmentStmt: - s = OperatorAssignmentStmt(self.from_operator(n.op), - self.visit(n.target), - self.visit(n.value)) - return self.set_line(s, n) + stmt = OperatorAssignmentStmt(self.from_operator(n.op), + self.visit(n.target), + self.visit(n.value)) + return self.set_line(stmt, n) # For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) def visit_For(self, n: ast27.For) -> ForStmt: @@ -552,26 +552,26 @@ def visit_For(self, n: ast27.For) -> ForStmt: target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) else: target_type = None - s = ForStmt(self.visit(n.target), - self.visit(n.iter), - self.as_required_block(n.body, n.lineno), - self.as_block(n.orelse, n.lineno), - target_type) - return self.set_line(s, n) + stmt = ForStmt(self.visit(n.target), + self.visit(n.iter), + self.as_required_block(n.body, n.lineno), + self.as_block(n.orelse, n.lineno), + target_type) + return self.set_line(stmt, n) # While(expr test, stmt* body, stmt* orelse) def visit_While(self, n: ast27.While) -> WhileStmt: - s = WhileStmt(self.visit(n.test), - self.as_required_block(n.body, n.lineno), - self.as_block(n.orelse, n.lineno)) - return self.set_line(s, n) + stmt = WhileStmt(self.visit(n.test), + self.as_required_block(n.body, n.lineno), + self.as_block(n.orelse, n.lineno)) + return self.set_line(stmt, n) # If(expr test, stmt* body, stmt* orelse) def visit_If(self, n: ast27.If) -> IfStmt: - s = IfStmt([self.visit(n.test)], - [self.as_required_block(n.body, n.lineno)], - self.as_block(n.orelse, n.lineno)) - return self.set_line(s, n) + stmt = IfStmt([self.visit(n.test)], + [self.as_required_block(n.body, n.lineno)], + self.as_block(n.orelse, n.lineno)) + return self.set_line(stmt, n) # With(withitem* items, stmt* body, string? type_comment) def visit_With(self, n: ast27.With) -> WithStmt: @@ -579,11 +579,11 @@ def visit_With(self, n: ast27.With) -> WithStmt: target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) else: target_type = None - s = WithStmt([self.visit(n.context_expr)], - [self.visit(n.optional_vars)], - self.as_required_block(n.body, n.lineno), - target_type) - return self.set_line(s, n) + stmt = WithStmt([self.visit(n.context_expr)], + [self.visit(n.optional_vars)], + self.as_required_block(n.body, n.lineno), + target_type) + return self.set_line(stmt, n) def visit_Raise(self, n: ast27.Raise) -> RaiseStmt: if n.type is None: @@ -597,20 +597,20 @@ def visit_Raise(self, n: ast27.Raise) -> RaiseStmt: else: e = TupleExpr([self.visit(n.type), self.visit(n.inst), self.visit(n.tback)]) - s = RaiseStmt(e, None) - return self.set_line(s, n) + stmt = RaiseStmt(e, None) + return self.set_line(stmt, n) # TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) def visit_TryExcept(self, n: ast27.TryExcept) -> TryStmt: - s = self.try_handler(n.body, n.handlers, n.orelse, [], n.lineno) - return self.set_line(s, n) + stmt = self.try_handler(n.body, n.handlers, n.orelse, [], n.lineno) + return self.set_line(stmt, n) def visit_TryFinally(self, n: ast27.TryFinally) -> TryStmt: if len(n.body) == 1 and isinstance(n.body[0], ast27.TryExcept): - s = self.try_handler([n.body[0]], [], [], n.finalbody, n.lineno) + stmt = self.try_handler([n.body[0]], [], [], n.finalbody, n.lineno) else: - s = self.try_handler(n.body, [], [], n.finalbody, n.lineno) - return self.set_line(s, n) + stmt = self.try_handler(n.body, [], [], n.finalbody, n.lineno) + return self.set_line(stmt, n) def try_handler(self, body: List[ast27.stmt], @@ -639,23 +639,23 @@ def try_handler(self, self.as_block(finalbody, lineno)) def visit_Print(self, n: ast27.Print) -> PrintStmt: - s = PrintStmt(self.translate_expr_list(n.values), n.nl, self.visit(n.dest)) - return self.set_line(s, n) + stmt = PrintStmt(self.translate_expr_list(n.values), n.nl, self.visit(n.dest)) + return self.set_line(stmt, n) def visit_Exec(self, n: ast27.Exec) -> ExecStmt: - s = ExecStmt(self.visit(n.body), + stmt = ExecStmt(self.visit(n.body), self.visit(n.globals), self.visit(n.locals)) - return self.set_line(s, n) + return self.set_line(stmt, n) def visit_Repr(self, n: ast27.Repr) -> BackquoteExpr: - s = BackquoteExpr(self.visit(n.value)) - return self.set_line(s, n) + stmt = BackquoteExpr(self.visit(n.value)) + return self.set_line(stmt, n) # Assert(expr test, expr? msg) def visit_Assert(self, n: ast27.Assert) -> AssertStmt: - s = AssertStmt(self.visit(n.test), self.visit(n.msg)) - return self.set_line(s, n) + stmt = AssertStmt(self.visit(n.test), self.visit(n.msg)) + return self.set_line(stmt, n) # Import(alias* names) def visit_Import(self, n: ast27.Import) -> Import: @@ -688,29 +688,29 @@ def visit_ImportFrom(self, n: ast27.ImportFrom) -> ImportBase: # Global(identifier* names) def visit_Global(self, n: ast27.Global) -> GlobalDecl: - s = GlobalDecl(n.names) - return self.set_line(s, n) + stmt = GlobalDecl(n.names) + return self.set_line(stmt, n) # Expr(expr value) def visit_Expr(self, n: ast27.Expr) -> ExpressionStmt: value = self.visit(n.value) - s = ExpressionStmt(value) - return self.set_line(s, n) + stmt = ExpressionStmt(value) + return self.set_line(stmt, n) # Pass def visit_Pass(self, n: ast27.Pass) -> PassStmt: - s = PassStmt() - return self.set_line(s, n) + stmt = PassStmt() + return self.set_line(stmt, n) # Break def visit_Break(self, n: ast27.Break) -> BreakStmt: - s = BreakStmt() - return self.set_line(s, n) + stmt = BreakStmt() + return self.set_line(stmt, n) # Continue def visit_Continue(self, n: ast27.Continue) -> ContinueStmt: - s = ContinueStmt() - return self.set_line(s, n) + stmt = ContinueStmt() + return self.set_line(stmt, n) # --- expr --- @@ -907,10 +907,10 @@ def visit_Str(self, n: ast27.Str) -> Expression: # because mypy considers str literals to be compatible with # unicode. if isinstance(n.s, bytes): - s = n.s + value = n.s # The following line is a bit hacky, but is the best way to maintain # compatibility with how mypy currently parses the contents of bytes literals. - contents = str(s)[2:-1] + contents = str(value)[2:-1] e = StrExpr(contents) # type: Union[StrExpr, UnicodeExpr] return self.set_line(e, n) else: @@ -923,6 +923,10 @@ def visit_Ellipsis(self, n: ast27.Ellipsis) -> EllipsisExpr: # Attribute(expr value, identifier attr, expr_context ctx) def visit_Attribute(self, n: Attribute) -> Expression: + # First create MemberExpr and then potentially replace with a SuperExpr + # to improve performance when compiled. The check for "super()" will be + # faster with native AST nodes. Note also that super expressions are + # less common than normal member expressions. member_expr = MemberExpr(self.visit(n.value), n.attr) obj = member_expr.expr if (isinstance(obj, CallExpr) and