diff --git a/src/lpython/parser/parser.yy b/src/lpython/parser/parser.yy index 362a9a0eef..43a8609c3f 100644 --- a/src/lpython/parser/parser.yy +++ b/src/lpython/parser/parser.yy @@ -105,6 +105,7 @@ void yyerror(YYLTYPE *yyloc, LFortran::Parser &p, const std::string &msg) %token TK_COMMENT %token TK_EOLCOMMENT %token TK_TYPE_COMMENT +%token TK_TYPE_IGNORE %token TK_POW "**" %token TK_FLOOR_DIV "//" %token TK_RIGHTSHIFT ">>" @@ -245,6 +246,7 @@ void yyerror(YYLTYPE *yyloc, LFortran::Parser &p, const std::string &msg) %type while_statement %type sep %type sep_one +%type type_comment %type string %type ternary_if_statement /* %type list_comprehension */ @@ -421,7 +423,7 @@ assignment_statement : target_list expr { $$ = ASSIGNMENT($1, $2, @$); } | target_list expr_list "," expr { $$ = ASSIGNMENT($1, TUPLE_01(TUPLE_($2, $4), @$), @$); } - | target_list expr TK_TYPE_COMMENT { $$ = ASSIGNMENT2($1, $2, $3, @$); } + | target_list expr type_comment { $$ = ASSIGNMENT2($1, $2, $3, @$); } ; augassign_statement @@ -486,7 +488,8 @@ dot_list import_statement : KW_IMPORT module_item_list { $$ = IMPORT_01($2, @$); } - | KW_IMPORT module_item_list TK_TYPE_COMMENT { $$ = IMPORT_01($2, @$); } + | KW_IMPORT module_item_list TK_TYPE_IGNORE { $$ = IMPORT_01($2, @$); + extract_type_comment(p, @$, $3); } | KW_FROM module KW_IMPORT module_item_list { $$ = IMPORT_02($2, $4, @$); } | KW_FROM module KW_IMPORT "(" module_item_list comma_opt ")" { @@ -541,9 +544,9 @@ for_statement $$ = FOR_01($2, TUPLE_03(A2LIST(p.m_a, $4), @$), $7, @$); } | KW_FOR for_target_list KW_IN expr ":" body_stmts KW_ELSE ":" body_stmts { $$ = FOR_02($2, $4, $6, $9, @$); } - | KW_FOR for_target_list KW_IN expr ":" TK_TYPE_COMMENT TK_NEWLINE + | KW_FOR for_target_list KW_IN expr ":" type_comment TK_NEWLINE statements { $$ = FOR_03($2, $4, $6, $8, @$); } - | KW_FOR for_target_list KW_IN expr ":" TK_TYPE_COMMENT TK_NEWLINE + | KW_FOR for_target_list KW_IN expr ":" type_comment TK_NEWLINE statements KW_ELSE ":" body_stmts { $$ = FOR_04($2, $4, $8, $11, $6, @$); } ; @@ -585,9 +588,9 @@ with_as_items with_statement : KW_WITH expr_list ":" body_stmts { $$ = WITH($2, $4, @$); } | KW_WITH with_as_items ":" body_stmts { $$ = WITH_02($2, $4, @$); } - | KW_WITH expr_list ":" TK_TYPE_COMMENT TK_NEWLINE + | KW_WITH expr_list ":" type_comment TK_NEWLINE statements { $$ = WITH_01($2, $6, $4, @$); } - | KW_WITH with_as_items ":" TK_TYPE_COMMENT TK_NEWLINE + | KW_WITH with_as_items ":" type_comment TK_NEWLINE statements { $$ = WITH_03($2, $6, $4, @$); } ; @@ -599,6 +602,14 @@ decorators_opt decorators : decorators "@" expr sep { $$ = $1; LIST_ADD($$, $3); } | "@" expr sep { LIST_NEW($$); LIST_ADD($$, $2); } + | decorators "@" expr TK_TYPE_IGNORE TK_NEWLINE { $$ = $1; + LIST_ADD($$, $3); extract_type_comment(p, @$, $4); } + | "@" expr TK_TYPE_IGNORE TK_NEWLINE { LIST_NEW($$); LIST_ADD($$, $2); + extract_type_comment(p, @$, $3); } + | decorators "@" expr TK_TYPE_IGNORE TK_NEWLINE sep { $$ = $1; + LIST_ADD($$, $3); extract_type_comment(p, @$, $4); } + | "@" expr TK_TYPE_IGNORE TK_NEWLINE sep { LIST_NEW($$); LIST_ADD($$, $2); + extract_type_comment(p, @$, $3); } ; parameter @@ -611,9 +622,9 @@ parameter defparameter_list : defparameter_list "," parameter { $$ = $1; LIST_ADD($$, $3); } | parameter { LIST_NEW($$); LIST_ADD($$, $1); } - | defparameter_list "," TK_TYPE_COMMENT parameter { $$ = $1; + | defparameter_list "," type_comment parameter { $$ = $1; LIST_ADD($$, $4); ADD_TYPE_COMMENT($1, $3, @$); } - | defparameter_list "," TK_TYPE_COMMENT { + | defparameter_list "," type_comment { $$ = $1; ADD_TYPE_COMMENT($1, $3, @$); } ; @@ -663,16 +674,16 @@ function_def | decorators_opt KW_DEF id "(" parameter_list_opt ")" "->" expr ":" body_stmts { $$ = FUNCTION_02($1, $3, $5, $8, $10, @$); } | decorators_opt KW_DEF id "(" parameter_list_opt ")" ":" - TK_TYPE_COMMENT TK_NEWLINE statements { + type_comment TK_NEWLINE statements { $$ = FUNCTION_03($1, $3, $5, $10, $8, @$); } | decorators_opt KW_DEF id "(" parameter_list_opt ")" "->" expr ":" - TK_TYPE_COMMENT TK_NEWLINE statements { + type_comment TK_NEWLINE statements { $$ = FUNCTION_04($1, $3, $5, $8, $12, $10, @$); } | decorators_opt KW_DEF id "(" parameter_list_opt ")" ":" - TK_NEWLINE TK_TYPE_COMMENT TK_NEWLINE statements { + TK_NEWLINE type_comment TK_NEWLINE statements { $$ = FUNCTION_03($1, $3, $5, $11, $9, @$); } | decorators_opt KW_DEF id "(" parameter_list_opt ")" "->" expr ":" - TK_NEWLINE TK_TYPE_COMMENT TK_NEWLINE statements { + TK_NEWLINE type_comment TK_NEWLINE statements { $$ = FUNCTION_04($1, $3, $5, $8, $13, $11, @$); } ; @@ -688,16 +699,16 @@ class_def | decorators_opt KW_CLASS id "(" keyword_items ")" ":" body_stmts { $$ = CLASS_04($1, $3, $5, $8, @$); } | decorators_opt KW_CLASS id "(" expr_list_opt ")" ":" - TK_TYPE_COMMENT TK_NEWLINE statements { + type_comment TK_NEWLINE statements { $$ = CLASS_05($1, $3, $5, $8, $10, @$); } | decorators_opt KW_CLASS id "(" expr_list "," keyword_items ")" - ":" TK_TYPE_COMMENT TK_NEWLINE statements { + ":" type_comment TK_NEWLINE statements { $$ = CLASS_06($1, $3, $5, $7, $10, $12, @$); } | decorators_opt KW_CLASS id "(" keyword_items "," expr_list ")" - ":" TK_TYPE_COMMENT TK_NEWLINE statements { + ":" type_comment TK_NEWLINE statements { $$ = CLASS_06($1, $3, $7, $5, $10, $12, @$); } | decorators_opt KW_CLASS id "(" keyword_items ")" ":" - TK_TYPE_COMMENT TK_NEWLINE statements { + type_comment TK_NEWLINE statements { $$ = CLASS_07($1, $3, $5, $8, $10, @$); } ; @@ -711,16 +722,16 @@ async_func_def | KW_ASYNC KW_DEF id "(" parameter_list_opt ")" "->" expr ":" body_stmts { $$ = ASYNC_FUNCTION_04($3, $5, $8, $10, @$); } | decorators KW_ASYNC KW_DEF id "(" parameter_list_opt ")" ":" - TK_TYPE_COMMENT TK_NEWLINE statements { + type_comment TK_NEWLINE statements { $$ = ASYNC_FUNCTION_05($1, $4, $6, $11, $9, @$); } | decorators KW_ASYNC KW_DEF id "(" parameter_list_opt ")" "->" expr ":" - TK_TYPE_COMMENT TK_NEWLINE statements { + type_comment TK_NEWLINE statements { $$ = ASYNC_FUNCTION_06($1, $4, $6, $9, $13, $11, @$); } | KW_ASYNC KW_DEF id "(" parameter_list_opt ")" ":" - TK_TYPE_COMMENT TK_NEWLINE statements { + type_comment TK_NEWLINE statements { $$ = ASYNC_FUNCTION_07($3, $5, $10, $8, @$); } | KW_ASYNC KW_DEF id "(" parameter_list_opt ")" "->" expr ":" - TK_TYPE_COMMENT TK_NEWLINE statements { + type_comment TK_NEWLINE statements { $$ = ASYNC_FUNCTION_08($3, $5, $8, $12, $10, @$); } ; @@ -731,10 +742,10 @@ async_for_stmt KW_ELSE ":" body_stmts { $$ = ASYNC_FOR_02($3, $5, $7, $10, @$); } | KW_ASYNC KW_FOR expr KW_IN expr ":" - TK_TYPE_COMMENT TK_NEWLINE statements { + type_comment TK_NEWLINE statements { $$ = ASYNC_FOR_03($3, $5, $9, $7, @$); } | KW_ASYNC KW_FOR expr KW_IN expr ":" - TK_TYPE_COMMENT TK_NEWLINE statements KW_ELSE ":" body_stmts { + type_comment TK_NEWLINE statements KW_ELSE ":" body_stmts { $$ = ASYNC_FOR_04($3, $5, $9, $12, $7, @$); } ; @@ -743,9 +754,9 @@ async_with_stmt $$ = ASYNC_WITH($3, $5, @$); } | KW_ASYNC KW_WITH with_as_items ":" body_stmts { $$ = ASYNC_WITH_02($3, $5, @$); } - | KW_ASYNC KW_WITH expr_list ":" TK_TYPE_COMMENT + | KW_ASYNC KW_WITH expr_list ":" type_comment TK_NEWLINE statements { $$ = ASYNC_WITH_01($3, $7, $5, @$); } - | KW_ASYNC KW_WITH with_as_items ":" TK_TYPE_COMMENT + | KW_ASYNC KW_WITH with_as_items ":" type_comment TK_NEWLINE statements { $$ = ASYNC_WITH_03($3, $7, $5, @$); } ; @@ -787,7 +798,7 @@ expr_list dict : expr ":" expr { $$ = DICT_EXPR_01($1, $3, @$); } - | expr ":" TK_TYPE_COMMENT expr { $$ = DICT_EXPR_02($1, $3, $4, @$); } + | expr ":" type_comment expr { $$ = DICT_EXPR_02($1, $3, $4, @$); } ; dict_list @@ -1015,6 +1026,11 @@ id : TK_NAME { $$ = SYMBOL($1, @$); } ; +type_comment + : TK_TYPE_COMMENT { $$ = $1; } + | TK_TYPE_IGNORE { $$ = $1; } + ; + sep : sep sep_one { $$ = $1; LIST_ADD($$, $2); } | sep_one { LIST_NEW($$); LIST_ADD($$, $1); } diff --git a/src/lpython/parser/tokenizer.re b/src/lpython/parser/tokenizer.re index b7781f5025..3349bda072 100644 --- a/src/lpython/parser/tokenizer.re +++ b/src/lpython/parser/tokenizer.re @@ -279,6 +279,7 @@ int Tokenizer::lex(Allocator &al, YYSTYPE &yylval, Location &loc, diag::Diagnost string2 = "'" ("\\'"|[^'\x00])* "'"; string3 = '"""' ( '\\"' | '"' [^"\x00] | '""' [^"\x00] | [^"\x00] )* '"""'; string4 = "'''" ( "\\'" | "'" [^'\x00] | "''" [^'\x00] | [^'\x00] )* "'''"; + type_ignore = "#" whitespace? "type:" whitespace? "ignore" [^\n\x00]*; type_comment = "#" whitespace? "type:" whitespace? [^\n\x00]*; comment = "#" [^\n\x00]*; // docstring = newline whitespace? string1 | string2; @@ -462,6 +463,14 @@ int Tokenizer::lex(Allocator &al, YYSTYPE &yylval, Location &loc, diag::Diagnost RET(TK_IMAG_NUM) } + type_ignore { + if (last_token == yytokentype::TK_COLON && !parenlevel) { + indent = true; + } + token(yylval.string); + RET(TK_TYPE_IGNORE); + } + type_comment { if (last_token == yytokentype::TK_COLON && !parenlevel) { indent = true; @@ -545,6 +554,7 @@ std::string token2text(const int token) T(TK_COMMENT, "comment") T(TK_EOLCOMMENT, "eolcomment") T(TK_TYPE_COMMENT, "type_comment") + T(TK_TYPE_IGNORE, "type_ignore") T(TK_POW, "**") T(TK_FLOOR_DIV, "//") @@ -679,6 +689,8 @@ std::string pickle_token(int token, const LFortran::YYSTYPE &yystype) t = t + " " + "\"" + yystype.string.str() + "\""; } else if (token == yytokentype::TK_TYPE_COMMENT) { t = t + " " + "\"" + yystype.string.str() + "\""; + } else if (token == yytokentype::TK_TYPE_IGNORE) { + t = t + " " + "\"" + yystype.string.str() + "\""; } t += ")"; return t; diff --git a/tests/parser/type_comment1.py b/tests/parser/type_comment1.py index d10dcbc111..1fa9ea896b 100644 --- a/tests/parser/type_comment1.py +++ b/tests/parser/type_comment1.py @@ -2,4 +2,18 @@ def ndarray_func(x): # type: (np.ndarray) -> np.ndarray - return x \ No newline at end of file + return x + +@decorator1 # type: ignore + +# Comment + +@decorator2 + +# Comment + +@decorator3 # type: ignore + +def test(x): + # type: (np.ndarray) -> np.ndarray + return x diff --git a/tests/reference/ast_new-type_comment1-710ea6c.json b/tests/reference/ast_new-type_comment1-710ea6c.json index a95a84c989..538138240a 100644 --- a/tests/reference/ast_new-type_comment1-710ea6c.json +++ b/tests/reference/ast_new-type_comment1-710ea6c.json @@ -2,11 +2,11 @@ "basename": "ast_new-type_comment1-710ea6c", "cmd": "lpython --show-ast --new-parser --no-color {infile} -o {outfile}", "infile": "tests/parser/type_comment1.py", - "infile_hash": "cdc0bdd0faec912e7f3db4e71c4a14aa281828e8b68faac91f5a78ee", + "infile_hash": "9c79bc041758b5401f4431a53b6b333999a4e7dfe9dfabac13d83178", "outfile": null, "outfile_hash": null, "stdout": "ast_new-type_comment1-710ea6c.stdout", - "stdout_hash": "98ccc7f80021238fff1a1a1395e64b5d2097f025e60e47821ec76fdb", + "stdout_hash": "c7019449158ebe30677a0808ad2fd8f93aebd2eee6cd90914c44cd98", "stderr": null, "stderr_hash": null, "returncode": 0 diff --git a/tests/reference/ast_new-type_comment1-710ea6c.stdout b/tests/reference/ast_new-type_comment1-710ea6c.stdout index 536733895a..1eaa508aad 100644 --- a/tests/reference/ast_new-type_comment1-710ea6c.stdout +++ b/tests/reference/ast_new-type_comment1-710ea6c.stdout @@ -1 +1 @@ -(Module [(Import [(pytest ())]) (FunctionDef ndarray_func ([] [(x () ())] [] [] [] [] []) [(Return (Name x Load))] [] () "(np.ndarray) -> np.ndarray")] []) +(Module [(Import [(pytest ())]) (FunctionDef ndarray_func ([] [(x () ())] [] [] [] [] []) [(Return (Name x Load))] [] () "(np.ndarray) -> np.ndarray") (FunctionDef test ([] [(x () ())] [] [] [] [] []) [(Return (Name x Load))] [(Name decorator1 Load) (Name decorator2 Load) (Name decorator3 Load)] () "(np.ndarray) -> np.ndarray")] [(TypeIgnore 0 "") (TypeIgnore 0 "") (TypeIgnore 0 "")])