Skip to content

Commit 447f2da

Browse files
Thirumalai-ShaktivelThirumalai-Shaktivel
authored
Merge pull request #979 from akshanshbhatt/pr_type_ignores
2 parents 2fda6cb + fdcc78c commit 447f2da

File tree

5 files changed

+71
-29
lines changed

5 files changed

+71
-29
lines changed

‎src/lpython/parser/parser.yy

Lines changed: 41 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ void yyerror(YYLTYPE *yyloc, LFortran::Parser &p, const std::string &msg)
105105
%token <string> TK_COMMENT
106106
%token <string> TK_EOLCOMMENT
107107
%token <string> TK_TYPE_COMMENT
108+
%token <string> TK_TYPE_IGNORE
108109
%token TK_POW "**"
109110
%token TK_FLOOR_DIV "//"
110111
%token TK_RIGHTSHIFT ">>"
@@ -245,6 +246,7 @@ void yyerror(YYLTYPE *yyloc, LFortran::Parser &p, const std::string &msg)
245246
%type <ast> while_statement
246247
%type <vec_ast> sep
247248
%type <ast> sep_one
249+
%type <string> type_comment
248250
%type <ast> string
249251
%type <ast> ternary_if_statement
250252
/* %type <ast> list_comprehension */
@@ -421,7 +423,7 @@ assignment_statement
421423
: target_list expr { $$ = ASSIGNMENT($1, $2, @$); }
422424
| target_list expr_list "," expr {
423425
$$ = ASSIGNMENT($1, TUPLE_01(TUPLE_($2, $4), @$), @$); }
424-
| target_list expr TK_TYPE_COMMENT { $$ = ASSIGNMENT2($1, $2, $3, @$); }
426+
| target_list expr type_comment { $$ = ASSIGNMENT2($1, $2, $3, @$); }
425427
;
426428

427429
augassign_statement
@@ -486,7 +488,8 @@ dot_list
486488

487489
import_statement
488490
: KW_IMPORT module_item_list { $$ = IMPORT_01($2, @$); }
489-
| KW_IMPORT module_item_list TK_TYPE_COMMENT { $$ = IMPORT_01($2, @$); }
491+
| KW_IMPORT module_item_list TK_TYPE_IGNORE { $$ = IMPORT_01($2, @$);
492+
extract_type_comment(p, @$, $3); }
490493
| KW_FROM module KW_IMPORT module_item_list {
491494
$$ = IMPORT_02($2, $4, @$); }
492495
| KW_FROM module KW_IMPORT "(" module_item_list comma_opt ")" {
@@ -541,9 +544,9 @@ for_statement
541544
$$ = FOR_01($2, TUPLE_03(A2LIST(p.m_a, $4), @$), $7, @$); }
542545
| KW_FOR for_target_list KW_IN expr ":" body_stmts KW_ELSE ":"
543546
body_stmts { $$ = FOR_02($2, $4, $6, $9, @$); }
544-
| KW_FOR for_target_list KW_IN expr ":" TK_TYPE_COMMENT TK_NEWLINE
547+
| KW_FOR for_target_list KW_IN expr ":" type_comment TK_NEWLINE
545548
statements { $$ = FOR_03($2, $4, $6, $8, @$); }
546-
| KW_FOR for_target_list KW_IN expr ":" TK_TYPE_COMMENT TK_NEWLINE
549+
| KW_FOR for_target_list KW_IN expr ":" type_comment TK_NEWLINE
547550
statements KW_ELSE ":" body_stmts {
548551
$$ = FOR_04($2, $4, $8, $11, $6, @$); }
549552
;
@@ -585,9 +588,9 @@ with_as_items
585588
with_statement
586589
: KW_WITH expr_list ":" body_stmts { $$ = WITH($2, $4, @$); }
587590
| KW_WITH with_as_items ":" body_stmts { $$ = WITH_02($2, $4, @$); }
588-
| KW_WITH expr_list ":" TK_TYPE_COMMENT TK_NEWLINE
591+
| KW_WITH expr_list ":" type_comment TK_NEWLINE
589592
statements { $$ = WITH_01($2, $6, $4, @$); }
590-
| KW_WITH with_as_items ":" TK_TYPE_COMMENT TK_NEWLINE
593+
| KW_WITH with_as_items ":" type_comment TK_NEWLINE
591594
statements { $$ = WITH_03($2, $6, $4, @$); }
592595
;
593596

@@ -599,6 +602,14 @@ decorators_opt
599602
decorators
600603
: decorators "@" expr sep { $$ = $1; LIST_ADD($$, $3); }
601604
| "@" expr sep { LIST_NEW($$); LIST_ADD($$, $2); }
605+
| decorators "@" expr TK_TYPE_IGNORE TK_NEWLINE { $$ = $1;
606+
LIST_ADD($$, $3); extract_type_comment(p, @$, $4); }
607+
| "@" expr TK_TYPE_IGNORE TK_NEWLINE { LIST_NEW($$); LIST_ADD($$, $2);
608+
extract_type_comment(p, @$, $3); }
609+
| decorators "@" expr TK_TYPE_IGNORE TK_NEWLINE sep { $$ = $1;
610+
LIST_ADD($$, $3); extract_type_comment(p, @$, $4); }
611+
| "@" expr TK_TYPE_IGNORE TK_NEWLINE sep { LIST_NEW($$); LIST_ADD($$, $2);
612+
extract_type_comment(p, @$, $3); }
602613
;
603614

604615
parameter
@@ -611,9 +622,9 @@ parameter
611622
defparameter_list
612623
: defparameter_list "," parameter { $$ = $1; LIST_ADD($$, $3); }
613624
| parameter { LIST_NEW($$); LIST_ADD($$, $1); }
614-
| defparameter_list "," TK_TYPE_COMMENT parameter { $$ = $1;
625+
| defparameter_list "," type_comment parameter { $$ = $1;
615626
LIST_ADD($$, $4); ADD_TYPE_COMMENT($1, $3, @$); }
616-
| defparameter_list "," TK_TYPE_COMMENT {
627+
| defparameter_list "," type_comment {
617628
$$ = $1; ADD_TYPE_COMMENT($1, $3, @$); }
618629
;
619630

@@ -663,16 +674,16 @@ function_def
663674
| decorators_opt KW_DEF id "(" parameter_list_opt ")" "->" expr ":"
664675
body_stmts { $$ = FUNCTION_02($1, $3, $5, $8, $10, @$); }
665676
| decorators_opt KW_DEF id "(" parameter_list_opt ")" ":"
666-
TK_TYPE_COMMENT TK_NEWLINE statements {
677+
type_comment TK_NEWLINE statements {
667678
$$ = FUNCTION_03($1, $3, $5, $10, $8, @$); }
668679
| decorators_opt KW_DEF id "(" parameter_list_opt ")" "->" expr ":"
669-
TK_TYPE_COMMENT TK_NEWLINE statements {
680+
type_comment TK_NEWLINE statements {
670681
$$ = FUNCTION_04($1, $3, $5, $8, $12, $10, @$); }
671682
| decorators_opt KW_DEF id "(" parameter_list_opt ")" ":"
672-
TK_NEWLINE TK_TYPE_COMMENT TK_NEWLINE statements {
683+
TK_NEWLINE type_comment TK_NEWLINE statements {
673684
$$ = FUNCTION_03($1, $3, $5, $11, $9, @$); }
674685
| decorators_opt KW_DEF id "(" parameter_list_opt ")" "->" expr ":"
675-
TK_NEWLINE TK_TYPE_COMMENT TK_NEWLINE statements {
686+
TK_NEWLINE type_comment TK_NEWLINE statements {
676687
$$ = FUNCTION_04($1, $3, $5, $8, $13, $11, @$); }
677688
;
678689

@@ -688,16 +699,16 @@ class_def
688699
| decorators_opt KW_CLASS id "(" keyword_items ")" ":" body_stmts {
689700
$$ = CLASS_04($1, $3, $5, $8, @$); }
690701
| decorators_opt KW_CLASS id "(" expr_list_opt ")" ":"
691-
TK_TYPE_COMMENT TK_NEWLINE statements {
702+
type_comment TK_NEWLINE statements {
692703
$$ = CLASS_05($1, $3, $5, $8, $10, @$); }
693704
| decorators_opt KW_CLASS id "(" expr_list "," keyword_items ")"
694-
":" TK_TYPE_COMMENT TK_NEWLINE statements {
705+
":" type_comment TK_NEWLINE statements {
695706
$$ = CLASS_06($1, $3, $5, $7, $10, $12, @$); }
696707
| decorators_opt KW_CLASS id "(" keyword_items "," expr_list ")"
697-
":" TK_TYPE_COMMENT TK_NEWLINE statements {
708+
":" type_comment TK_NEWLINE statements {
698709
$$ = CLASS_06($1, $3, $7, $5, $10, $12, @$); }
699710
| decorators_opt KW_CLASS id "(" keyword_items ")" ":"
700-
TK_TYPE_COMMENT TK_NEWLINE statements {
711+
type_comment TK_NEWLINE statements {
701712
$$ = CLASS_07($1, $3, $5, $8, $10, @$); }
702713
;
703714

@@ -711,16 +722,16 @@ async_func_def
711722
| KW_ASYNC KW_DEF id "(" parameter_list_opt ")" "->" expr ":"
712723
body_stmts { $$ = ASYNC_FUNCTION_04($3, $5, $8, $10, @$); }
713724
| decorators KW_ASYNC KW_DEF id "(" parameter_list_opt ")" ":"
714-
TK_TYPE_COMMENT TK_NEWLINE statements {
725+
type_comment TK_NEWLINE statements {
715726
$$ = ASYNC_FUNCTION_05($1, $4, $6, $11, $9, @$); }
716727
| decorators KW_ASYNC KW_DEF id "(" parameter_list_opt ")" "->" expr ":"
717-
TK_TYPE_COMMENT TK_NEWLINE statements {
728+
type_comment TK_NEWLINE statements {
718729
$$ = ASYNC_FUNCTION_06($1, $4, $6, $9, $13, $11, @$); }
719730
| KW_ASYNC KW_DEF id "(" parameter_list_opt ")" ":"
720-
TK_TYPE_COMMENT TK_NEWLINE statements {
731+
type_comment TK_NEWLINE statements {
721732
$$ = ASYNC_FUNCTION_07($3, $5, $10, $8, @$); }
722733
| KW_ASYNC KW_DEF id "(" parameter_list_opt ")" "->" expr ":"
723-
TK_TYPE_COMMENT TK_NEWLINE statements {
734+
type_comment TK_NEWLINE statements {
724735
$$ = ASYNC_FUNCTION_08($3, $5, $8, $12, $10, @$); }
725736
;
726737

@@ -731,10 +742,10 @@ async_for_stmt
731742
KW_ELSE ":" body_stmts {
732743
$$ = ASYNC_FOR_02($3, $5, $7, $10, @$); }
733744
| KW_ASYNC KW_FOR expr KW_IN expr ":"
734-
TK_TYPE_COMMENT TK_NEWLINE statements {
745+
type_comment TK_NEWLINE statements {
735746
$$ = ASYNC_FOR_03($3, $5, $9, $7, @$); }
736747
| KW_ASYNC KW_FOR expr KW_IN expr ":"
737-
TK_TYPE_COMMENT TK_NEWLINE statements KW_ELSE ":" body_stmts {
748+
type_comment TK_NEWLINE statements KW_ELSE ":" body_stmts {
738749
$$ = ASYNC_FOR_04($3, $5, $9, $12, $7, @$); }
739750
;
740751

@@ -743,9 +754,9 @@ async_with_stmt
743754
$$ = ASYNC_WITH($3, $5, @$); }
744755
| KW_ASYNC KW_WITH with_as_items ":" body_stmts {
745756
$$ = ASYNC_WITH_02($3, $5, @$); }
746-
| KW_ASYNC KW_WITH expr_list ":" TK_TYPE_COMMENT
757+
| KW_ASYNC KW_WITH expr_list ":" type_comment
747758
TK_NEWLINE statements { $$ = ASYNC_WITH_01($3, $7, $5, @$); }
748-
| KW_ASYNC KW_WITH with_as_items ":" TK_TYPE_COMMENT
759+
| KW_ASYNC KW_WITH with_as_items ":" type_comment
749760
TK_NEWLINE statements { $$ = ASYNC_WITH_03($3, $7, $5, @$); }
750761
;
751762

@@ -787,7 +798,7 @@ expr_list
787798

788799
dict
789800
: expr ":" expr { $$ = DICT_EXPR_01($1, $3, @$); }
790-
| expr ":" TK_TYPE_COMMENT expr { $$ = DICT_EXPR_02($1, $3, $4, @$); }
801+
| expr ":" type_comment expr { $$ = DICT_EXPR_02($1, $3, $4, @$); }
791802
;
792803

793804
dict_list
@@ -1015,6 +1026,11 @@ id
10151026
: TK_NAME { $$ = SYMBOL($1, @$); }
10161027
;
10171028

1029+
type_comment
1030+
: TK_TYPE_COMMENT { $$ = $1; }
1031+
| TK_TYPE_IGNORE { $$ = $1; }
1032+
;
1033+
10181034
sep
10191035
: sep sep_one { $$ = $1; LIST_ADD($$, $2); }
10201036
| sep_one { LIST_NEW($$); LIST_ADD($$, $1); }

‎src/lpython/parser/tokenizer.re

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,7 @@ int Tokenizer::lex(Allocator &al, YYSTYPE &yylval, Location &loc, diag::Diagnost
279279
string2 = "'" ("\\'"|[^'\x00])* "'";
280280
string3 = '"""' ( '\\"' | '"' [^"\x00] | '""' [^"\x00] | [^"\x00] )* '"""';
281281
string4 = "'''" ( "\\'" | "'" [^'\x00] | "''" [^'\x00] | [^'\x00] )* "'''";
282+
type_ignore = "#" whitespace? "type:" whitespace? "ignore" [^\n\x00]*;
282283
type_comment = "#" whitespace? "type:" whitespace? [^\n\x00]*;
283284
comment = "#" [^\n\x00]*;
284285
// docstring = newline whitespace? string1 | string2;
@@ -462,6 +463,14 @@ int Tokenizer::lex(Allocator &al, YYSTYPE &yylval, Location &loc, diag::Diagnost
462463
RET(TK_IMAG_NUM)
463464
}
464465
466+
type_ignore {
467+
if (last_token == yytokentype::TK_COLON && !parenlevel) {
468+
indent = true;
469+
}
470+
token(yylval.string);
471+
RET(TK_TYPE_IGNORE);
472+
}
473+
465474
type_comment {
466475
if (last_token == yytokentype::TK_COLON && !parenlevel) {
467476
indent = true;
@@ -545,6 +554,7 @@ std::string token2text(const int token)
545554
T(TK_COMMENT, "comment")
546555
T(TK_EOLCOMMENT, "eolcomment")
547556
T(TK_TYPE_COMMENT, "type_comment")
557+
T(TK_TYPE_IGNORE, "type_ignore")
548558

549559
T(TK_POW, "**")
550560
T(TK_FLOOR_DIV, "//")
@@ -679,6 +689,8 @@ std::string pickle_token(int token, const LFortran::YYSTYPE &yystype)
679689
t = t + " " + "\"" + yystype.string.str() + "\"";
680690
} else if (token == yytokentype::TK_TYPE_COMMENT) {
681691
t = t + " " + "\"" + yystype.string.str() + "\"";
692+
} else if (token == yytokentype::TK_TYPE_IGNORE) {
693+
t = t + " " + "\"" + yystype.string.str() + "\"";
682694
}
683695
t += ")";
684696
return t;

‎tests/parser/type_comment1.py

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,18 @@
22

33
def ndarray_func(x):
44
# type: (np.ndarray) -> np.ndarray
5-
return x
5+
return x
6+
7+
@decorator1 # type: ignore
8+
9+
# Comment
10+
11+
@decorator2
12+
13+
# Comment
14+
15+
@decorator3 # type: ignore
16+
17+
def test(x):
18+
# type: (np.ndarray) -> np.ndarray
19+
return x

‎tests/reference/ast_new-type_comment1-710ea6c.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@
22
"basename": "ast_new-type_comment1-710ea6c",
33
"cmd": "lpython --show-ast --new-parser --no-color {infile} -o {outfile}",
44
"infile": "tests/parser/type_comment1.py",
5-
"infile_hash": "cdc0bdd0faec912e7f3db4e71c4a14aa281828e8b68faac91f5a78ee",
5+
"infile_hash": "9c79bc041758b5401f4431a53b6b333999a4e7dfe9dfabac13d83178",
66
"outfile": null,
77
"outfile_hash": null,
88
"stdout": "ast_new-type_comment1-710ea6c.stdout",
9-
"stdout_hash": "98ccc7f80021238fff1a1a1395e64b5d2097f025e60e47821ec76fdb",
9+
"stdout_hash": "c7019449158ebe30677a0808ad2fd8f93aebd2eee6cd90914c44cd98",
1010
"stderr": null,
1111
"stderr_hash": null,
1212
"returncode": 0

‎tests/reference/ast_new-type_comment1-710ea6c.stdout

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
(Module [(Import [(pytest ())]) (FunctionDef ndarray_func ([] [(x () ())] [] [] [] [] []) [(Return (Name x Load))] [] () "(np.ndarray) -> np.ndarray")] [])
1+
(Module [(Import [(pytest ())]) (FunctionDef ndarray_func ([] [(x () ())] [] [] [] [] []) [(Return (Name x Load))] [] () "(np.ndarray) -> np.ndarray") (FunctionDef test ([] [(x () ())] [] [] [] [] []) [(Return (Name x Load))] [(Name decorator1 Load) (Name decorator2 Load) (Name decorator3 Load)] () "(np.ndarray) -> np.ndarray")] [(TypeIgnore 0 "") (TypeIgnore 0 "") (TypeIgnore 0 "")])

0 commit comments

Comments
 (0)