Merge pull request #619 from akshanshbhatt/pr_comments

Thirumalai-Shaktivel · web-flow · commit 9e3d33cbd332 · 2022-06-17T09:10:32.000+05:30
diff --git a/src/lpython/parser/parser.yy b/src/lpython/parser/parser.yy
@@ -290,8 +290,9 @@ statements1
     ;
 
 statement
-    : single_line_statement sep
+    : single_line_statement sep { $$ = $1; }
     | multi_line_statement
+    | multi_line_statement sep { $$ = $1; }
     ;
 
 single_line_statement
diff --git a/src/lpython/parser/tokenizer.h b/src/lpython/parser/tokenizer.h
@@ -23,6 +23,7 @@ class Tokenizer
 
     bool indent = false; // Next line is expected to be indented
     int dedent = 0; // Allowed values: 0, 1, 2, see the code below the meaning of this state variable
+    bool colon_actual_last_token = false; // If the actual last token was a colon
     long int last_indent_length = 0;
     std::vector<uint64_t> indent_length;
 
diff --git a/src/lpython/parser/tokenizer.re b/src/lpython/parser/tokenizer.re
@@ -356,8 +356,11 @@ int Tokenizer::lex(Allocator &al, YYSTYPE &yylval, Location &loc, diag::Diagnost
 
             // Tokens
             newline {
+                if(cur[0] == '#') { RET(TK_NEWLINE); }
                 if(parenlevel) { continue; }
-                if (last_token == yytokentype::TK_COLON) {
+                if (last_token == yytokentype::TK_COLON
+                        || colon_actual_last_token) {
+                    colon_actual_last_token = false;
                     indent = true;
                 } else if (cur[0] != ' ' && cur[0] != '\n'
                         && last_indent_length > cur-tok) {
@@ -379,7 +382,12 @@ int Tokenizer::lex(Allocator &al, YYSTYPE &yylval, Location &loc, diag::Diagnost
             "+" { RET(TK_PLUS) }
             "-" { RET(TK_MINUS) }
             "=" { RET(TK_EQUAL) }
-            ":" { RET(TK_COLON) }
+            ":" { 
+                    if(cur[0] == '\n'){
+                        colon_actual_last_token = true; 
+                    }
+                    RET(TK_COLON);
+                }
             ";" { RET(TK_SEMICOLON) }
             "/" { RET(TK_SLASH) }
             "%" { RET(TK_PERCENT) }
@@ -449,7 +457,8 @@ int Tokenizer::lex(Allocator &al, YYSTYPE &yylval, Location &loc, diag::Diagnost
                 RET(TK_IMAG_NUM)
             }
 
-            comment newline {
+            comment {
+                if(last_token == -1) { RET(TK_COMMENT); }
                 if(parenlevel) { continue; }
                 line_num++; cur_line=cur;
                 token(yylval.string);
diff --git a/tests/reference/ast_new-comment2-f0984d5.json b/tests/reference/ast_new-comment2-f0984d5.json
@@ -0,0 +1,13 @@
+{
+    "basename": "ast_new-comment2-f0984d5",
+    "cmd": "lpython --show-ast --new-parser --no-color {infile} -o {outfile}",
+    "infile": "tests/tokens/comment2.py",
+    "infile_hash": "8bbf936b113be965ec25f99d010959c3d6551e0271fa70c76cff1693",
+    "outfile": null,
+    "outfile_hash": null,
+    "stdout": "ast_new-comment2-f0984d5.stdout",
+    "stdout_hash": "1668ba80cbd288c4926c8e48593f374a1e49963dff66747738f82fd8",
+    "stderr": null,
+    "stderr_hash": null,
+    "returncode": 0
+}
diff --git a/tests/reference/ast_new-comment2-f0984d5.stdout b/tests/reference/ast_new-comment2-f0984d5.stdout
@@ -0,0 +1 @@
+(Module [(FunctionDef main ([] [] [] [] [] [] []) [(If (Name foo Load) [(If (Name bar Load) [(Pass)] [])] [(Pass)]) (Return ())] [] () ()) (FunctionDef test ([] [] [] [] [] [] []) [(Expr (Call (Name print Load) [] []))] [] () ()) (FunctionDef main ([] [] [] [] [] [] []) [(Expr (Call (Name print Load) [] []))] [] () ()) (FunctionDef main ([] [] [] [] [] [] []) [(Expr (Call (Name print Load) [] [])) (Pass)] [] () ()) (FunctionDef main ([] [] [] [] [] [] []) [(If (Name foo Load) [(Pass) (Pass) (If (Name bar Load) [(Pass)] []) (Pass)] []) (Pass)] [] () ())] [])
diff --git a/tests/reference/tokens-comment1-2f8ab90.json b/tests/reference/tokens-comment1-2f8ab90.json
@@ -6,7 +6,7 @@
     "outfile": null,
     "outfile_hash": null,
     "stdout": "tokens-comment1-2f8ab90.stdout",
-    "stdout_hash": "1dd4691dd42cfd3ed3e6c7bfb783aa2b0977cf977bd68729f5264224",
+    "stdout_hash": "9e9a900b941381aadb0d8f29662fa337e918a769daba0c019d7ccdee",
     "stderr": null,
     "stderr_hash": null,
     "returncode": 0
diff --git a/tests/reference/tokens-comment1-2f8ab90.stdout b/tests/reference/tokens-comment1-2f8ab90.stdout
@@ -1,25 +1,34 @@
-(TOKEN "eolcomment") 0:10
+(TOKEN "comment") 0:9
+(NEWLINE) 10:10
 (KEYWORD "def") 11:13
 (TOKEN "identifier" test) 15:18
 (TOKEN "(") 19:19
 (TOKEN ")") 20:20
 (TOKEN ":") 21:21
-(TOKEN "eolcomment") 23:34
-(TOKEN "comment") 39:50
+(TOKEN "eolcomment") 23:33
+(NEWLINE) 34:34
+(TOKEN "comment") 39:49
+(NEWLINE) 50:50
 (TOKEN "indent") 51:54
 (TOKEN "identifier" print) 55:59
 (TOKEN "(") 60:60
 (TOKEN ")") 61:61
 (NEWLINE) 62:62
-(TOKEN "dedent") 62:62
-(TOKEN "comment") 63:74
+(TOKEN "comment") 63:73
+(NEWLINE) 74:74
+(TOKEN "dedent") 74:74
 (TOKEN "identifier" test) 75:78
 (TOKEN "(") 79:79
 (TOKEN ")") 80:80
-(TOKEN "eolcomment") 82:93
-(TOKEN "comment") 94:105
+(TOKEN "eolcomment") 82:92
+(NEWLINE) 93:93
+(TOKEN "comment") 94:104
+(NEWLINE) 105:105
 (NEWLINE) 106:106
-(TOKEN "comment") 107:110
-(TOKEN "comment") 111:123
-(TOKEN "comment") 124:134
+(TOKEN "comment") 107:109
+(NEWLINE) 110:110
+(TOKEN "comment") 111:122
+(NEWLINE) 123:123
+(TOKEN "comment") 124:133
+(NEWLINE) 134:134
 (EOF) 135:135
diff --git a/tests/reference/tokens-comment2-b289dad.json b/tests/reference/tokens-comment2-b289dad.json
@@ -0,0 +1,13 @@
+{
+    "basename": "tokens-comment2-b289dad",
+    "cmd": "lpython --no-color --show-tokens {infile} -o {outfile}",
+    "infile": "tests/tokens/comment2.py",
+    "infile_hash": "8bbf936b113be965ec25f99d010959c3d6551e0271fa70c76cff1693",
+    "outfile": null,
+    "outfile_hash": null,
+    "stdout": "tokens-comment2-b289dad.stdout",
+    "stdout_hash": "69a0edcb42300fd719c704671b6aa3f815691c7af05048cba629b84c",
+    "stderr": null,
+    "stderr_hash": null,
+    "returncode": 0
+}
diff --git a/tests/reference/tokens-comment2-b289dad.stdout b/tests/reference/tokens-comment2-b289dad.stdout
@@ -0,0 +1,125 @@
+(KEYWORD "def") 0:2
+(TOKEN "identifier" main) 4:7
+(TOKEN "(") 8:8
+(TOKEN ")") 9:9
+(TOKEN ":") 10:10
+(NEWLINE) 11:11
+(TOKEN "comment") 12:14
+(NEWLINE) 15:15
+(TOKEN "comment") 20:22
+(NEWLINE) 23:23
+(TOKEN "indent") 24:27
+(KEYWORD "if") 28:29
+(TOKEN "identifier" foo) 31:33
+(TOKEN ":") 34:34
+(TOKEN "eolcomment") 36:38
+(NEWLINE) 39:39
+(TOKEN "comment") 48:50
+(NEWLINE) 51:51
+(TOKEN "comment") 56:58
+(NEWLINE) 59:59
+(TOKEN "comment") 60:61
+(NEWLINE) 62:62
+(TOKEN "indent") 63:70
+(KEYWORD "if") 71:72
+(TOKEN "identifier" bar) 74:76
+(TOKEN ":") 77:77
+(TOKEN "eolcomment") 79:81
+(NEWLINE) 82:82
+(TOKEN "indent") 83:94
+(KEYWORD "pass") 95:98
+(NEWLINE) 99:99
+(TOKEN "dedent") 100:103
+(TOKEN "dedent") 103:103
+(KEYWORD "else") 104:107
+(TOKEN ":") 108:108
+(NEWLINE) 109:109
+(TOKEN "indent") 110:117
+(KEYWORD "pass") 118:121
+(NEWLINE) 122:122
+(TOKEN "dedent") 123:126
+(KEYWORD "return") 127:132
+(NEWLINE) 133:133
+(NEWLINE) 134:134
+(TOKEN "dedent") 134:134
+(KEYWORD "def") 135:137
+(TOKEN "identifier" test) 139:142
+(TOKEN "(") 143:143
+(TOKEN ")") 144:144
+(TOKEN ":") 145:145
+(NEWLINE) 146:146
+(TOKEN "indent") 147:150
+(TOKEN "identifier" print) 151:155
+(TOKEN "(") 156:156
+(TOKEN ")") 157:157
+(NEWLINE) 158:158
+(NEWLINE) 159:159
+(TOKEN "comment") 160:168
+(NEWLINE) 169:169
+(NEWLINE) 170:170
+(TOKEN "dedent") 170:170
+(KEYWORD "def") 171:173
+(TOKEN "identifier" main) 175:178
+(TOKEN "(") 179:179
+(TOKEN ")") 180:180
+(TOKEN ":") 181:181
+(NEWLINE) 182:182
+(TOKEN "indent") 183:186
+(TOKEN "identifier" print) 187:191
+(TOKEN "(") 192:192
+(TOKEN ")") 193:193
+(NEWLINE) 194:194
+(TOKEN "comment") 199:207
+(NEWLINE) 208:208
+(NEWLINE) 209:209
+(TOKEN "dedent") 209:209
+(KEYWORD "def") 210:212
+(TOKEN "identifier" main) 214:217
+(TOKEN "(") 218:218
+(TOKEN ")") 219:219
+(TOKEN ":") 220:220
+(NEWLINE) 221:221
+(TOKEN "indent") 222:225
+(TOKEN "identifier" print) 226:230
+(TOKEN "(") 231:231
+(TOKEN ")") 232:232
+(NEWLINE) 233:233
+(TOKEN "comment") 234:242
+(NEWLINE) 243:243
+(KEYWORD "pass") 248:251
+(NEWLINE) 252:252
+(NEWLINE) 253:253
+(TOKEN "dedent") 253:253
+(KEYWORD "def") 254:256
+(TOKEN "identifier" main) 258:261
+(TOKEN "(") 262:262
+(TOKEN ")") 263:263
+(TOKEN ":") 264:264
+(NEWLINE) 265:265
+(TOKEN "indent") 266:269
+(KEYWORD "if") 270:271
+(TOKEN "identifier" foo) 273:275
+(TOKEN ":") 276:276
+(NEWLINE) 277:277
+(TOKEN "indent") 278:285
+(KEYWORD "pass") 286:289
+(NEWLINE) 290:290
+(TOKEN "comment") 291:299
+(NEWLINE) 300:300
+(KEYWORD "pass") 309:312
+(NEWLINE) 313:313
+(KEYWORD "if") 322:323
+(TOKEN "identifier" bar) 325:327
+(TOKEN ":") 328:328
+(NEWLINE) 329:329
+(TOKEN "indent") 330:341
+(KEYWORD "pass") 342:345
+(NEWLINE) 346:346
+(TOKEN "dedent") 347:354
+(KEYWORD "pass") 355:358
+(NEWLINE) 359:359
+(TOKEN "dedent") 360:363
+(KEYWORD "pass") 364:367
+(NEWLINE) 368:368
+(TOKEN "dedent") 368:368
+(EOF) 369:369
diff --git a/tests/tests.toml b/tests/tests.toml
@@ -293,6 +293,11 @@ tokens = true
 filename = "tokens/comment1.py"
 tokens = true
 
+[[test]]
+filename = "tokens/comment2.py"
+tokens = true
+ast_new = true
+
 [[test]]
 filename = "tokens/symbols1.py"
 tokens = true
diff --git a/tests/tokens/comment2.py b/tests/tokens/comment2.py
@@ -0,0 +1,36 @@
+def main():
+# a
+    # b
+    if foo: # c
+        # d
+    # e
+#f
+        if bar: # g
+            pass
+    else:
+        pass
+    return
+
+def test():
+    print()
+
+# Comment
+
+def main():
+    print()
+    # Comment
+
+def main():
+    print()
+# Comment
+    pass
+
+def main():
+    if foo:
+        pass
+# Comment
+        pass
+        if bar:
+            pass
+        pass
+    pass

Original file line number	Diff line number	Diff line change
`@@ -290,8 +290,9 @@ statements1`
`290`	`290`	`;`
`291`	`291`
`292`	`292`	`statement`
`293`		`- : single_line_statement sep`
	`293`	`+ : single_line_statement sep { $$ = $1; }`
`294`	`294`	`\| multi_line_statement`
	`295`	`+ \| multi_line_statement sep { $$ = $1; }`
`295`	`296`	`;`
`296`	`297`
`297`	`298`	`single_line_statement`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+(Module [(FunctionDef main ([] [] [] [] [] [] []) [(If (Name foo Load) [(If (Name bar Load) [(Pass)] [])] [(Pass)]) (Return ())] [] () ()) (FunctionDef test ([] [] [] [] [] [] []) [(Expr (Call (Name print Load) [] []))] [] () ()) (FunctionDef main ([] [] [] [] [] [] []) [(Expr (Call (Name print Load) [] []))] [] () ()) (FunctionDef main ([] [] [] [] [] [] []) [(Expr (Call (Name print Load) [] [])) (Pass)] [] () ()) (FunctionDef main ([] [] [] [] [] [] []) [(If (Name foo Load) [(Pass) (Pass) (If (Name bar Load) [(Pass)] []) (Pass)] []) (Pass)] [] () ())] [])