Merge pull request #664 from certik/str_fix

Unescape strings in the new parser

Author: certik

src/lpython/parser/semantics.h

@@ -423,11 +423,24 @@ char* concat_string(Allocator &al, ast_t *a, char *b) {
     return LFortran::s2c(al, std::string(s) + std::string(b));
 }
 
+char* unescape(Allocator &al, LFortran::Str &s) {
+    std::string x;
+    for (size_t idx=0; idx < s.size(); idx++) {
+        if (s.p[idx] == '\\' && s.p[idx+1] == 'n') {
+            x += "\n";
+            idx++;
+        } else {
+            x += s.p[idx];
+        }
+    }
+    return LFortran::s2c(al, x);
+}
+
 #define SYMBOL(x, l) make_Name_t(p.m_a, l, \
         x.c_str(p.m_a), expr_contextType::Load)
 // `x.int_n` is of type BigInt but we store the int64_t directly in AST
 #define INTEGER(x, l) make_ConstantInt_t(p.m_a, l, x, nullptr)
-#define STRING1(x, l) make_ConstantStr_t(p.m_a, l, x.c_str(p.m_a), nullptr)
+#define STRING1(x, l) make_ConstantStr_t(p.m_a, l, unescape(p.m_a, x), nullptr)
 #define STRING2(x, y, l) make_ConstantStr_t(p.m_a, l, concat_string(p.m_a, x, y.c_str(p.m_a)), nullptr)
 #define STRING3(id, x, l) PREFIX_STRING(p.m_a, l, name2char(id), x.c_str(p.m_a))
 #define FLOAT(x, l) make_ConstantFloat_t(p.m_a, l, x, nullptr)

tests/parser/string1.py

@@ -2,6 +2,8 @@
 
 print(f"{__file__} executed in {elapsed} seconds.")
 
+"\nsometext\nanotherline\n"
+
 # TODO: Support format specifiers
 
 # print(f"{__file__} executed in {elapsed:0.2f} seconds.")

tests/reference/ast_new-string1-96b90b3.json

@@ -2,11 +2,11 @@
     "basename": "ast_new-string1-96b90b3",
     "cmd": "lpython --show-ast --new-parser --no-color {infile} -o {outfile}",
     "infile": "tests/parser/string1.py",
-    "infile_hash": "e51ab7a60ad39354a13d987f0ed3bdb8f51b8e6d2a677a9d7b720749",
+    "infile_hash": "83fc48a45a9a7373d7a8bc4340620acefc0eb1a0258f5df5c5f5bf2a",
     "outfile": null,
     "outfile_hash": null,
     "stdout": "ast_new-string1-96b90b3.stdout",
-    "stdout_hash": "00a9b4899bd6ba7938b503da224840f51e2e7ba2e41121292809bdf2",
+    "stdout_hash": "43d6ec3f9816a431342762957a9ee4e0bc5c6eb362d38e958e910842",
     "stderr": null,
     "stderr_hash": null,
     "returncode": 0

tests/reference/ast_new-string1-96b90b3.stdout

@@ -1 +1,4 @@
-(Module [(Expr (JoinedStr [(ConstantStr "Hello, " ()) (FormattedValue (Name first_name Load) -1 ()) (ConstantStr " " ()) (FormattedValue (Name last_name Load) -1 ()) (ConstantStr ". You are " ()) (FormattedValue (Name age Load) -1 ()) (ConstantStr " years old." ())])) (Expr (Call (Name print Load) [(JoinedStr [(FormattedValue (Name __file__ Load) -1 ()) (ConstantStr " executed in " ()) (FormattedValue (Name elapsed Load) -1 ()) (ConstantStr " seconds." ())])] []))] [])
+(Module [(Expr (JoinedStr [(ConstantStr "Hello, " ()) (FormattedValue (Name first_name Load) -1 ()) (ConstantStr " " ()) (FormattedValue (Name last_name Load) -1 ()) (ConstantStr ". You are " ()) (FormattedValue (Name age Load) -1 ()) (ConstantStr " years old." ())])) (Expr (Call (Name print Load) [(JoinedStr [(FormattedValue (Name __file__ Load) -1 ()) (ConstantStr " executed in " ()) (FormattedValue (Name elapsed Load) -1 ()) (ConstantStr " seconds." ())])] [])) (Expr (ConstantStr "
+sometext
+anotherline
+" ()))] [])