Continue the location information for the imported files.

Author: Thirumalai-Shaktivel

src/lpython/parser/parser.cpp

@@ -15,11 +15,11 @@
 namespace LFortran {
 
 Result<LPython::AST::Module_t*> parse(Allocator &al, const std::string &s,
-        diag::Diagnostics &diagnostics)
+        uint32_t prev_loc, diag::Diagnostics &diagnostics)
 {
     Parser p(al, diagnostics);
     try {
-        p.parse(s);
+        p.parse(s, prev_loc);
     } catch (const parser_local::TokenizerError &e) {
         Error error;
         diagnostics.diagnostics.push_back(e.d);
@@ -42,15 +42,15 @@ Result<LPython::AST::Module_t*> parse(Allocator &al, const std::string &s,
         p.result.p, p.result.size(), p.type_ignore.p, p.type_ignore.size());
 }
 
-void Parser::parse(const std::string &input)
+void Parser::parse(const std::string &input, uint32_t prev_loc)
 {
     inp = input;
     if (inp.size() > 0) {
         if (inp[inp.size()-1] != '\n') inp.append("\n");
     } else {
         inp.append("\n");
     }
-    m_tokenizer.set_string(inp);
+    m_tokenizer.set_string(inp, prev_loc);
     if (yyparse(*this) == 0) {
         return;
     }
@@ -116,13 +116,14 @@ Result<LPython::AST::ast_t*> parse_python_file(Allocator &al,
         const std::string &/*runtime_library_dir*/,
         const std::string &infile,
         diag::Diagnostics &diagnostics,
+        uint32_t prev_loc,
         bool new_parser) {
     LPython::AST::ast_t* ast;
     // We will be using the new parser from now on
     new_parser = true;
     LFORTRAN_ASSERT(new_parser)
     std::string input = read_file(infile);
-    Result<LPython::AST::Module_t*> res = parse(al, input, diagnostics);
+    Result<LPython::AST::Module_t*> res = parse(al, input, prev_loc, diagnostics);
     if (res.ok) {
         ast = (LPython::AST::ast_t*)res.result;
     } else {

src/lpython/parser/parser.h

@@ -26,21 +26,21 @@ class Parser
         type_ignore.reserve(al, 4);
     }
 
-    void parse(const std::string &input);
+    void parse(const std::string &input, uint32_t prev_loc);
     void handle_yyerror(const Location &loc, const std::string &msg);
 };
 
 
 // Parses Python code to AST
 Result<LPython::AST::Module_t*> parse(Allocator &al,
-    const std::string &s,
+    const std::string &s, uint32_t prev_loc,
     diag::Diagnostics &diagnostics);
 
 Result<LPython::AST::ast_t*> parse_python_file(Allocator &al,
         const std::string &runtime_library_dir,
         const std::string &infile,
         diag::Diagnostics &diagnostics,
-        bool new_parser);
+        uint32_t prev_loc, bool new_parser);
 
 } // namespace LFortran
 

src/lpython/parser/tokenizer.h

@@ -18,6 +18,7 @@ class Tokenizer
     unsigned char *cur_line;
     unsigned int line_num;
     unsigned char *string_start;
+    uint32_t prev_loc; // The previous file ended at this location.
 
     int last_token=-1;
 
@@ -34,7 +35,7 @@ class Tokenizer
 public:
     // Set the string to tokenize. The caller must ensure `str` will stay valid
     // as long as `lex` is being called.
-    void set_string(const std::string &str);
+    void set_string(const std::string &str, uint32_t prev_loc_);
 
     // Get next token. Token ID is returned as function result, the semantic
     // value is put into `yylval`.
@@ -71,8 +72,8 @@ class Tokenizer
     // Return the current token's location
     void token_loc(Location &loc) const
     {
-        loc.first = tok-string_start;
-        loc.last = cur-string_start-1;
+        loc.first = prev_loc + (tok-string_start);
+        loc.last = prev_loc + (cur-string_start-1);
     }
 
     void record_paren(Location &loc, char c);

src/lpython/parser/tokenizer.re

@@ -122,14 +122,15 @@ void lex_imag(Allocator &al, const unsigned char *s,
     }
 }
 
-void Tokenizer::set_string(const std::string &str)
+void Tokenizer::set_string(const std::string &str, uint32_t prev_loc_)
 {
     // The input string must be NULL terminated, otherwise the tokenizer will
     // not detect the end of string. After C++11, the std::string is guaranteed
     // to end with \0, but we check this here just in case.
     LFORTRAN_ASSERT(str[str.size()] == '\0');
     cur = (unsigned char *)(&str[0]);
     string_start = cur;
+    prev_loc = prev_loc_;
     cur_line = cur;
     line_num = 1;
 }
@@ -777,7 +778,7 @@ Result<std::vector<int>> tokens(Allocator &al, const std::string &input,
         std::vector<Location> *locations)
 {
     Tokenizer t;
-    t.set_string(input);
+    t.set_string(input, 0);
     std::vector<int> tst;
     int token = yytokentype::END_OF_FILE + 1; // Something different from EOF
     while (token != yytokentype::END_OF_FILE) {