XX Create new structure FileLocations to handle imported file errors

Author: Thirumalai-Shaktivel

src/bin/lpython.cpp

@@ -380,9 +380,11 @@ class ASRToLLVMVisitor : public ASR::BaseVisitor<ASRToLLVMVisitor>
     void debug_get_line_column(const uint32_t &loc_first,
             uint32_t &line, uint32_t &column) {
         LocationManager lm;
-        lm.in_filename = infile;
+        LocationManager::FileLocations fl;
+        fl.in_filename = infile;
+        lm.files.push_back(fl);
         lm.init_simple(LFortran::read_file(infile));
-        lm.pos_to_linecol(lm.output_to_input_pos(loc_first, false), line, column);
+        lm.pos_to_linecol(lm.output_to_input_pos(loc_first, false), line, column, fl.in_filename);
     }
 
     template <typename T>

src/libasr/codegen/asr_to_llvm.cpp

@@ -62,7 +62,7 @@ bool Diagnostics::has_error() const {
 }
 
 std::string Diagnostics::render(const std::string &input,
-        const LocationManager &lm, const CompilerOptions &compiler_options) {
+        LocationManager &lm, const CompilerOptions &compiler_options) {
     std::string out;
     for (auto &d : this->diagnostics) {
         if (compiler_options.no_warnings && d.level != Level::Error) {
@@ -109,12 +109,14 @@ std::string get_line(std::string str, int n)
 }
 
 void populate_span(diag::Span &s, const LocationManager &lm,
-        const std::string &input) {
+        const std::string &/*input*/) {
     lm.pos_to_linecol(lm.output_to_input_pos(s.loc.first, false),
-        s.first_line, s.first_column);
+        s.first_line, s.first_column, s.filename);
     lm.pos_to_linecol(lm.output_to_input_pos(s.loc.last, true),
-        s.last_line, s.last_column);
-    s.filename = lm.in_filename;
+        s.last_line, s.last_column, s.filename);
+    std::string input;
+// std::cout << s.filename << "\n";
+    read_file(s.filename, input);
     for (uint32_t i = s.first_line; i <= s.last_line; i++) {
         s.source_code.push_back(get_line(input, i));
     }
@@ -178,7 +180,7 @@ std::string render_diagnostic_human(const Diagnostic &d, bool use_colors) {
     std::string type_color = "";
     switch (d.level) {
         case (Level::Error):
-            primary_color = red_bold;    
+            primary_color = red_bold;
             type_color = primary_color;
             switch (d.stage) {
                 case (Stage::CPreprocessor):
@@ -205,17 +207,17 @@ std::string render_diagnostic_human(const Diagnostic &d, bool use_colors) {
             }
             break;
         case (Level::Warning):
-            primary_color = yellow_bold;    
+            primary_color = yellow_bold;
             type_color = primary_color;
             message_type = "warning";
             break;
         case (Level::Note):
-            primary_color = bold;    
+            primary_color = bold;
             type_color = primary_color;
             message_type = "note";
             break;
         case (Level::Help):
-            primary_color = bold;    
+            primary_color = bold;
             type_color = primary_color;
             message_type = "help";
             break;
@@ -286,6 +288,18 @@ std::string render_diagnostic_human(const Diagnostic &d, bool use_colors) {
                 }
                 // and start a new one:
                 s0 = s2;
+                if (s0.filename != s.filename) {
+                    out << std::endl;
+                    // TODO: print the primary line+column here, not the first label:
+                    out << std::string(line_num_width, ' ') << blue_bold;
+                    out << "-->" << reset << " " << s0.filename << ":";
+                    out << s0.first_line << ":" << s0.first_column;
+                    if (s0.first_line != s0.last_line) {
+                        out << " - " << s0.last_line << ":" << s0.last_column;
+                    }
+                    out << std::endl;
+                }
+
                 if (s0.first_line == s0.last_line) {
                     out << std::string(line_num_width+1, ' ') << blue_bold << "|"
                         << reset << std::endl;

src/libasr/diagnostics.cpp

@@ -114,7 +114,7 @@ struct Diagnostics {
     std::vector<Diagnostic> diagnostics;
 
     std::string render(const std::string &input,
-            const LocationManager &lm, const CompilerOptions &compiler_options);
+            LocationManager &lm, const CompilerOptions &compiler_options);
 
     // Returns true iff diagnostics contains at least one error message
     bool has_error() const;

src/libasr/diagnostics.h

@@ -3,6 +3,7 @@
 
 #include <cstdint>
 #include <vector>
+// #include <iostream>
 
 namespace LFortran
 {
@@ -94,50 +95,67 @@ struct LocationManager {
     //
     //
     //
-    std::vector<uint32_t> out_start; // consecutive intervals in the output code
-    std::vector<uint32_t> in_start; // start + size in the original code
-    std::vector<uint32_t> in_newlines; // position of all \n in the original code
+    struct FileLocations {
+        std::vector<uint32_t> out_start; // consecutive intervals in the output code
+        std::vector<uint32_t> in_start; // start + size in the original code
+        std::vector<uint32_t> in_newlines; // position of all \n in the original code
 
-    // For preprocessor (if preprocessor==true).
-    // TODO: design a common structure, that also works with #include, that
-    // has these mappings for each file
-    bool preprocessor = false;
-    std::string in_filename;
-    uint32_t current_line=0;
-    std::vector<uint32_t> out_start0; // consecutive intervals in the output code
-    std::vector<uint32_t> in_start0; // start + size in the original code
-    std::vector<uint32_t> in_size0; // Size of the `in` interval
-    std::vector<uint32_t> interval_type0; // 0 .... 1:1; 1 ... many to many;
-    std::vector<uint32_t> in_newlines0; // position of all \n in the original code
-//    std::vector<uint32_t> filename_id; // file name for each interval, ID
-//    std::vector<std::string> filenames; // filenames lookup for an ID
+        // For preprocessor (if preprocessor==true).
+        // TODO: design a common structure, that also works with #include, that
+        // has these mappings for each file
+        bool preprocessor = false;
+        std::string in_filename;
+        uint32_t current_line=0;
+        std::vector<uint32_t> out_start0; // consecutive intervals in the output code
+        std::vector<uint32_t> in_start0; // start + size in the original code
+        std::vector<uint32_t> in_size0; // Size of the `in` interval
+        std::vector<uint32_t> interval_type0; // 0 .... 1:1; 1 ... many to many;
+        std::vector<uint32_t> in_newlines0; // position of all \n in the original code
+    };
+    std::vector<FileLocations> files;
+    std::vector<uint32_t> file_ends; // position of all ends of files
+    // For a given Location we use the `file_ends` and bisection to determine
+    // the file (index) which the location is from. Then we use this index into
+    // the `files` vector and use `in_newlines` and other information to
+    // determine the line and column inside this file, and the `in_filename`
+    // field to determine the filename. This happens when the diagnostic is
+    // printed. The `pos_to_linecol` function below should be modified to
+    // return line, column and the filename:
+    //
+    // void pos_to_linecol(uint32_t position, uint32_t &line, uint32_t &col,
+    //     std::string &filename) const;
 
     // Converts a position in the output code to a position in the original code
     // Every character in the output code has a corresponding location in the
     // original code, so this function always succeeds
     uint32_t output_to_input_pos(uint32_t out_pos, bool show_last) const {
-        if (out_start.size() == 0) return 0;
-        uint32_t interval = bisection(out_start, out_pos)-1;
-        uint32_t rel_pos = out_pos - out_start[interval];
-        uint32_t in_pos = in_start[interval] + rel_pos;
-        if (preprocessor) {
+// std::cout << "out: " << out_pos << "\n";
+        uint32_t index = bisection(file_ends, out_pos);
+        if (index == file_ends.size()) index -= 1;
+// std::cout << index << ":\n";
+        if (files[index].out_start.size() == 0) return 0;
+        uint32_t interval = bisection(files[index].out_start, out_pos)-1;
+        uint32_t rel_pos = out_pos - files[index].out_start[interval];
+        uint32_t in_pos = files[index].in_start[interval] + rel_pos;
+// std::cout <<  "in: "<< in_pos << "\n";
+        if (files[index].preprocessor) {
             // If preprocessor was used, do one more remapping
-            uint32_t interval0 = bisection(out_start0, in_pos)-1;
-            if (interval_type0[interval0] == 0) {
+            uint32_t interval0 = bisection(files[index].out_start0, in_pos)-1;
+            if (files[index].interval_type0[interval0] == 0) {
                 // 1:1 interval
-                uint32_t rel_pos0 = in_pos - out_start0[interval0];
-                uint32_t in_pos0 = in_start0[interval0] + rel_pos0;
+                uint32_t rel_pos0 = in_pos - files[index].out_start0[interval0];
+                uint32_t in_pos0 = files[index].in_start0[interval0] + rel_pos0;
                 return in_pos0;
             } else {
                 // many to many interval
                 uint32_t in_pos0;
-                if (in_pos == out_start0[interval0+1]-1 || show_last) {
+                if (in_pos == files[index].out_start0[interval0+1]-1 || show_last) {
                     // The end of the interval in "out" code
                     // Return the end of the interval in "in" code
-                    in_pos0 = in_start0[interval0]+in_size0[interval0]-1;
+                    in_pos0 = files[index].in_start0[interval0]+files[index].in_size0[interval0]-1;
                 } else {
                     // Otherwise return the beginning of the interval in "in"
-                    in_pos0 = in_start0[interval0];
+                    in_pos0 = files[index].in_start0[interval0];
                 }
                 return in_pos0;
             }
@@ -150,12 +168,26 @@ struct LocationManager {
     // `position` starts from 0
     // `line` and `col` starts from 1
     // `in_newlines` starts from 0
-    void pos_to_linecol(uint32_t position, uint32_t &line, uint32_t &col) const {
+    void pos_to_linecol(uint32_t position, uint32_t &line, uint32_t &col,
+            std::string &filename) const {
+        uint32_t index = bisection(file_ends, position);
+        if (index == file_ends.size()) index -= 1;
+// std::cout << position << ": Pos1\n";
+        filename = files[index].in_filename;
+// std::cout << "----------------------------\n";
+// for (auto &x: file_ends) std::cout << x << "\n";
+// std::cout << "----------------------------\n";
+// std::cout << file_ends[index]  << ": "<< file_ends[index - 1] << ": " << position - file_ends[index - 1] << "\n";
+        if (index > 0) position -= file_ends[index - 1];
+// std::cout << index << "_\n";
+// std::cout << position << ": Pos2\n";
+// std::cout << filename << ": Name\n";
+// exit(0);
         const std::vector<uint32_t> *newlines;
-        if (preprocessor) {
-            newlines = &in_newlines0;
+        if (files[index].preprocessor) {
+            newlines = &files[index].in_newlines0;
         } else {
-            newlines = &in_newlines;
+            newlines = &files[index].in_newlines;
         }
         int32_t interval = bisection(*newlines, position);
         if (interval >= 1 && position == (*newlines)[interval-1]) {
@@ -179,9 +211,9 @@ struct LocationManager {
 
     void init_simple(const std::string &input) {
         uint32_t n = input.size();
-        out_start = {0, n};
-        in_start = {0, n};
-        get_newlines(input, in_newlines);
+        files.back().out_start = {0, n};
+        files.back().in_start = {0, n};
+        get_newlines(input, files.back().in_newlines);
     }
 
 };

src/libasr/location.h

@@ -15,11 +15,11 @@
 namespace LFortran {
 
 Result<LPython::AST::Module_t*> parse(Allocator &al, const std::string &s,
-        diag::Diagnostics &diagnostics)
+        uint32_t prev_loc, diag::Diagnostics &diagnostics)
 {
     Parser p(al, diagnostics);
     try {
-        p.parse(s);
+        p.parse(s, prev_loc);
     } catch (const parser_local::TokenizerError &e) {
         Error error;
         diagnostics.diagnostics.push_back(e.d);
@@ -42,15 +42,15 @@ Result<LPython::AST::Module_t*> parse(Allocator &al, const std::string &s,
         p.result.p, p.result.size(), p.type_ignore.p, p.type_ignore.size());
 }
 
-void Parser::parse(const std::string &input)
+void Parser::parse(const std::string &input, uint32_t prev_loc)
 {
     inp = input;
     if (inp.size() > 0) {
         if (inp[inp.size()-1] != '\n') inp.append("\n");
     } else {
         inp.append("\n");
     }
-    m_tokenizer.set_string(inp);
+    m_tokenizer.set_string(inp, prev_loc);
     if (yyparse(*this) == 0) {
         return;
     }
@@ -116,13 +116,14 @@ Result<LPython::AST::ast_t*> parse_python_file(Allocator &al,
         const std::string &/*runtime_library_dir*/,
         const std::string &infile,
         diag::Diagnostics &diagnostics,
+        uint32_t prev_loc,
         bool new_parser) {
     LPython::AST::ast_t* ast;
     // We will be using the new parser from now on
     new_parser = true;
     LFORTRAN_ASSERT(new_parser)
     std::string input = read_file(infile);
-    Result<LPython::AST::Module_t*> res = parse(al, input, diagnostics);
+    Result<LPython::AST::Module_t*> res = parse(al, input, prev_loc, diagnostics);
     if (res.ok) {
         ast = (LPython::AST::ast_t*)res.result;
     } else {

src/lpython/parser/parser.cpp

@@ -26,21 +26,21 @@ class Parser
         type_ignore.reserve(al, 4);
     }
 
-    void parse(const std::string &input);
+    void parse(const std::string &input, uint32_t prev_loc);
     void handle_yyerror(const Location &loc, const std::string &msg);
 };
 
 
 // Parses Python code to AST
 Result<LPython::AST::Module_t*> parse(Allocator &al,
-    const std::string &s,
+    const std::string &s, uint32_t prev_loc,
     diag::Diagnostics &diagnostics);
 
 Result<LPython::AST::ast_t*> parse_python_file(Allocator &al,
         const std::string &runtime_library_dir,
         const std::string &infile,
         diag::Diagnostics &diagnostics,
-        bool new_parser);
+        uint32_t prev_loc, bool new_parser);
 
 } // namespace LFortran
 

src/lpython/parser/parser.h

@@ -18,6 +18,7 @@ class Tokenizer
     unsigned char *cur_line;
     unsigned int line_num;
     unsigned char *string_start;
+    uint32_t prev_loc;
 
     int last_token=-1;
 
@@ -34,7 +35,7 @@ class Tokenizer
 public:
     // Set the string to tokenize. The caller must ensure `str` will stay valid
     // as long as `lex` is being called.
-    void set_string(const std::string &str);
+    void set_string(const std::string &str, uint32_t prev_loc_);
 
     // Get next token. Token ID is returned as function result, the semantic
     // value is put into `yylval`.
@@ -71,8 +72,8 @@ class Tokenizer
     // Return the current token's location
     void token_loc(Location &loc) const
     {
-        loc.first = tok-string_start;
-        loc.last = cur-string_start-1;
+        loc.first = prev_loc + (tok-string_start);
+        loc.last = prev_loc + (cur-string_start-1);
     }
 
     void record_paren(Location &loc, char c);

src/lpython/parser/tokenizer.h

@@ -122,14 +122,15 @@ void lex_imag(Allocator &al, const unsigned char *s,
     }
 }
 
-void Tokenizer::set_string(const std::string &str)
+void Tokenizer::set_string(const std::string &str, uint32_t prev_loc_)
 {
     // The input string must be NULL terminated, otherwise the tokenizer will
     // not detect the end of string. After C++11, the std::string is guaranteed
     // to end with \0, but we check this here just in case.
     LFORTRAN_ASSERT(str[str.size()] == '\0');
     cur = (unsigned char *)(&str[0]);
     string_start = cur;
+    prev_loc = prev_loc_;
     cur_line = cur;
     line_num = 1;
 }
@@ -777,7 +778,7 @@ Result<std::vector<int>> tokens(Allocator &al, const std::string &input,
         std::vector<Location> *locations)
 {
     Tokenizer t;
-    t.set_string(input);
+    t.set_string(input, 0);
     std::vector<int> tst;
     int token = yytokentype::END_OF_FILE + 1; // Something different from EOF
     while (token != yytokentype::END_OF_FILE) {