some code cleanup and improvements

Author: vtjnash

base/Base.jl

@@ -541,7 +541,7 @@ in_sysimage(pkgid::PkgId) = pkgid in _sysimage_modules
 for match = _methods(+, (Int, Int), -1, get_world_counter())
     m = match.method
     delete!(push!(Set{Method}(), m), m)
-    Core.Compiler.copy(Core.Compiler.retrieve_code_info(Core.Compiler.specialize_method(match), typemax(UInt)))
+    copy(Core.Compiler.retrieve_code_info(Core.Compiler.specialize_method(match), typemax(UInt)))
 
     empty!(Set())
     push!(push!(Set{Union{GlobalRef,Symbol}}(), :two), GlobalRef(Base, :two))

base/compiler/ssair/ir.jl

@@ -179,35 +179,44 @@ mutable struct DebugInfoStream
     def::Union{MethodInstance,Symbol,Nothing}
     linetable::Union{Nothing,Core.DebugInfo}
     edges::Vector{Any} # Vector{Core.DebugInfo}
+    firstline::Int32 # the starting line for this block (specified by having an index of 0)
     codelocs::Vector{Int32} # for each statement:
         # index into linetable (if defined), else a line number (in the file represented by def)
         # then index into edges
         # then index into edges[linetable]
-    function DebugInfoStream(def::Union{MethodInstance,Nothing}, linetable::Union{Nothing,DebugInfo}, edges::Vector{Any}, codelocs::Vector{Int32})
-        return new(def, linetable, edges, codelocs)
+    function DebugInfoStream(codelocs::Vector{Int32})
+        return new(nothing, nothing, [], 0, codelocs)
     end
     #DebugInfoStream(def::Union{MethodInstance,Nothing}, di::DebugInfo, nstmts::Int) =
     #    if debuginfo_file1(di.def) === debuginfo_file1(di.def)
-    #        new(def, di.linetable, Core.svec(di.edges...),
-    #            ccall(:jl_uncompress_codelocs, Any, (Any, Int), di.codelocs, nstmts))
+    #        new(def, di.linetable, Core.svec(di.edges...), getdebugidx(di, 0),
+    #            ccall(:jl_uncompress_codelocs, Any, (Any, Int), di.codelocs, nstmts)::Vector{Int32})
     #    else
     function DebugInfoStream(def::Union{MethodInstance,Nothing}, di::DebugInfo, nstmts::Int)
         codelocs = zeros(Int32, nstmts * 3)
         for i = 1:nstmts
             codelocs[3i - 2] = i
         end
-        return new(def, di, Vector{Any}(), codelocs)
+        return new(def, di, Vector{Any}(), 0, codelocs)
     end
-    global copy(di::DebugInfoStream) = new(di.def, di.linetable, di.edges, di.codelocs)
+    global copy(di::DebugInfoStream) = new(di.def, di.linetable, di.edges, di.firstline, di.codelocs)
 end
 
 Core.DebugInfo(di::DebugInfoStream, nstmts::Int) =
     Core.DebugInfo(something(di.def), di.linetable, Core.svec(di.edges...),
-        ccall(:jl_compress_codelocs, Any, (Any, Int), di.codelocs, nstmts)::String)
+        ccall(:jl_compress_codelocs, Any, (Int32, Any, Int), di.firstline, di.codelocs, nstmts)::String)
 
-getdebugidx(debuginfo::Core.DebugInfo, pc::Int) = ccall(:jl_uncompress1_codeloc, NTuple{3,Int32}, (Any, Int), debuginfo.codelocs, (pc - 1))
-getdebugidx(debuginfo::DebugInfoStream, pc::Int) =
-    3 <= 3pc <= length(debuginfo.codelocs) ? (debuginfo.codelocs[3pc - 2], debuginfo.codelocs[3pc - 1], debuginfo.codelocs[3pc - 0]) : (Int32(-1), Int32(0), Int32(0))
+getdebugidx(debuginfo::Core.DebugInfo, pc::Int) = ccall(:jl_uncompress1_codeloc, NTuple{3,Int32}, (Any, Int), debuginfo.codelocs, pc)
+
+function getdebugidx(debuginfo::DebugInfoStream, pc::Int)
+    if 3 <= 3pc <= length(debuginfo.codelocs)
+        return (debuginfo.codelocs[3pc - 2], debuginfo.codelocs[3pc - 1], debuginfo.codelocs[3pc - 0])
+    elseif pc == 0
+        return (Int32(debuginfo.firstline), Int32(0), Int32(0))
+    else
+        return (Int32(-1), Int32(0), Int32(0))
+    end
+end
 
 
 # SSA values that need renaming
@@ -446,7 +455,7 @@ from the frontend or one of the caches.
 """
 function IRCode()
     stmts = InstructionStream(1)
-    debuginfo = DebugInfoStream(nothing, nothing, Any[], stmts.line)
+    debuginfo = DebugInfoStream(stmts.line)
     stmts.line[1] = 1
     ir = IRCode(stmts, CFG([BasicBlock(1:1, Int[], Int[])], Int[1]), debuginfo, Any[], Expr[], VarState[])
     ir[SSAValue(1)][:stmt] = ReturnNode(nothing)

base/compiler/ssair/show.jl

@@ -341,42 +341,16 @@ function debuginfo_file1(debuginfo::Union{Core.DebugInfo,DebugInfoStream})
     return :var"<unknown>"
 end
 
-# utility function to extract the file name from a DebugInfo object
-function debuginfo_file(debuginfo::Union{Core.DebugInfo,DebugInfoStream})
-    linetable = debuginfo.linetable
-    while linetable != nothing
-        debuginfo = linetable
-        linetable = debuginfo.linetable
-    end
-    return debuginfo_file1(debuginfo)
-end
-
-# utility function to extract the first line number of a block of code (corresponding to debuginfo_file)
+# utility function to extract the first line number and file of a block of code
 function debuginfo_firstline(debuginfo::Union{Core.DebugInfo,DebugInfoStream})
     linetable = debuginfo.linetable
     while linetable != nothing
         debuginfo = linetable
         linetable = debuginfo.linetable
     end
-    codeloc = getdebugidx(debuginfo, 1)
-    return codeloc[1]
-end
-
-## utility function to extract the line number at a particular program counter (ignoring inlining)
-## corresponds to debuginfo_file. return <= 0 if there is no line number change caused by this statement
-function debuginfo_line(debuginfo::Union{Core.DebugInfo,DebugInfoStream}, pc::Int)
-    while pc > 0
-        codeloc = getdebugidx(debuginfo, pc)
-        pc::Int = codeloc[1]
-        debuginfo = debuginfo.linetable
-        if debuginfo === nothing || pc <= 0
-            return pc
-        end
-    end
-    return -1
+    codeloc = getdebugidx(debuginfo, 0)
+    return debuginfo_file1(debuginfo), codeloc[1]
 end
-## alternative definition of debuginfo_firstline which gets the line for the first existing pc, rather that the first line
-#debuginfo_firstline(debuginfo::Union{Core.DebugInfo,DebugInfoStream}) = deubginfo_line(debuginfo, 1)
 
 struct LineInfoNode
     method # ::Union{Method,MethodInstance,Symbol}

base/compiler/ssair/verify.jl

@@ -384,12 +384,10 @@ end
 
 function verify_linetable(di::DebugInfoStream, nstmts::Int, print::Bool=true)
     @assert 3nstmts == length(di.codelocs)
-    # jwn
-    #for i in 1:nstmts
-    #    @assert di.codelocs[3i-2] <= length(di.linetable)
-    #    di.codelocs[3i-2] == 0 && continue
-    #    @assert di.codelocs[3i-1] <= length(di.edges)
-    #    di.codelocs[3i-1] == 0 && continue
-    #    @assert di.codelocs[3i-0] <= length(di.edges[di.codelocs[3i-1]].linetable)
-    #end
+    for i in 1:nstmts
+        edge = di.codelocs[3i-1]
+        if !(edge == 0 || get(di.edges, edge, nothing) isa DebugInfo)
+            @verify_error "Malformed debuginfo index into edges"
+        end
+    end
 end

base/show.jl

@@ -1339,7 +1339,8 @@ function show_mi(io::IO, l::Core.MethodInstance, from_stackframe::Bool=false)
         # to print a little more identifying information.
         if !from_stackframe
             di = mi.uninferred.debuginfo
-            file, line = string(IRShow.debuginfo_file(di)), IRShow.debuginfo_firstline(di)
+            file, line = IRShow.debuginfo_firstline(di)
+            file = string(file)
             line = isempty(file) || line < 0 ? "<unknown>" : "$file:$line"
             print(io, " from ", def, " starting at ", line)
         end
@@ -1364,7 +1365,8 @@ function show(io::IO, mi_info::Core.Compiler.Timings.InferenceFrameInfo)
         end
     else
         di = mi.uninferred.debuginfo
-        file, line = string(IRShow.debuginfo_file(di)), IRShow.debuginfo_firstline(di)
+        file, line = IRShow.debuginfo_firstline(di)
+        file = string(file)
         line = isempty(file) || line < 0 ? "<unknown>" : "$file:$line"
         print(io, "Toplevel InferenceFrameInfo thunk from ", def, " starting at ", line)
     end

base/stacktraces.jl

@@ -143,7 +143,7 @@ function lookup(ip::Union{Base.InterpreterIP,Core.Compiler.InterpreterIP})
         file = empty_sym
         line = Int32(0)
     end
-    pc::Int = max(ip.stmt, 0) # n.b. ip.stmt is 0-indexed
+    pc::Int = max(ip.stmt + 1, 0) # n.b. ip.stmt is 0-indexed
     debuginfo = codeinfo.debuginfo
     codeloc = @ccall jl_uncompress1_codeloc(debuginfo.codelocs::Any, pc::Int)::NTuple{3,Int32}
     if (codeloc[1] == 0 && codeloc[2] == 0) || codeloc[1] < 0
@@ -157,8 +157,8 @@ function lookup(ip::Union{Base.InterpreterIP,Core.Compiler.InterpreterIP})
             debuginfo.def isa Symbol || (def = debuginfo.def)
             codeloc = @ccall jl_uncompress1_codeloc(debuginfo.codelocs::Any, pc::Int)::NTuple{3,Int32}
             line = codeloc[1]
-            line < 0 && (line = 0) # broken debug info?
-            if debuginfo.linetable === nothing || line == 0
+            if debuginfo.linetable === nothing || pc <= 0 || line < 0
+                line < 0 && (line = 0) # broken debug info?
                 push!(scopes, StackFrame(normalize_method_name(def), debuginfo_file1(debuginfo), line, inlined ? nothing : code, false, inlined, 0))
             else
                 append_scopes!(scopes, line - 1, debuginfo.linetable::Core.DebugInfo, def, inlined)
@@ -169,8 +169,6 @@ function lookup(ip::Union{Base.InterpreterIP,Core.Compiler.InterpreterIP})
             inl_to == 0 && break
             debuginfo = debuginfo.edges[inl_to]
             pc::Int = codeloc[3]
-            pc == 0 && break # TODO: use toplevel line?
-            pc -= 1
         end
     end
     append_scopes!(scopes, pc, debuginfo, def, false)

doc/src/devdocs/ast.md

@@ -779,6 +779,7 @@ mutable struct DebugInfoStream
     def::Union{Method,MethodInstance,Symbol}
     linetable::Union{Nothing,DebugInfo}
     edges::Vector{DebugInfo}
+    firstline::Int32 # the starting line for this block (specified by an index of 0)
     codelocs::Vector{Int32} # for each statement:
         # index into linetable (if defined), else a line number (in the file represented by def)
         # then index into edges
@@ -793,15 +794,30 @@ end
 
     Another debuginfo that this was derived from. If `def` is not a Symbol, then it replaces
     the current function for metadata. The codelocs line number also becomes an index into
-    this codelocs instead of being a line number itself (including its edges).
+    this codelocs instead of being a line number itself, as described below.
 
-  * `edges` : Vector of unique DebugInfo for every inlined function
+  * `edges` : Vector of the unique DebugInfo for every inlined function
+
+  * `firstline` (when uncompressed to DebugInfoStream)
+
+    The line number associated with the `begin` statement (or other keyword such as
+    `function` or `quote`) that delinated where this code definition "starts".
 
   * `codelocs` (when uncompressed to DebugInfoStream)
 
-    A vector of indicies, with 3 values for each statement in the IR, that describe the stacktrace from that point:
-     - the integer index into the `linetable`, giving the original location associated with each statement.
+    A vector of indices, with 3 values for each statement in the IR plus one for the
+    starting point of the block, that describe the stacktrace from that point:
+     1. the integer index into the `linetable.codelocs` field, giving the original location
+        associated with each statement (including its edges), or zero indicating to use
+        `linetable.firstline` as the line number.
        or
        the line number itself if the `linetable` field is `nothing`
-     - the integer index into edges, giving the DebugInfo inlined there.
-     - the integer index into edges[codelocs], giving the recursion point.
+     2. the integer index into edges, giving the DebugInfo inlined there (or zero if there
+        are no edges).
+     3. (if entry 2 is non-zero) the integer index into edges[].codelocs, giving the
+        recursion point, or zero indicating to use `edges[].firstline` as the line number.
+
+   Special codes include:
+     - (zero, zero, *) : no change to the line number or edges
+     - (zero, *, *) : no line number, just edges (usually because of macro-expansion into
+       top-level code)

src/codegen.cpp

@@ -7588,19 +7588,13 @@ static jl_llvm_functions_t
         ctx.file = jl_symbol_name(lam->def.method->file);
     }
     else if ((jl_value_t*)src->debuginfo != jl_nothing) {
-        // look for the file and line info of the first actual statement
-        // as sometimes lowering emits inlining info that we want to ignore
+        // look for the file and line info of the original start of this block, as reported by lowering
         jl_debuginfo_t *debuginfo = src->debuginfo;
+        while ((jl_value_t*)debuginfo->linetable != jl_nothing)
+            debuginfo = debuginfo->linetable;
         ctx.file = jl_debuginfo_file(debuginfo);
-        if (jl_array_nrows(debuginfo->codelocs) > 0) {
-            struct jl_codeloc_t lineidx = jl_uncompress1_codeloc(debuginfo->codelocs, 0);
-            jl_debuginfo_t *linetable = debuginfo->linetable;
-            while (lineidx.line > 0 && (jl_value_t*)linetable != jl_nothing) {
-                lineidx = jl_uncompress1_codeloc(linetable->codelocs, lineidx.line - 1);
-                linetable = linetable->linetable;
-            }
-            toplineno = std::max((int32_t)0, lineidx.line);
-        }
+        struct jl_codeloc_t lineidx = jl_uncompress1_codeloc(debuginfo->codelocs, 0);
+        toplineno = std::max((int32_t)0, lineidx.line);
     }
     if (ctx.file.empty())
         ctx.file = "<missing>";
@@ -8340,27 +8334,25 @@ static jl_llvm_functions_t
     new_lineinfo.push_back(topinfo);
     auto update_lineinfo = [&] (size_t pc) {
         jl_debuginfo_t *debuginfo = src->debuginfo;
-        struct jl_codeloc_t lineidx = jl_uncompress1_codeloc(debuginfo->codelocs, pc);
+        struct jl_codeloc_t lineidx = jl_uncompress1_codeloc(debuginfo->codelocs, pc + 1);
         if (lineidx.line == 0 && lineidx.to == 0)
             return false; // do not change anything
         prev_lineinfo.resize(0);
         std::swap(prev_lineinfo, new_lineinfo);
-        std::function<void(jl_debuginfo_t *debuginfo, jl_value_t *func, size_t to, size_t pc)> append_lineinfo =
+        std::function<void(jl_debuginfo_t*, jl_value_t*, size_t, size_t)> append_lineinfo =
                 [&] (jl_debuginfo_t *debuginfo, jl_value_t *func, size_t to, size_t pc) -> void {
             while (1) {
-                if (pc + 1 == 0)
-                    break; // TODO: this should be the toplevel node
                 if (!jl_is_symbol(debuginfo->def)) // this is a path
                     func = debuginfo->def; // this is inlined
                 struct jl_codeloc_t lineidx = jl_uncompress1_codeloc(debuginfo->codelocs, pc);
                 size_t i = lineidx.line;
-                if (i < 0)
-                    i = 0; // pc out of range: broken debuginfo?
-                if (i > 0 && (jl_value_t*)debuginfo->linetable != jl_nothing) {
+                if (pc > 0 && i >= 0 && (jl_value_t*)debuginfo->linetable != jl_nothing) {
                     // indirection node
-                    append_lineinfo(debuginfo->linetable, func, to, i - 1);
+                    append_lineinfo(debuginfo->linetable, func, to, i);
                 }
                 else {
+                    if (i < 0)
+                        i = 0; // pc out of range: broken debuginfo?
                     // actual node
                     DebugLineTable info;
                     info.edgeid = to;
@@ -8407,12 +8399,12 @@ static jl_llvm_functions_t
                 to = lineidx.to;
                 if (to == 0)
                     break;
-                pc = lineidx.pc - 1;
+                pc = lineidx.pc;
                 debuginfo = (jl_debuginfo_t*)jl_svecref(debuginfo->edges, to - 1);
                 func = NULL;
             }
         };
-        append_lineinfo(debuginfo, (jl_value_t*)lam, 0, pc);
+        append_lineinfo(debuginfo, (jl_value_t*)lam, 0, pc + 1);
         assert(new_lineinfo.size() > 0);
         return true;
     };
@@ -8570,10 +8562,21 @@ static jl_llvm_functions_t
             bool is_tracked = in_tracked_path(file);
             if (do_coverage(is_user_code, is_tracked)) {
                 for (size_t pc = 0; 1; pc++) {
-                    struct jl_codeloc_t lineidx = jl_uncompress1_codeloc(debuginfo->codelocs, pc);
+                    struct jl_codeloc_t lineidx = jl_uncompress1_codeloc(debuginfo->codelocs, pc + 1);
                     if (lineidx.line == -1)
                         break;
-                    jl_coverage_alloc_line(file, lineidx.line);
+                    jl_debuginfo_t *linetable = debuginfo->linetable;
+                    while (lineidx.line >= 0 && (jl_value_t*)linetable != jl_nothing) {
+                        if (lineidx.line == 0) {
+                            lineidx = jl_uncompress1_codeloc(linetable->codelocs, lineidx.line);
+                            break;
+                        }
+                        lineidx = jl_uncompress1_codeloc(linetable->codelocs, lineidx.line);
+                        linetable = linetable->linetable;
+                    }
+                    if (lineidx.line > 0) {
+                        jl_coverage_alloc_line(file, lineidx.line);
+                    }
                 }
             }
         };