Profile-based inlining enabled

Author: Raj Barik

src/cmd/compile/internal/base/flag.go

@@ -121,6 +121,8 @@ type CmdFlags struct {
 	TraceProfile       string       "help:\"write an execution trace to `file`\""
 	TrimPath           string       "help:\"remove `prefix` from recorded source file paths\""
 	WB                 bool         "help:\"enable write barrier\"" // TODO: remove
+	ProfileUse         string       "help:\"read profile from `file`\""
+	InlineHotThreshold string       "help:\"Threshold percentage for determining hot methods for inlining\""
 
 	// Configuration derived from flags; not a flag itself.
 	Cfg struct {

src/cmd/compile/internal/gc/main.go

@@ -16,6 +16,7 @@ import (
 	"cmd/compile/internal/ir"
 	"cmd/compile/internal/logopt"
 	"cmd/compile/internal/noder"
+	"cmd/compile/internal/pgo"
 	"cmd/compile/internal/pkginit"
 	"cmd/compile/internal/reflectdata"
 	"cmd/compile/internal/ssa"
@@ -232,10 +233,28 @@ func Main(archInit func(*ssagen.ArchInfo)) {
 		typecheck.AllImportedBodies()
 	}
 
+	// Read cpu profile file and build cross-package pprof-graph and per-package weighted-call-graph. Pprof-graph is built one-time.
+	base.Timer.Start("fe", "profileuse")
+	if base.Flag.ProfileUse != "" {
+		if pgo.PProfGraph == nil {
+			pgo.PProfGraph = pgo.BuildGlobalPProfGraph(base.Flag.ProfileUse, &pgo.Options{
+				CallTree:    false,
+				SampleValue: func(v []int64) int64 { return v[1] },
+			})
+		}
+		pgo.WeightedCG = pgo.BuildWeightedCallGraphPerPkg()
+	}
+
 	// Inlining
 	base.Timer.Start("fe", "inlining")
 	if base.Flag.LowerL != 0 {
+		if pgo.WeightedCG != nil {
+			inline.InlinePrologue()
+		}
 		inline.InlinePackage()
+		if pgo.WeightedCG != nil {
+			inline.InlineEpilogue()
+		}
 	}
 	noder.MakeWrappers(typecheck.Target) // must happen after inlining
 

src/cmd/compile/internal/inline/inl.go

@@ -29,11 +29,13 @@ package inline
 import (
 	"fmt"
 	"go/constant"
+	"strconv"
 	"strings"
 
 	"cmd/compile/internal/base"
 	"cmd/compile/internal/ir"
 	"cmd/compile/internal/logopt"
+	"cmd/compile/internal/pgo"
 	"cmd/compile/internal/typecheck"
 	"cmd/compile/internal/types"
 	"cmd/internal/obj"
@@ -42,17 +44,98 @@ import (
 
 // Inlining budget parameters, gathered in one place
 const (
-	inlineMaxBudget       = 80
-	inlineExtraAppendCost = 0
+	inlineMaxBudget = 80
+	// Budget increased due to hotness.
+	inlineHotCalleeMaxBudget = 160
+	inlineExtraAppendCost    = 0
 	// default is to inline if there's at most one call. -l=4 overrides this by using 1 instead.
 	inlineExtraCallCost  = 57              // 57 was benchmarked to provided most benefit with no bad surprises; see https://github.com/golang/go/issues/19348#issuecomment-439370742
 	inlineExtraPanicCost = 1               // do not penalize inlining panics.
 	inlineExtraThrowCost = inlineMaxBudget // with current (2018-05/1.11) code, inlining runtime.throw does not help.
 
 	inlineBigFunctionNodes   = 5000 // Functions with this many nodes are considered "big".
 	inlineBigFunctionMaxCost = 20   // Max cost of inlinee when inlining into a "big" function.
+
 )
 
+var (
+	// Per-caller data structure to track the list of hot call sites. This gets rewritten every caller leaving it to GC for cleanup.
+	listOfHotCallSites = make(map[pgo.CallSiteInfo]struct{})
+
+	// List of all hot call sites.
+	candHotEdgeMap = make(map[string]struct{})
+
+	// List of inlined call sites.
+	inlinedCallSites = make(map[pgo.CallSiteInfo]struct{})
+
+	// Threshold for Hot callsite inlining.
+	inlineHotThresholdPercent = float64(2)
+)
+
+// weightInPercentage converts profile weights to a percentage.
+func weightInPercentage(value int64, total int64) float64 {
+	var ratio float64
+	// percentage is computed at the (weight/totalweights) * 100
+	// e.g. if edge weight is 30 and the sum of all the edges weight is 126
+	// the percentage will be 23.8%
+	if total != 0 {
+		ratio = (float64(value) / float64(total)) * 100
+	}
+	return ratio
+}
+
+// InlinePrologue records the hot callsites from ir-graph.
+func InlinePrologue() {
+	if s, err := strconv.ParseFloat(base.Flag.InlineHotThreshold, 64); err == nil {
+		inlineHotThresholdPercent = s
+		if base.Flag.LowerM != 0 {
+			fmt.Printf("hot-thres=%v\n", inlineHotThresholdPercent)
+		}
+	}
+	ir.VisitFuncsBottomUp(typecheck.Target.Decls, func(list []*ir.Func, recursive bool) {
+		for _, f := range list {
+			name := ir.PkgFuncName(f)
+			if n, ok := pgo.WeightedCG.IRNodes[name]; ok {
+				nodeweight := weightInPercentage(n.Flat, pgo.GlobalTotalNodeWeight)
+
+				for _, e := range pgo.WeightedCG.OutEdges[n] {
+					if e.Weight != 0 {
+						weightpercent := weightInPercentage(e.Weight, pgo.GlobalTotalEdgeWeight)
+						if weightpercent > inlineHotThresholdPercent {
+							splits := strings.Split(e.CallSite, ":")
+							line2, _ := strconv.ParseInt(splits[len(splits)-2], 0, 64)
+							lineno := fmt.Sprintf("%v", line2)
+							canonicalName := ir.PkgFuncName(n.AST) + "-" + lineno + "-" + ir.PkgFuncName(e.Dst.AST)
+							candHotEdgeMap[canonicalName] = struct{}{}
+							if base.Flag.LowerM != 0 {
+								fmt.Printf("hot-inline cand=%v\n", canonicalName)
+							}
+						}
+					}
+				}
+				if nodeweight > inlineHotThresholdPercent {
+					n.HotNode = true
+					if base.Flag.LowerM != 0 {
+						fmt.Printf("hot-node=%v\n", name)
+					}
+				}
+			}
+		}
+	})
+}
+
+// InlineEpilogue updates IRGraph after inlining.
+func InlineEpilogue() {
+	ir.VisitFuncsBottomUp(typecheck.Target.Decls, func(list []*ir.Func, recursive bool) {
+		for _, f := range list {
+			name := ir.PkgFuncName(f)
+			if n, ok := pgo.WeightedCG.IRNodes[name]; ok {
+				pgo.RedirectEdges(n, inlinedCallSites)
+			}
+		}
+	})
+}
+
 // InlinePackage finds functions that can be inlined and clones them before walk expands them.
 func InlinePackage() {
 	ir.VisitFuncsBottomUp(typecheck.Target.Decls, func(list []*ir.Func, recursive bool) {
@@ -81,6 +164,9 @@ func CanInline(fn *ir.Func) {
 		base.Fatalf("CanInline no nname %+v", fn)
 	}
 
+	// Initialize an empty list of hot callsites for this caller.
+	listOfHotCallSites = make(map[pgo.CallSiteInfo]struct{})
+
 	var reason string // reason, if any, that the function was not inlined
 	if base.Flag.LowerM > 1 || logopt.Enabled() {
 		defer func() {
@@ -252,6 +338,23 @@ func (v *hairyVisitor) tooHairy(fn *ir.Func) bool {
 		return true
 	}
 	if v.budget < 0 {
+		if pgo.WeightedCG != nil {
+			// If the cost of hot function is greater than inlineHotCalleeMaxBudget,
+			// the inliner won't inline this function.
+			if inlineMaxBudget-v.budget < inlineHotCalleeMaxBudget {
+				// In some cases, there is no hot calling edges in the function.
+				// However, if the function is hot function, it is still inline
+				// candidate.
+				if n, ok := pgo.WeightedCG.IRNodes[ir.PkgFuncName(fn)]; ok {
+					if n.HotNode == true {
+						if base.Flag.LowerM != 0 {
+							fmt.Printf("hot-node enabled increased budget for func=%v\n", ir.PkgFuncName(fn))
+						}
+						return false
+					}
+				}
+			}
+		}
 		v.reason = fmt.Sprintf("function too complex: cost %d exceeds budget %d", inlineMaxBudget-v.budget, inlineMaxBudget)
 		return true
 	}
@@ -315,6 +418,20 @@ func (v *hairyVisitor) doNode(n ir.Node) bool {
 			}
 		}
 
+		// Determine if callee edge is a hot callee or not.
+		if pgo.WeightedCG != nil && ir.CurFunc != nil {
+			if fn := inlCallee(n.X); fn != nil && typecheck.HaveInlineBody(fn) {
+				lineno := fmt.Sprintf("%v", ir.Line(n))
+				splits := strings.Split(lineno, ":")
+				l, _ := strconv.ParseInt(splits[len(splits)-2], 0, 64)
+				linenum := fmt.Sprintf("%v", l)
+				canonicalName := ir.PkgFuncName(ir.CurFunc) + "-" + linenum + "-" + ir.PkgFuncName(fn)
+				if _, o := candHotEdgeMap[canonicalName]; o {
+					listOfHotCallSites[pgo.CallSiteInfo{ir.Line(n), ir.CurFunc}] = struct{}{}
+				}
+			}
+		}
+
 		if ir.IsIntrinsicCall(n) {
 			// Treat like any other node.
 			break
@@ -716,7 +833,15 @@ func mkinlcall(n *ir.CallExpr, fn *ir.Func, maxCost int32, inlCalls *[]*ir.Inlin
 			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(ir.CurFunc),
 				fmt.Sprintf("cost %d of %s exceeds max large caller cost %d", fn.Inl.Cost, ir.PkgFuncName(fn), maxCost))
 		}
-		return n
+
+		// If the callsite is hot and it is under the inlineHotCalleeMaxBudget budget, then inline it, or else bail.
+		if _, ok := listOfHotCallSites[pgo.CallSiteInfo{ir.Line(n), ir.CurFunc}]; ok {
+			if fn.Inl.Cost > inlineHotCalleeMaxBudget {
+				return n
+			}
+		} else {
+			return n
+		}
 	}
 
 	if fn == ir.CurFunc {
@@ -817,6 +942,13 @@ func mkinlcall(n *ir.CallExpr, fn *ir.Func, maxCost int32, inlCalls *[]*ir.Inlin
 		fmt.Printf("%v: Before inlining: %+v\n", ir.Line(n), n)
 	}
 
+	if _, ok := inlinedCallSites[pgo.CallSiteInfo{ir.Line(n), ir.CurFunc}]; !ok {
+		inlinedCallSites[pgo.CallSiteInfo{ir.Line(n), ir.CurFunc}] = struct{}{}
+	}
+	if base.Flag.LowerM != 0 {
+		fmt.Printf("Line %v: is definitely inlined\n", ir.Line(n))
+	}
+
 	res := NewInline(n, fn, inlIndex)
 	if res == nil {
 		return n