Skip to content

Commit 0672dff

Browse files
y1yang0y1yang0
committed
cmd/compile: hoist some loop invariants
Conservatively hoist some loop invariants outside the loop. Updates #15808
1 parent 20e9b7f commit 0672dff

File tree

8 files changed

+757
-453
lines changed

8 files changed

+757
-453
lines changed

‎src/cmd/compile/internal/ssa/branchelim.go

+9-2
Original file line numberDiff line numberDiff line change
@@ -424,6 +424,14 @@ func shouldElimIfElse(no, yes, post *Block, arch string) bool {
424424
}
425425
}
426426

427+
func hasSideEffect(v *Value) bool {
428+
if v.Op == OpPhi || isDivMod(v.Op) || isPtrArithmetic(v.Op) || v.Type.IsMemory() ||
429+
v.MemoryArg() != nil || opcodeTable[v.Op].hasSideEffects {
430+
return true
431+
}
432+
return false
433+
}
434+
427435
// canSpeculativelyExecute reports whether every value in the block can
428436
// be evaluated without causing any observable side effects (memory
429437
// accesses, panics and so on) except for execution time changes. It
@@ -436,8 +444,7 @@ func canSpeculativelyExecute(b *Block) bool {
436444
// don't fuse memory ops, Phi ops, divides (can panic),
437445
// or anything else with side-effects
438446
for _, v := range b.Values {
439-
if v.Op == OpPhi || isDivMod(v.Op) || isPtrArithmetic(v.Op) || v.Type.IsMemory() ||
440-
v.MemoryArg() != nil || opcodeTable[v.Op].hasSideEffects {
447+
if hasSideEffect(v) {
441448
return false
442449
}
443450
}

‎src/cmd/compile/internal/ssa/compile.go

+1
Original file line numberDiff line numberDiff line change
@@ -485,6 +485,7 @@ var passes = [...]pass{
485485
{name: "writebarrier", fn: writebarrier, required: true}, // expand write barrier ops
486486
{name: "insert resched checks", fn: insertLoopReschedChecks,
487487
disabled: !buildcfg.Experiment.PreemptibleLoops}, // insert resched checks in loops.
488+
{name: "hoist loop invariant", fn: hoistLoopInvariant},
488489
{name: "lower", fn: lower, required: true},
489490
{name: "addressing modes", fn: addressingModes, required: false},
490491
{name: "late lower", fn: lateLower, required: true},

‎src/cmd/compile/internal/ssa/graphkit.go

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
// Copyright 2023 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
package ssa
6+
7+
// ----------------------------------------------------------------------------
8+
// Graph transformation
9+
10+
// replaceUses replaces all uses of old in b with new.
11+
func (b *Block) replaceUses(old, new *Value) {
12+
for _, v := range b.Values {
13+
for i, a := range v.Args {
14+
if a == old {
15+
v.SetArg(i, new)
16+
}
17+
}
18+
}
19+
for i, v := range b.ControlValues() {
20+
if v == old {
21+
b.ReplaceControl(i, new)
22+
}
23+
}
24+
}
25+
26+
// moveTo moves v to dst, adjusting the appropriate Block.Values slices.
27+
// The caller is responsible for ensuring that this is safe.
28+
// i is the index of v in v.Block.Values.
29+
func (v *Value) moveTo(dst *Block, i int) {
30+
if dst.Func.scheduled {
31+
v.Fatalf("moveTo after scheduling")
32+
}
33+
src := v.Block
34+
if src.Values[i] != v {
35+
v.Fatalf("moveTo bad index %d", v, i)
36+
}
37+
if src == dst {
38+
return
39+
}
40+
v.Block = dst
41+
dst.Values = append(dst.Values, v)
42+
last := len(src.Values) - 1
43+
src.Values[i] = src.Values[last]
44+
src.Values[last] = nil
45+
src.Values = src.Values[:last]
46+
}

‎src/cmd/compile/internal/ssa/hoistloopiv.go

+132
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
// Copyright 2023 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
package ssa
6+
7+
import "fmt"
8+
9+
const MaxLoopBlockSize = 8
10+
11+
func printInvariant(val *Value, block *Block, domBlock *Block) {
12+
fmt.Printf("== Hoist %v(%v) from b%v to b%v in %v\n",
13+
val.Op.String(), val.String(),
14+
block.ID, domBlock.ID, block.Func.Name)
15+
fmt.Printf(" %v\n", val.LongString())
16+
}
17+
18+
func isCandidate(block *Block, val *Value) bool {
19+
if len(val.Args) == 0 {
20+
// not a profitable expression, e.g. constant
21+
return false
22+
}
23+
if block.Likely == BranchUnlikely {
24+
// all values are excluded as candidate when branch becomes unlikely to reach
25+
return false
26+
}
27+
return true
28+
}
29+
30+
func isInsideLoop(loopBlocks []*Block, v *Value) bool {
31+
for _, block := range loopBlocks {
32+
for _, val := range block.Values {
33+
if val == v {
34+
return true
35+
}
36+
}
37+
}
38+
return false
39+
}
40+
41+
// tryHoist hoists profitable loop invariant to block that dominates the entire loop.
42+
// Value is considered as loop invariant if all its inputs are defined outside the loop
43+
// or all its inputs are loop invariants. Since loop invariant will immediately moved
44+
// to dominator block of loop, the first rule actually already implies the second rule
45+
func tryHoist(loopnest *loopnest, loop *loop, loopBlocks []*Block) {
46+
for _, block := range loopBlocks {
47+
// if basic block is located in a nested loop rather than directly in the
48+
// current loop, it will not be processed.
49+
if loopnest.b2l[block.ID] != loop {
50+
continue
51+
}
52+
for i := 0; i < len(block.Values); i++ {
53+
var val *Value = block.Values[i]
54+
if !isCandidate(block, val) {
55+
continue
56+
}
57+
// value can hoist because it may causes observable side effects
58+
if hasSideEffect(val) {
59+
continue
60+
}
61+
// consider the following operation as pinned anyway
62+
switch val.Op {
63+
case OpInlMark,
64+
OpAtomicLoad8, OpAtomicLoad32, OpAtomicLoad64,
65+
OpAtomicLoadPtr, OpAtomicLoadAcq32, OpAtomicLoadAcq64:
66+
continue
67+
}
68+
// input def is inside loop, consider as variant
69+
isInvariant := true
70+
loopnest.assembleChildren()
71+
for _, arg := range val.Args {
72+
if isInsideLoop(loopBlocks, arg) {
73+
isInvariant = false
74+
break
75+
}
76+
}
77+
if isInvariant {
78+
for valIdx, v := range block.Values {
79+
if val != v {
80+
continue
81+
}
82+
domBlock := loopnest.sdom.Parent(loop.header)
83+
if block.Func.pass.debug >= 1 {
84+
printInvariant(val, block, domBlock)
85+
}
86+
val.moveTo(domBlock, valIdx)
87+
i--
88+
break
89+
}
90+
}
91+
}
92+
}
93+
}
94+
95+
// hoistLoopInvariant hoists expressions that computes the same value
96+
// while has no effect outside loop
97+
func hoistLoopInvariant(f *Func) {
98+
loopnest := f.loopnest()
99+
if loopnest.hasIrreducible {
100+
return
101+
}
102+
if len(loopnest.loops) == 0 {
103+
return
104+
}
105+
for _, loop := range loopnest.loops {
106+
loopBlocks := loopnest.findLoopBlocks(loop)
107+
if len(loopBlocks) >= MaxLoopBlockSize {
108+
continue
109+
}
110+
111+
// check if it's too complicated for such optmization
112+
tooComplicated := false
113+
Out:
114+
for _, block := range loopBlocks {
115+
for _, val := range block.Values {
116+
if val.Op.IsCall() || val.Op.HasSideEffects() {
117+
tooComplicated = true
118+
break Out
119+
}
120+
switch val.Op {
121+
case OpLoad, OpStore:
122+
tooComplicated = true
123+
break Out
124+
}
125+
}
126+
}
127+
// try to hoist loop invariant outside the loop
128+
if !tooComplicated {
129+
tryHoist(loopnest, loop, loopBlocks)
130+
}
131+
}
132+
}

‎src/cmd/compile/internal/ssa/hoistloopiv_test.go

+111
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
// Copyright 2023 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
package ssa
6+
7+
import (
8+
"cmd/compile/internal/types"
9+
"testing"
10+
)
11+
12+
func checkValueMotion(t *testing.T, fun fun, valName, expectedBlock string) {
13+
for _, b := range fun.f.Blocks {
14+
for _, v := range b.Values {
15+
if v == fun.values[valName] {
16+
if fun.blocks[expectedBlock] != b {
17+
t.Errorf("Error: %v\n", v.LongString())
18+
}
19+
}
20+
}
21+
}
22+
}
23+
24+
// var d int
25+
// var p = 15
26+
//
27+
// for i := 0; i < 10; i++ {
28+
// t := 1 * p
29+
// d = i + t
30+
// }
31+
//
32+
// t should be hoisted to dominator block of loop header
33+
func TestHoistLoopIVSimple(t *testing.T) {
34+
c := testConfig(t)
35+
fun := c.Fun("b1",
36+
Bloc("b1",
37+
Valu("mem", OpInitMem, types.TypeMem, 0, nil),
38+
Valu("zero", OpConst64, c.config.Types.Int64, 0, nil),
39+
Valu("one", OpConst64, c.config.Types.Int64, 1, nil),
40+
Valu("ten", OpConst64, c.config.Types.Int64, 10, nil),
41+
Valu("p", OpConst64, c.config.Types.Int64, 15, nil),
42+
Goto("b2")),
43+
Bloc("b2",
44+
Valu("i", OpPhi, c.config.Types.Int64, 0, nil, "one", "i2"),
45+
Valu("d", OpPhi, c.config.Types.Int64, 0, nil, "zero", "d2"),
46+
Valu("cmp", OpLess64, c.config.Types.Bool, 0, nil, "i", "ten"),
47+
If("cmp", "b3", "b4")),
48+
Bloc("b3",
49+
Valu("loopiv", OpMul64, c.config.Types.Int64, 0, nil, "one", "p"),
50+
Valu("d2", OpAdd64, c.config.Types.Int64, 0, nil, "loopiv", "d"),
51+
Valu("i2", OpAdd64, c.config.Types.Int64, 0, nil, "i", "one"),
52+
Goto("b2")),
53+
Bloc("b4",
54+
Exit("mem")))
55+
56+
CheckFunc(fun.f)
57+
hoistLoopInvariant(fun.f)
58+
CheckFunc(fun.f)
59+
checkValueMotion(t, fun, "loopiv", "b1")
60+
}
61+
62+
func BenchmarkHoistIV1Opt(b *testing.B) {
63+
var d = 0
64+
var a = 3
65+
66+
for i := 0; i < b.N; i++ {
67+
d = i + (a*10 - a + 3)
68+
}
69+
_ = d
70+
}
71+
72+
func BenchmarkHoistIV1Manual(b *testing.B) {
73+
var d = 0
74+
var a = 3
75+
val := (a*10 - a + 3)
76+
for i := 0; i < b.N; i++ {
77+
d = i + val
78+
}
79+
_ = d
80+
}
81+
82+
//go:noinline
83+
func hoistLoopIV2Opt(n, d int) {
84+
t := 0
85+
for i := 0; i < n*d; i++ {
86+
t += 1
87+
}
88+
_ = t
89+
}
90+
91+
//go:noinline
92+
func hoistLoopIV2Manual(n, d int) {
93+
t := 0
94+
val := n * d
95+
for i := 0; i < val; i++ {
96+
t += 1
97+
}
98+
_ = t
99+
}
100+
101+
func BenchmarkHoistIV2Opt(b *testing.B) {
102+
for i := 0; i < b.N; i++ {
103+
hoistLoopIV2Opt(i%10, i%5)
104+
}
105+
}
106+
107+
func BenchmarkHoistIV2Manual(b *testing.B) {
108+
for i := 0; i < b.N; i++ {
109+
hoistLoopIV2Manual(i%10, i%5)
110+
}
111+
}

0 commit comments

Comments
 (0)