Skip to content

Commit 54789ef

Browse files
josharianjosharian
committed
cmd/internal/obj/arm64: make function prologue more predictable
Static branch prediction guesses that forward branches aren't taken. Since stacks are rarely grown, make the forward branch mean grow. While we're here, remove the debug-only instruction saving the frame size in the temp register. Sample disassembly for func f() { _ = [128]byte{} } Before: 0x4008248 ldr x1, [x28, #0x10] 0x400824c sub x2, sp, #0x90 0x4008250 cmp x2, x1 0x4008254 b.hi 0x4008268 0x4008258 mov x3, x30 0x400825c movz x27, #0x90 0x4008260 bl runtime.morestack_noctxt 0x4008264 b main.f 0x4008268 sub sp, sp, #0x90 0x400826c add x16, sp, #0x10 0x4008270 str xzr, [x16] 0x4008274 str xzr, [x16, #0x8] 0x4008278 str xzr, [x16, #0x10] 0x400827c str xzr, [x16, #0x18] 0x4008280 str xzr, [x16, #0x20] 0x4008284 str xzr, [x16, #0x28] 0x4008288 str xzr, [x16, #0x30] 0x400828c str xzr, [x16, #0x38] 0x4008290 str xzr, [x16, #0x40] 0x4008294 str xzr, [x16, #0x48] 0x4008298 str xzr, [x16, #0x50] 0x400829c str xzr, [x16, #0x58] 0x40082a0 str xzr, [x16, #0x60] 0x40082a4 str xzr, [x16, #0x68] 0x40082a8 str xzr, [x16, #0x70] 0x40082ac str xzr, [x16, #0x78] 0x40082b0 add sp, sp, #0x90 0x40082b4 ret After: 0x4004bc8 ldr x1, [x28, #0x10] 0x4004bcc sub x2, sp, #0x90 0x4004bd0 cmp x2, x1 0x4004bd4 b.ls 0x4004c28 0x4004bd8 sub sp, sp, #0x90 0x4004bdc add x16, sp, #0x10 0x4004be0 str xzr, [x16] 0x4004be4 str xzr, [x16, #0x8] 0x4004be8 str xzr, [x16, #0x10] 0x4004bec str xzr, [x16, #0x18] 0x4004bf0 str xzr, [x16, #0x20] 0x4004bf4 str xzr, [x16, #0x28] 0x4004bf8 str xzr, [x16, #0x30] 0x4004bfc str xzr, [x16, #0x38] 0x4004c00 str xzr, [x16, #0x40] 0x4004c04 str xzr, [x16, #0x48] 0x4004c08 str xzr, [x16, #0x50] 0x4004c0c str xzr, [x16, #0x58] 0x4004c10 str xzr, [x16, #0x60] 0x4004c14 str xzr, [x16, #0x68] 0x4004c18 str xzr, [x16, #0x70] 0x4004c1c str xzr, [x16, #0x78] 0x4004c20 add sp, sp, #0x90 0x4004c24 ret 0x4004c28 mov x3, x30 0x4004c2c bl runtime.morestack_noctxt 0x4004c30 b main.f Updates #10587. Package sort benchmarks using an iPhone 6: name old time/op new time/op delta SearchWrappers 355ns ± 1% 328ns ± 1% -7.57% (p=0.000 n=25+19) SortString1K 580µs ± 1% 577µs ± 1% -0.48% (p=0.000 n=25+25) StableString1K 1.04ms ± 0% 1.04ms ± 0% ~ (p=0.851 n=24+25) SortInt1K 251µs ± 1% 247µs ± 1% -1.52% (p=0.000 n=23+25) StableInt1K 267µs ± 2% 261µs ± 2% -2.02% (p=0.000 n=25+25) SortInt64K 23.8ms ± 1% 23.6ms ± 0% -0.97% (p=0.000 n=25+23) StableInt64K 22.8ms ± 0% 22.4ms ± 1% -1.76% (p=0.000 n=24+25) Sort1e2 123µs ± 1% 124µs ± 1% ~ (p=0.256 n=23+23) Stable1e2 248µs ± 1% 247µs ± 1% -0.69% (p=0.000 n=23+25) Sort1e4 24.3ms ± 2% 24.6ms ± 5% +1.36% (p=0.017 n=22+25) Stable1e4 77.2ms ± 6% 76.2ms ± 5% -1.36% (p=0.020 n=25+25) Sort1e6 3.95s ± 8% 3.95s ± 8% ~ (p=0.863 n=25+25) Stable1e6 15.7s ± 1% 15.5s ± 1% -1.11% (p=0.000 n=22+23) Change-Id: I377b3817af2ed27ddeecf24edef97fad91fc1afc Reviewed-on: https://go-review.googlesource.com/10500 Reviewed-by: Russ Cox <[email protected]> Run-TryBot: Josh Bleecher Snyder <[email protected]> Reviewed-by: Dave Cheney <[email protected]>
1 parent 5353cde commit 54789ef

File tree

1 file changed

+43
-42
lines changed

1 file changed

+43
-42
lines changed

‎src/cmd/internal/obj/arm64/obj7.go

+43-42
Original file line numberDiff line numberDiff line change
@@ -152,60 +152,61 @@ func stacksplit(ctxt *obj.Link, p *obj.Prog, framesize int32) *obj.Prog {
152152
p.Reg = REG_R2
153153
}
154154

155-
// BHI done
156-
p = obj.Appendp(ctxt, p)
157-
q1 := p
155+
// BLS do-morestack
156+
bls := obj.Appendp(ctxt, p)
157+
bls.As = ABLS
158+
bls.To.Type = obj.TYPE_BRANCH
158159

159-
p.As = ABHI
160-
p.To.Type = obj.TYPE_BRANCH
160+
var last *obj.Prog
161+
for last = ctxt.Cursym.Text; last.Link != nil; last = last.Link {
162+
}
161163

162164
// MOV LR, R3
163-
p = obj.Appendp(ctxt, p)
164-
165-
p.As = AMOVD
166-
p.From.Type = obj.TYPE_REG
167-
p.From.Reg = REGLINK
168-
p.To.Type = obj.TYPE_REG
169-
p.To.Reg = REG_R3
165+
movlr := obj.Appendp(ctxt, last)
166+
movlr.As = AMOVD
167+
movlr.From.Type = obj.TYPE_REG
168+
movlr.From.Reg = REGLINK
169+
movlr.To.Type = obj.TYPE_REG
170+
movlr.To.Reg = REG_R3
170171
if q != nil {
171-
q.Pcond = p
172+
q.Pcond = movlr
173+
}
174+
bls.Pcond = movlr
175+
176+
debug := movlr
177+
if false {
178+
debug = obj.Appendp(ctxt, debug)
179+
debug.As = AMOVD
180+
debug.From.Type = obj.TYPE_CONST
181+
debug.From.Offset = int64(framesize)
182+
debug.To.Type = obj.TYPE_REG
183+
debug.To.Reg = REGTMP
172184
}
173-
174-
// TODO(minux): only for debug
175-
p = obj.Appendp(ctxt, p)
176-
p.As = AMOVD
177-
p.From.Type = obj.TYPE_CONST
178-
p.From.Offset = int64(framesize)
179-
p.To.Type = obj.TYPE_REG
180-
p.To.Reg = REGTMP
181185

182186
// BL runtime.morestack(SB)
183-
p = obj.Appendp(ctxt, p)
184-
185-
p.As = ABL
186-
p.To.Type = obj.TYPE_BRANCH
187-
if ctxt.Cursym.Cfunc != 0 {
188-
p.To.Sym = obj.Linklookup(ctxt, "runtime.morestackc", 0)
189-
} else if ctxt.Cursym.Text.From3.Offset&obj.NEEDCTXT == 0 {
190-
p.To.Sym = obj.Linklookup(ctxt, "runtime.morestack_noctxt", 0)
191-
} else {
192-
p.To.Sym = obj.Linklookup(ctxt, "runtime.morestack", 0)
187+
call := obj.Appendp(ctxt, debug)
188+
call.As = ABL
189+
call.To.Type = obj.TYPE_BRANCH
190+
morestack := "runtime.morestack"
191+
switch {
192+
case ctxt.Cursym.Cfunc != 0:
193+
morestack = "runtime.morestackc"
194+
case ctxt.Cursym.Text.From3.Offset&obj.NEEDCTXT == 0:
195+
morestack = "runtime.morestack_noctxt"
193196
}
197+
call.To.Sym = obj.Linklookup(ctxt, morestack, 0)
194198

195199
// B start
196-
p = obj.Appendp(ctxt, p)
197-
198-
p.As = AB
199-
p.To.Type = obj.TYPE_BRANCH
200-
p.Pcond = ctxt.Cursym.Text.Link
201-
202-
// placeholder for q1's jump target
203-
p = obj.Appendp(ctxt, p)
200+
jmp := obj.Appendp(ctxt, call)
201+
jmp.As = AB
202+
jmp.To.Type = obj.TYPE_BRANCH
203+
jmp.Pcond = ctxt.Cursym.Text.Link
204204

205-
p.As = obj.ANOP
206-
q1.Pcond = p
205+
// placeholder for bls's jump target
206+
// p = obj.Appendp(ctxt, p)
207+
// p.As = obj.ANOP
207208

208-
return p
209+
return bls
209210
}
210211

211212
func progedit(ctxt *obj.Link, p *obj.Prog) {

0 commit comments

Comments
 (0)