Skip to content

Commit 8a85a2e

Browse files
mateusz834prattmic
mateusz834
authored and
prattmic
committed
runtime, internal/runtime/maps: speed-up empty/zero map lookups
This lets the inliner do a better job optimizing the mapKeyError call. goos: linux goarch: amd64 pkg: runtime cpu: AMD Ryzen 5 4600G with Radeon Graphics │ /tmp/before2 │ /tmp/after3 │ │ sec/op │ sec/op vs base │ MapAccessZero/Key=int64-12 1.875n ± 0% 1.875n ± 0% ~ (p=0.506 n=25) MapAccessZero/Key=int32-12 1.875n ± 0% 1.875n ± 0% ~ (p=0.082 n=25) MapAccessZero/Key=string-12 1.902n ± 1% 1.902n ± 1% ~ (p=0.256 n=25) MapAccessZero/Key=mediumType-12 2.816n ± 0% 1.958n ± 0% -30.47% (p=0.000 n=25) MapAccessZero/Key=bigType-12 2.815n ± 0% 1.935n ± 0% -31.26% (p=0.000 n=25) MapAccessEmpty/Key=int64-12 1.942n ± 0% 2.109n ± 0% +8.60% (p=0.000 n=25) MapAccessEmpty/Key=int32-12 2.110n ± 0% 1.940n ± 0% -8.06% (p=0.000 n=25) MapAccessEmpty/Key=string-12 2.024n ± 0% 2.109n ± 0% +4.20% (p=0.000 n=25) MapAccessEmpty/Key=mediumType-12 3.157n ± 0% 2.344n ± 0% -25.75% (p=0.000 n=25) MapAccessEmpty/Key=bigType-12 3.054n ± 0% 2.115n ± 0% -30.75% (p=0.000 n=25) geomean 2.305n 2.011n -12.75% Change-Id: Iee83930884dc4c8a791a711aa189a1c93b68d536 Reviewed-on: https://go-review.googlesource.com/c/go/+/663495 Reviewed-by: Dmitri Shuralyov <[email protected]> LUCI-TryBot-Result: Go LUCI <[email protected]> Reviewed-by: Michael Pratt <[email protected]>
1 parent 7d0cb2a commit 8a85a2e

File tree

10 files changed

+138
-97
lines changed

10 files changed

+138
-97
lines changed

‎src/cmd/link/internal/loader/loader.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2359,8 +2359,8 @@ var blockedLinknames = map[string][]string{
23592359
"crypto/internal/sysrand.fatal": {"crypto/internal/sysrand"},
23602360
"crypto/rand.fatal": {"crypto/rand"},
23612361
"internal/runtime/maps.errNilAssign": {"internal/runtime/maps"},
2362+
"internal/runtime/maps.typeString": {"internal/runtime/maps"},
23622363
"internal/runtime/maps.fatal": {"internal/runtime/maps"},
2363-
"internal/runtime/maps.mapKeyError": {"internal/runtime/maps"},
23642364
"internal/runtime/maps.newarray": {"internal/runtime/maps"},
23652365
"internal/runtime/maps.newobject": {"internal/runtime/maps"},
23662366
"internal/runtime/maps.typedmemclr": {"internal/runtime/maps"},

‎src/internal/abi/iface.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,9 @@ type EmptyInterface struct {
2525
Type *Type
2626
Data unsafe.Pointer
2727
}
28+
29+
// EmptyInterface describes the layout of an interface that contains any methods.
30+
type NonEmptyInterface struct {
31+
ITab *ITab
32+
Data unsafe.Pointer
33+
}

‎src/internal/runtime/maps/map.go

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -806,3 +806,89 @@ func (m *Map) Clone(typ *abi.SwissMapType) *Map {
806806

807807
return m
808808
}
809+
810+
func OldMapKeyError(t *abi.OldMapType, p unsafe.Pointer) error {
811+
if !t.HashMightPanic() {
812+
return nil
813+
}
814+
return mapKeyError2(t.Key, p)
815+
}
816+
817+
func mapKeyError(t *abi.SwissMapType, p unsafe.Pointer) error {
818+
if !t.HashMightPanic() {
819+
return nil
820+
}
821+
return mapKeyError2(t.Key, p)
822+
}
823+
824+
func mapKeyError2(t *abi.Type, p unsafe.Pointer) error {
825+
if t.TFlag&abi.TFlagRegularMemory != 0 {
826+
return nil
827+
}
828+
switch t.Kind() {
829+
case abi.Float32, abi.Float64, abi.Complex64, abi.Complex128, abi.String:
830+
return nil
831+
case abi.Interface:
832+
i := (*abi.InterfaceType)(unsafe.Pointer(t))
833+
var t *abi.Type
834+
var pdata *unsafe.Pointer
835+
if len(i.Methods) == 0 {
836+
a := (*abi.EmptyInterface)(p)
837+
t = a.Type
838+
if t == nil {
839+
return nil
840+
}
841+
pdata = &a.Data
842+
} else {
843+
a := (*abi.NonEmptyInterface)(p)
844+
if a.ITab == nil {
845+
return nil
846+
}
847+
t = a.ITab.Type
848+
pdata = &a.Data
849+
}
850+
851+
if t.Equal == nil {
852+
return unhashableTypeError{t}
853+
}
854+
855+
if t.Kind_&abi.KindDirectIface != 0 {
856+
return mapKeyError2(t, unsafe.Pointer(pdata))
857+
} else {
858+
return mapKeyError2(t, *pdata)
859+
}
860+
case abi.Array:
861+
a := (*abi.ArrayType)(unsafe.Pointer(t))
862+
for i := uintptr(0); i < a.Len; i++ {
863+
if err := mapKeyError2(a.Elem, unsafe.Pointer(uintptr(p)+i*a.Elem.Size_)); err != nil {
864+
return err
865+
}
866+
}
867+
return nil
868+
case abi.Struct:
869+
s := (*abi.StructType)(unsafe.Pointer(t))
870+
for _, f := range s.Fields {
871+
if f.Name.IsBlank() {
872+
continue
873+
}
874+
if err := mapKeyError2(f.Typ, unsafe.Pointer(uintptr(p)+f.Offset)); err != nil {
875+
return err
876+
}
877+
}
878+
return nil
879+
default:
880+
// Should never happen, keep this case for robustness.
881+
return unhashableTypeError{t}
882+
}
883+
}
884+
885+
type unhashableTypeError struct{ typ *abi.Type }
886+
887+
func (unhashableTypeError) RuntimeError() {}
888+
889+
func (e unhashableTypeError) Error() string { return "hash of unhashable type: " + typeString(e.typ) }
890+
891+
// Pushed from runtime
892+
//
893+
//go:linkname typeString
894+
func typeString(typ *abi.Type) string

‎src/internal/runtime/maps/runtime_noswiss.go

Lines changed: 0 additions & 17 deletions
This file was deleted.

‎src/internal/runtime/maps/runtime_swiss.go

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,6 @@ import (
1717

1818
// Functions below pushed from runtime.
1919

20-
//go:linkname mapKeyError
21-
func mapKeyError(typ *abi.SwissMapType, p unsafe.Pointer) error
22-
2320
// Pushed from runtime in order to use runtime.plainError
2421
//
2522
//go:linkname errNilAssign

‎src/runtime/alg.go

Lines changed: 0 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -250,74 +250,6 @@ func typehash(t *_type, p unsafe.Pointer, h uintptr) uintptr {
250250
}
251251
}
252252

253-
func mapKeyError(t *maptype, p unsafe.Pointer) error {
254-
if !t.HashMightPanic() {
255-
return nil
256-
}
257-
return mapKeyError2(t.Key, p)
258-
}
259-
260-
func mapKeyError2(t *_type, p unsafe.Pointer) error {
261-
if t.TFlag&abi.TFlagRegularMemory != 0 {
262-
return nil
263-
}
264-
switch t.Kind_ & abi.KindMask {
265-
case abi.Float32, abi.Float64, abi.Complex64, abi.Complex128, abi.String:
266-
return nil
267-
case abi.Interface:
268-
i := (*interfacetype)(unsafe.Pointer(t))
269-
var t *_type
270-
var pdata *unsafe.Pointer
271-
if len(i.Methods) == 0 {
272-
a := (*eface)(p)
273-
t = a._type
274-
if t == nil {
275-
return nil
276-
}
277-
pdata = &a.data
278-
} else {
279-
a := (*iface)(p)
280-
if a.tab == nil {
281-
return nil
282-
}
283-
t = a.tab.Type
284-
pdata = &a.data
285-
}
286-
287-
if t.Equal == nil {
288-
return errorString("hash of unhashable type " + toRType(t).string())
289-
}
290-
291-
if isDirectIface(t) {
292-
return mapKeyError2(t, unsafe.Pointer(pdata))
293-
} else {
294-
return mapKeyError2(t, *pdata)
295-
}
296-
case abi.Array:
297-
a := (*arraytype)(unsafe.Pointer(t))
298-
for i := uintptr(0); i < a.Len; i++ {
299-
if err := mapKeyError2(a.Elem, add(p, i*a.Elem.Size_)); err != nil {
300-
return err
301-
}
302-
}
303-
return nil
304-
case abi.Struct:
305-
s := (*structtype)(unsafe.Pointer(t))
306-
for _, f := range s.Fields {
307-
if f.Name.IsBlank() {
308-
continue
309-
}
310-
if err := mapKeyError2(f.Typ, add(p, f.Offset)); err != nil {
311-
return err
312-
}
313-
}
314-
return nil
315-
default:
316-
// Should never happen, keep this case for robustness.
317-
return errorString("hash of unhashable type " + toRType(t).string())
318-
}
319-
}
320-
321253
//go:linkname reflect_typehash reflect.typehash
322254
func reflect_typehash(t *_type, p unsafe.Pointer, h uintptr) uintptr {
323255
return typehash(t, p, h)

‎src/runtime/map_benchmark_test.go

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1191,3 +1191,39 @@ func BenchmarkMapSmallAccessMiss(b *testing.B) {
11911191
b.Run("Key=string/Elem=string", smallBenchSizes(benchmarkMapAccessMiss[string, string]))
11921192
b.Run("Key=smallType/Elem=int32", smallBenchSizes(benchmarkMapAccessMiss[smallType, int32]))
11931193
}
1194+
1195+
func mapAccessZeroBenchmark[K comparable](b *testing.B) {
1196+
var m map[K]uint64
1197+
var key K
1198+
for i := 0; i < b.N; i++ {
1199+
sink = m[key]
1200+
}
1201+
}
1202+
1203+
func BenchmarkMapAccessZero(b *testing.B) {
1204+
b.Run("Key=int64", mapAccessZeroBenchmark[int64])
1205+
b.Run("Key=int32", mapAccessZeroBenchmark[int32])
1206+
b.Run("Key=string", mapAccessZeroBenchmark[string])
1207+
b.Run("Key=mediumType", mapAccessZeroBenchmark[mediumType])
1208+
b.Run("Key=bigType", mapAccessZeroBenchmark[bigType])
1209+
}
1210+
1211+
func mapAccessEmptyBenchmark[K mapBenchmarkKeyType](b *testing.B) {
1212+
m := make(map[K]uint64)
1213+
for i, v := range genValues[K](0, 1000) {
1214+
m[v] = uint64(i)
1215+
}
1216+
clear(m)
1217+
var key K
1218+
for i := 0; i < b.N; i++ {
1219+
sink = m[key]
1220+
}
1221+
}
1222+
1223+
func BenchmarkMapAccessEmpty(b *testing.B) {
1224+
b.Run("Key=int64", mapAccessEmptyBenchmark[int64])
1225+
b.Run("Key=int32", mapAccessEmptyBenchmark[int32])
1226+
b.Run("Key=string", mapAccessEmptyBenchmark[string])
1227+
b.Run("Key=mediumType", mapAccessEmptyBenchmark[mediumType])
1228+
b.Run("Key=bigType", mapAccessEmptyBenchmark[bigType])
1229+
}

‎src/runtime/map_noswiss.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ import (
5959
"internal/abi"
6060
"internal/goarch"
6161
"internal/runtime/atomic"
62+
"internal/runtime/maps"
6263
"internal/runtime/math"
6364
"internal/runtime/sys"
6465
"unsafe"
@@ -426,7 +427,7 @@ func mapaccess1(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer {
426427
asanread(key, t.Key.Size_)
427428
}
428429
if h == nil || h.count == 0 {
429-
if err := mapKeyError(t, key); err != nil {
430+
if err := maps.OldMapKeyError(t, key); err != nil {
430431
panic(err) // see issue 23734
431432
}
432433
return unsafe.Pointer(&zeroVal[0])
@@ -496,7 +497,7 @@ func mapaccess2(t *maptype, h *hmap, key unsafe.Pointer) (unsafe.Pointer, bool)
496497
asanread(key, t.Key.Size_)
497498
}
498499
if h == nil || h.count == 0 {
499-
if err := mapKeyError(t, key); err != nil {
500+
if err := maps.OldMapKeyError(t, key); err != nil {
500501
panic(err) // see issue 23734
501502
}
502503
return unsafe.Pointer(&zeroVal[0]), false
@@ -757,7 +758,7 @@ func mapdelete(t *maptype, h *hmap, key unsafe.Pointer) {
757758
asanread(key, t.Key.Size_)
758759
}
759760
if h == nil || h.count == 0 {
760-
if err := mapKeyError(t, key); err != nil {
761+
if err := maps.OldMapKeyError(t, key); err != nil {
761762
panic(err) // see issue 23734
762763
}
763764
return

‎src/runtime/map_swiss.go

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,6 @@ type maptype = abi.SwissMapType
2424
//go:linkname maps_errNilAssign internal/runtime/maps.errNilAssign
2525
var maps_errNilAssign error = plainError("assignment to entry in nil map")
2626

27-
//go:linkname maps_mapKeyError internal/runtime/maps.mapKeyError
28-
func maps_mapKeyError(t *abi.SwissMapType, p unsafe.Pointer) error {
29-
return mapKeyError(t, p)
30-
}
31-
3227
func makemap64(t *abi.SwissMapType, hint int64, m *maps.Map) *maps.Map {
3328
if int64(int(hint)) != hint {
3429
hint = 0

‎src/runtime/type.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,11 @@ import (
1414
"unsafe"
1515
)
1616

17+
//go:linkname maps_typeString internal/runtime/maps.typeString
18+
func maps_typeString(typ *abi.Type) string {
19+
return toRType(typ).string()
20+
}
21+
1722
type nameOff = abi.NameOff
1823
type typeOff = abi.TypeOff
1924
type textOff = abi.TextOff

0 commit comments

Comments
 (0)