Skip to content

Commit 8585f9f

Browse files
martischmartisch
committed
runtime: refactor insertion slot tracking for fast hashmap functions
* Avoid calculating insertk until needed. * Avoid a pointer into b.tophash and just track the insertion index. This avoids b.tophash being marked as escaping to heap. * Calculate val only once at the end of the mapassign functions. Function sizes decrease slightly, e.g. for mapassign_faststr: before "".mapassign_faststr STEXT size=1166 args=0x28 locals=0x78 after "".mapassign_faststr STEXT size=1080 args=0x28 locals=0x68 name old time/op new time/op delta MapAssign/Int32/256-4 19.4ns ± 4% 19.5ns ±11% ~ (p=0.973 n=20+20) MapAssign/Int32/65536-4 32.5ns ± 2% 32.4ns ± 3% ~ (p=0.078 n=20+19) MapAssign/Int64/256-4 20.3ns ± 6% 17.6ns ± 5% -13.01% (p=0.000 n=20+20) MapAssign/Int64/65536-4 33.3ns ± 2% 33.3ns ± 1% ~ (p=0.444 n=20+20) MapAssign/Str/256-4 22.3ns ± 3% 22.4ns ± 3% ~ (p=0.343 n=20+20) MapAssign/Str/65536-4 44.9ns ± 1% 43.9ns ± 1% -2.39% (p=0.000 n=20+19) Change-Id: I2627bb8a961d366d9473b5922fa129176319eb22 Reviewed-on: https://go-review.googlesource.com/74870 Run-TryBot: Martin Möhrmann <[email protected]> TryBot-Result: Gobot Gobot <[email protected]> Reviewed-by: Keith Randall <[email protected]>
1 parent 2f1f607 commit 8585f9f

File tree

1 file changed

+45
-42
lines changed

1 file changed

+45
-42
lines changed

‎src/runtime/hashmap_fast.go

Lines changed: 45 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -374,24 +374,25 @@ again:
374374
}
375375
b := (*bmap)(unsafe.Pointer(uintptr(h.buckets) + bucket*uintptr(t.bucketsize)))
376376

377-
var inserti *uint8
377+
var insertb *bmap
378+
var inserti uintptr
378379
var insertk unsafe.Pointer
379-
var val unsafe.Pointer
380+
380381
for {
381382
for i := uintptr(0); i < bucketCnt; i++ {
382383
if b.tophash[i] == empty {
383-
if inserti == nil {
384-
inserti = &b.tophash[i]
385-
insertk = add(unsafe.Pointer(b), dataOffset+i*4)
386-
val = add(unsafe.Pointer(b), dataOffset+bucketCnt*4+i*uintptr(t.valuesize))
384+
if insertb == nil {
385+
inserti = i
386+
insertb = b
387387
}
388388
continue
389389
}
390390
k := *((*uint32)(add(unsafe.Pointer(b), dataOffset+i*4)))
391391
if k != key {
392392
continue
393393
}
394-
val = add(unsafe.Pointer(b), dataOffset+bucketCnt*4+i*uintptr(t.valuesize))
394+
inserti = i
395+
insertb = b
395396
goto done
396397
}
397398
ovf := b.overflow(t)
@@ -410,25 +411,25 @@ again:
410411
goto again // Growing the table invalidates everything, so try again
411412
}
412413

413-
if inserti == nil {
414+
if insertb == nil {
414415
// all current buckets are full, allocate a new one.
415-
newb := h.newoverflow(t, b)
416-
inserti = &newb.tophash[0]
417-
insertk = add(unsafe.Pointer(newb), dataOffset)
418-
val = add(insertk, bucketCnt*4)
416+
insertb = h.newoverflow(t, b)
417+
inserti = 0 // not necessary, but avoids needlessly spilling inserti
419418
}
419+
insertb.tophash[inserti&(bucketCnt-1)] = tophash(hash) // mask inserti to avoid bounds checks
420420

421+
insertk = add(unsafe.Pointer(insertb), dataOffset+inserti*4)
421422
// store new key at insert position
422423
if sys.PtrSize == 4 && t.key.kind&kindNoPointers == 0 && writeBarrier.enabled {
423424
writebarrierptr((*uintptr)(insertk), uintptr(key))
424425
} else {
425426
*(*uint32)(insertk) = key
426427
}
427428

428-
*inserti = tophash(hash)
429429
h.count++
430430

431431
done:
432+
val := add(unsafe.Pointer(insertb), dataOffset+bucketCnt*4+inserti*uintptr(t.valuesize))
432433
if h.flags&hashWriting == 0 {
433434
throw("concurrent map writes")
434435
}
@@ -463,24 +464,25 @@ again:
463464
}
464465
b := (*bmap)(unsafe.Pointer(uintptr(h.buckets) + bucket*uintptr(t.bucketsize)))
465466

466-
var inserti *uint8
467+
var insertb *bmap
468+
var inserti uintptr
467469
var insertk unsafe.Pointer
468-
var val unsafe.Pointer
470+
469471
for {
470472
for i := uintptr(0); i < bucketCnt; i++ {
471473
if b.tophash[i] == empty {
472-
if inserti == nil {
473-
inserti = &b.tophash[i]
474-
insertk = add(unsafe.Pointer(b), dataOffset+i*8)
475-
val = add(unsafe.Pointer(b), dataOffset+bucketCnt*8+i*uintptr(t.valuesize))
474+
if insertb == nil {
475+
insertb = b
476+
inserti = i
476477
}
477478
continue
478479
}
479480
k := *((*uint64)(add(unsafe.Pointer(b), dataOffset+i*8)))
480481
if k != key {
481482
continue
482483
}
483-
val = add(unsafe.Pointer(b), dataOffset+bucketCnt*8+i*uintptr(t.valuesize))
484+
insertb = b
485+
inserti = i
484486
goto done
485487
}
486488
ovf := b.overflow(t)
@@ -499,14 +501,14 @@ again:
499501
goto again // Growing the table invalidates everything, so try again
500502
}
501503

502-
if inserti == nil {
504+
if insertb == nil {
503505
// all current buckets are full, allocate a new one.
504-
newb := h.newoverflow(t, b)
505-
inserti = &newb.tophash[0]
506-
insertk = add(unsafe.Pointer(newb), dataOffset)
507-
val = add(insertk, bucketCnt*8)
506+
insertb = h.newoverflow(t, b)
507+
inserti = 0 // not necessary, but avoids needlessly spilling inserti
508508
}
509+
insertb.tophash[inserti&(bucketCnt-1)] = tophash(hash) // mask inserti to avoid bounds checks
509510

511+
insertk = add(unsafe.Pointer(insertb), dataOffset+inserti*8)
510512
// store new key at insert position
511513
if t.key.kind&kindNoPointers == 0 && writeBarrier.enabled {
512514
if sys.PtrSize == 8 {
@@ -520,18 +522,18 @@ again:
520522
*(*uint64)(insertk) = key
521523
}
522524

523-
*inserti = tophash(hash)
524525
h.count++
525526

526527
done:
528+
val := add(unsafe.Pointer(insertb), dataOffset+bucketCnt*8+inserti*uintptr(t.valuesize))
527529
if h.flags&hashWriting == 0 {
528530
throw("concurrent map writes")
529531
}
530532
h.flags &^= hashWriting
531533
return val
532534
}
533535

534-
func mapassign_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer {
536+
func mapassign_faststr(t *maptype, h *hmap, s string) unsafe.Pointer {
535537
if h == nil {
536538
panic(plainError("assignment to entry in nil map"))
537539
}
@@ -542,8 +544,8 @@ func mapassign_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer {
542544
if h.flags&hashWriting != 0 {
543545
throw("concurrent map writes")
544546
}
545-
key := stringStructOf(&ky)
546-
hash := t.key.alg.hash(noescape(unsafe.Pointer(&ky)), uintptr(h.hash0))
547+
key := stringStructOf(&s)
548+
hash := t.key.alg.hash(noescape(unsafe.Pointer(&s)), uintptr(h.hash0))
547549

548550
// Set hashWriting after calling alg.hash for consistency with mapassign.
549551
h.flags |= hashWriting
@@ -560,16 +562,16 @@ again:
560562
b := (*bmap)(unsafe.Pointer(uintptr(h.buckets) + bucket*uintptr(t.bucketsize)))
561563
top := tophash(hash)
562564

563-
var inserti *uint8
565+
var insertb *bmap
566+
var inserti uintptr
564567
var insertk unsafe.Pointer
565-
var val unsafe.Pointer
568+
566569
for {
567570
for i := uintptr(0); i < bucketCnt; i++ {
568571
if b.tophash[i] != top {
569-
if b.tophash[i] == empty && inserti == nil {
570-
inserti = &b.tophash[i]
571-
insertk = add(unsafe.Pointer(b), dataOffset+i*2*sys.PtrSize)
572-
val = add(unsafe.Pointer(b), dataOffset+bucketCnt*2*sys.PtrSize+i*uintptr(t.valuesize))
572+
if b.tophash[i] == empty && insertb == nil {
573+
insertb = b
574+
inserti = i
573575
}
574576
continue
575577
}
@@ -581,7 +583,8 @@ again:
581583
continue
582584
}
583585
// already have a mapping for key. Update it.
584-
val = add(unsafe.Pointer(b), dataOffset+bucketCnt*2*sys.PtrSize+i*uintptr(t.valuesize))
586+
inserti = i
587+
insertb = b
585588
goto done
586589
}
587590
ovf := b.overflow(t)
@@ -600,20 +603,20 @@ again:
600603
goto again // Growing the table invalidates everything, so try again
601604
}
602605

603-
if inserti == nil {
606+
if insertb == nil {
604607
// all current buckets are full, allocate a new one.
605-
newb := h.newoverflow(t, b)
606-
inserti = &newb.tophash[0]
607-
insertk = add(unsafe.Pointer(newb), dataOffset)
608-
val = add(insertk, bucketCnt*2*sys.PtrSize)
608+
insertb = h.newoverflow(t, b)
609+
inserti = 0 // not necessary, but avoids needlessly spilling inserti
609610
}
611+
insertb.tophash[inserti&(bucketCnt-1)] = top // mask inserti to avoid bounds checks
610612

613+
insertk = add(unsafe.Pointer(insertb), dataOffset+inserti*2*sys.PtrSize)
611614
// store new key at insert position
612615
*((*stringStruct)(insertk)) = *key
613-
*inserti = top
614616
h.count++
615617

616618
done:
619+
val := add(unsafe.Pointer(insertb), dataOffset+bucketCnt*2*sys.PtrSize+inserti*uintptr(t.valuesize))
617620
if h.flags&hashWriting == 0 {
618621
throw("concurrent map writes")
619622
}

0 commit comments

Comments
 (0)