Skip to content

Commit b47f2fe

Browse files
randall77randall77
committed
runtime,hash/maphash: reuse hashSets to save memory pressure
Might help with OOMs on 32-bit platforms Change-Id: Idd5129c61ecdfeedd5a9a18fce85dbba27cab946 Reviewed-on: https://go-review.googlesource.com/c/go/+/574475 Reviewed-by: Emmanuel Odeke <[email protected]> Reviewed-by: Keith Randall <[email protected]> LUCI-TryBot-Result: Go LUCI <[email protected]> Reviewed-by: Than McIntosh <[email protected]>
1 parent d3e5e9f commit b47f2fe

File tree

2 files changed

+58
-54
lines changed

2 files changed

+58
-54
lines changed

‎src/hash/maphash/smhasher_test.go

Lines changed: 27 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,8 @@ func (s *hashSet) check(t *testing.T) {
112112
if float64(collisions) > expected+SLOP*(3*stddev+1) {
113113
t.Errorf("unexpected number of collisions: got=%d mean=%f stddev=%f", collisions, expected, stddev)
114114
}
115+
// Reset for reuse
116+
s.list = s.list[:0]
115117
}
116118

117119
// a string plus adding zeros must make distinct hashes
@@ -212,8 +214,8 @@ func TestSmhasherCyclic(t *testing.T) {
212214
r := rand.New(rand.NewSource(1234))
213215
const REPEAT = 8
214216
const N = 1000000
217+
h := newHashSet()
215218
for n := 4; n <= 12; n++ {
216-
h := newHashSet()
217219
b := make([]byte, REPEAT*n)
218220
for i := 0; i < N; i++ {
219221
b[0] = byte(i * 79 % 97)
@@ -238,18 +240,18 @@ func TestSmhasherSparse(t *testing.T) {
238240
if testing.Short() {
239241
t.Skip("Skipping in short mode")
240242
}
241-
sparse(t, 32, 6)
242-
sparse(t, 40, 6)
243-
sparse(t, 48, 5)
244-
sparse(t, 56, 5)
245-
sparse(t, 64, 5)
246-
sparse(t, 96, 4)
247-
sparse(t, 256, 3)
248-
sparse(t, 2048, 2)
249-
}
250-
func sparse(t *testing.T, n int, k int) {
251-
b := make([]byte, n/8)
252243
h := newHashSet()
244+
sparse(t, h, 32, 6)
245+
sparse(t, h, 40, 6)
246+
sparse(t, h, 48, 5)
247+
sparse(t, h, 56, 5)
248+
sparse(t, h, 64, 5)
249+
sparse(t, h, 96, 4)
250+
sparse(t, h, 256, 3)
251+
sparse(t, h, 2048, 2)
252+
}
253+
func sparse(t *testing.T, h *hashSet, n int, k int) {
254+
b := make([]byte, n/8)
253255
setbits(h, b, 0, k)
254256
h.check(t)
255257
}
@@ -276,15 +278,15 @@ func TestSmhasherPermutation(t *testing.T) {
276278
if testing.Short() {
277279
t.Skip("Skipping in short mode")
278280
}
279-
permutation(t, []uint32{0, 1, 2, 3, 4, 5, 6, 7}, 8)
280-
permutation(t, []uint32{0, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 8)
281-
permutation(t, []uint32{0, 1}, 20)
282-
permutation(t, []uint32{0, 1 << 31}, 20)
283-
permutation(t, []uint32{0, 1, 2, 3, 4, 5, 6, 7, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 6)
281+
h := newHashSet()
282+
permutation(t, h, []uint32{0, 1, 2, 3, 4, 5, 6, 7}, 8)
283+
permutation(t, h, []uint32{0, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 8)
284+
permutation(t, h, []uint32{0, 1}, 20)
285+
permutation(t, h, []uint32{0, 1 << 31}, 20)
286+
permutation(t, h, []uint32{0, 1, 2, 3, 4, 5, 6, 7, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 6)
284287
}
285-
func permutation(t *testing.T, s []uint32, n int) {
288+
func permutation(t *testing.T, h *hashSet, s []uint32, n int) {
286289
b := make([]byte, n*4)
287-
h := newHashSet()
288290
genPerm(h, b, s, 0)
289291
h.check(t)
290292
}
@@ -418,8 +420,8 @@ func windowed(t *testing.T, k key) {
418420
}
419421
const BITS = 16
420422

423+
h := newHashSet()
421424
for r := 0; r < k.bits(); r++ {
422-
h := newHashSet()
423425
for i := 0; i < 1<<BITS; i++ {
424426
k.clear()
425427
for j := 0; j < BITS; j++ {
@@ -438,18 +440,18 @@ func TestSmhasherText(t *testing.T) {
438440
if testing.Short() {
439441
t.Skip("Skipping in short mode")
440442
}
441-
text(t, "Foo", "Bar")
442-
text(t, "FooBar", "")
443-
text(t, "", "FooBar")
443+
h := newHashSet()
444+
text(t, h, "Foo", "Bar")
445+
text(t, h, "FooBar", "")
446+
text(t, h, "", "FooBar")
444447
}
445-
func text(t *testing.T, prefix, suffix string) {
448+
func text(t *testing.T, h *hashSet, prefix, suffix string) {
446449
const N = 4
447450
const S = "ABCDEFGHIJKLMNOPQRSTabcdefghijklmnopqrst0123456789"
448451
const L = len(S)
449452
b := make([]byte, len(prefix)+N+len(suffix))
450453
copy(b, prefix)
451454
copy(b[len(prefix)+N:], suffix)
452-
h := newHashSet()
453455
c := b[len(prefix):]
454456
for i := 0; i < L; i++ {
455457
c[0] = S[i]

‎src/runtime/hash_test.go

Lines changed: 31 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,8 @@ func (s *HashSet) check(t *testing.T) {
121121
if float64(collisions) > expected+SLOP*(3*stddev+1) {
122122
t.Errorf("unexpected number of collisions: got=%d mean=%f stddev=%f threshold=%f", collisions, expected, stddev, expected+SLOP*(3*stddev+1))
123123
}
124+
// Reset for reuse
125+
s.list = s.list[:0]
124126
}
125127

126128
// a string plus adding zeros must make distinct hashes
@@ -230,8 +232,8 @@ func TestSmhasherCyclic(t *testing.T) {
230232
r := rand.New(rand.NewSource(1234))
231233
const REPEAT = 8
232234
const N = 1000000
235+
h := newHashSet()
233236
for n := 4; n <= 12; n++ {
234-
h := newHashSet()
235237
b := make([]byte, REPEAT*n)
236238
for i := 0; i < N; i++ {
237239
b[0] = byte(i * 79 % 97)
@@ -256,18 +258,18 @@ func TestSmhasherSparse(t *testing.T) {
256258
if testing.Short() {
257259
t.Skip("Skipping in short mode")
258260
}
259-
sparse(t, 32, 6)
260-
sparse(t, 40, 6)
261-
sparse(t, 48, 5)
262-
sparse(t, 56, 5)
263-
sparse(t, 64, 5)
264-
sparse(t, 96, 4)
265-
sparse(t, 256, 3)
266-
sparse(t, 2048, 2)
267-
}
268-
func sparse(t *testing.T, n int, k int) {
269-
b := make([]byte, n/8)
270261
h := newHashSet()
262+
sparse(t, h, 32, 6)
263+
sparse(t, h, 40, 6)
264+
sparse(t, h, 48, 5)
265+
sparse(t, h, 56, 5)
266+
sparse(t, h, 64, 5)
267+
sparse(t, h, 96, 4)
268+
sparse(t, h, 256, 3)
269+
sparse(t, h, 2048, 2)
270+
}
271+
func sparse(t *testing.T, h *HashSet, n int, k int) {
272+
b := make([]byte, n/8)
271273
setbits(h, b, 0, k)
272274
h.check(t)
273275
}
@@ -297,15 +299,15 @@ func TestSmhasherPermutation(t *testing.T) {
297299
if race.Enabled {
298300
t.Skip("Too long for race mode")
299301
}
300-
permutation(t, []uint32{0, 1, 2, 3, 4, 5, 6, 7}, 8)
301-
permutation(t, []uint32{0, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 8)
302-
permutation(t, []uint32{0, 1}, 20)
303-
permutation(t, []uint32{0, 1 << 31}, 20)
304-
permutation(t, []uint32{0, 1, 2, 3, 4, 5, 6, 7, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 6)
302+
h := newHashSet()
303+
permutation(t, h, []uint32{0, 1, 2, 3, 4, 5, 6, 7}, 8)
304+
permutation(t, h, []uint32{0, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 8)
305+
permutation(t, h, []uint32{0, 1}, 20)
306+
permutation(t, h, []uint32{0, 1 << 31}, 20)
307+
permutation(t, h, []uint32{0, 1, 2, 3, 4, 5, 6, 7, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 6)
305308
}
306-
func permutation(t *testing.T, s []uint32, n int) {
309+
func permutation(t *testing.T, h *HashSet, s []uint32, n int) {
307310
b := make([]byte, n*4)
308-
h := newHashSet()
309311
genPerm(h, b, s, 0)
310312
h.check(t)
311313
}
@@ -542,14 +544,15 @@ func TestSmhasherWindowed(t *testing.T) {
542544
if race.Enabled {
543545
t.Skip("Too long for race mode")
544546
}
547+
h := newHashSet()
545548
t.Logf("32 bit keys")
546-
windowed(t, &Int32Key{})
549+
windowed(t, h, &Int32Key{})
547550
t.Logf("64 bit keys")
548-
windowed(t, &Int64Key{})
551+
windowed(t, h, &Int64Key{})
549552
t.Logf("string keys")
550-
windowed(t, &BytesKey{make([]byte, 128)})
553+
windowed(t, h, &BytesKey{make([]byte, 128)})
551554
}
552-
func windowed(t *testing.T, k Key) {
555+
func windowed(t *testing.T, h *HashSet, k Key) {
553556
if GOARCH == "wasm" {
554557
t.Skip("Too slow on wasm")
555558
}
@@ -566,7 +569,6 @@ func windowed(t *testing.T, k Key) {
566569
const BITS = 16
567570

568571
for r := 0; r < k.bits(); r++ {
569-
h := newHashSet()
570572
for i := 0; i < 1<<BITS; i++ {
571573
k.clear()
572574
for j := 0; j < BITS; j++ {
@@ -585,18 +587,18 @@ func TestSmhasherText(t *testing.T) {
585587
if testing.Short() {
586588
t.Skip("Skipping in short mode")
587589
}
588-
text(t, "Foo", "Bar")
589-
text(t, "FooBar", "")
590-
text(t, "", "FooBar")
590+
h := newHashSet()
591+
text(t, h, "Foo", "Bar")
592+
text(t, h, "FooBar", "")
593+
text(t, h, "", "FooBar")
591594
}
592-
func text(t *testing.T, prefix, suffix string) {
595+
func text(t *testing.T, h *HashSet, prefix, suffix string) {
593596
const N = 4
594597
const S = "ABCDEFGHIJKLMNOPQRSTabcdefghijklmnopqrst0123456789"
595598
const L = len(S)
596599
b := make([]byte, len(prefix)+N+len(suffix))
597600
copy(b, prefix)
598601
copy(b[len(prefix)+N:], suffix)
599-
h := newHashSet()
600602
c := b[len(prefix):]
601603
for i := 0; i < L; i++ {
602604
c[0] = S[i]

0 commit comments

Comments
 (0)