sql/sqlbase: TestKeyAddress failed under stress

[cockroach-teamcity]

SHA: https://github.com/cockroachdb/cockroach/commits/382709bd132741c65ef9759c02b08d52b7510d86

Parameters:

COCKROACH_PROPOSER_EVALUATED_KV=true
TAGS=
GOFLAGS=

Stress build found a failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=163184&tab=buildLog

runtime stack:
runtime.throw(0x1aea9b9, 0x1a)
	/usr/local/go/src/runtime/panic.go:596 +0x95 fp=0x2b2d7ac00a70 sp=0x2b2d7ac00a50
runtime.notewakeup(0x2b4e2d8)
	/usr/local/go/src/runtime/lock_futex.go:133 +0xb4 fp=0x2b2d7ac00a98 sp=0x2b2d7ac00a70
runtime.gchelper()
	/usr/local/go/src/runtime/mgc.go:1923 +0xf5 fp=0x2b2d7ac00ac8 sp=0x2b2d7ac00a98
runtime.stopm()
	/usr/local/go/src/runtime/proc.go:1653 +0xdd fp=0x2b2d7ac00af0 sp=0x2b2d7ac00ac8
runtime.findrunnable(0xc420022600, 0x0)
	/usr/local/go/src/runtime/proc.go:2102 +0x2e4 fp=0x2b2d7ac00b88 sp=0x2b2d7ac00af0
runtime.schedule()
	/usr/local/go/src/runtime/proc.go:2222 +0x14c fp=0x2b2d7ac00bc8 sp=0x2b2d7ac00b88
runtime.park_m(0xc42039b040)
	/usr/local/go/src/runtime/proc.go:2285 +0xab fp=0x2b2d7ac00c00 sp=0x2b2d7ac00bc8
runtime.mcall(0x2b2d77b842c0)
	/usr/local/go/src/runtime/asm_amd64.s:269 +0x5b fp=0x2b2d7ac00c10 sp=0x2b2d7ac00c00

goroutine 1 [chan receive]:
runtime.gopark(0x1b3acc8, 0xc42032a0b8, 0x1ad9fc1, 0xc, 0xc42001e117, 0x3)
	/usr/local/go/src/runtime/proc.go:271 +0x13a fp=0xc4201c5af0 sp=0xc4201c5ac0
runtime.goparkunlock(0xc42032a0b8, 0x1ad9fc1, 0xc, 0x17, 0x3)
	/usr/local/go/src/runtime/proc.go:277 +0x5e fp=0xc4201c5b30 sp=0xc4201c5af0
runtime.chanrecv(0x18d1680, 0xc42032a060, 0x0, 0xc4201c5c01, 0x77492a)
	/usr/local/go/src/runtime/chan.go:513 +0x371 fp=0xc4201c5bd0 sp=0xc4201c5b30
runtime.chanrecv1(0x18d1680, 0xc42032a060, 0x0)
	/usr/local/go/src/runtime/chan.go:395 +0x35 fp=0xc4201c5c08 sp=0xc4201c5bd0
testing.(*T).Run(0xc4203aeb60, 0x1adc03f, 0xe, 0x1b38ef0, 0xc4200bfc01)
	/usr/local/go/src/testing/testing.go:698 +0x2f4 fp=0xc4201c5cb0 sp=0xc4201c5c08
testing.runTests.func1(0xc4203aeb60)
	/usr/local/go/src/testing/testing.go:882 +0x67 fp=0xc4201c5d00 sp=0xc4201c5cb0
testing.tRunner(0xc4203aeb60, 0xc4200bfdb0)
	/usr/local/go/src/testing/testing.go:657 +0x96 fp=0xc4201c5d28 sp=0xc4201c5d00
testing.runTests(0xc4203a8e20, 0x2650ee0, 0x16, 0x16, 0x2b4c640)
	/usr/local/go/src/testing/testing.go:888 +0x2c1 fp=0xc4201c5de0 sp=0xc4201c5d28
testing.(*M).Run(0xc4201c5f20, 0x2b90f40)
	/usr/local/go/src/testing/testing.go:822 +0xfc fp=0xc4201c5ed0 sp=0xc4201c5de0
github.com/cockroachdb/cockroach/pkg/sql/sqlbase_test.TestMain(0xc4200bff20)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/main_test.go:34 +0x8e fp=0xc4201c5f00 sp=0xc4201c5ed0
main.main()
	github.com/cockroachdb/cockroach/pkg/sql/sqlbase/_test/_testmain.go:84 +0xf7 fp=0xc4201c5f88 sp=0xc4201c5f00
runtime.main()
	/usr/local/go/src/runtime/proc.go:185 +0x20a fp=0xc4201c5fe0 sp=0xc4201c5f88
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc4201c5fe8 sp=0xc4201c5fe0

goroutine 17 [syscall, locked to thread]:
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc420042fe8 sp=0xc420042fe0

goroutine 2 [force gc (idle)]:
runtime.gopark(0x1b3acc8, 0x2b4b650, 0x1add4b8, 0xf, 0x1b3ab14, 0x1)
	/usr/local/go/src/runtime/proc.go:271 +0x13a fp=0xc420030768 sp=0xc420030738
runtime.goparkunlock(0x2b4b650, 0x1add4b8, 0xf, 0xc420000114, 0x1)
	/usr/local/go/src/runtime/proc.go:277 +0x5e fp=0xc4200307a8 sp=0xc420030768
runtime.forcegchelper()
	/usr/local/go/src/runtime/proc.go:226 +0x9e fp=0xc4200307e0 sp=0xc4200307a8
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc4200307e8 sp=0xc4200307e0
created by runtime.init.4
	/usr/local/go/src/runtime/proc.go:215 +0x35

goroutine 3 [runnable]:
runtime.gopark(0x1b3acc8, 0x2b4b940, 0x1adac0a, 0xd, 0x6db714, 0x1)
	/usr/local/go/src/runtime/proc.go:271 +0x13a fp=0xc420030f58 sp=0xc420030f28
runtime.goparkunlock(0x2b4b940, 0x1adac0a, 0xd, 0x14, 0x1)
	/usr/local/go/src/runtime/proc.go:277 +0x5e fp=0xc420030f98 sp=0xc420030f58
runtime.bgsweep(0xc42005e000)
	/usr/local/go/src/runtime/mgcsweep.go:72 +0x12e fp=0xc420030fd8 sp=0xc420030f98
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc420030fe0 sp=0xc420030fd8
created by runtime.gcenable
	/usr/local/go/src/runtime/mgc.go:212 +0x61

goroutine 4 [finalizer wait]:
runtime.gopark(0x1b3acc8, 0x2b91370, 0x1adc36b, 0xe, 0x14, 0x1)
	/usr/local/go/src/runtime/proc.go:271 +0x13a fp=0xc420031718 sp=0xc4200316e8
runtime.goparkunlock(0x2b91370, 0x1adc36b, 0xe, 0x14, 0x1)
	/usr/local/go/src/runtime/proc.go:277 +0x5e fp=0xc420031758 sp=0xc420031718
runtime.runfinq()
	/usr/local/go/src/runtime/mfinal.go:161 +0xb2 fp=0xc4200317e0 sp=0xc420031758
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc4200317e8 sp=0xc4200317e0
created by runtime.createfing
	/usr/local/go/src/runtime/mfinal.go:142 +0x62

goroutine 5 [chan receive]:
runtime.gopark(0x1b3acc8, 0xc4202d0058, 0x1ad9fc1, 0xc, 0xc4202dc017, 0x3)
	/usr/local/go/src/runtime/proc.go:271 +0x13a fp=0xc420031e70 sp=0xc420031e40
runtime.goparkunlock(0xc4202d0058, 0x1ad9fc1, 0xc, 0x17, 0x3)
	/usr/local/go/src/runtime/proc.go:277 +0x5e fp=0xc420031eb0 sp=0xc420031e70
runtime.chanrecv(0x18d1340, 0xc4202d0000, 0xc420031fb0, 0x6fc23ac01, 0xc4202d4000)
	/usr/local/go/src/runtime/chan.go:513 +0x371 fp=0xc420031f50 sp=0xc420031eb0
runtime.chanrecv2(0x18d1340, 0xc4202d0000, 0xc420031fb0, 0x0)
	/usr/local/go/src/runtime/chan.go:400 +0x35 fp=0xc420031f88 sp=0xc420031f50
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).flushDaemon(0x2b4dec0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1012 +0x77 fp=0xc420031fd8 sp=0xc420031f88
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc420031fe0 sp=0xc420031fd8
created by github.com/cockroachdb/cockroach/pkg/util/log.init.1
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:583 +0xd2

goroutine 6 [chan receive]:
runtime.gopark(0x1b3acc8, 0xc42023cfb8, 0x1ad9fc1, 0xc, 0xc4203c6017, 0x3)
	/usr/local/go/src/runtime/proc.go:271 +0x13a fp=0xc420032688 sp=0xc420032658
runtime.goparkunlock(0xc42023cfb8, 0x1ad9fc1, 0xc, 0x17, 0x3)
	/usr/local/go/src/runtime/proc.go:277 +0x5e fp=0xc4200326c8 sp=0xc420032688
runtime.chanrecv(0x18d20c0, 0xc42023cf60, 0xc4200327c0, 0x1, 0x0)
	/usr/local/go/src/runtime/chan.go:513 +0x371 fp=0xc420032768 sp=0xc4200326c8
runtime.chanrecv2(0x18d20c0, 0xc42023cf60, 0xc4200327c0, 0x0)
	/usr/local/go/src/runtime/chan.go:400 +0x35 fp=0xc4200327a0 sp=0xc420032768
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0x2b4dec0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1039 +0x5b fp=0xc4200327d8 sp=0xc4200327a0
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc4200327e0 sp=0xc4200327d8
created by github.com/cockroachdb/cockroach/pkg/util/log.init.1
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:584 +0xf6

goroutine 18 [syscall]:
runtime.notetsleepg(0x2b4c338, 0x12a05a637, 0x1)
	/usr/local/go/src/runtime/lock_futex.go:205 +0x42 fp=0xc42002c760 sp=0xc42002c730
runtime.timerproc()
	/usr/local/go/src/runtime/time.go:209 +0x327 fp=0xc42002c7e0 sp=0xc42002c760
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc42002c7e8 sp=0xc42002c7e0
created by runtime.addtimerLocked
	/usr/local/go/src/runtime/time.go:116 +0xed

goroutine 9 [GC worker (idle)]:
runtime.gopark(0x1b3ab48, 0xc420377ec0, 0x1addf05, 0x10, 0x258ac614, 0x0)
	/usr/local/go/src/runtime/proc.go:271 +0x13a fp=0xc420033750 sp=0xc420033720
runtime.gcBgMarkWorker(0xc420020000)
	/usr/local/go/src/runtime/mgc.go:1491 +0x138 fp=0xc4200337d8 sp=0xc420033750
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc4200337e0 sp=0xc4200337d8
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1412 +0x98

goroutine 10 [GC worker (idle)]:
runtime.gopark(0x1b3ab48, 0xc420377ed0, 0x1addf05, 0x10, 0x14, 0x0)
	/usr/local/go/src/runtime/proc.go:271 +0x13a fp=0xc420033f50 sp=0xc420033f20
runtime.gcBgMarkWorker(0xc420021300)
	/usr/local/go/src/runtime/mgc.go:1491 +0x138 fp=0xc420033fd8 sp=0xc420033f50
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc420033fe0 sp=0xc420033fd8
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1412 +0x98

goroutine 11 [GC worker (idle)]:
runtime.gopark(0x1b3ab48, 0xc420377ee0, 0x1addf05, 0x10, 0x14, 0x0)
	/usr/local/go/src/runtime/proc.go:271 +0x13a fp=0xc4205d6750 sp=0xc4205d6720
runtime.gcBgMarkWorker(0xc420022600)
	/usr/local/go/src/runtime/mgc.go:1491 +0x138 fp=0xc4205d67d8 sp=0xc4205d6750
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc4205d67e0 sp=0xc4205d67d8
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1412 +0x98

goroutine 12 [GC worker (idle)]:
runtime.gopark(0x1b3ab48, 0xc420377ef0, 0x1addf05, 0x10, 0x14, 0x0)
	/usr/local/go/src/runtime/proc.go:271 +0x13a fp=0xc4205d6f50 sp=0xc4205d6f20
runtime.gcBgMarkWorker(0xc420023900)
	/usr/local/go/src/runtime/mgc.go:1491 +0x138 fp=0xc4205d6fd8 sp=0xc4205d6f50
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc4205d6fe0 sp=0xc4205d6fd8
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1412 +0x98

goroutine 19 [GC worker (idle)]:
runtime.gopark(0x1b3ab48, 0xc4205da000, 0x1addf05, 0x10, 0x14, 0x0)
	/usr/local/go/src/runtime/proc.go:271 +0x13a fp=0xc42002cf50 sp=0xc42002cf20
runtime.gcBgMarkWorker(0xc420024c00)
	/usr/local/go/src/runtime/mgc.go:1491 +0x138 fp=0xc42002cfd8 sp=0xc42002cf50
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc42002cfe0 sp=0xc42002cfd8
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1412 +0x98

goroutine 20 [GC worker (idle)]:
runtime.gopark(0x1b3ab48, 0xc4205da010, 0x1addf05, 0x10, 0x14, 0x0)
	/usr/local/go/src/runtime/proc.go:271 +0x13a fp=0xc42002d750 sp=0xc42002d720
runtime.gcBgMarkWorker(0xc420026000)
	/usr/local/go/src/runtime/mgc.go:1491 +0x138 fp=0xc42002d7d8 sp=0xc42002d750
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc42002d7e0 sp=0xc42002d7d8
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1412 +0x98

goroutine 21 [GC worker (idle)]:
runtime.gopark(0x1b3ab48, 0xc4205da020, 0x1addf05, 0x10, 0x14, 0x0)
	/usr/local/go/src/runtime/proc.go:271 +0x13a fp=0xc42002df50 sp=0xc42002df20
runtime.gcBgMarkWorker(0xc420027300)
	/usr/local/go/src/runtime/mgc.go:1491 +0x138 fp=0xc42002dfd8 sp=0xc42002df50
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc42002dfe0 sp=0xc42002dfd8
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1412 +0x98

goroutine 22 [GC worker (idle)]:
runtime.gopark(0x1b3ab48, 0xc4205da030, 0x1addf05, 0x10, 0x14, 0x0)
	/usr/local/go/src/runtime/proc.go:271 +0x13a fp=0xc42002e750 sp=0xc42002e720
runtime.gcBgMarkWorker(0xc420028600)
	/usr/local/go/src/runtime/mgc.go:1491 +0x138 fp=0xc42002e7d8 sp=0xc42002e750
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc42002e7e0 sp=0xc42002e7d8
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1412 +0x98

goroutine 50 [running]:
	goroutine running on other thread; stack unavailable
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:697 +0x2ca


ERROR: exit status 2

Makefile:177: recipe for target 'stress' failed
make: *** [stress] Error 1

[a-robinson]

Seems like a bug in the go runtime?

[02:43:10][Step 2/2] === RUN   TestEncDatumRowAlloc
[02:43:10][Step 2/2] notewakeup - double wakeup (1)
[02:43:10][Step 2/2] fatal error: notewakeup - double wakeup

The most recent references I can find to this are from 2014, though. Closing unless we see again.

[a-robinson]

(the reference being golang/go#5139)

[tamird]

Do you plan to report this upstream? Doesn't seem that the Go team is aware that this is still an issue.

…

On Mon, Feb 27, 2017 at 10:46 AM, Alex Robinson ***@***.***> wrote: (the reference being golang/go#5139 <golang/go#5139>) — You are receiving this because you are subscribed to this thread. Reply to this email directly, view it on GitHub <#13814 (comment)>, or mute the thread <https://github.com/notifications/unsubscribe-auth/ABdsPHtvOSZqKs3bURXB6zbhOIHbdOKMks5rgu_wgaJpZM4MMyVp> .

[a-robinson]

Sure, although we have no idea whether it's reproducible and it's far from a minimal test setup. Opened golang/go#19305

[tamird]

Yep. Thanks.

…

On Mon, Feb 27, 2017 at 12:56 PM, Alex Robinson ***@***.***> wrote: Sure, although we have no idea whether it's reproducible and it's far from a minimal test setup. Opened golang/go#19305 <golang/go#19305> — You are receiving this because you commented. Reply to this email directly, view it on GitHub <#13814 (comment)>, or mute the thread <https://github.com/notifications/unsubscribe-auth/ABdsPOasIy_NmeAZr22N4p71q_eQz9COks5rgw42gaJpZM4MMyVp> .