sql/tests: TestRandomSyntaxSQLSmith failed [stack overflow in `sql.(*routineGenerator).startInternal`]

[cockroach-teamcity]

sql/tests.TestRandomSyntaxSQLSmith failed with artifacts on master @ 9bfbb28ab9c20847b872d2a92ec7201e570eddb0:

runtime.selectgo(0xc03b267e88, 0xc03b267e50, 0xc031dcb4a0?, 0x0, 0x0?, 0x1)
	GOROOT/src/runtime/select.go:335 +0x7a5 fp=0xc03b267e18 sp=0xc03b267cf0 pc=0x467f05
github.com/cockroachdb/cockroach/pkg/kv/kvserver/storeliveness.(*SupportManager).startLoop(0xc0313981e0, {0x87f1448, 0xc031dcb4a0})
	pkg/kv/kvserver/storeliveness/support_manager.go:262 +0x2c9 fp=0xc03b267f58 sp=0xc03b267e18 pc=0x4f59a69
github.com/cockroachdb/cockroach/pkg/kv/kvserver/storeliveness.(*SupportManager).Start.func1({0x87f1448, 0xc031dcb4a0})
	pkg/kv/kvserver/storeliveness/support_manager.go:189 +0x8a fp=0xc03b267fc0 sp=0xc03b267f58 pc=0x4f5924a
github.com/cockroachdb/cockroach/pkg/kv/kvserver/storeliveness.(*SupportManager).Start.gowrap1()
	pkg/kv/kvserver/storeliveness/support_manager.go:190 +0x28 fp=0xc03b267fe0 sp=0xc03b267fc0 pc=0x4f59188
runtime.goexit({})
	src/runtime/asm_amd64.s:1700 +0x1 fp=0xc03b267fe8 sp=0xc03b267fe0 pc=0x495dc1
created by github.com/cockroachdb/cockroach/pkg/kv/kvserver/storeliveness.(*SupportManager).Start in goroutine 12958383
	pkg/kv/kvserver/storeliveness/support_manager.go:187 +0x130

goroutine 12959781 gp=0xc0357edc00 m=nil [select]:
runtime.gopark(0xc0098d3e78?, 0x2?, 0x69?, 0xb1?, 0xc0098d3e04?)
	GOROOT/src/runtime/proc.go:424 +0xce fp=0xc0098d3c78 sp=0xc0098d3c58 pc=0x48cbee
runtime.selectgo(0xc0098d3e78, 0xc0098d3e00, 0x8?, 0x0, 0x1?, 0x1)
	GOROOT/src/runtime/select.go:335 +0x7a5 fp=0xc0098d3da0 sp=0xc0098d3c78 pc=0x467f05
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc005b1fdc0, {0x87efc98, 0xc01775a690}, {0x8852120, 0xc02758dd10}, 0xc0215fa0e0)
	pkg/kv/kvclient/rangefeed/rangefeed.go:434 +0xe5 fp=0xc0098d3ea8 sp=0xc0098d3da0 pc=0x26ed4e5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func3({0x87efc98?, 0xc01775a690?})
	pkg/kv/kvclient/rangefeed/rangefeed.go:391 +0xf8 fp=0xc0098d3f08 sp=0xc0098d3ea8 pc=0x26ece78
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait.Group.GoCtx.func1()
	pkg/util/ctxgroup/ctxgroup.go:200 +0x8b fp=0xc0098d3f78 sp=0xc0098d3f08 pc=0x1c02ceb
golang.org/x/sync/errgroup.(*Group).Go.func1()
	external/org_golang_x_sync/errgroup/errgroup.go:93 +0x50 fp=0xc0098d3fe0 sp=0xc0098d3f78 pc=0x1c01a70
runtime.goexit({})
	src/runtime/asm_amd64.s:1700 +0x1 fp=0xc0098d3fe8 sp=0xc0098d3fe0 pc=0x495dc1
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 12959813
	external/org_golang_x_sync/errgroup/errgroup.go:78 +0x96

goroutine 12958615 gp=0xc0357eddc0 m=nil [sync.Cond.Wait]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	GOROOT/src/runtime/proc.go:424 +0xce fp=0xc0154b5e08 sp=0xc0154b5de8 pc=0x48cbee
runtime.goparkunlock(...)
	GOROOT/src/runtime/proc.go:430
sync.runtime_notifyListWait(0xc0629e6990, 0x67f)
	GOROOT/src/runtime/sema.go:587 +0x159 fp=0xc0154b5e58 sp=0xc0154b5e08 pc=0x48e8b9
sync.(*Cond).Wait(0x6b949c0?)
	GOROOT/src/sync/cond.go:71 +0x85 fp=0xc0154b5e98 sp=0xc0154b5e58 pc=0x4a16a5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc02af3b810, {0x87f1448, 0xc02edddcc0}, {0x881c320, 0xc031df6008}, 0xc031db6008)
	pkg/kv/kvserver/scheduler.go:378 +0x385 fp=0xc0154b5f18 sp=0xc0154b5e98 pc=0x5085865
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x87f1448, 0xc02edddcc0}, 0xc031dcb4a0?)
	pkg/kv/kvserver/scheduler.go:314 +0xe9 fp=0xc0154b5fb8 sp=0xc0154b5f18 pc=0x5085029
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.gowrap1()
	pkg/kv/kvserver/scheduler.go:328 +0x2c fp=0xc0154b5fe0 sp=0xc0154b5fb8 pc=0x5084f0c
runtime.goexit({})
	src/runtime/asm_amd64.s:1700 +0x1 fp=0xc0154b5fe8 sp=0xc0154b5fe0 pc=0x495dc1
created by github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start in goroutine 12958383
	pkg/kv/kvserver/scheduler.go:328 +0x197

Help

See also: How To Investigate a Go Test Failure (internal)

/cc @cockroachdb/sql-foundations

This test on roachdash | Improve this report!

Jira issue: CRDB-55286

[rafiss]

This stack overflow came up before in #155036 -- @yuzefovich I think you were looking for another way to reproduce this. Unfortunately, I think this test framework may not have logged the query. Maybe it can be reproduced more easily by modifying the randomization to be biased in favor of plpgsql/DO.

runtime: goroutine stack exceeds 1000000000-byte limit
runtime: sp=0xc154a7cba8 stack=[0xc154a7c000, 0xc174a7c000]
fatal error: stack overflow

runtime stack:
runtime.throw({0x7812a0b?, 0x7fdbaebfe310?})
        GOROOT/src/runtime/panic.go:1073 +0x48 fp=0x7fdbaebfe2d0 sp=0x7fdbaebfe2a0 pc=0x48cac8
runtime.newstack()
        GOROOT/src/runtime/stack.go:1117 +0x5bd fp=0x7fdbaebfe410 sp=0x7fdbaebfe2d0 pc=0x46ebdd
runtime.morestack()
        src/runtime/asm_amd64.s:621 +0x7a fp=0x7fdbaebfe418 sp=0x7fdbaebfe410 pc=0x493f1a

goroutine 12971048 gp=0xc0092d1500 m=14 mp=0xc00353c008 [running]:
github.com/cockroachdb/cockroach/pkg/sql/rowexec.NewProcessor({0x87efc98, 0xc27feedd10}, 0xc282012508, 0x0, 0x1?, 0xc282016b60, 0xc28201a3f0, {0x0, 0x0, 0x0}, ...)
        pkg/sql/rowexec/processors.go:99 +0x2ba5 fp=0xc154a7cbb8 sp=0xc154a7cbb0 pc=0x3238825
github.com/cockroachdb/cockroach/pkg/sql/colexec/colbuilder.opResult.createAndWrapRowSource.func1({0x0?, 0x0?, 0x0?})
        pkg/sql/colexec/colbuilder/execplan.go:595 +0xaa fp=0xc154a7cc70 sp=0xc154a7cbb8 pc=0x3e49fca
github.com/cockroachdb/cockroach/pkg/sql/colexec/colbuilder.wrapRowSources({0x87efc98, 0xc27feedd10}, 0xc282012508, 0xc23597a700, {0xd4de3e0, 0x0, 0x0?}, {0xd4de3e0, 0x0, 0x0}, ...)
        pkg/sql/colexec/colbuilder/execplan.go:107 +0x52c fp=0xc154a7cdc8 sp=0xc154a7cc70 pc=0x3e473ac
github.com/cockroachdb/cockroach/pkg/sql/colexec/colbuilder.opResult.createAndWrapRowSource({0x0?}, {0x87efc98, 0xc27feedd10}, 0xc282012508, 0xc23597a700, {0xd4de3e0, 0x0, 0x0}, {0xd4de3e0, 0x0, ...}, ...)
        pkg/sql/colexec/colbuilder/execplan.go:583 +0x2e5 fp=0xc154a7cf28 sp=0xc154a7cdc8 pc=0x3e49785
github.com/cockroachdb/cockroach/pkg/sql/colexec/colbuilder.NewColOperator({0x87efc98, 0xc27feedd10}, 0xc282012508, 0xc23597a700)
        pkg/sql/colexec/colbuilder/execplan.go:821 +0x96f6 fp=0xc154a7dc78 sp=0xc154a7cf28 pc=0x3e54a76
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlowCreator).setupFlow.func1()
        pkg/sql/colflow/vectorized_flow.go:1193 +0x688 fp=0xc154a7df88 sp=0xc154a7dc78 pc=0x3e71a28
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc03bba1360?)
        pkg/sql/colexecerror/error.go:162 +0x4d fp=0xc154a7dfc8 sp=0xc154a7df88 pc=0xbf13cd
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlowCreator).setupFlow(0xc2819d8780, {0x87efc98?, 0xc27feedd10?}, {0xc282016b48?, 0x0?, 0x10000?}, 0x0?)
        pkg/sql/colflow/vectorized_flow.go:1131 +0xa5 fp=0xc154a7e040 sp=0xc154a7dfc8 pc=0x3e712c5
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Setup(0xc239a35c00, {0x87f1448?, 0xc281ff88c0?}, 0xc28198ba48, 0x1)
        pkg/sql/colflow/vectorized_flow.go:256 +0x2ae fp=0xc154a7e1c0 sp=0xc154a7e040 pc=0x3e6a34e
github.com/cockroachdb/cockroach/pkg/sql/distsql.(*ServerImpl).setupFlow(0xc03fd94408, {0x87f1448, 0xc281ff88c0}, 0xc282022008, 0xc023a92580, 0xc283389230, 0xc28198ba00, {0x87ac7c0, 0xc2819efb08}, {0x0, ...}, ...)
        pkg/sql/distsql/server.go:435 +0x18c5 fp=0xc154a7e8b0 sp=0xc154a7e1c0 pc=0x3e8c585
github.com/cockroachdb/cockroach/pkg/sql/distsql.(*ServerImpl).SetupLocalSyncFlow(0xc03fd94408, {0x87f1448, 0xc281ff8780}, 0xc023a92580,
 0xc28198ba00, {0x87ac7c0, 0xc2819efb08}, {0x0, 0x0}, {0xc282019008, ...})
        pkg/sql/distsql/server.go:605 +0x145 fp=0xc154a7e968 sp=0xc154a7e8b0 pc=0x3e8e385
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).setupFlows(0xc03f8e4dc0, {0x87f1448, 0xc281ff8780}, 0xc282019008, 0xc281964a9
0, 0x0, 0xc2820200c0, 0xc2819efb08, {0xc282019008, 0xc0314ffe00, ...}, ...)
        pkg/sql/distsql_running.go:530 +0xabf fp=0xc154a7f000 sp=0xc154a7e968 pc=0x431ec1f
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc03f8e4dc0, {0x87f1448, 0xc281ff8780}, 0xc281964a90, 0xc024664420, 0xc2
81765480, 0xc2819efb08, 0xc282019008, 0xc154a7f788)
        pkg/sql/distsql_running.go:991 +0xa07 fp=0xc154a7f5f0 sp=0xc154a7f000 pc=0x4320907
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc03f8e4dc0, {0x87f1448, 0xc281ff8780}, 0xc282019008, 0xc281964a9
0, 0xc024664420, {{0x8833030, 0xc281268ba0}, 0x0}, 0xc2819efb08, ...)
        pkg/sql/distsql_running.go:2252 +0x238 fp=0xc154a7f718 sp=0xc154a7f5f0 pc=0x43273d8
github.com/cockroachdb/cockroach/pkg/sql.runPlanInsidePlan.func1(0x0?, 0x0?, {0x87f1448?, 0xc281ff8780?}, 0x87f1448?, 0xc281ff8780?, 0xc
282019008?, 0xc282006808?)
        pkg/sql/apply_join.go:334 +0xb0 fp=0xc154a7f7b0 sp=0xc154a7f718 pc=0x4207b70
github.com/cockroachdb/cockroach/pkg/sql.runPlanInsidePlan({0x87f1448, 0xc281ff8780}, {{0x87f1448, 0xc281ff8780}, 0xc282005558, 0xc28200
5008}, 0xc28201a240, {0x87f1e20, 0xc2819e8300}, {0x87ac6a8, ...}, ...)
        pkg/sql/apply_join.go:337 +0x59e fp=0xc154a7f918 sp=0xc154a7f7b0 pc=0x42075fe
github.com/cockroachdb/cockroach/pkg/sql.(*routineGenerator).startInternal.func1({0x6b2b0a0, 0xc28201a240}, {0x0, 0x0}, 0x1)
        pkg/sql/routine.go:354 +0x445 fp=0xc154a7fa28 sp=0xc154a7f918 pc=0x442a2e5
github.com/cockroachdb/cockroach/pkg/sql/opt/exec/execbuilder.(*Builder).buildUDF.(*Builder).buildRoutinePlanGenerator.func2({0x87f1448,
 0xc281ff85a0}, {0x77298a0, 0xc2819e82e0}, {0x6e82d20, 0xc2819e8300}, {0xc2819e6750, 0x1, 0x1}, 0xc27feedcc0)
        pkg/sql/opt/exec/execbuilder/scalar.go:1327 +0x9d9 fp=0xc154a7fcc8 sp=0xc154a7fa28 pc=0x405cbf9
github.com/cockroachdb/cockroach/pkg/sql.(*routineGenerator).startInternal(0xc281ff86e0, {0x87f1448, 0xc281ff85a0}, 0xc024664420)
        pkg/sql/routine.go:310 +0x3cd fp=0xc154a7fdc8 sp=0xc154a7fcc8 pc=0x4429bcd
github.com/cockroachdb/cockroach/pkg/sql.(*routineGenerator).Start(0xc281ff86e0, {0x87f1448, 0xc281ff85a0}, 0xc024664420)
        pkg/sql/routine.go:260 +0x10f fp=0xc154a7fec8 sp=0xc154a7fdc8 pc=0x442968f
github.com/cockroachdb/cockroach/pkg/sql.(*planner).EvalRoutineExpr(0xc282005008, {0x87f1448, 0xc281ff85a0}, 0xc281765280, {0xc2819e6750
, 0x1, 0x1})
        pkg/sql/routine.go:162 +0x385 fp=0xc154a80030 sp=0xc154a7fec8 pc=0x4429045
github.com/cockroachdb/cockroach/pkg/sql/sem/eval.(*evaluator).EvalRoutineExpr(0xc282018808, {0x87f1448, 0xc281ff85a0}, 0xc281765280)
        pkg/sql/sem/eval/expr.go:648 +0x84 fp=0xc154a80078 sp=0xc154a80030 pc=0x2343104
github.com/cockroachdb/cockroach/pkg/sql/sem/tree.(*RoutineExpr).Eval(0xc2819d7a10?, {0x87f1448?, 0xc281ff85a0?}, {0x88af850?, 0xc282018
808?})
        bazel-out/k8-fastbuild/bin/pkg/sql/sem/tree/eval_expr_generated.go:362 +0x31 fp=0xc154a800a8 sp=0xc154a80078 pc=0x1864531
github.com/cockroachdb/cockroach/pkg/sql/sem/eval.Expr(...)
        pkg/sql/sem/eval/expr.go:22
github.com/cockroachdb/cockroach/pkg/sql.(*valuesNode).startExec(0xc281268ae0, {{0x87f1448, 0xc281ff85a0}, 0xc282018808, 0xc282005008})
        pkg/sql/values.go:86 +0x293 fp=0xc154a80188 sp=0xc154a800a8 pc=0x44d4873
github.com/cockroachdb/cockroach/pkg/sql.startExec({{0x87f1448?, 0xc281ff85a0?}, 0xc282018808?, 0xc282005008?}, {0x8833030, 0xc281268ae0
})
        pkg/sql/plan.go:541 +0x2a2 fp=0xc154a80250 sp=0xc154a80188 pc=0x43ea4a2
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc2819ff508, {0x87f1448, 0xc281ff8500})
        pkg/sql/plan_node_to_row_source.go:180 +0x10d fp=0xc154a802b0 sp=0xc154a80250 pc=0x43edb4d
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start.func1()
        pkg/sql/colflow/flow_coordinator.go:111 +0x2e fp=0xc154a802d8 sp=0xc154a802b0 pc=0x3e64d0e
 ESCOC
6b48?, 0x0?, 0x10000?}, 0x0?)
0 sp=0xc154a7dfc8 pc=0x3e712c5
etup(0xc239a35c00, {0x87f1448?, 0xc281ff88c0?}, 0xc28198ba48, 0x1)
0 sp=0xc154a7e040 pc=0x3e6a34e
Flow(0xc03fd94408, {0x87f1448, 0xc281ff88c0}, 0xc282022008, 0xc023a92580, 0xc283389230, 0xc28198ba00, {0x87ac7c0, 0xc2819efb08}, {0x0, >
154a7e1c0 pc=0x3e8c585
LocalSyncFlow(0xc03fd94408, {0x87f1448, 0xc281ff8780}, 0xc023a92580, 0xc28198ba00, {0x87ac7c0, 0xc2819efb08}, {0x0, 0x0}, {0xc282019008>
54a7e8b0 pc=0x3e8e385
s(0xc03f8e4dc0, {0x87f1448, 0xc281ff8780}, 0xc282019008, 0xc281964a90, 0x0, 0xc2820200c0, 0xc2819efb08, {0xc282019008, 0xc0314ffe00, ..>
154a7e968 pc=0x431ec1f
f8e4dc0, {0x87f1448, 0xc281ff8780}, 0xc281964a90, 0xc024664420, 0xc281765480, 0xc2819efb08, 0xc282019008, 0xc154a7f788)
154a7f000 pc=0x4320907
n(0xc03f8e4dc0, {0x87f1448, 0xc281ff8780}, 0xc282019008, 0xc281964a90, 0xc024664420, {{0x8833030, 0xc281268ba0}, 0x0}, 0xc2819efb08, ..>
c154a7f5f0 pc=0x43273d8
?, 0x0?, {0x87f1448?, 0xc281ff8780?}, 0x87f1448?, 0xc281ff8780?, 0xc282019008?, 0xc282006808?)
718 pc=0x4207b70
8, 0xc281ff8780}, {{0x87f1448, 0xc281ff8780}, 0xc282005558, 0xc282005008}, 0xc28201a240, {0x87f1e20, 0xc2819e8300}, {0x87ac6a8, ...}, .>
f7b0 pc=0x42075fe
ternal.func1({0x6b2b0a0, 0xc28201a240}, {0x0, 0x0}, 0x1)
8 pc=0x442a2e5
der).buildUDF.(*Builder).buildRoutinePlanGenerator.func2({0x87f1448, 0xc281ff85a0}, {0x77298a0, 0xc2819e82e0}, {0x6e82d20, 0xc2819e8300>
a7fcc8 sp=0xc154a7fa28 pc=0x405cbf9
ternal(0xc281ff86e0, {0x87f1448, 0xc281ff85a0}, 0xc024664420)
8 pc=0x4429bcd
xc281ff86e0, {0x87f1448, 0xc281ff85a0}, 0xc024664420)
8 pc=0x442968f
0xc282005008, {0x87f1448, 0xc281ff85a0}, 0xc281765280, {0xc2819e6750, 0x1, 0x1})
8 pc=0x4429045
outineExpr(0xc282018808, {0x87f1448, 0xc281ff85a0}, 0xc281765280)
a80030 pc=0x2343104
l(0xc2819d7a10?, {0x87f1448?, 0xc281ff85a0?}, {0x88af850?, 0xc282018808?})
ted.go:362 +0x31 fp=0xc154a800a8 sp=0xc154a80078 pc=0x1864531


281268ae0, {{0x87f1448, 0xc281ff85a0}, 0xc282018808, 0xc282005008})
pc=0x44d4873
281ff85a0?}, 0xc282018808?, 0xc282005008?}, {0x8833030, 0xc281268ae0})
c=0x43ea4a2
t(0xc2819ff508, {0x87f1448, 0xc281ff8500})
0 sp=0xc154a80250 pc=0x43edb4d
Start.func1()
8 sp=0xc154a802b0 pc=0x3e64d0e
dRuntimeError(0xc28175ed20?)
xc154a802d8 pc=0xbf13cd
Start(0xc28175ed20, {0x87efc98?, 0xc27feedc70?})
8 sp=0xc154a80318 pc=0x3e64c86
Helper).Run(0xc28175ed20, {0x87efc98?, 0xc27feedc70?}, {0x87ac7c0, 0xc2819ef208})
8 sp=0xc154a80388 pc=0x2c8aa8a
Run(0xc154a80498?, {0x87efc98?, 0xc27feedc70?}, {0x87ac7c0?, 0xc2819ef208?})
x3e73632
xc282012288, {0x87efc98, 0xc27feedc70}, 0x20?)
54a80400 pc=0x338dd96
un(0x887f740?, {0x87efc98?, 0xc27feedc70?}, 0x20?)
 ESCOD
6b48?, 0x0?, 0x10000?}, 0x0?)
        pkg/sql/colflow/vectorized_flow.go:1131 +0xa5 fp=0xc154a7e040 sp=0xc154a7dfc8 pc=0x3e712c5
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Setup(0xc239a35c00, {0x87f1448?, 0xc281ff88c0?}, 0xc28198ba48, 0x1)
        pkg/sql/colflow/vectorized_flow.go:256 +0x2ae fp=0xc154a7e1c0 sp=0xc154a7e040 pc=0x3e6a34e
github.com/cockroachdb/cockroach/pkg/sql/distsql.(*ServerImpl).setupFlow(0xc03fd94408, {0x87f1448, 0xc281ff88c0}, 0xc282022008, 0xc023a92580, 0xc283389230, 0xc28198ba00, {0x87ac7c0, 0xc2819efb08}, {0x0, ...}, ...)
        pkg/sql/distsql/server.go:435 +0x18c5 fp=0xc154a7e8b0 sp=0xc154a7e1c0 pc=0x3e8c585
github.com/cockroachdb/cockroach/pkg/sql/distsql.(*ServerImpl).SetupLocalSyncFlow(0xc03fd94408, {0x87f1448, 0xc281ff8780}, 0xc023a92580, 0xc28198ba00, {0x87ac7c0, 0xc2819efb08}, {0x0, 0x0}, {0xc282019008, ...})
        pkg/sql/distsql/server.go:605 +0x145 fp=0xc154a7e968 sp=0xc154a7e8b0 pc=0x3e8e385
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).setupFlows(0xc03f8e4dc0, {0x87f1448, 0xc281ff8780}, 0xc282019008, 0xc281964a90, 0x0, 0xc2820200c0, 0xc2819efb08, {0xc282019008, 0xc0314ffe00, ...}, ...)
        pkg/sql/distsql_running.go:530 +0xabf fp=0xc154a7f000 sp=0xc154a7e968 pc=0x431ec1f
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc03f8e4dc0, {0x87f1448, 0xc281ff8780}, 0xc281964a90, 0xc024664420, 0xc281765480, 0xc2819efb08, 0xc282019008, 0xc154a7f788)
        pkg/sql/distsql_running.go:991 +0xa07 fp=0xc154a7f5f0 sp=0xc154a7f000 pc=0x4320907
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc03f8e4dc0, {0x87f1448, 0xc281ff8780}, 0xc282019008, 0xc281964a90, 0xc024664420, {{0x8833030, 0xc281268ba0}, 0x0}, 0xc2819efb08, ...)
        pkg/sql/distsql_running.go:2252 +0x238 fp=0xc154a7f718 sp=0xc154a7f5f0 pc=0x43273d8
github.com/cockroachdb/cockroach/pkg/sql.runPlanInsidePlan.func1(0x0?, 0x0?, {0x87f1448?, 0xc281ff8780?}, 0x87f1448?, 0xc281ff8780?, 0xc282019008?, 0xc282006808?)
        pkg/sql/apply_join.go:334 +0xb0 fp=0xc154a7f7b0 sp=0xc154a7f718 pc=0x4207b70
github.com/cockroachdb/cockroach/pkg/sql.runPlanInsidePlan({0x87f1448, 0xc281ff8780}, {{0x87f1448, 0xc281ff8780}, 0xc282005558, 0xc282005008}, 0xc28201a240, {0x87f1e20, 0xc2819e8300}, {0x87ac6a8, ...}, ...)
        pkg/sql/apply_join.go:337 +0x59e fp=0xc154a7f918 sp=0xc154a7f7b0 pc=0x42075fe
github.com/cockroachdb/cockroach/pkg/sql.(*routineGenerator).startInternal.func1({0x6b2b0a0, 0xc28201a240}, {0x0, 0x0}, 0x1)
        pkg/sql/routine.go:354 +0x445 fp=0xc154a7fa28 sp=0xc154a7f918 pc=0x442a2e5
github.com/cockroachdb/cockroach/pkg/sql/opt/exec/execbuilder.(*Builder).buildUDF.(*Builder).buildRoutinePlanGenerator.func2({0x87f1448, 0xc281ff85a0}, {0x77298a0, 0xc2819e82e0}, {0x6e82d20, 0xc2819e8300}, {0xc2819e6750, 0x1, 0x1}, 0xc27feedcc0)
        pkg/sql/opt/exec/execbuilder/scalar.go:1327 +0x9d9 fp=0xc154a7fcc8 sp=0xc154a7fa28 pc=0x405cbf9
github.com/cockroachdb/cockroach/pkg/sql.(*routineGenerator).startInternal(0xc281ff86e0, {0x87f1448, 0xc281ff85a0}, 0xc024664420)
        pkg/sql/routine.go:310 +0x3cd fp=0xc154a7fdc8 sp=0xc154a7fcc8 pc=0x4429bcd
github.com/cockroachdb/cockroach/pkg/sql.(*routineGenerator).Start(0xc281ff86e0, {0x87f1448, 0xc281ff85a0}, 0xc024664420)
        pkg/sql/routine.go:260 +0x10f fp=0xc154a7fec8 sp=0xc154a7fdc8 pc=0x442968f
github.com/cockroachdb/cockroach/pkg/sql.(*planner).EvalRoutineExpr(0xc282005008, {0x87f1448, 0xc281ff85a0}, 0xc281765280, {0xc2819e6750, 0x1, 0x1})
        pkg/sql/routine.go:162 +0x385 fp=0xc154a80030 sp=0xc154a7fec8 pc=0x4429045
github.com/cockroachdb/cockroach/pkg/sql/sem/eval.(*evaluator).EvalRoutineExpr(0xc282018808, {0x87f1448, 0xc281ff85a0}, 0xc281765280)
        pkg/sql/sem/eval/expr.go:648 +0x84 fp=0xc154a80078 sp=0xc154a80030 pc=0x2343104
github.com/cockroachdb/cockroach/pkg/sql/sem/tree.(*RoutineExpr).Eval(0xc2819d7a10?, {0x87f1448?, 0xc281ff85a0?}, {0x88af850?, 0xc282018808?})
        bazel-out/k8-fastbuild/bin/pkg/sql/sem/tree/eval_expr_generated.go:362 +0x31 fp=0xc154a800a8 sp=0xc154a80078 pc=0x1864531
github.com/cockroachdb/cockroach/pkg/sql/sem/eval.Expr(...)
        pkg/sql/sem/eval/expr.go:22
github.com/cockroachdb/cockroach/pkg/sql.(*valuesNode).startExec(0xc281268ae0, {{0x87f1448, 0xc281ff85a0}, 0xc282018808, 0xc282005008})
        pkg/sql/values.go:86 +0x293 fp=0xc154a80188 sp=0xc154a800a8 pc=0x44d4873
github.com/cockroachdb/cockroach/pkg/sql.startExec({{0x87f1448?, 0xc281ff85a0?}, 0xc282018808?, 0xc282005008?}, {0x8833030, 0xc281268ae0})
        pkg/sql/plan.go:541 +0x2a2 fp=0xc154a80250 sp=0xc154a80188 pc=0x43ea4a2
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc2819ff508, {0x87f1448, 0xc281ff8500})
        pkg/sql/plan_node_to_row_source.go:180 +0x10d fp=0xc154a802b0 sp=0xc154a80250 pc=0x43edb4d
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start.func1()
        pkg/sql/colflow/flow_coordinator.go:111 +0x2e fp=0xc154a802d8 sp=0xc154a802b0 pc=0x3e64d0e
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc28175ed20?)
        pkg/sql/colexecerror/error.go:162 +0x4d fp=0xc154a80318 sp=0xc154a802d8 pc=0xbf13cd
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start(0xc28175ed20, {0x87efc98?, 0xc27feedc70?})
        pkg/sql/colflow/flow_coordinator.go:110 +0x66 fp=0xc154a80388 sp=0xc154a80318 pc=0x3e64c86
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run(0xc28175ed20, {0x87efc98?, 0xc27feedc70?}, {0x87ac7c0, 0
xc2819ef208})
        pkg/sql/execinfra/processorsbase.go:751 +0x4a fp=0xc154a803c8 sp=0xc154a80388 pc=0x2c8aa8a
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Run(0xc154a80498?, {0x87efc98?, 0xc27feedc70?}, {0x87ac7c0?, 0xc2819
ef208?})
        <autogenerated>:1 +0x32 fp=0xc154a80400 sp=0xc154a803c8 pc=0x3e73632
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc282012288, {0x87efc98, 0xc27feedc70}, 0x20?)
        pkg/sql/flowinfra/flow.go:574 +0x216 fp=0xc154a804d0 sp=0xc154a80400 pc=0x338dd96
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0x887f740?, {0x87efc98?, 0xc27feedc70?}, 0x20?)
        pkg/sql/colflow/vectorized_flow.go:301 +0x215 fp=0xc154a80568 sp=0xc154a804d0 pc=0x3e6a8f5
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc03f8e4dc0, {0x87efc98, 0xc27feedc70}, 0xc2819649c0, 0xc024664420, 0xc2
81765300, 0xc2819ef208, 0xc282018008, 0xc154a80cf0)
        pkg/sql/distsql_running.go:1038 +0xdd0 fp=0xc154a80b58 sp=0xc154a80568 pc=0x4320cd0
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc03f8e4dc0, {0x87f1448, 0xc281ff8320}, 0xc282018008, 0xc2819649c
0, 0xc024664420, {{0x8833030, 0xc281268ae0}, 0x0}, 0xc2819ef208, ...)
        pkg/sql/distsql_running.go:2252 +0x238 fp=0xc154a80c80 sp=0xc154a80b58 pc=0x43273d8
github.com/cockroachdb/cockroach/pkg/sql.runPlanInsidePlan.func1(0x0?, 0x0?, {0x87f1448?, 0xc281ff8320?}, 0x87f1448?, 0xc281ff8320?, 0xc
282018008?, 0xc282005008?)
        pkg/sql/apply_join.go:334 +0xb0 fp=0xc154a80d18 sp=0xc154a80c80 pc=0x4207b70
github.com/cockroachdb/cockroach/pkg/sql.runPlanInsidePlan({0x87f1448, 0xc281ff8320}, {{0x87f1448, 0xc281ff8320}, 0xc282003d58, 0xc28200
3808}, 0xc281993ef0, {0x87f1e20, 0xc2819e8200}, {0x87ac6a8, ...}, ...)
        pkg/sql/apply_join.go:337 +0x59e fp=0xc154a80e80 sp=0xc154a80d18 pc=0x42075fe
github.com/cockroachdb/cockroach/pkg/sql.(*routineGenerator).startInternal.func1({0x6b2b0a0, 0xc281993ef0}, {0x0, 0x0}, 0x1)
        pkg/sql/routine.go:354 +0x445 fp=0xc154a80f90 sp=0xc154a80e80 pc=0x442a2e5
github.com/cockroachdb/cockroach/pkg/sql/opt/exec/execbuilder.(*Builder).buildUDF.(*Builder).buildRoutinePlanGenerator.func2({0x87f1448,
 0xc281ff8140}, {0x77298a0, 0xc2819e81e0}, {0x6e82d20, 0xc2819e8200}, {0xc2819e6560, 0x1, 0x1}, 0xc27feedc20)
        pkg/sql/opt/exec/execbuilder/scalar.go:1327 +0x9d9 fp=0xc154a81230 sp=0xc154a80f90 pc=0x405cbf9
github.com/cockroachdb/cockroach/pkg/sql.(*routineGenerator).startInternal(0xc281ff8280, {0x87f1448, 0xc281ff8140}, 0xc024664420)
        pkg/sql/routine.go:310 +0x3cd fp=0xc154a81330 sp=0xc154a81230 pc=0x4429bcd
github.com/cockroachdb/cockroach/pkg/sql.(*routineGenerator).Start(0xc281ff8280, {0x87f1448, 0xc281ff8140}, 0xc024664420)
        pkg/sql/routine.go:260 +0x10f fp=0xc154a81430 sp=0xc154a81330 pc=0x442968f
github.com/cockroachdb/cockroach/pkg/sql.(*planner).EvalRoutineExpr(0xc282003808, {0x87f1448, 0xc281ff8140}, 0xc281765100, {0xc2819e6560
, 0x1, 0x1})
        pkg/sql/routine.go:162 +0x385 fp=0xc154a81598 sp=0xc154a81430 pc=0x4429045
github.com/cockroachdb/cockroach/pkg/sql/sem/eval.(*evaluator).EvalRoutineExpr(0xc2819f9808, {0x87f1448, 0xc281ff8140}, 0xc281765100)
        pkg/sql/sem/eval/expr.go:648 +0x84 fp=0xc154a815e0 sp=0xc154a81598 pc=0x2343104
github.com/cockroachdb/cockroach/pkg/sql/sem/tree.(*RoutineExpr).Eval(0xc2819d74a0?, {0x87f1448?, 0xc281ff8140?}, {0x88af850?, 0xc2819f9
808?})
        bazel-out/k8-fastbuild/bin/pkg/sql/sem/tree/eval_expr_generated.go:362 +0x31 fp=0xc154a81610 sp=0xc154a815e0 pc=0x1864531
github.com/cockroachdb/cockroach/pkg/sql/sem/eval.Expr(...)
        pkg/sql/sem/eval/expr.go:22
github.com/cockroachdb/cockroach/pkg/sql.(*valuesNode).startExec(0xc281268a20, {{0x87f1448, 0xc281ff8140}, 0xc2819f9808, 0xc282003808})
        pkg/sql/values.go:86 +0x293 fp=0xc154a816f0 sp=0xc154a81610 pc=0x44d4873
github.com/cockroachdb/cockroach/pkg/sql.startExec({{0x87f1448?, 0xc281ff8140?}, 0xc2819f9808?, 0xc282003808?}, {0x8833030, 0xc281268a20
})
        pkg/sql/plan.go:541 +0x2a2 fp=0xc154a817b8 sp=0xc154a816f0 pc=0x43ea4a2
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc2819fe708, {0x87f1448, 0xc281ff80a0})
        pkg/sql/plan_node_to_row_source.go:180 +0x10d fp=0xc154a81818 sp=0xc154a817b8 pc=0x43edb4d
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start.func1()
        pkg/sql/colflow/flow_coordinator.go:111 +0x2e fp=0xc154a81840 sp=0xc154a81818 pc=0x3e64d0e
...2155164 frames elided...
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc0220e9860?)
        pkg/sql/colexecerror/error.go:162 +0x4d fp=0xc174a77f90 sp=0xc174a77f50 pc=0xbf13cd
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start(0xc0220e9860, {0x87efc98?, 0xc02466c8c0?})
        pkg/sql/colflow/flow_coordinator.go:110 +0x66 fp=0xc174a78000 sp=0xc174a77f90 pc=0x3e64c86
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run(0xc0220e9860, {0x87efc98?, 0xc02466c8c0?}, {0x87ac7c0, 0
xc0248fcd88})
        pkg/sql/execinfra/processorsbase.go:751 +0x4a fp=0xc174a78040 sp=0xc174a78000 pc=0x2c8aa8a
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Run(0xc02116a110?, {0x87efc98?, 0xc02466c8c0?}, {0x87ac7c0?, 0xc0248
fcd88?})
        <autogenerated>:1 +0x32 fp=0xc174a78078 sp=0xc174a78040 pc=0x3e73632
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc022651688, {0x87efc98, 0xc02466c8c0}, 0x20?)
        pkg/sql/flowinfra/flow.go:574 +0x216 fp=0xc174a78148 sp=0xc174a78078 pc=0x338dd96
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0x887f740?, {0x87efc98?, 0xc02466c8c0?}, 0x20?)
        pkg/sql/colflow/vectorized_flow.go:301 +0x215 fp=0xc174a781e0 sp=0xc174a78148 pc=0x3e6a8f5
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc03f8e4dc0, {0x87efc98, 0xc02466c8c0}, 0xc02467cb60, 0xc024664420, 0xc0
2466e500, 0xc0248fcd88, 0xc024663008, 0xc02116a968)
        pkg/sql/distsql_running.go:1038 +0xdd0 fp=0xc174a787d0 sp=0xc174a781e0 pc=0x4320cd0
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc03f8e4dc0, {0x87f1448, 0xc02465fa40}, 0xc024663008, 0xc02467cb6
0, 0xc024664420, {{0x8833030, 0xc02289f560}, 0x0}, 0xc0248fcd88, ...)
        pkg/sql/distsql_running.go:2252 +0x238 fp=0xc174a788f8 sp=0xc174a787d0 pc=0x43273d8
github.com/cockroachdb/cockroach/pkg/sql.runPlanInsidePlan.func1(0x0?, 0x0?, {0x87f1448?, 0xc02465fa40?}, 0x87f1448?, 0xc02465fa40?, 0xc
024663008?, 0xc024ab8008?)
        pkg/sql/apply_join.go:334 +0xb0 fp=0xc174a78990 sp=0xc174a788f8 pc=0x4207b70
github.com/cockroachdb/cockroach/pkg/sql.runPlanInsidePlan({0x87f1448, 0xc02465fa40}, {{0x87f1448, 0xc02465fa40}, 0xc00a3f0648, 0xc00a3f
00f8}, 0xc0244fcfc0, {0x87f1e20, 0xc0246685e0}, {0x87ac6a8, ...}, ...)
        pkg/sql/apply_join.go:337 +0x59e fp=0xc174a78af8 sp=0xc174a78990 pc=0x42075fe
github.com/cockroachdb/cockroach/pkg/sql.(*routineGenerator).startInternal.func1({0x6b2b0a0, 0xc0244fcfc0}, {0x0, 0x0}, 0x1)
        pkg/sql/routine.go:354 +0x445 fp=0xc174a78c08 sp=0xc174a78af8 pc=0x442a2e5
github.com/cockroachdb/cockroach/pkg/sql/opt/exec/execbuilder.(*Builder).buildCall.(*Builder).buildRoutinePlanGenerator.func2({0x87f1448
, 0xc02465f900}, {0x77298a0, 0xc0246685c0}, {0x6e82d20, 0xc0246685e0}, {0x0, 0x0, 0x0}, 0xc02466c870)
        pkg/sql/opt/exec/execbuilder/scalar.go:1327 +0x9d9 fp=0xc174a78ea8 sp=0xc174a78c08 pc=0x404ac59
github.com/cockroachdb/cockroach/pkg/sql.(*routineGenerator).startInternal(0xc02465f9a0, {0x87f1448, 0xc02465f900}, 0xc024664420)
        pkg/sql/routine.go:310 +0x3cd fp=0xc174a78fa8 sp=0xc174a78ea8 pc=0x4429bcd
github.com/cockroachdb/cockroach/pkg/sql.(*routineGenerator).Start(0xc02465f9a0, {0x87f1448, 0xc02465f900}, 0xc024664420)
        pkg/sql/routine.go:260 +0x10f fp=0xc174a790a8 sp=0xc174a78fa8 pc=0x442968f
github.com/cockroachdb/cockroach/pkg/sql.(*planner).EvalRoutineExpr(0xc00a3f00f8, {0x87f1448, 0xc02465f900}, 0xc02466e100, {0x0, 0x0, 0x
0})
        pkg/sql/routine.go:162 +0x385 fp=0xc174a79210 sp=0xc174a790a8 pc=0x4429045
github.com/cockroachdb/cockroach/pkg/sql/sem/eval.(*evaluator).EvalRoutineExpr(0xc024662808, {0x87f1448, 0xc02465f900}, 0xc02466e100)
        pkg/sql/sem/eval/expr.go:648 +0x84 fp=0xc174a79258 sp=0xc174a79210 pc=0x2343104
github.com/cockroachdb/cockroach/pkg/sql/sem/tree.(*RoutineExpr).Eval(...)
        bazel-out/k8-fastbuild/bin/pkg/sql/sem/tree/eval_expr_generated.go:362
github.com/cockroachdb/cockroach/pkg/sql/sem/eval.Expr(...)
        pkg/sql/sem/eval/expr.go:22
github.com/cockroachdb/cockroach/pkg/sql.(*callNode).startExec(0xc024668520, {{0x87f1448, 0xc02465f900}, 0xc024662808, 0xc00a3f00f8})
        pkg/sql/routine.go:42 +0x4a fp=0xc174a792d8 sp=0xc174a79258 pc=0x44289ca
github.com/cockroachdb/cockroach/pkg/sql.startExec({{0x87f1448?, 0xc02465f900?}, 0xc024662808?, 0xc00a3f00f8?}, {0x8833f30, 0xc024668520
})
        pkg/sql/plan.go:541 +0x2a2 fp=0xc174a793a0 sp=0xc174a792d8 pc=0x43ea4a2
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc02465ca88, {0x87f1448, 0xc02465f860})
        pkg/sql/plan_node_to_row_source.go:180 +0x10d fp=0xc174a79400 sp=0xc174a793a0 pc=0x43edb4d
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Init(0xc022239808, {0x87f1448, 0xc02465f7c0})
        pkg/sql/colexec/columnarizer.go:178 +0xe8 fp=0xc174a79458 sp=0xc174a79400 pc=0x35b81a8
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*invariantsChecker).Init(0xc0248f81c0, {0x87f1448?, 0xc02465f7c0?})
        pkg/sql/colexec/invariants_checker.go:62 +0x49 fp=0xc174a79480 sp=0xc174a79458 pc=0x35bd749
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*batchInfoCollector).init(...)
        pkg/sql/colflow/stats.go:96
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*batchInfoCollector).init-fm()
        <autogenerated>:1 +0x30 fp=0xc174a794a8 sp=0xc174a79480 pc=0x3e79650
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc0248f8200?)
        pkg/sql/colexecerror/error.go:162 +0x4d fp=0xc174a794e8 sp=0xc174a794a8 pc=0xbf13cd
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*batchInfoCollector).Init(0xc0248ea5a0, {0x87f1448?, 0xc02465f7c0?})
        pkg/sql/colflow/stats.go:105 +0x71 fp=0xc174a79530 sp=0xc174a794e8 pc=0x3e68751
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedStatsCollectorImpl).Init(0x0?, {0x87f1448?, 0xc02465f7c0?})
        <autogenerated>:1 +0x28 fp=0xc174a79558 sp=0xc174a79530 pc=0x3e79048
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run.(*BatchFlowCoordinator).init.func2()
        pkg/sql/colflow/flow_coordinator.go:236 +0x28 fp=0xc174a79580 sp=0xc174a79558 pc=0x3e65888
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x87efc98?)
        pkg/sql/colexecerror/error.go:162 +0x4d fp=0xc174a795c0 sp=0xc174a79580 pc=0xbf13cd
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).init(...)
        pkg/sql/colflow/flow_coordinator.go:235
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc021c494a0, {0x87efc98?, 0xc02466c7d0?})
        pkg/sql/colflow/flow_coordinator.go:269 +0xe5 fp=0xc174a796a8 sp=0xc174a795c0 pc=0x3e65565
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc0220aaec0, {0x87efc98, 0xc02466c7d0}, 0x0)
        pkg/sql/colflow/vectorized_flow.go:316 +0x1e6 fp=0xc174a79740 sp=0xc174a796a8 pc=0x3e6a8c6
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc03f8e4dc0, {0x87efc98, 0xc02466c7d0}, 0xc02467ca90, 0xc024664420, 0xc0
2466e300, 0xc0220a7208, 0xc00a3f0648, 0xc02116bec8)
        pkg/sql/distsql_running.go:1038 +0xdd0 fp=0xc174a79d30 sp=0xc174a79740 pc=0x4320cd0
        pkg/sql/distsql_running.go:1038 +0xdd0 fp=0xc174a79d30 sp=0xc174a79740 pc=0x4320cd0
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc03f8e4dc0, {0x87f1448, 0xc02465f5e0}, 0xc00a3f0648, 0xc02467ca9
0, 0xc024664420, {{0x8833f30, 0xc024668520}, 0x0}, 0xc0220a7208, ...)
        pkg/sql/distsql_running.go:2252 +0x238 fp=0xc174a79e58 sp=0xc174a79d30 pc=0x43273d8
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll.func3(0x10?, 0xc02116bf50?, {0x87f1448?, 0xc02465f5e0?}, 0xc004
423510?, 0x8833f30?, 0xc024668520?)
        pkg/sql/distsql_running.go:1956 +0xaf fp=0xc174a79ef0 sp=0xc174a79e58 pc=0x432514f
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0xc03f8e4dc0, {0x87f1448, 0xc02465f5e0}, 0xc00a3f0648, 0xc02467
ca90, 0xc00a3f00f8, 0xc0220a7208, 0x0)
        pkg/sql/distsql_running.go:1959 +0x206 fp=0xc174a7a048 sp=0xc174a79ef0 pc=0x4324aa6
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc00a3ef908, {0x87f1448, 0xc02465f5e0}, 0xc00a3f00f8, 0x
3?, {0x7fdbb21ddc30, 0xc0238e3f60}, 0x0, 0xc024664558, {0x8792b40, ...})
        pkg/sql/conn_executor_exec.go:3361 +0x477 fp=0xc174a7a160 sp=0xc174a7a048 pc=0x4274c37
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc00a3ef908, {0x87f1448, 0xc02465f5e0}, 0xc00a3f00f8
, {0x7fdbb21ddc30, 0xc0238e3f60})
        pkg/sql/conn_executor_exec.go:2897 +0x10cd fp=0xc174a7a608 sp=0xc174a7a160 pc=0x42715ed
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc00a3ef908, {0x87efc98?, 0xc02466c550?}, {{0x8818820, 0xc
004422ef0}, {0x0, 0x0, 0x0}, {0xc024333b14, 0xba}, ...}, ...)
        pkg/sql/conn_executor_exec.go:1067 +0x39cc fp=0xc174a7b1e0 sp=0xc174a7a608 pc=0x42625ac
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func2({0x87efc98?, 0xc02466c550?})
        pkg/sql/conn_executor_exec.go:175 +0xb0 fp=0xc174a7b290 sp=0xc174a7b1e0 pc=0x425dd50
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0xccaebe0?, {0x87efc98?, 0xc02466c550?}, {0x8818820?, 0xc0044
22ef0?}, 0x42b68d?, 0x10000314ffe00?)
        pkg/sql/conn_executor_exec.go:4394 +0x2ec fp=0xc174a7b3d8 sp=0xc174a7b290 pc=0x427c34c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc00a3ef908, {0x87efc98, 0xc02466c550}, {{0x8818820, 0xc004422ef0}, {
0x0, 0x0, 0x0}, {0xc024333b14, 0xba}, ...}, ...)
        pkg/sql/conn_executor_exec.go:174 +0x9a5 fp=0xc174a7b698 sp=0xc174a7b3d8 pc=0x425d4e5
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x8818820, 0xc004422ef0}, {0x0, 0x0, 0x0}, {0xc024333b14, 0xba
}, 0x0, 0x0}, 0x1a575532469, ...}, ...)
        pkg/sql/conn_executor.go:2374 +0x3dd fp=0xc174a7b760 sp=0xc174a7b698 pc=0x424ac7d
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00a3ef908)
        pkg/sql/conn_executor.go:2379 +0x888 fp=0xc174a7bb70 sp=0xc174a7b760 pc=0x4248048
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00a3ef908, {0x87f1448, 0xc024320320}, 0xc038b64000?, 0x0?, 0xc0231ec920?
)
        pkg/sql/conn_executor.go:2287 +0x21b fp=0xc174a7bc48 sp=0xc174a7bb70 pc=0x424737b
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc03fc22d80?, {0x87f1448?, 0xc024320320?}, {0xc03a12c240?}, 0x2?, 0x0?)
        pkg/sql/conn_executor.go:1063 +0xb9 fp=0xc174a7bcf8 sp=0xc174a7bc48 pc=0x4241ef9
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommands(0xc0238e3808, {0x87f1448, 0xc024320320}, {0x0, 0x4, {0x1, {0x77d
1912, 0x3}, {0xc02c0312f0, 0xf}, ...}, ...}, ...)
        pkg/sql/pgwire/conn.go:253 +0x4b5 fp=0xc174a7bef8 sp=0xc174a7bcf8 pc=0x5856f15
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveImpl.func4()
        pkg/sql/pgwire/server.go:1226 +0xec fp=0xc174a7bfe0 sp=0xc174a7bef8 pc=0x586cfec
runtime.goexit({})
        src/runtime/asm_amd64.s:1700 +0x1 fp=0xc174a7bfe8 sp=0xc174a7bfe0 pc=0x495dc1
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveImpl in goroutine 12971046
        pkg/sql/pgwire/server.go:1223 +0x5f2

[yuzefovich]

The reproduction is actually quite simple. On 9bfbb28 on a binary compiled with crdb_test flag this crashes quickly:

DO $$
BEGIN
    WHILE true LOOP
        NULL;
    END LOOP;
END;
$$;

It's interesting to note that non-crdb-test binary on the same SHA nor 25.3.2 experiences that. I think it's because of this metamorphic randomization: "tail-call-optimization-enabled" with value false which makes sense.