subChannel triggerDelete lead to dead loop

[YangXin-Sheep]

grpc version: v1.61.0
go version: 1.20.2
linux version: Debian GNU/Linux 11

runtime: goroutine stack exceeds 1000000000-byte limit
runtime: sp=0xc047d1c380 stack=[0xc047d1c000, 0xc067d1c000]
fatal error: stack overflow

runtime stack:
runtime.throw({0x20428c6?, 0x36fb300?})
/usr/local/go/src/runtime/panic.go:1047 +0x5d fp=0x7fa0fa7fa440 sp=0x7fa0fa7fa410 pc=0x4882fd
runtime.newstack()
/usr/local/go/src/runtime/stack.go:1105 +0x5bd fp=0x7fa0fa7fa5f0 sp=0x7fa0fa7fa440 pc=0x4a275d
runtime.morestack()
/usr/local/go/src/runtime/asm_amd64.s:574 +0x8b fp=0x7fa0fa7fa5f8 sp=0x7fa0fa7fa5f0 pc=0x4bd22b

goroutine 197 [running]:
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:353 +0x185 fp=0xc047d1c390 sp=0xc047d1c388 pc=0x84c7e5
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1c3c8 sp=0xc047d1c390 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1c3f0 sp=0xc047d1c3c8 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1c428 sp=0xc047d1c3f0 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1c460 sp=0xc047d1c428 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1c488 sp=0xc047d1c460 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1c4c0 sp=0xc047d1c488 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1c4f8 sp=0xc047d1c4c0 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1c520 sp=0xc047d1c4f8 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1c558 sp=0xc047d1c520 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1c590 sp=0xc047d1c558 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1c5b8 sp=0xc047d1c590 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1c5f0 sp=0xc047d1c5b8 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1c628 sp=0xc047d1c5f0 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1c650 sp=0xc047d1c628 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1c688 sp=0xc047d1c650 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1c6c0 sp=0xc047d1c688 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1c6e8 sp=0xc047d1c6c0 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1c720 sp=0xc047d1c6e8 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1c758 sp=0xc047d1c720 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1c780 sp=0xc047d1c758 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1c7b8 sp=0xc047d1c780 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1c7f0 sp=0xc047d1c7b8 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1c818 sp=0xc047d1c7f0 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1c850 sp=0xc047d1c818 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1c888 sp=0xc047d1c850 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1c8b0 sp=0xc047d1c888 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1c8e8 sp=0xc047d1c8b0 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1c920 sp=0xc047d1c8e8 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1c948 sp=0xc047d1c920 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1c980 sp=0xc047d1c948 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1c9b8 sp=0xc047d1c980 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1c9e0 sp=0xc047d1c9b8 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1ca18 sp=0xc047d1c9e0 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1ca50 sp=0xc047d1ca18 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1ca78 sp=0xc047d1ca50 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1cab0 sp=0xc047d1ca78 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1cae8 sp=0xc047d1cab0 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1cb10 sp=0xc047d1cae8 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1cb48 sp=0xc047d1cb10 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1cb80 sp=0xc047d1cb48 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1cba8 sp=0xc047d1cb80 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1cbe0 sp=0xc047d1cba8 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1cc18 sp=0xc047d1cbe0 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1cc40 sp=0xc047d1cc18 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1cc78 sp=0xc047d1cc40 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1ccb0 sp=0xc047d1cc78 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1ccd8 sp=0xc047d1ccb0 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1cd10 sp=0xc047d1ccd8 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1cd48 sp=0xc047d1cd10 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1cd70 sp=0xc047d1cd48 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1cda8 sp=0xc047d1cd70 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1cde0 sp=0xc047d1cda8 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1ce08 sp=0xc047d1cde0 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1ce40 sp=0xc047d1ce08 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1ce78 sp=0xc047d1ce40 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1cea0 sp=0xc047d1ce78 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1ced8 sp=0xc047d1cea0 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1cf10 sp=0xc047d1ced8 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1cf38 sp=0xc047d1cf10 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1cf70 sp=0xc047d1cf38 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1cfa8 sp=0xc047d1cf70 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1cfd0 sp=0xc047d1cfa8 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1d008 sp=0xc047d1cfd0 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1d040 sp=0xc047d1d008 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1d068 sp=0xc047d1d040 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1d0a0 sp=0xc047d1d068 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1d0d8 sp=0xc047d1d0a0 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1d100 sp=0xc047d1d0d8 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1d138 sp=0xc047d1d100 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1d170 sp=0xc047d1d138 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1d198 sp=0xc047d1d170 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1d1d0 sp=0xc047d1d198 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1d208 sp=0xc047d1d1d0 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1d230 sp=0xc047d1d208 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1d268 sp=0xc047d1d230 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1d2a0 sp=0xc047d1d268 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1d2c8 sp=0xc047d1d2a0 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1d300 sp=0xc047d1d2c8 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1d338 sp=0xc047d1d300 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1d360 sp=0xc047d1d338 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1d398 sp=0xc047d1d360 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1d3d0 sp=0xc047d1d398 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1d3f8 sp=0xc047d1d3d0 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1d430 sp=0xc047d1d3f8 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1d468 sp=0xc047d1d430 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1d490 sp=0xc047d1d468 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1d4c8 sp=0xc047d1d490 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1d500 sp=0xc047d1d4c8 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1d528 sp=0xc047d1d500 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1d560 sp=0xc047d1d528 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1d598 sp=0xc047d1d560 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1d5c0 sp=0xc047d1d598 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1d5f8 sp=0xc047d1d5c0 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1d630 sp=0xc047d1d5f8 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1d658 sp=0xc047d1d630 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1d690 sp=0xc047d1d658 pc=0x84c7bd
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfIfReady(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:389 +0x25 fp=0xc047d1d6c8 sp=0xc047d1d690 pc=0x84c825
google.golang.org/grpc/internal/channelz.(*subChannel).deleteChild(0xc00028bc80, 0xc009d1ef60?)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:334 +0x39 fp=0xc047d1d6f0 sp=0xc047d1d6c8 pc=0x84c5d9
google.golang.org/grpc/internal/channelz.(*subChannel).deleteSelfFromTree(0xc00028bc80)
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/channelz/types.go:357 +0x15d fp=0xc047d1d728 sp=0xc047d1d6f0 pc=0x84c7bd
...additional frames elided...
created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer
/go/pkg/mod/gitlab.ttyuyin.com/mp-engineering/grpc@v1.60.1/internal/grpcsync/callback_serializer.go:52 +0x138

[dfawley]

Do you have a reliable way to reproduce this?

I have #6969 out to address some underlying issues in channelz, and want to do more work there to simplify and remove any possibility of this kind of thing.

[YangXin-Sheep]

Do you have a reliable way to reproduce this?

I have #6969 out to address some underlying issues in channelz, and want to do more work there to simplify and remove any possibility of this kind of thing.

This exception just occurred at service start，but service restart several times it disappeared.
I just found log INFO: [channelz] attempt to delete child with id 3 from a parent (id=5) that doesn't currently exist

[noPumpkin]

Do you have a reliable way to reproduce this?

I have #6969 out to address some underlying issues in channelz, and want to do more work there to simplify and remove any possibility of this kind of thing.

we add a log in this method

func (sc *subChannel) deleteSelfFromTree() (deleted bool) {
	if !sc.closeCalled || len(sc.sockets) != 0 {
		return false
	}
	logger.Infof("deleteSelfFromTree | pid:%d id:%d", sc.pid, sc.id) // log
	sc.cm.findEntry(sc.pid).deleteChild(sc.id)
	return true
}

print this log when service start
INFO: [core] [Channel #7 SubChannel #7] Subchannel created

keep printing this log when service dead loop
INFO: [channelz] deleteSelfFromTree | pid:7 id:7

[dfawley]

Can you cause this problem to occur without modifying our code to add logs?

If I can't write a test for it, it's a lot harder to fix.

[github-actions]

This issue is labeled as requiring an update from the reporter, and no update has been received after 6 days. If no update is provided in the next 7 days, this issue will be automatically closed.