Panic in applier unit tests: DATA RACE / send on closed channel #6337
Description
This happened while testing commit 7c2558f, which is part of #6334. The changes in the commit seem completely unrelated, hence I think this needs further investigation. Here's the link to the relevant actions run: https://github.com/k0sproject/k0s/actions/runs/17290458710/job/49076364425?pr=6334#step:8:476
Not sure if the data race or the send on closed channel is the root problem and if one of them triggered the other...
Excerpt from test logs
time="2025-08-28T08:40:14Z" level=info msg="Starting watch loop" component=applier-manager
time="2025-08-28T08:40:14Z" level=info msg="Running stack" component=applier-manager stack=/tmp/TestManager_AppliesStacks1552291229/001/manifests/before
time="2025-08-28T08:40:15Z" level=info msg="Applying manifests" component=applier-before
time="2025-08-28T08:40:15Z" level=info msg="Running stack" component=applier-manager stack=/tmp/TestManager_AppliesStacks1552291229/001/manifests/after
time="2025-08-28T08:40:16Z" level=info msg="Applying manifests" component=applier-after
time="2025-08-28T08:40:16Z" level=info msg="Watch loop done (applier manager is stopping)" component=applier-manager
time="2025-08-28T08:40:16Z" level=info msg="Stack done (applier manager is stopping)" component=applier-manager stack=/tmp/TestManager_AppliesStacks1552291229/001/manifests/before
time="2025-08-28T08:40:16Z" level=info msg="Stack done (applier manager is stopping)" component=applier-manager stack=/tmp/TestManager_AppliesStacks1552291229/001/manifests/after
time="2025-08-28T08:40:16Z" level=info msg="Starting watch loop" component=applier-manager
time="2025-08-28T08:40:16Z" level=info msg="Watch loop done (applier manager is stopping)" component=applier-manager
time="2025-08-28T08:40:16Z" level=info msg="Starting watch loop" component=applier-manager
time="2025-08-28T08:40:16Z" level=info msg="Running stack" component=applier-manager stack=/tmp/TestManager3742153756/001/stack1
time="2025-08-28T08:40:17Z" level=info msg="Applying manifests" component=applier-stack1
time="2025-08-28T08:40:18Z" level=info msg="Applying manifests" component=applier-stack1
time="2025-08-28T08:40:18Z" level=info msg="Running stack" component=applier-manager stack=/tmp/TestManager3742153756/001/stack2
time="2025-08-28T08:40:18Z" level=info msg="Watch loop done (lost the lead)" component=applier-manager
time="2025-08-28T08:40:18Z" level=info msg="Stack done (lost the lead)" component=applier-manager stack=/tmp/TestManager3742153756/001/stack2
time="2025-08-28T08:40:18Z" level=info msg="Stack done (lost the lead)" component=applier-manager stack=/tmp/TestManager3742153756/001/stack1
==================
WARNING: DATA RACE
Read at 0x00c000110710 by goroutine 1513:
runtime.chansend()
/usr/local/go/src/runtime/chan.go:176 +0x0
github.com/k0sproject/k0s/pkg/applier.(*StackApplier).Run.func2()
/go/src/github.com/k0sproject/k0s/pkg/applier/stackapplier.go:75 +0x59
Previous write at 0x00c000110710 by goroutine 1512:
runtime.closechan()
/usr/local/go/src/runtime/chan.go:414 +0x0
github.com/fsnotify/fsnotify.(*inotify).readEvents.func1()
/run/k0s-build/go/mod/github.com/fsnotify/[email protected]/backend_inotify.go:348 +0x84
runtime.deferreturn()
/usr/local/go/src/runtime/panic.go:610 +0x5d
github.com/fsnotify/fsnotify.newBackend.gowrap1()
/run/k0s-build/go/mod/github.com/fsnotify/[email protected]/backend_inotify.go:155 +0x33
Goroutine 1513 (running) created at:
github.com/k0sproject/k0s/pkg/applier.(*StackApplier).Run()
/go/src/github.com/k0sproject/k0s/pkg/applier/stackapplier.go:75 +0x38a
github.com/k0sproject/k0s/pkg/applier.(*Manager).createStack.func1.1()
/go/src/github.com/k0sproject/k0s/pkg/applier/manager.go:186 +0xbe
k8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1()
/run/k0s-build/go/mod/k8s.io/[email protected]/pkg/util/wait/backoff.go:255 +0x9c
k8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext()
/run/k0s-build/go/mod/k8s.io/[email protected]/pkg/util/wait/backoff.go:256 +0xed
k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext()
/run/k0s-build/go/mod/k8s.io/[email protected]/pkg/util/wait/backoff.go:223 +0x108
k8s.io/apimachinery/pkg/util/wait.UntilWithContext()
/run/k0s-build/go/mod/k8s.io/[email protected]/pkg/util/wait/backoff.go:172 +0x197
github.com/k0sproject/k0s/pkg/applier.(*Manager).createStack.func1()
/go/src/github.com/k0sproject/k0s/pkg/applier/manager.go:184 +0xd6
Goroutine 1512 (finished) created at:
github.com/fsnotify/fsnotify.newBackend()
/run/k0s-build/go/mod/github.com/fsnotify/[email protected]/backend_inotify.go:155 +0x485
github.com/fsnotify/fsnotify.NewWatcher()
/run/k0s-build/go/mod/github.com/fsnotify/[email protected]/fsnotify.go:254 +0x64
github.com/k0sproject/k0s/pkg/applier.(*StackApplier).Run()
/go/src/github.com/k0sproject/k0s/pkg/applier/stackapplier.go:58 +0x71
github.com/k0sproject/k0s/pkg/applier.(*Manager).createStack.func1.1()
/go/src/github.com/k0sproject/k0s/pkg/applier/manager.go:186 +0xbe
k8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1()
/run/k0s-build/go/mod/k8s.io/[email protected]/pkg/util/wait/backoff.go:255 +0x9c
k8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext()
/run/k0s-build/go/mod/k8s.io/[email protected]/pkg/util/wait/backoff.go:256 +0xed
k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext()
/run/k0s-build/go/mod/k8s.io/[email protected]/pkg/util/wait/backoff.go:223 +0x108
k8s.io/apimachinery/pkg/util/wait.UntilWithContext()
/run/k0s-build/go/mod/k8s.io/[email protected]/pkg/util/wait/backoff.go:172 +0x197
github.com/k0sproject/k0s/pkg/applier.(*Manager).createStack.func1()
/go/src/github.com/k0sproject/k0s/pkg/applier/manager.go:184 +0xd6
==================
panic: send on closed channel
goroutine 1337 [running]:
github.com/k0sproject/k0s/pkg/applier.(*StackApplier).Run.func2()
/go/src/github.com/k0sproject/k0s/pkg/applier/stackapplier.go:75 +0x5a
created by github.com/k0sproject/k0s/pkg/applier.(*StackApplier).Run in goroutine 1335
/go/src/github.com/k0sproject/k0s/pkg/applier/stackapplier.go:75 +0x38b
FAIL github.com/k0sproject/k0s/pkg/applier 4.762s