Skip to content

Commit e82ab40

Browse files
sethiyashsethiyash
authored
Merge pull request #1415 from carvel-dev/expose-metrics
Expose metrics to report time taken in fetch/template/deploy phase of app, pkgi, pkgr
2 parents 7b7d396 + b292117 commit e82ab40

27 files changed

+636
-189
lines changed

‎cmd/controller/run.go‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -95,10 +95,6 @@ func Run(opts Options, runLog logr.Logger) error {
9595
return fmt.Errorf("Building packaging client: %s", err)
9696
}
9797

98-
runLog.Info("setting up metrics")
99-
appMetrics := metrics.NewAppMetrics()
100-
appMetrics.RegisterAllMetrics()
101-
10298
var server *apiserver.APIServer
10399
if opts.StartAPIServer {
104100
// assign bindPort to env var KAPPCTRL_API_PORT if available
@@ -187,6 +183,11 @@ func Run(opts Options, runLog logr.Logger) error {
187183
kubeconf := kubeconfig.NewKubeconfig(coreClient, runLog)
188184
compInfo := componentinfo.NewComponentInfo(coreClient, kubeconf, Version)
189185

186+
runLog.Info("setting up metrics")
187+
appMetrics := metrics.NewMetrics()
188+
appMetrics.ReconcileTimeMetrics.RegisterAllMetrics()
189+
appMetrics.ReconcileCountMetrics.RegisterAllMetrics()
190+
190191
cacheFolderApps := memdir.NewTmpDir("cache-appcr")
191192
err = cacheFolderApps.Create()
192193
if err != nil {
@@ -227,7 +228,8 @@ func Run(opts Options, runLog logr.Logger) error {
227228
pkgToPkgInstallHandler := pkginstall.NewPackageInstallVersionHandler(
228229
kcClient, opts.PackagingGlobalNS, runLog.WithName("handler"))
229230

230-
reconciler := pkginstall.NewReconciler(kcClient, pkgClient, coreClient, pkgToPkgInstallHandler, runLog.WithName("pkgi"), compInfo, kcConfig)
231+
reconciler := pkginstall.NewReconciler(kcClient, pkgClient, coreClient, pkgToPkgInstallHandler,
232+
runLog.WithName("pkgi"), compInfo, kcConfig, appMetrics)
231233

232234
ctrl, err := controller.New("pkgi", mgr, controller.Options{
233235
Reconciler: reconciler,
@@ -254,6 +256,7 @@ func Run(opts Options, runLog logr.Logger) error {
254256
CoreClient: coreClient,
255257
AppClient: kcClient,
256258
KcConfig: kcConfig,
259+
AppMetrics: appMetrics,
257260
CmdRunner: sidecarCmdExec,
258261
Kubeconf: kubeconf,
259262
CacheFolder: cacheFolderPkgRepoApps,

‎config/config/agg-api.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,5 +23,10 @@ spec:
2323
- port: 443
2424
protocol: TCP
2525
targetPort: api
26+
name: main
27+
- port: 8080
28+
protocol: TCP
29+
targetPort: metrics
30+
name: metrics
2631
selector:
2732
app: kapp-controller

‎config/config/deployment.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@ spec:
5252
- containerPort: #@ data.values.apiPort
5353
name: api
5454
protocol: TCP
55+
- containerPort: #@ data.values.metricsPort
56+
name: metrics
57+
protocol: TCP
5558
securityContext:
5659
allowPrivilegeEscalation: false
5760
readOnlyRootFilesystem: true

‎config/values-schema.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ dangerousEnablePprof: false
1717
tlsCipherSuites: ""
1818
#@schema/desc "API port"
1919
apiPort: 8443
20+
#@schema/desc "Metrics port"
21+
metricsPort: 8080
2022
#@schema/desc "The coreDNSIP will be injected into /etc/resolv.conf of kapp-controller pod"
2123
coreDNSIP: ""
2224
#@schema/desc "HostNetwork of kapp-controller deployment."

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ require (
3030
github.com/cppforlife/go-cli-ui v0.0.0-20220425131040-94f26b16bc14
3131
github.com/go-logr/logr v1.2.4
3232
github.com/k14s/semver/v4 v4.0.1-0.20210701191048-266d47ac6115
33+
github.com/prometheus/client_model v0.4.0
3334
github.com/spf13/cobra v1.6.1
3435
golang.org/x/sync v0.3.0
3536
gopkg.in/yaml.v2 v2.4.0
@@ -86,7 +87,6 @@ require (
8687
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
8788
github.com/pkg/errors v0.9.1 // indirect
8889
github.com/pmezard/go-difflib v1.0.0 // indirect
89-
github.com/prometheus/client_model v0.4.0 // indirect
9090
github.com/prometheus/common v0.42.0 // indirect
9191
github.com/prometheus/procfs v0.9.0 // indirect
9292
github.com/spf13/pflag v1.0.5 // indirect

‎pkg/app/app.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ type App struct {
5757

5858
log logr.Logger
5959
opts Opts
60-
appMetrics *metrics.AppMetrics
60+
appMetrics *metrics.Metrics
6161

6262
pendingStatusUpdate bool
6363
flushAllStatusUpdates bool
@@ -66,7 +66,7 @@ type App struct {
6666

6767
func NewApp(app v1alpha1.App, hooks Hooks,
6868
fetchFactory fetch.Factory, templateFactory template.Factory,
69-
deployFactory deploy.Factory, log logr.Logger, opts Opts, appMetrics *metrics.AppMetrics, compInfo ComponentInfo) *App {
69+
deployFactory deploy.Factory, log logr.Logger, opts Opts, appMetrics *metrics.Metrics, compInfo ComponentInfo) *App {
7070

7171
return &App{app: app, appPrev: *(app.DeepCopy()), hooks: hooks,
7272
fetchFactory: fetchFactory, templateFactory: templateFactory,
@@ -76,6 +76,9 @@ func NewApp(app v1alpha1.App, hooks Hooks,
7676
func (a *App) Name() string { return a.app.Name }
7777
func (a *App) Namespace() string { return a.app.Namespace }
7878

79+
// Kind return kind of App
80+
func (a *App) Kind() string { return "App" }
81+
7982
func (a *App) Status() v1alpha1.AppStatus { return a.app.Status }
8083

8184
func (a *App) StatusAsYAMLBytes() ([]byte, error) {

‎pkg/app/app_factory.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ type CRDAppFactory struct {
2626
CoreClient kubernetes.Interface
2727
AppClient kcclient.Interface
2828
KcConfig *config.Config
29-
AppMetrics *metrics.AppMetrics
29+
AppMetrics *metrics.Metrics
3030
VendirConfigHook func(vendirconf.Config) vendirconf.Config
3131
KbldAllowBuild bool
3232
CmdRunner exec.CmdRunner

‎pkg/app/app_reconcile.go‎

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ func (a *App) Reconcile(force bool) (reconcile.Result, error) {
2121

2222
var err error
2323

24-
a.appMetrics.InitMetrics(a.Name(), a.Namespace())
24+
a.appMetrics.ReconcileCountMetrics.InitMetrics(a.Kind(), a.Name(), a.Namespace())
2525

2626
timerOpts := ReconcileTimerOpts{
2727
DefaultSyncPeriod: a.opts.DefaultSyncPeriod,
@@ -103,6 +103,13 @@ func (a *App) reconcileDeploy() error {
103103
}
104104

105105
func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult {
106+
reconcileStartTime := time.Now()
107+
a.appMetrics.IsFirstReconcile = a.appMetrics.ReconcileCountMetrics.GetReconcileAttemptCounterValue(a.Kind(), a.Name(), a.Namespace()) == 1
108+
defer func() {
109+
a.appMetrics.ReconcileTimeMetrics.RegisterOverallTime(a.Kind(), a.Name(), a.Namespace(), a.appMetrics.IsFirstReconcile,
110+
time.Since(reconcileStartTime))
111+
}()
112+
106113
tmpDir := memdir.NewTmpDir("fetch-template-deploy")
107114

108115
err := tmpDir.Create()
@@ -129,6 +136,9 @@ func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult {
129136
UpdatedAt: metav1.NewTime(time.Now().UTC()),
130137
}
131138

139+
a.appMetrics.ReconcileTimeMetrics.RegisterFetchTime(a.Kind(), a.Name(), a.Namespace(), a.appMetrics.IsFirstReconcile,
140+
a.app.Status.Fetch.UpdatedAt.Sub(a.app.Status.Fetch.StartedAt.Time))
141+
132142
err := a.updateStatus("marking fetch completed")
133143
if err != nil {
134144
return exec.NewCmdRunResultWithErr(err)
@@ -139,6 +149,8 @@ func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult {
139149
}
140150
}
141151

152+
templateStartTime := time.Now()
153+
142154
tplResult := a.template(assetsPath)
143155

144156
a.app.Status.Template = &v1alpha1.AppStatusTemplate{
@@ -148,6 +160,9 @@ func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult {
148160
UpdatedAt: metav1.NewTime(time.Now().UTC()),
149161
}
150162

163+
a.appMetrics.ReconcileTimeMetrics.RegisterTemplateTime(a.Kind(), a.Name(), a.Namespace(), a.appMetrics.IsFirstReconcile,
164+
a.app.Status.Template.UpdatedAt.Sub(templateStartTime))
165+
151166
err = a.updateStatus("marking template completed")
152167
if err != nil {
153168
return exec.NewCmdRunResultWithErr(err)
@@ -202,6 +217,9 @@ func (a *App) updateLastDeploy(result exec.CmdRunResult) exec.CmdRunResult {
202217
},
203218
}
204219

220+
a.appMetrics.ReconcileTimeMetrics.RegisterDeployTime(a.Kind(), a.Name(), a.Namespace(), a.appMetrics.IsFirstReconcile,
221+
a.Status().Deploy.UpdatedAt.Sub(a.Status().Deploy.StartedAt.Time))
222+
205223
return result
206224
}
207225

@@ -253,7 +271,7 @@ func (a *App) setReconciling() {
253271
Status: corev1.ConditionTrue,
254272
})
255273

256-
a.appMetrics.RegisterReconcileAttempt(a.app.Name, a.app.Namespace)
274+
a.appMetrics.ReconcileCountMetrics.RegisterReconcileAttempt(a.Kind(), a.Name(), a.Namespace())
257275
a.app.Status.FriendlyDescription = "Reconciling"
258276
}
259277

@@ -269,7 +287,7 @@ func (a *App) setReconcileCompleted(result exec.CmdRunResult) {
269287
a.app.Status.ConsecutiveReconcileFailures++
270288
a.app.Status.ConsecutiveReconcileSuccesses = 0
271289
a.app.Status.FriendlyDescription = fmt.Sprintf("Reconcile failed: %s", result.ErrorStr())
272-
a.appMetrics.RegisterReconcileFailure(a.app.Name, a.app.Namespace)
290+
a.appMetrics.ReconcileCountMetrics.RegisterReconcileFailure(a.Kind(), a.Name(), a.Namespace())
273291
a.setUsefulErrorMessage(result)
274292
} else {
275293
a.app.Status.Conditions = append(a.app.Status.Conditions, v1alpha1.Condition{
@@ -280,7 +298,7 @@ func (a *App) setReconcileCompleted(result exec.CmdRunResult) {
280298
a.app.Status.ConsecutiveReconcileSuccesses++
281299
a.app.Status.ConsecutiveReconcileFailures = 0
282300
a.app.Status.FriendlyDescription = "Reconcile succeeded"
283-
a.appMetrics.RegisterReconcileSuccess(a.app.Name, a.app.Namespace)
301+
a.appMetrics.ReconcileCountMetrics.RegisterReconcileSuccess(a.Kind(), a.Name(), a.Namespace())
284302
a.app.Status.UsefulErrorMessage = ""
285303
}
286304
}
@@ -293,7 +311,7 @@ func (a *App) setDeleting() {
293311
Status: corev1.ConditionTrue,
294312
})
295313

296-
a.appMetrics.RegisterReconcileDeleteAttempt(a.app.Name, a.app.Namespace)
314+
a.appMetrics.ReconcileCountMetrics.RegisterReconcileDeleteAttempt(a.Kind(), a.Name(), a.Namespace())
297315
a.app.Status.FriendlyDescription = "Deleting"
298316
}
299317

@@ -309,10 +327,10 @@ func (a *App) setDeleteCompleted(result exec.CmdRunResult) {
309327
a.app.Status.ConsecutiveReconcileFailures++
310328
a.app.Status.ConsecutiveReconcileSuccesses = 0
311329
a.app.Status.FriendlyDescription = fmt.Sprintf("Delete failed: %s", result.ErrorStr())
312-
a.appMetrics.RegisterReconcileDeleteFailed(a.app.Name, a.app.Namespace)
330+
a.appMetrics.ReconcileCountMetrics.RegisterReconcileDeleteFailed(a.Kind(), a.Name(), a.Namespace())
313331
a.setUsefulErrorMessage(result)
314332
} else {
315-
a.appMetrics.DeleteMetrics(a.app.Name, a.app.Namespace)
333+
a.appMetrics.ReconcileCountMetrics.DeleteMetrics(a.Kind(), a.Name(), a.Namespace())
316334
}
317335
}
318336

‎pkg/app/app_reconcile_test.go‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ import (
2929

3030
func Test_NoInspectReconcile_IfNoDeployAttempted(t *testing.T) {
3131
log := logf.Log.WithName("kc")
32-
var appMetrics = metrics.NewAppMetrics()
3332

3433
// The url under fetch is invalid, which will cause this
3534
// app to fail before deploy.
@@ -52,7 +51,7 @@ func Test_NoInspectReconcile_IfNoDeployAttempted(t *testing.T) {
5251
tmpFac := template.NewFactory(k8scs, fetchFac, false, exec.NewPlainCmdRunner())
5352
deployFac := deploy.NewFactory(k8scs, kubeconfig.NewKubeconfig(k8scs, log), nil, exec.NewPlainCmdRunner(), log)
5453

55-
crdApp := NewCRDApp(&app, log, appMetrics, kappcs, fetchFac, tmpFac, deployFac, FakeComponentInfo{}, Opts{MinimumSyncPeriod: 30 * time.Second})
54+
crdApp := NewCRDApp(&app, log, metrics.NewMetrics(), kappcs, fetchFac, tmpFac, deployFac, FakeComponentInfo{}, Opts{MinimumSyncPeriod: 30 * time.Second})
5655
_, err := crdApp.Reconcile(false)
5756
assert.Nil(t, err, "unexpected error with reconciling", err)
5857

@@ -86,7 +85,6 @@ func Test_NoInspectReconcile_IfNoDeployAttempted(t *testing.T) {
8685

8786
func Test_NoInspectReconcile_IfInspectNotEnabled(t *testing.T) {
8887
log := logf.Log.WithName("kc")
89-
var appMetrics = metrics.NewAppMetrics()
9088

9189
app := v1alpha1.App{
9290
ObjectMeta: metav1.ObjectMeta{
@@ -119,7 +117,7 @@ func Test_NoInspectReconcile_IfInspectNotEnabled(t *testing.T) {
119117
tmpFac := template.NewFactory(k8scs, fetchFac, false, exec.NewPlainCmdRunner())
120118
deployFac := deploy.NewFactory(k8scs, kubeconfig.NewKubeconfig(k8scs, log), nil, exec.NewPlainCmdRunner(), log)
121119

122-
crdApp := NewCRDApp(&app, log, appMetrics, kappcs, fetchFac, tmpFac, deployFac, FakeComponentInfo{}, Opts{MinimumSyncPeriod: 30 * time.Second})
120+
crdApp := NewCRDApp(&app, log, metrics.NewMetrics(), kappcs, fetchFac, tmpFac, deployFac, FakeComponentInfo{}, Opts{MinimumSyncPeriod: 30 * time.Second})
123121
_, err := crdApp.Reconcile(false)
124122
assert.Nil(t, err, "unexpected error with reconciling", err)
125123

@@ -164,7 +162,6 @@ func Test_NoInspectReconcile_IfInspectNotEnabled(t *testing.T) {
164162

165163
func Test_TemplateError_DisplayedInStatus_UsefulErrorMessageProperty(t *testing.T) {
166164
log := logf.Log.WithName("kc")
167-
var appMetrics = metrics.NewAppMetrics()
168165

169166
fetchInline := map[string]string{
170167
"file.yml": `foo: #@ data.values.nothere`,
@@ -191,7 +188,7 @@ func Test_TemplateError_DisplayedInStatus_UsefulErrorMessageProperty(t *testing.
191188
tmpFac := template.NewFactory(k8scs, fetchFac, false, exec.NewPlainCmdRunner())
192189
deployFac := deploy.NewFactory(k8scs, kubeconfig.NewKubeconfig(k8scs, log), nil, exec.NewPlainCmdRunner(), log)
193190

194-
crdApp := NewCRDApp(&app, log, appMetrics, kappcs, fetchFac, tmpFac, deployFac, FakeComponentInfo{}, Opts{MinimumSyncPeriod: 30 * time.Second})
191+
crdApp := NewCRDApp(&app, log, metrics.NewMetrics(), kappcs, fetchFac, tmpFac, deployFac, FakeComponentInfo{}, Opts{MinimumSyncPeriod: 30 * time.Second})
195192
_, err := crdApp.Reconcile(false)
196193
assert.Nil(t, err, "Unexpected error with reconciling", err)
197194

‎pkg/app/app_template_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ func Test_BuildAdditionalDownwardAPIValues_MemoizedCallCount(t *testing.T) {
6161
K8sAPIsCount: &k8sAPIsCallCount,
6262
KCVersionCount: &kcVersionCallCount,
6363
}
64-
app := NewApp(appEmpty, Hooks{}, fetchFac, tmpFac, deployFac, log, Opts{}, metrics.NewAppMetrics(), fakeInfo)
64+
app := NewApp(appEmpty, Hooks{}, fetchFac, tmpFac, deployFac, log, Opts{}, metrics.NewMetrics(), fakeInfo)
6565

6666
dir, err := os.MkdirTemp("", "temp")
6767
assert.NoError(t, err)

0 commit comments

Comments
 (0)