Skip to content

Commit 2892d1c

Browse files
zeripathzeripath
committed
Cache statistics and provide estimation methods
Currently whenever the prometheus metrics endpoint or `/admin` endpoint are viewed the statistics are recalculated immediately - using COUNT rather than a less expensive method. This PR provides a mechanism to cache these statistics, avoids generating all of the metrics on the admin page and provides an estimation method for the plain table counts. Fix go-gitea#17506 Signed-off-by: Andrew Thornton <[email protected]>
1 parent 1f05417 commit 2892d1c

File tree

9 files changed

+165
-76
lines changed

9 files changed

+165
-76
lines changed

‎docs/content/doc/advanced/config-cheat-sheet.en-us.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,8 @@ The following configuration set `Content-Type: application/vnd.android.package-a
197197
- `REPO_PAGING_NUM`: **50**: Number of repos that are shown in one page.
198198
- `NOTICE_PAGING_NUM`: **25**: Number of notices that are shown in one page.
199199
- `ORG_PAGING_NUM`: **50**: Number of organizations that are shown in one page.
200+
- `ESTIMATE_COUNTS`: **false**: Estimate counts for summary statistics instead counting directly.
201+
- `STATISTICS_TTL`: **5m**: Cache summary statistics for this period of time.
200202

201203
### UI - Metadata (`ui.meta`)
202204

@@ -974,6 +976,8 @@ Default templates for project boards:
974976
- `ENABLED_ISSUE_BY_LABEL`: **false**: Enable issue by label metrics with format `gitea_issues_by_label{label="bug"} 2`.
975977
- `ENABLED_ISSUE_BY_REPOSITORY`: **false**: Enable issue by repository metrics with format `gitea_issues_by_repository{repository="org/repo"} 5`.
976978
- `TOKEN`: **\<empty\>**: You need to specify the token, if you want to include in the authorization the metrics . The same token need to be used in prometheus parameters `bearer_token` or `bearer_token_file`.
979+
- `ESTIMATE_COUNTS`: **false**: Estimate counts for statistics instead counting directly.
980+
- `STATISTICS_TTL`: **5m**: Cache summary statistics for this period of time.
977981

978982
## API (`api`)
979983

‎models/db/context.go

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"code.gitea.io/gitea/modules/setting"
1212

1313
"xorm.io/builder"
14+
"xorm.io/xorm/schemas"
1415
)
1516

1617
// DefaultContext is the default context to run xorm queries in
@@ -168,3 +169,24 @@ func CountByBean(ctx context.Context, bean interface{}) (int64, error) {
168169
func TableName(bean interface{}) string {
169170
return x.TableName(bean)
170171
}
172+
173+
// EstimateTotal returns an estimate of total number of rows in table
174+
func EstimateTotal(bean interface{}) (int64, error) {
175+
tablename := x.TableName(bean)
176+
switch x.Dialect().URI().DBType {
177+
case schemas.MYSQL:
178+
var rows int64
179+
_, err := x.SQL("SELECT table_rows FROM information_schema.tables WHERE tables.table_name = ? AND tables.table_schema = ?;", tablename, x.Dialect().URI().DBName).Get(&rows)
180+
return rows, err
181+
case schemas.POSTGRES:
182+
var rows int64
183+
_, err := x.SQL("SELECT reltuples AS estimate FROM pg_class WHERE relname = ?;", tablename).Get(&rows)
184+
return rows, err
185+
case schemas.MSSQL:
186+
var rows int64
187+
_, err := x.SQL("sp_spaceused ?;", tablename).Get(&rows)
188+
return rows, err
189+
default:
190+
return x.Count(tablename)
191+
}
192+
}

‎models/statistic.go

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
package models
66

77
import (
8+
"time"
9+
810
asymkey_model "code.gitea.io/gitea/models/asymkey"
911
"code.gitea.io/gitea/models/auth"
1012
"code.gitea.io/gitea/models/db"
@@ -31,6 +33,7 @@ type Statistic struct {
3133
IssueByLabel []IssueByLabelCount
3234
IssueByRepository []IssueByRepositoryCount
3335
}
36+
Time time.Time
3437
}
3538

3639
// IssueByLabelCount contains the number of issue group by label
@@ -47,23 +50,31 @@ type IssueByRepositoryCount struct {
4750
}
4851

4952
// GetStatistic returns the database statistics
50-
func GetStatistic() (stats Statistic) {
53+
func GetStatistic(estimate, metrics bool) (stats Statistic) {
5154
e := db.GetEngine(db.DefaultContext)
5255
stats.Counter.User = user_model.CountUsers()
5356
stats.Counter.Org = organization.CountOrganizations()
54-
stats.Counter.PublicKey, _ = e.Count(new(asymkey_model.PublicKey))
5557
stats.Counter.Repo = repo_model.CountRepositories(true)
56-
stats.Counter.Watch, _ = e.Count(new(repo_model.Watch))
57-
stats.Counter.Star, _ = e.Count(new(repo_model.Star))
58-
stats.Counter.Action, _ = e.Count(new(Action))
59-
stats.Counter.Access, _ = e.Count(new(Access))
58+
if estimate {
59+
stats.Counter.PublicKey, _ = db.EstimateTotal(new(asymkey_model.PublicKey))
60+
stats.Counter.Watch, _ = db.EstimateTotal(new(repo_model.Watch))
61+
stats.Counter.Star, _ = db.EstimateTotal(new(repo_model.Star))
62+
stats.Counter.Action, _ = db.EstimateTotal(new(Action))
63+
stats.Counter.Access, _ = db.EstimateTotal(new(Access))
64+
} else {
65+
stats.Counter.PublicKey, _ = e.Count(new(asymkey_model.PublicKey))
66+
stats.Counter.Watch, _ = e.Count(new(repo_model.Watch))
67+
stats.Counter.Star, _ = e.Count(new(repo_model.Star))
68+
stats.Counter.Action, _ = e.Count(new(Action))
69+
stats.Counter.Access, _ = e.Count(new(Access))
70+
}
6071

6172
type IssueCount struct {
6273
Count int64
6374
IsClosed bool
6475
}
6576

66-
if setting.Metrics.EnabledIssueByLabel {
77+
if metrics && setting.Metrics.EnabledIssueByLabel {
6778
stats.Counter.IssueByLabel = []IssueByLabelCount{}
6879

6980
_ = e.Select("COUNT(*) AS count, l.name AS label").
@@ -73,7 +84,7 @@ func GetStatistic() (stats Statistic) {
7384
Find(&stats.Counter.IssueByLabel)
7485
}
7586

76-
if setting.Metrics.EnabledIssueByRepository {
87+
if metrics && setting.Metrics.EnabledIssueByRepository {
7788
stats.Counter.IssueByRepository = []IssueByRepositoryCount{}
7889

7990
_ = e.Select("COUNT(*) AS count, r.owner_name, r.name AS repository").
@@ -110,5 +121,6 @@ func GetStatistic() (stats Statistic) {
110121
stats.Counter.Attachment, _ = e.Count(new(repo_model.Attachment))
111122
stats.Counter.Project, _ = e.Count(new(project_model.Project))
112123
stats.Counter.ProjectBoard, _ = e.Count(new(project_model.Board))
124+
stats.Time = time.Now()
113125
return
114126
}

‎modules/metrics/collector.go

Lines changed: 60 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
package metrics
66

77
import (
8-
"code.gitea.io/gitea/models"
8+
"code.gitea.io/gitea/modules/setting"
99

1010
"github.com/prometheus/client_golang/prometheus"
1111
)
@@ -43,6 +43,8 @@ type Collector struct {
4343
Users *prometheus.Desc
4444
Watches *prometheus.Desc
4545
Webhooks *prometheus.Desc
46+
47+
StatisticsTime *prometheus.Desc
4648
}
4749

4850
// NewCollector returns a new Collector with all prometheus.Desc initialized
@@ -225,152 +227,152 @@ func (c Collector) Describe(ch chan<- *prometheus.Desc) {
225227

226228
// Collect returns the metrics with values
227229
func (c Collector) Collect(ch chan<- prometheus.Metric) {
228-
stats := models.GetStatistic()
230+
stats := GetStatistic(setting.Metrics.EstimateCounts, setting.Metrics.StatisticTTL)
229231

230-
ch <- prometheus.MustNewConstMetric(
232+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
231233
c.Accesses,
232234
prometheus.GaugeValue,
233235
float64(stats.Counter.Access),
234-
)
235-
ch <- prometheus.MustNewConstMetric(
236+
))
237+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
236238
c.Actions,
237239
prometheus.GaugeValue,
238240
float64(stats.Counter.Action),
239-
)
240-
ch <- prometheus.MustNewConstMetric(
241+
))
242+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
241243
c.Attachments,
242244
prometheus.GaugeValue,
243245
float64(stats.Counter.Attachment),
244-
)
245-
ch <- prometheus.MustNewConstMetric(
246+
))
247+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
246248
c.Comments,
247249
prometheus.GaugeValue,
248250
float64(stats.Counter.Comment),
249-
)
250-
ch <- prometheus.MustNewConstMetric(
251+
))
252+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
251253
c.Follows,
252254
prometheus.GaugeValue,
253255
float64(stats.Counter.Follow),
254-
)
255-
ch <- prometheus.MustNewConstMetric(
256+
))
257+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
256258
c.HookTasks,
257259
prometheus.GaugeValue,
258260
float64(stats.Counter.HookTask),
259-
)
260-
ch <- prometheus.MustNewConstMetric(
261+
))
262+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
261263
c.Issues,
262264
prometheus.GaugeValue,
263265
float64(stats.Counter.Issue),
264-
)
266+
))
265267
for _, il := range stats.Counter.IssueByLabel {
266-
ch <- prometheus.MustNewConstMetric(
268+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
267269
c.IssuesByLabel,
268270
prometheus.GaugeValue,
269271
float64(il.Count),
270272
il.Label,
271-
)
273+
))
272274
}
273275
for _, ir := range stats.Counter.IssueByRepository {
274-
ch <- prometheus.MustNewConstMetric(
276+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
275277
c.IssuesByRepository,
276278
prometheus.GaugeValue,
277279
float64(ir.Count),
278280
ir.OwnerName+"/"+ir.Repository,
279-
)
281+
))
280282
}
281-
ch <- prometheus.MustNewConstMetric(
283+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
282284
c.IssuesClosed,
283285
prometheus.GaugeValue,
284286
float64(stats.Counter.IssueClosed),
285-
)
286-
ch <- prometheus.MustNewConstMetric(
287+
))
288+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
287289
c.IssuesOpen,
288290
prometheus.GaugeValue,
289291
float64(stats.Counter.IssueOpen),
290-
)
291-
ch <- prometheus.MustNewConstMetric(
292+
))
293+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
292294
c.Labels,
293295
prometheus.GaugeValue,
294296
float64(stats.Counter.Label),
295-
)
296-
ch <- prometheus.MustNewConstMetric(
297+
))
298+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
297299
c.LoginSources,
298300
prometheus.GaugeValue,
299301
float64(stats.Counter.AuthSource),
300-
)
301-
ch <- prometheus.MustNewConstMetric(
302+
))
303+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
302304
c.Milestones,
303305
prometheus.GaugeValue,
304306
float64(stats.Counter.Milestone),
305-
)
306-
ch <- prometheus.MustNewConstMetric(
307+
))
308+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
307309
c.Mirrors,
308310
prometheus.GaugeValue,
309311
float64(stats.Counter.Mirror),
310-
)
311-
ch <- prometheus.MustNewConstMetric(
312+
))
313+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
312314
c.Oauths,
313315
prometheus.GaugeValue,
314316
float64(stats.Counter.Oauth),
315-
)
316-
ch <- prometheus.MustNewConstMetric(
317+
))
318+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
317319
c.Organizations,
318320
prometheus.GaugeValue,
319321
float64(stats.Counter.Org),
320-
)
321-
ch <- prometheus.MustNewConstMetric(
322+
))
323+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
322324
c.Projects,
323325
prometheus.GaugeValue,
324326
float64(stats.Counter.Project),
325-
)
326-
ch <- prometheus.MustNewConstMetric(
327+
))
328+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
327329
c.ProjectBoards,
328330
prometheus.GaugeValue,
329331
float64(stats.Counter.ProjectBoard),
330-
)
331-
ch <- prometheus.MustNewConstMetric(
332+
))
333+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
332334
c.PublicKeys,
333335
prometheus.GaugeValue,
334336
float64(stats.Counter.PublicKey),
335-
)
336-
ch <- prometheus.MustNewConstMetric(
337+
))
338+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
337339
c.Releases,
338340
prometheus.GaugeValue,
339341
float64(stats.Counter.Release),
340-
)
341-
ch <- prometheus.MustNewConstMetric(
342+
))
343+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
342344
c.Repositories,
343345
prometheus.GaugeValue,
344346
float64(stats.Counter.Repo),
345-
)
346-
ch <- prometheus.MustNewConstMetric(
347+
))
348+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
347349
c.Stars,
348350
prometheus.GaugeValue,
349351
float64(stats.Counter.Star),
350-
)
351-
ch <- prometheus.MustNewConstMetric(
352+
))
353+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
352354
c.Teams,
353355
prometheus.GaugeValue,
354356
float64(stats.Counter.Team),
355-
)
356-
ch <- prometheus.MustNewConstMetric(
357+
))
358+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
357359
c.UpdateTasks,
358360
prometheus.GaugeValue,
359361
float64(stats.Counter.UpdateTask),
360-
)
361-
ch <- prometheus.MustNewConstMetric(
362+
))
363+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
362364
c.Users,
363365
prometheus.GaugeValue,
364366
float64(stats.Counter.User),
365-
)
366-
ch <- prometheus.MustNewConstMetric(
367+
))
368+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
367369
c.Watches,
368370
prometheus.GaugeValue,
369371
float64(stats.Counter.Watch),
370-
)
371-
ch <- prometheus.MustNewConstMetric(
372+
))
373+
ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
372374
c.Webhooks,
373375
prometheus.GaugeValue,
374376
float64(stats.Counter.Webhook),
375-
)
377+
))
376378
}

‎modules/metrics/statistics.go

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
// Copyright 2022 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package metrics
6+
7+
import (
8+
"strconv"
9+
"sync"
10+
"time"
11+
12+
"code.gitea.io/gitea/models"
13+
"code.gitea.io/gitea/modules/cache"
14+
"code.gitea.io/gitea/modules/setting"
15+
)
16+
17+
var statisticsLock sync.Mutex
18+
19+
func GetStatistic(estimate bool, statisticsTTL time.Duration, metrics bool) models.Statistic {
20+
if statisticsTTL > 0 {
21+
c := cache.GetCache()
22+
if c != nil {
23+
statisticsLock.Lock()
24+
defer statisticsLock.Unlock()
25+
cacheKey := "models/statistic.Statistic." + strconv.FormatBool(estimate) + strconv.FormatBool(metrics)
26+
27+
if stats, ok := c.Get(cacheKey).(*models.Statistic); ok {
28+
return *stats
29+
}
30+
31+
stats := models.GetStatistic(estimate, metrics)
32+
c.Put(cacheKey, &stats, setting.DurationToCacheTTL(statisticsTTL))
33+
return stats
34+
}
35+
}
36+
37+
return models.GetStatistic(estimate, metrics)
38+
}

0 commit comments

Comments
 (0)