Merge branch 'master' into cleanup/gitlab-exporter-long-xacts

Sticksman · web-flow · commit 1a3fdaf500ff · 2023-07-10T12:17:33.000-07:00
diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml
@@ -1,3 +1,5 @@
+---
+# This action is synced from https://github.com/prometheus/prometheus
 name: golangci-lint
 on:
   push:
@@ -27,4 +29,4 @@ jobs:
       - name: Lint
         uses: golangci/golangci-lint-action@v3.4.0
         with:
-          version: v1.51.2
+          version: v1.53.3
diff --git a/.golangci.yml b/.golangci.yml
@@ -1,6 +1,7 @@
 ---
 linters:
   enable:
+  - misspell
   - revive
 
 issues:
@@ -14,3 +15,9 @@ linters-settings:
     exclude-functions:
       # Never check for logger errors.
       - (github.com/go-kit/log.Logger).Log
+  revive:
+    rules:
+      # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unused-parameter
+      - name: unused-parameter
+        severity: warning
+        disabled: true
diff --git a/.yamllint b/.yamllint
@@ -20,5 +20,4 @@ rules:
       config/testdata/section_key_dup.bad.yml
   line-length: disable
   truthy:
-    ignore: |
-      .github/workflows/*.yml
+    check-keys: false
diff --git a/Makefile.common b/Makefile.common
@@ -55,7 +55,7 @@ ifneq ($(shell command -v gotestsum > /dev/null),)
 endif
 endif
 
-PROMU_VERSION ?= 0.14.0
+PROMU_VERSION ?= 0.15.0
 PROMU_URL     := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz
 
 SKIP_GOLANGCI_LINT :=
diff --git a/collector/collector_test.go b/collector/collector_test.go
@@ -50,7 +50,12 @@ func sanitizeQuery(q string) string {
 	q = strings.Join(strings.Fields(q), " ")
 	q = strings.Replace(q, "(", "\\(", -1)
 	q = strings.Replace(q, ")", "\\)", -1)
+	q = strings.Replace(q, "[", "\\[", -1)
+	q = strings.Replace(q, "]", "\\]", -1)
+	q = strings.Replace(q, "{", "\\{", -1)
+	q = strings.Replace(q, "}", "\\}", -1)
 	q = strings.Replace(q, "*", "\\*", -1)
+	q = strings.Replace(q, "^", "\\^", -1)
 	q = strings.Replace(q, "$", "\\$", -1)
 	return q
 }
diff --git a/collector/pg_database.go b/collector/pg_database.go
@@ -115,10 +115,7 @@ func (c PGDatabaseCollector) Update(ctx context.Context, instance *instance, ch
 			prometheus.GaugeValue, sizeMetric, datname,
 		)
 	}
-	if err := rows.Err(); err != nil {
-		return err
-	}
-	return nil
+	return rows.Err()
 }
 
 func sliceContains(slice []string, s string) bool {
diff --git a/collector/pg_process_idle.go b/collector/pg_process_idle.go
@@ -18,6 +18,7 @@ import (
 	"database/sql"
 
 	"github.com/go-kit/log"
+	"github.com/lib/pq"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -39,7 +40,7 @@ func NewPGProcessIdleCollector(config collectorConfig) (Collector, error) {
 var pgProcessIdleSeconds = prometheus.NewDesc(
 	prometheus.BuildFQName(namespace, processIdleSubsystem, "seconds"),
 	"Idle time of server processes",
-	[]string{"application_name"},
+	[]string{"state", "application_name"},
 	prometheus.Labels{},
 )
 
@@ -49,15 +50,17 @@ func (PGProcessIdleCollector) Update(ctx context.Context, instance *instance, ch
 		`WITH
 			metrics AS (
 				SELECT
+				state,
 				application_name,
 				SUM(EXTRACT(EPOCH FROM (CURRENT_TIMESTAMP - state_change))::bigint)::float AS process_idle_seconds_sum,
 				COUNT(*) AS process_idle_seconds_count
 				FROM pg_stat_activity
-				WHERE state = 'idle'
-				GROUP BY application_name
+				WHERE state ~ '^idle'
+				GROUP BY state, application_name
 			),
 			buckets AS (
 				SELECT
+				state,
 				application_name,
 				le,
 				SUM(
@@ -69,35 +72,42 @@ func (PGProcessIdleCollector) Update(ctx context.Context, instance *instance, ch
 				FROM
 				pg_stat_activity,
 				UNNEST(ARRAY[1, 2, 5, 15, 30, 60, 90, 120, 300]) AS le
-				GROUP BY application_name, le
-				ORDER BY application_name, le
+				GROUP BY state, application_name, le
+				ORDER BY state, application_name, le
 			)
 			SELECT
+			state,
 			application_name,
 			process_idle_seconds_sum as seconds_sum,
 			process_idle_seconds_count as seconds_count,
 			ARRAY_AGG(le) AS seconds,
 			ARRAY_AGG(bucket) AS seconds_bucket
-			FROM metrics JOIN buckets USING (application_name)
-			GROUP BY 1, 2, 3;`)
+			FROM metrics JOIN buckets USING (state, application_name)
+			GROUP BY 1, 2, 3, 4;`)
 
+	var state sql.NullString
 	var applicationName sql.NullString
-	var secondsSum sql.NullInt64
+	var secondsSum sql.NullFloat64
 	var secondsCount sql.NullInt64
-	var seconds []uint64
-	var secondsBucket []uint64
+	var seconds []float64
+	var secondsBucket []int64
 
-	err := row.Scan(&applicationName, &secondsSum, &secondsCount, &seconds, &secondsBucket)
+	err := row.Scan(&state, &applicationName, &secondsSum, &secondsCount, pq.Array(&seconds), pq.Array(&secondsBucket))
+	if err != nil {
+		return err
+	}
 
 	var buckets = make(map[float64]uint64, len(seconds))
 	for i, second := range seconds {
 		if i >= len(secondsBucket) {
 			break
 		}
-		buckets[float64(second)] = secondsBucket[i]
+		buckets[second] = uint64(secondsBucket[i])
 	}
-	if err != nil {
-		return err
+
+	stateLabel := "unknown"
+	if state.Valid {
+		stateLabel = state.String
 	}
 
 	applicationNameLabel := "unknown"
@@ -111,12 +121,12 @@ func (PGProcessIdleCollector) Update(ctx context.Context, instance *instance, ch
 	}
 	secondsSumMetric := 0.0
 	if secondsSum.Valid {
-		secondsSumMetric = float64(secondsSum.Int64)
+		secondsSumMetric = secondsSum.Float64
 	}
 	ch <- prometheus.MustNewConstHistogram(
 		pgProcessIdleSeconds,
 		secondsCountMetric, secondsSumMetric, buckets,
-		applicationNameLabel,
+		stateLabel, applicationNameLabel,
 	)
 	return nil
 }
diff --git a/collector/pg_replication.go b/collector/pg_replication.go
@@ -29,7 +29,7 @@ type PGReplicationCollector struct {
 }
 
 func NewPGReplicationCollector(collectorConfig) (Collector, error) {
-	return &PGPostmasterCollector{}, nil
+	return &PGReplicationCollector{}, nil
 }
 
 var (
diff --git a/collector/pg_replication_slot.go b/collector/pg_replication_slot.go
@@ -126,8 +126,5 @@ func (PGReplicationSlotCollector) Update(ctx context.Context, instance *instance
 			prometheus.GaugeValue, isActiveValue, slotNameLabel,
 		)
 	}
-	if err := rows.Err(); err != nil {
-		return err
-	}
-	return nil
+	return rows.Err()
 }
diff --git a/collector/pg_stat_bgwriter_test.go b/collector/pg_stat_bgwriter_test.go
@@ -51,7 +51,7 @@ func TestPGStatBGWriterCollector(t *testing.T) {
 	}
 
 	rows := sqlmock.NewRows(columns).
-		AddRow(354, 4945, 289097744, 1242257, 3275602074, 89320867, 450139, 2034563757, 0, 2725688749, srT)
+		AddRow(354, 4945, 289097744, 1242257, int64(3275602074), 89320867, 450139, 2034563757, 0, int64(2725688749), srT)
 	mock.ExpectQuery(sanitizeQuery(statBGWriterQuery)).WillReturnRows(rows)
 
 	ch := make(chan prometheus.Metric)
diff --git a/collector/pg_stat_database_test.go b/collector/pg_stat_database_test.go
@@ -67,12 +67,12 @@ func TestPGStatDatabaseCollector(t *testing.T) {
 			4945,
 			289097744,
 			1242257,
-			3275602074,
+			int64(3275602074),
 			89320867,
 			450139,
 			2034563757,
 			0,
-			2725688749,
+			int64(2725688749),
 			23,
 			52,
 			74,
@@ -263,12 +263,12 @@ func TestPGStatDatabaseCollectorRowLeakTest(t *testing.T) {
 			4945,
 			289097744,
 			1242257,
-			3275602074,
+			int64(3275602074),
 			89320867,
 			450139,
 			2034563757,
 			0,
-			2725688749,
+			int64(2725688749),
 			23,
 			52,
 			74,
diff --git a/collector/pg_statio_user_tables.go b/collector/pg_statio_user_tables.go
@@ -218,8 +218,5 @@ func (PGStatIOUserTablesCollector) Update(ctx context.Context, instance *instanc
 			datnameLabel, schemanameLabel, relnameLabel,
 		)
 	}
-	if err := rows.Err(); err != nil {
-		return err
-	}
-	return nil
+	return rows.Err()
 }
diff --git a/collector/pg_wal.go b/collector/pg_wal.go
@@ -0,0 +1,84 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collector
+
+import (
+	"context"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const walSubsystem = "wal"
+
+func init() {
+	registerCollector(walSubsystem, defaultEnabled, NewPGWALCollector)
+}
+
+type PGWALCollector struct {
+}
+
+func NewPGWALCollector(config collectorConfig) (Collector, error) {
+	return &PGWALCollector{}, nil
+}
+
+var (
+	pgWALSegments = prometheus.NewDesc(
+		prometheus.BuildFQName(
+			namespace,
+			walSubsystem,
+			"segments",
+		),
+		"Number of WAL segments",
+		[]string{}, nil,
+	)
+	pgWALSize = prometheus.NewDesc(
+		prometheus.BuildFQName(
+			namespace,
+			walSubsystem,
+			"size_bytes",
+		),
+		"Total size of WAL segments",
+		[]string{}, nil,
+	)
+
+	pgWALQuery = `
+		SELECT
+			COUNT(*) AS segments,
+			SUM(size) AS size
+		FROM pg_ls_waldir()
+		WHERE name ~ '^[0-9A-F]{24}$'`
+)
+
+func (c PGWALCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error {
+	db := instance.getDB()
+	row := db.QueryRowContext(ctx,
+		pgWALQuery,
+	)
+
+	var segments uint64
+	var size uint64
+	err := row.Scan(&segments, &size)
+	if err != nil {
+		return err
+	}
+	ch <- prometheus.MustNewConstMetric(
+		pgWALSegments,
+		prometheus.GaugeValue, float64(segments),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		pgWALSize,
+		prometheus.GaugeValue, float64(size),
+	)
+	return nil
+}
diff --git a/collector/pg_wal_test.go b/collector/pg_wal_test.go
@@ -0,0 +1,63 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package collector
+
+import (
+	"context"
+	"testing"
+
+	"github.com/DATA-DOG/go-sqlmock"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestPgWALCollector(t *testing.T) {
+	db, mock, err := sqlmock.New()
+	if err != nil {
+		t.Fatalf("Error opening a stub db connection: %s", err)
+	}
+	defer db.Close()
+
+	inst := &instance{db: db}
+
+	columns := []string{"segments", "size"}
+	rows := sqlmock.NewRows(columns).
+		AddRow(47, 788529152)
+	mock.ExpectQuery(sanitizeQuery(pgWALQuery)).WillReturnRows(rows)
+
+	ch := make(chan prometheus.Metric)
+	go func() {
+		defer close(ch)
+		c := PGWALCollector{}
+
+		if err := c.Update(context.Background(), inst, ch); err != nil {
+			t.Errorf("Error calling PGWALCollector.Update: %s", err)
+		}
+	}()
+
+	expected := []MetricResult{
+		{labels: labelMap{}, value: 47, metricType: dto.MetricType_GAUGE},
+		{labels: labelMap{}, value: 788529152, metricType: dto.MetricType_GAUGE},
+	}
+
+	convey.Convey("Metrics comparison", t, func() {
+		for _, expect := range expected {
+			m := readMetric(<-ch)
+			convey.So(expect, convey.ShouldResemble, m)
+		}
+	})
+	if err := mock.ExpectationsWereMet(); err != nil {
+		t.Errorf("there were unfulfilled exceptions: %s", err)
+	}
+}
diff --git a/go.mod b/go.mod
diff --git a/go.sum b/go.sum

Original file line number	Diff line number	Diff line change
`@@ -115,10 +115,7 @@ func (c PGDatabaseCollector) Update(ctx context.Context, instance *instance, ch`
`115`	`115`	`prometheus.GaugeValue, sizeMetric, datname,`
`116`	`116`	`)`
`117`	`117`	`}`
`118`		`- if err := rows.Err(); err != nil {`
`119`		`- return err`
`120`		`- }`
`121`		`- return nil`
	`118`	`+ return rows.Err()`
`122`	`119`	`}`
`123`	`120`
`124`	`121`	`func sliceContains(slice []string, s string) bool {`
Original file line number	Diff line number	Diff line change
`@@ -29,7 +29,7 @@ type PGReplicationCollector struct {`
`29`	`29`	`}`
`30`	`30`
`31`	`31`	`func NewPGReplicationCollector(collectorConfig) (Collector, error) {`
`32`		`- return &PGPostmasterCollector{}, nil`
	`32`	`+ return &PGReplicationCollector{}, nil`
`33`	`33`	`}`
`34`	`34`
`35`	`35`	`var (`
Original file line number	Diff line number	Diff line change
`@@ -126,8 +126,5 @@ func (PGReplicationSlotCollector) Update(ctx context.Context, instance *instance`
`126`	`126`	`prometheus.GaugeValue, isActiveValue, slotNameLabel,`
`127`	`127`	`)`
`128`	`128`	`}`
`129`		`- if err := rows.Err(); err != nil {`
`130`		`- return err`
`131`		`- }`
`132`		`- return nil`
	`129`	`+ return rows.Err()`
`133`	`130`	`}`
Original file line number	Diff line number	Diff line change
`@@ -51,7 +51,7 @@ func TestPGStatBGWriterCollector(t *testing.T) {`
`51`	`51`	`}`
`52`	`52`
`53`	`53`	`rows := sqlmock.NewRows(columns).`
`54`		`- AddRow(354, 4945, 289097744, 1242257, 3275602074, 89320867, 450139, 2034563757, 0, 2725688749, srT)`
	`54`	`+ AddRow(354, 4945, 289097744, 1242257, int64(3275602074), 89320867, 450139, 2034563757, 0, int64(2725688749), srT)`
`55`	`55`	`mock.ExpectQuery(sanitizeQuery(statBGWriterQuery)).WillReturnRows(rows)`
`56`	`56`
`57`	`57`	`ch := make(chan prometheus.Metric)`