Skip to content

Commit ef972b9

Browse files
committed
Add exemplars to counter and histogram
Signed-off-by: beorn7 <[email protected]>
1 parent 803ef2a commit ef972b9

File tree

8 files changed

+229
-60
lines changed

8 files changed

+229
-60
lines changed

examples/random/main.go

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package main
1818

1919
import (
2020
"flag"
21+
"fmt"
2122
"log"
2223
"math"
2324
"math/rand"
@@ -89,7 +90,11 @@ func main() {
8990
for {
9091
v := (rand.NormFloat64() * *normDomain) + *normMean
9192
rpcDurations.WithLabelValues("normal").Observe(v)
92-
rpcDurationsHistogram.Observe(v)
93+
rpcDurationsHistogram.ObserveWithExemplar(
94+
// Demonstrate exemplar support with a dummy ID. This would be
95+
// something like a trace ID in a real application.
96+
v, prometheus.Labels{"dummyID": fmt.Sprint(rand.Intn(100000))},
97+
)
9398
time.Sleep(time.Duration(75*oscillationFactor()) * time.Millisecond)
9499
}
95100
}()
@@ -103,6 +108,12 @@ func main() {
103108
}()
104109

105110
// Expose the registered metrics via HTTP.
106-
http.Handle("/metrics", promhttp.Handler())
111+
http.Handle("/metrics", promhttp.HandlerFor(
112+
prometheus.DefaultGatherer,
113+
promhttp.HandlerOpts{
114+
// Opt into OpenMetrics to support exemplars.
115+
EnableOpenMetrics: true,
116+
},
117+
))
107118
log.Fatal(http.ListenAndServe(*addr, nil))
108119
}

go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ require (
55
github.com/cespare/xxhash/v2 v2.1.1
66
github.com/golang/protobuf v1.3.2
77
github.com/json-iterator/go v1.1.8
8-
github.com/prometheus/client_model v0.1.0
9-
github.com/prometheus/common v0.7.0
8+
github.com/prometheus/client_model v0.2.0
9+
github.com/prometheus/common v0.9.0
1010
github.com/prometheus/procfs v0.0.8
1111
golang.org/x/sys v0.0.0-20191220142924-d4481acd189f
1212
)

go.sum

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -58,12 +58,12 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f
5858
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
5959
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
6060
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
61-
github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE=
62-
github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
61+
github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
62+
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
6363
github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
6464
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
65-
github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
66-
github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
65+
github.com/prometheus/common v0.9.0 h1:yg//x/8DqN+PxXTBFMwVCopGqDn3wSxmbF/3PCuu1bk=
66+
github.com/prometheus/common v0.9.0/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
6767
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
6868
github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
6969
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -97,4 +97,4 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
9797
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
9898
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
9999
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
100-
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
100+
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

prometheus/counter.go

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"errors"
1818
"math"
1919
"sync/atomic"
20+
"time"
2021

2122
dto "github.com/prometheus/client_model/go"
2223
)
@@ -40,6 +41,16 @@ type Counter interface {
4041
// Add adds the given value to the counter. It panics if the value is <
4142
// 0.
4243
Add(float64)
44+
// AddWithExemplar works like Add but also replaces the currently saved
45+
// exemplar (possibly none) with a new one, created from the provided
46+
// value, the current time as timestamp, and the provided labels. Empty
47+
// Labels will lead to a valid (label-less) exemplar. But if Labels is
48+
// nil, the current exemplar is left in place. This method panics if the
49+
// value is < 0, if any of the provided labels are invalid, or if the
50+
// provided labels contain more than 64 runes in total. AddWithExemplar
51+
// should not be called in a hot path as it is significantly more costly
52+
// than Add.
53+
AddWithExemplar(value float64, exemplar Labels)
4354
}
4455

4556
// CounterOpts is an alias for Opts. See there for doc comments.
@@ -78,6 +89,7 @@ type counter struct {
7889
desc *Desc
7990

8091
labelPairs []*dto.LabelPair
92+
exemplar atomic.Value // *dto.Exemplar
8193
}
8294

8395
func (c *counter) Desc() *Desc {
@@ -88,6 +100,7 @@ func (c *counter) Add(v float64) {
88100
if v < 0 {
89101
panic(errors.New("counter cannot decrease in value"))
90102
}
103+
91104
ival := uint64(v)
92105
if float64(ival) == v {
93106
atomic.AddUint64(&c.valInt, ival)
@@ -103,6 +116,11 @@ func (c *counter) Add(v float64) {
103116
}
104117
}
105118

119+
func (c *counter) AddWithExemplar(v float64, e Labels) {
120+
c.Add(v)
121+
c.updateExemplar(v, e)
122+
}
123+
106124
func (c *counter) Inc() {
107125
atomic.AddUint64(&c.valInt, 1)
108126
}
@@ -112,7 +130,23 @@ func (c *counter) Write(out *dto.Metric) error {
112130
ival := atomic.LoadUint64(&c.valInt)
113131
val := fval + float64(ival)
114132

115-
return populateMetric(CounterValue, val, c.labelPairs, out)
133+
var exemplar *dto.Exemplar
134+
if e := c.exemplar.Load(); e != nil {
135+
exemplar = e.(*dto.Exemplar)
136+
}
137+
138+
return populateMetric(CounterValue, val, c.labelPairs, exemplar, out)
139+
}
140+
141+
func (c *counter) updateExemplar(v float64, l Labels) {
142+
if l == nil {
143+
return
144+
}
145+
e, err := newExemplar(v, time.Now(), l)
146+
if err != nil {
147+
panic(err)
148+
}
149+
c.exemplar.Store(e)
116150
}
117151

118152
// CounterVec is a Collector that bundles a set of Counters that all share the

prometheus/gauge.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ func (g *gauge) Sub(val float64) {
123123

124124
func (g *gauge) Write(out *dto.Metric) error {
125125
val := math.Float64frombits(atomic.LoadUint64(&g.valBits))
126-
return populateMetric(GaugeValue, val, g.labelPairs, out)
126+
return populateMetric(GaugeValue, val, g.labelPairs, nil, out)
127127
}
128128

129129
// GaugeVec is a Collector that bundles a set of Gauges that all share the same

prometheus/histogram.go

Lines changed: 83 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"sort"
2121
"sync"
2222
"sync/atomic"
23+
"time"
2324

2425
"github.com/golang/protobuf/proto"
2526

@@ -47,6 +48,16 @@ type Histogram interface {
4748

4849
// Observe adds a single observation to the histogram.
4950
Observe(float64)
51+
// ObserveWithExemplar works like Observe but also replaces the
52+
// currently saved exemplar for the relevant bucket (possibly none) with
53+
// a new one, created from the provided value, the current time as
54+
// timestamp, and the provided Labels. Empty Labels will lead to a valid
55+
// (label-less) exemplar. But if Labels is nil, the current exemplar in
56+
// the relevant bucket is left in place. This method panics if any of
57+
// the provided labels are invalid or if the provided labels contain
58+
// more than 64 runes in total. ObserveWithExemplar should not be called
59+
// in a hot path as it is significantly more costly than Observe.
60+
ObserveWithExemplar(value float64, exemplar Labels)
5061
}
5162

5263
// bucketLabel is used for the label that defines the upper bound of a
@@ -205,9 +216,10 @@ func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogr
205216
}
206217
}
207218
// Finally we know the final length of h.upperBounds and can make buckets
208-
// for both counts:
219+
// for both counts as well as exemplars:
209220
h.counts[0].buckets = make([]uint64, len(h.upperBounds))
210221
h.counts[1].buckets = make([]uint64, len(h.upperBounds))
222+
h.exemplars = make([]atomic.Value, len(h.upperBounds)+1)
211223

212224
h.init(h) // Init self-collection.
213225
return h
@@ -254,43 +266,21 @@ type histogram struct {
254266

255267
upperBounds []float64
256268
labelPairs []*dto.LabelPair
269+
exemplars []atomic.Value // One more than buckets (to include +Inf), each a *dto.Exemplar.
257270
}
258271

259272
func (h *histogram) Desc() *Desc {
260273
return h.desc
261274
}
262275

263276
func (h *histogram) Observe(v float64) {
264-
// TODO(beorn7): For small numbers of buckets (<30), a linear search is
265-
// slightly faster than the binary search. If we really care, we could
266-
// switch from one search strategy to the other depending on the number
267-
// of buckets.
268-
//
269-
// Microbenchmarks (BenchmarkHistogramNoLabels):
270-
// 11 buckets: 38.3 ns/op linear - binary 48.7 ns/op
271-
// 100 buckets: 78.1 ns/op linear - binary 54.9 ns/op
272-
// 300 buckets: 154 ns/op linear - binary 61.6 ns/op
273-
i := sort.SearchFloat64s(h.upperBounds, v)
274-
275-
// We increment h.countAndHotIdx so that the counter in the lower
276-
// 63 bits gets incremented. At the same time, we get the new value
277-
// back, which we can use to find the currently-hot counts.
278-
n := atomic.AddUint64(&h.countAndHotIdx, 1)
279-
hotCounts := h.counts[n>>63]
277+
h.observe(v, h.findBucket(v))
278+
}
280279

281-
if i < len(h.upperBounds) {
282-
atomic.AddUint64(&hotCounts.buckets[i], 1)
283-
}
284-
for {
285-
oldBits := atomic.LoadUint64(&hotCounts.sumBits)
286-
newBits := math.Float64bits(math.Float64frombits(oldBits) + v)
287-
if atomic.CompareAndSwapUint64(&hotCounts.sumBits, oldBits, newBits) {
288-
break
289-
}
290-
}
291-
// Increment count last as we take it as a signal that the observation
292-
// is complete.
293-
atomic.AddUint64(&hotCounts.count, 1)
280+
func (h *histogram) ObserveWithExemplar(v float64, e Labels) {
281+
i := h.findBucket(v)
282+
h.observe(v, i)
283+
h.updateExemplar(v, i, e)
294284
}
295285

296286
func (h *histogram) Write(out *dto.Metric) error {
@@ -329,6 +319,18 @@ func (h *histogram) Write(out *dto.Metric) error {
329319
CumulativeCount: proto.Uint64(cumCount),
330320
UpperBound: proto.Float64(upperBound),
331321
}
322+
if e := h.exemplars[i].Load(); e != nil {
323+
his.Bucket[i].Exemplar = e.(*dto.Exemplar)
324+
}
325+
}
326+
// If there is an exemplar for the +Inf bucket, we have to add that bucket explicitly.
327+
if e := h.exemplars[len(h.upperBounds)].Load(); e != nil {
328+
b := &dto.Bucket{
329+
CumulativeCount: proto.Uint64(count),
330+
UpperBound: proto.Float64(math.Inf(1)),
331+
Exemplar: e.(*dto.Exemplar),
332+
}
333+
his.Bucket = append(his.Bucket, b)
332334
}
333335

334336
out.Histogram = his
@@ -352,6 +354,57 @@ func (h *histogram) Write(out *dto.Metric) error {
352354
return nil
353355
}
354356

357+
// findBucket returns the index of the bucket for the provided value, or
358+
// len(h.upperBounds) for the +Inf bucket.
359+
func (h *histogram) findBucket(v float64) int {
360+
// TODO(beorn7): For small numbers of buckets (<30), a linear search is
361+
// slightly faster than the binary search. If we really care, we could
362+
// switch from one search strategy to the other depending on the number
363+
// of buckets.
364+
//
365+
// Microbenchmarks (BenchmarkHistogramNoLabels):
366+
// 11 buckets: 38.3 ns/op linear - binary 48.7 ns/op
367+
// 100 buckets: 78.1 ns/op linear - binary 54.9 ns/op
368+
// 300 buckets: 154 ns/op linear - binary 61.6 ns/op
369+
return sort.SearchFloat64s(h.upperBounds, v)
370+
}
371+
372+
// observe is the implementation for Observe without the findBucket part.
373+
func (h *histogram) observe(v float64, bucket int) {
374+
// We increment h.countAndHotIdx so that the counter in the lower
375+
// 63 bits gets incremented. At the same time, we get the new value
376+
// back, which we can use to find the currently-hot counts.
377+
n := atomic.AddUint64(&h.countAndHotIdx, 1)
378+
hotCounts := h.counts[n>>63]
379+
380+
if bucket < len(h.upperBounds) {
381+
atomic.AddUint64(&hotCounts.buckets[bucket], 1)
382+
}
383+
for {
384+
oldBits := atomic.LoadUint64(&hotCounts.sumBits)
385+
newBits := math.Float64bits(math.Float64frombits(oldBits) + v)
386+
if atomic.CompareAndSwapUint64(&hotCounts.sumBits, oldBits, newBits) {
387+
break
388+
}
389+
}
390+
// Increment count last as we take it as a signal that the observation
391+
// is complete.
392+
atomic.AddUint64(&hotCounts.count, 1)
393+
}
394+
395+
// updateExemplar replaces the exemplar for the provided bucket. With empty
396+
// labels, it's a no-op. It panics if any of the labels is invalid.
397+
func (h *histogram) updateExemplar(v float64, bucket int, l Labels) {
398+
if l == nil {
399+
return
400+
}
401+
e, err := newExemplar(v, time.Now(), l)
402+
if err != nil {
403+
panic(err)
404+
}
405+
h.exemplars[bucket].Store(e)
406+
}
407+
355408
// HistogramVec is a Collector that bundles a set of Histograms that all share the
356409
// same Desc, but have different values for their variable labels. This is used
357410
// if you want to count the same thing partitioned by various dimensions

0 commit comments

Comments
 (0)