Skip to content

Commit 18f74ec

Browse files
committed
feat: Support zstd encoding
This allows endpoints to respond with zstd compressed metric data, if the requester supports it. For backwards compatibility, gzip compression will take precedence. Signed-off-by: Manuel Rüger <[email protected]>
1 parent e133e49 commit 18f74ec

File tree

4 files changed

+174
-10
lines changed

4 files changed

+174
-10
lines changed

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ require (
77
github.com/cespare/xxhash/v2 v2.2.0
88
github.com/davecgh/go-spew v1.1.1
99
github.com/json-iterator/go v1.1.12
10+
github.com/klauspost/compress v1.17.8
1011
github.com/prometheus/client_model v0.6.0
1112
github.com/prometheus/common v0.48.0
1213
github.com/prometheus/procfs v0.13.0

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2E
1717
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
1818
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
1919
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
20+
github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
21+
github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
2022
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
2123
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
2224
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=

prometheus/promhttp/http.go

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ import (
4242
"sync"
4343
"time"
4444

45+
"github.com/klauspost/compress/zstd"
4546
"github.com/prometheus/common/expfmt"
4647

4748
"github.com/prometheus/client_golang/prometheus"
@@ -169,15 +170,31 @@ func HandlerForTransactional(reg prometheus.TransactionalGatherer, opts HandlerO
169170
header.Set(contentTypeHeader, string(contentType))
170171

171172
w := io.Writer(rsp)
172-
if !opts.DisableCompression && gzipAccepted(req.Header) {
173-
header.Set(contentEncodingHeader, "gzip")
174-
gz := gzipPool.Get().(*gzip.Writer)
175-
defer gzipPool.Put(gz)
173+
if !opts.DisableCompression {
174+
// Gzip takes precedence over zstd
175+
// TODO(mrueg): Replace klauspost/compress with stdlib implementation once https://github.com/golang/go/issues/62513 is implemented.
176+
if EncodingAccepted(req.Header, "zstd") {
177+
header.Set(contentEncodingHeader, "zstd")
178+
z, err := zstd.NewWriter(rsp, zstd.WithEncoderLevel(zstd.SpeedFastest))
179+
if err != nil {
180+
return
181+
}
182+
z.Reset(w)
183+
defer z.Close()
184+
185+
w = z
186+
}
187+
if EncodingAccepted(req.Header, "gzip") {
188+
header.Set(contentEncodingHeader, "gzip")
189+
gz := gzipPool.Get().(*gzip.Writer)
190+
defer gzipPool.Put(gz)
191+
192+
gz.Reset(w)
193+
defer gz.Close()
176194

177-
gz.Reset(w)
178-
defer gz.Close()
195+
w = gz
196+
}
179197

180-
w = gz
181198
}
182199

183200
enc := expfmt.NewEncoder(w, contentType)
@@ -381,13 +398,13 @@ type HandlerOpts struct {
381398
ProcessStartTime time.Time
382399
}
383400

384-
// gzipAccepted returns whether the client will accept gzip-encoded content.
385-
func gzipAccepted(header http.Header) bool {
401+
// EncodingAccepted returns whether the client will accept encoded content.
402+
func EncodingAccepted(header http.Header, encoding string) bool {
386403
a := header.Get(acceptEncodingHeader)
387404
parts := strings.Split(a, ",")
388405
for _, part := range parts {
389406
part = strings.TrimSpace(part)
390-
if part == "gzip" || strings.HasPrefix(part, "gzip;") {
407+
if part == encoding || strings.HasPrefix(part, encoding+";") {
391408
return true
392409
}
393410
}

prometheus/promhttp/http_test.go

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -331,3 +331,147 @@ func TestHandlerTimeout(t *testing.T) {
331331

332332
close(c.Block) // To not leak a goroutine.
333333
}
334+
335+
func TestEncodingAccepted(t *testing.T) {
336+
testCases := []struct {
337+
name string
338+
header http.Header
339+
encodingType string
340+
expected bool
341+
}{
342+
{
343+
name: "test with gzip accept-encoding",
344+
header: http.Header{"Accept-Encoding": {"gzip"}},
345+
encodingType: "gzip",
346+
expected: true,
347+
},
348+
{
349+
name: "test with zstd accept-encoding",
350+
header: http.Header{"Accept-Encoding": {"zstd"}},
351+
encodingType: "zstd",
352+
expected: true,
353+
},
354+
{
355+
name: "test with zstd + gzip accept-encoding",
356+
header: http.Header{"Accept-Encoding": {"zstd;gzip"}},
357+
encodingType: "zstd",
358+
expected: true,
359+
},
360+
{
361+
name: "test with gzip accept-encoding and zstd allowed",
362+
header: http.Header{"Accept-Encoding": {"gzip"}},
363+
encodingType: "zstd",
364+
expected: false,
365+
},
366+
{
367+
name: "test with plain encoding",
368+
header: http.Header{"Accept-Encoding": {"plain"}},
369+
encodingType: "zstd",
370+
expected: false,
371+
},
372+
{
373+
name: "test with encoding header",
374+
header: http.Header{},
375+
encodingType: "zstd",
376+
expected: false,
377+
},
378+
}
379+
380+
for _, test := range testCases {
381+
if actual := EncodingAccepted(test.header, test.encodingType); test.expected != actual {
382+
t.Fatalf("%v: expected %v, actual %v", test.name, test.expected, actual)
383+
}
384+
}
385+
}
386+
387+
func BenchmarkEncoding(b *testing.B) {
388+
benchmarks := []struct {
389+
name string
390+
encodingType string
391+
}{
392+
{
393+
name: "test with gzip encoding",
394+
encodingType: "gzip",
395+
},
396+
{
397+
name: "test with zstd encoding",
398+
encodingType: "zstd",
399+
},
400+
{
401+
name: "test with no encoding",
402+
encodingType: "identity",
403+
},
404+
}
405+
sizes := []struct {
406+
name string
407+
metricCount int
408+
labelCount int
409+
labelLength int
410+
metricLength int
411+
}{
412+
{
413+
name: "small",
414+
metricCount: 50,
415+
labelCount: 5,
416+
labelLength: 5,
417+
metricLength: 5,
418+
},
419+
{
420+
name: "medium",
421+
metricCount: 500,
422+
labelCount: 10,
423+
labelLength: 5,
424+
metricLength: 10,
425+
},
426+
{
427+
name: "large",
428+
metricCount: 5000,
429+
labelCount: 10,
430+
labelLength: 5,
431+
metricLength: 10,
432+
},
433+
{
434+
name: "extra-large",
435+
metricCount: 50000,
436+
labelCount: 20,
437+
labelLength: 5,
438+
metricLength: 10,
439+
},
440+
}
441+
442+
for _, size := range sizes {
443+
reg := prometheus.NewRegistry()
444+
handler := HandlerFor(reg, HandlerOpts{})
445+
446+
// Generate Metrics
447+
// Original source: https://github.com/prometheus-community/avalanche/blob/main/metrics/serve.go
448+
labelKeys := make([]string, size.labelCount)
449+
for idx := 0; idx < size.labelCount; idx++ {
450+
labelKeys[idx] = fmt.Sprintf("label_key_%s_%v", strings.Repeat("k", size.labelLength), idx)
451+
}
452+
labelValues := make([]string, size.labelCount)
453+
for idx := 0; idx < size.labelCount; idx++ {
454+
labelValues[idx] = fmt.Sprintf("label_val_%s_%v", strings.Repeat("v", size.labelLength), idx)
455+
}
456+
metrics := make([]*prometheus.GaugeVec, size.metricCount)
457+
for idx := 0; idx < size.metricCount; idx++ {
458+
gauge := prometheus.NewGaugeVec(prometheus.GaugeOpts{
459+
Name: fmt.Sprintf("avalanche_metric_%s_%v_%v", strings.Repeat("m", size.metricLength), 0, idx),
460+
Help: "A tasty metric morsel",
461+
}, append([]string{"series_id", "cycle_id"}, labelKeys...))
462+
reg.MustRegister(gauge)
463+
metrics[idx] = gauge
464+
}
465+
466+
for _, benchmark := range benchmarks {
467+
b.Run(benchmark.name+"_"+size.name, func(b *testing.B) {
468+
for i := 0; i < b.N; i++ {
469+
writer := httptest.NewRecorder()
470+
request, _ := http.NewRequest("GET", "/", nil)
471+
request.Header.Add("Accept-Encoding", benchmark.encodingType)
472+
handler.ServeHTTP(writer, request)
473+
}
474+
})
475+
}
476+
}
477+
}

0 commit comments

Comments
 (0)