Skip to content

Commit f9c1994

Browse files
codesomecodesome
authored
Merge pull request #440 from zenador/sparsehistograms
Optimise JSON marshalling for sparse histograms
2 parents ca1f99b + 18a4214 commit f9c1994

File tree

8 files changed

+469
-320
lines changed

8 files changed

+469
-320
lines changed

‎go.mod‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ go 1.17
55
require (
66
github.com/go-kit/log v0.2.1
77
github.com/golang/protobuf v1.5.2
8+
github.com/json-iterator/go v1.1.12
89
github.com/julienschmidt/httprouter v1.3.0
910
github.com/matttproud/golang_protobuf_extensions v1.0.4
1011
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f
@@ -23,6 +24,8 @@ require (
2324
github.com/cespare/xxhash/v2 v2.1.2 // indirect
2425
github.com/go-logfmt/logfmt v0.5.1 // indirect
2526
github.com/jpillora/backoff v1.0.0 // indirect
27+
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
28+
github.com/modern-go/reflect2 v1.0.2 // indirect
2629
github.com/prometheus/procfs v0.8.0 // indirect
2730
github.com/stretchr/testify v1.8.0 // indirect
2831
golang.org/x/sys v0.3.0 // indirect

‎go.sum‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX
139139
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
140140
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
141141
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
142+
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
142143
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
143144
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
144145
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
@@ -159,9 +160,11 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
159160
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
160161
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
161162
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
163+
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
162164
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
163165
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
164166
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
167+
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
165168
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
166169
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
167170
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=

‎model/value.go‎

Lines changed: 4 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -100,37 +100,19 @@ func (s Sample) MarshalJSON() ([]byte, error) {
100100
return json.Marshal(&v)
101101
}
102102

103-
type sampleHistogramPairPtr struct {
104-
Timestamp Time
105-
Histogram *SampleHistogram
106-
}
107-
108-
func (s *sampleHistogramPairPtr) UnmarshalJSON(buf []byte) error {
109-
tmp := []interface{}{&s.Timestamp, &s.Histogram}
110-
wantLen := len(tmp)
111-
if err := json.Unmarshal(buf, &tmp); err != nil {
112-
return err
113-
}
114-
if gotLen := len(tmp); gotLen != wantLen {
115-
return fmt.Errorf("wrong number of fields: %d != %d", gotLen, wantLen)
116-
}
117-
return nil
118-
}
119-
120103
// UnmarshalJSON implements json.Unmarshaler.
121-
// TODO: simplify and remove the need for both sampleHistogramPairPtr and SampleHistogramPair
122104
func (s *Sample) UnmarshalJSON(b []byte) error {
123105
v := struct {
124-
Metric Metric `json:"metric"`
125-
Value SamplePair `json:"value"`
126-
Histogram sampleHistogramPairPtr `json:"histogram"`
106+
Metric Metric `json:"metric"`
107+
Value SamplePair `json:"value"`
108+
Histogram SampleHistogramPair `json:"histogram"`
127109
}{
128110
Metric: s.Metric,
129111
Value: SamplePair{
130112
Timestamp: s.Timestamp,
131113
Value: s.Value,
132114
},
133-
Histogram: sampleHistogramPairPtr{
115+
Histogram: SampleHistogramPair{
134116
Timestamp: s.Timestamp,
135117
Histogram: s.Histogram,
136118
},

‎model/value_float.go‎

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,15 @@ import (
1818
"fmt"
1919
"math"
2020
"strconv"
21+
"unsafe"
22+
23+
jsoniter "github.com/json-iterator/go"
2124
)
2225

26+
func init() {
27+
jsoniter.RegisterTypeEncoderFunc("model.SamplePair", marshalSamplePairJSON, marshalJSONIsEmpty)
28+
}
29+
2330
var (
2431
// ZeroSamplePair is the pseudo zero-value of SamplePair used to signal a
2532
// non-existing sample pair. It is a SamplePair with timestamp Earliest and
@@ -71,17 +78,18 @@ type SamplePair struct {
7178
Value SampleValue
7279
}
7380

74-
// MarshalJSON implements json.Marshaler.
81+
// marshalSamplePairJSON writes `[ts, "val"]`.
82+
func marshalSamplePairJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) {
83+
p := *((*SamplePair)(ptr))
84+
stream.WriteArrayStart()
85+
MarshalTimestamp(int64(p.Timestamp), stream)
86+
stream.WriteMore()
87+
MarshalValue(float64(p.Value), stream)
88+
stream.WriteArrayEnd()
89+
}
90+
7591
func (s SamplePair) MarshalJSON() ([]byte, error) {
76-
t, err := json.Marshal(s.Timestamp)
77-
if err != nil {
78-
return nil, err
79-
}
80-
v, err := json.Marshal(s.Value)
81-
if err != nil {
82-
return nil, err
83-
}
84-
return []byte(fmt.Sprintf("[%s,%s]", t, v)), nil
92+
return jsoniter.ConfigCompatibleWithStandardLibrary.Marshal(s)
8593
}
8694

8795
// UnmarshalJSON implements json.Unmarshaler.

‎model/value_float_test.go‎

Lines changed: 47 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,42 @@ import (
2020
"testing"
2121
)
2222

23+
var (
24+
samplePairMatrixPlain = `[{"metric":{"__name__":"test_metric"},"values":[[1234.567,"123.1"],[12345.678,"123.12"]]},{"metric":{"foo":"bar"},"values":[[2234.567,"223.1"],[22345.678,"223.12"]]}]`
25+
samplePairMatrixValue = Matrix{
26+
&SampleStream{
27+
Metric: Metric{
28+
MetricNameLabel: "test_metric",
29+
},
30+
Values: []SamplePair{
31+
{
32+
Value: 123.1,
33+
Timestamp: 1234567,
34+
},
35+
{
36+
Value: 123.12,
37+
Timestamp: 12345678,
38+
},
39+
},
40+
},
41+
&SampleStream{
42+
Metric: Metric{
43+
"foo": "bar",
44+
},
45+
Values: []SamplePair{
46+
{
47+
Value: 223.1,
48+
Timestamp: 2234567,
49+
},
50+
{
51+
Value: 223.12,
52+
Timestamp: 22345678,
53+
},
54+
},
55+
},
56+
}
57+
)
58+
2359
func TestEqualValues(t *testing.T) {
2460
tests := map[string]struct {
2561
in1, in2 SampleValue
@@ -231,39 +267,8 @@ func TestMatrixJSON(t *testing.T) {
231267
value: Matrix{},
232268
},
233269
{
234-
plain: `[{"metric":{"__name__":"test_metric"},"values":[[1234.567,"123.1"],[12345.678,"123.12"]]},{"metric":{"foo":"bar"},"values":[[2234.567,"223.1"],[22345.678,"223.12"]]}]`,
235-
value: Matrix{
236-
&SampleStream{
237-
Metric: Metric{
238-
MetricNameLabel: "test_metric",
239-
},
240-
Values: []SamplePair{
241-
{
242-
Value: 123.1,
243-
Timestamp: 1234567,
244-
},
245-
{
246-
Value: 123.12,
247-
Timestamp: 12345678,
248-
},
249-
},
250-
},
251-
&SampleStream{
252-
Metric: Metric{
253-
"foo": "bar",
254-
},
255-
Values: []SamplePair{
256-
{
257-
Value: 223.1,
258-
Timestamp: 2234567,
259-
},
260-
{
261-
Value: 223.12,
262-
Timestamp: 22345678,
263-
},
264-
},
265-
},
266-
},
270+
plain: samplePairMatrixPlain,
271+
value: samplePairMatrixValue,
267272
},
268273
}
269274

@@ -291,3 +296,12 @@ func TestMatrixJSON(t *testing.T) {
291296
}
292297
}
293298
}
299+
300+
func BenchmarkJSONMarshallingSamplePairMatrix(b *testing.B) {
301+
for i := 0; i < b.N; i++ {
302+
_, err := json.Marshal(samplePairMatrixValue)
303+
if err != nil {
304+
b.Fatal("error marshalling")
305+
}
306+
}
307+
}

‎model/value_histogram.go‎

Lines changed: 23 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,16 @@ import (
1818
"fmt"
1919
"strconv"
2020
"strings"
21+
"unsafe"
22+
23+
jsoniter "github.com/json-iterator/go"
2124
)
2225

26+
func init() {
27+
jsoniter.RegisterTypeEncoderFunc("model.HistogramBucket", marshalHistogramBucketJSON, marshalJSONIsEmpty)
28+
jsoniter.RegisterTypeEncoderFunc("model.SampleHistogramPair", marshalSampleHistogramPairJSON, marshalJSONIsEmpty)
29+
}
30+
2331
type FloatString float64
2432

2533
func (v FloatString) String() string {
@@ -49,24 +57,10 @@ type HistogramBucket struct {
4957
Count FloatString
5058
}
5159

52-
func (s HistogramBucket) MarshalJSON() ([]byte, error) {
53-
b, err := json.Marshal(s.Boundaries)
54-
if err != nil {
55-
return nil, err
56-
}
57-
l, err := json.Marshal(s.Lower)
58-
if err != nil {
59-
return nil, err
60-
}
61-
u, err := json.Marshal(s.Upper)
62-
if err != nil {
63-
return nil, err
64-
}
65-
c, err := json.Marshal(s.Count)
66-
if err != nil {
67-
return nil, err
68-
}
69-
return []byte(fmt.Sprintf("[%s,%s,%s,%s]", b, l, u, c)), nil
60+
// marshalHistogramBucketJSON writes fmt.Sprintf("[%s,%s,%s,%s]", b.Boundaries, b.Lower, b.Upper, b.Count).
61+
func marshalHistogramBucketJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) {
62+
b := *((*HistogramBucket)(ptr))
63+
MarshalHistogramBucket(b, stream)
7064
}
7165

7266
func (s *HistogramBucket) UnmarshalJSON(buf []byte) error {
@@ -139,19 +133,21 @@ type SampleHistogramPair struct {
139133
Histogram *SampleHistogram
140134
}
141135

136+
// marshalSampleHistogramPairJSON writes `[ts, "val"]`.
137+
func marshalSampleHistogramPairJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) {
138+
p := *((*SampleHistogramPair)(ptr))
139+
stream.WriteArrayStart()
140+
MarshalTimestamp(int64(p.Timestamp), stream)
141+
stream.WriteMore()
142+
MarshalHistogram(*p.Histogram, stream)
143+
stream.WriteArrayEnd()
144+
}
145+
142146
func (s SampleHistogramPair) MarshalJSON() ([]byte, error) {
143-
t, err := json.Marshal(s.Timestamp)
144-
if err != nil {
145-
return nil, err
146-
}
147147
if s.Histogram == nil {
148148
return nil, fmt.Errorf("histogram is nil")
149149
}
150-
v, err := json.Marshal(s.Histogram)
151-
if err != nil {
152-
return nil, err
153-
}
154-
return []byte(fmt.Sprintf("[%s,%s]", t, v)), nil
150+
return jsoniter.ConfigCompatibleWithStandardLibrary.Marshal(s)
155151
}
156152

157153
func (s *SampleHistogramPair) UnmarshalJSON(buf []byte) error {

0 commit comments

Comments
 (0)