From a075803426656fa71e0a5009a907b1d9b6870d00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=8C=B2=20Harry=20=F0=9F=8C=8A=20John=20=F0=9F=8F=94?= Date: Wed, 15 Feb 2023 08:59:38 -0800 Subject: [PATCH 1/4] Pass request through MergeResponse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 🌲 Harry 🌊 John 🏔 --- pkg/querier/tripperware/instantquery/instant_query.go | 2 +- pkg/querier/tripperware/instantquery/instant_query_test.go | 2 +- pkg/querier/tripperware/query.go | 2 +- pkg/querier/tripperware/queryrange/query_range.go | 2 +- pkg/querier/tripperware/queryrange/query_range_test.go | 2 +- pkg/querier/tripperware/queryrange/results_cache.go | 6 +++--- pkg/querier/tripperware/queryrange/split_by_interval.go | 2 +- .../tripperware/queryrange/split_by_interval_test.go | 2 +- pkg/querier/tripperware/shard_by.go | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/querier/tripperware/instantquery/instant_query.go b/pkg/querier/tripperware/instantquery/instant_query.go index fd1f3559db3..91b9d8f1bb2 100644 --- a/pkg/querier/tripperware/instantquery/instant_query.go +++ b/pkg/querier/tripperware/instantquery/instant_query.go @@ -245,7 +245,7 @@ func (instantQueryCodec) EncodeResponse(ctx context.Context, res tripperware.Res return &resp, nil } -func (instantQueryCodec) MergeResponse(ctx context.Context, responses ...tripperware.Response) (tripperware.Response, error) { +func (instantQueryCodec) MergeResponse(ctx context.Context, _ tripperware.Request, responses ...tripperware.Response) (tripperware.Response, error) { sp, _ := opentracing.StartSpanFromContext(ctx, "PrometheusInstantQueryResponse.MergeResponse") sp.SetTag("response_count", len(responses)) defer sp.Finish() diff --git a/pkg/querier/tripperware/instantquery/instant_query_test.go b/pkg/querier/tripperware/instantquery/instant_query_test.go index efdf968fea9..4ddb902b14b 100644 --- a/pkg/querier/tripperware/instantquery/instant_query_test.go +++ b/pkg/querier/tripperware/instantquery/instant_query_test.go @@ -308,7 +308,7 @@ func TestMergeResponse(t *testing.T) { require.NoError(t, err) resps = append(resps, dr) } - resp, err := InstantQueryCodec.MergeResponse(context.Background(), resps...) + resp, err := InstantQueryCodec.MergeResponse(context.Background(), nil, resps...) assert.Equal(t, err, tc.expectedErr) if err != nil { return diff --git a/pkg/querier/tripperware/query.go b/pkg/querier/tripperware/query.go index 6c6d0dd3bbd..d8144ad17ac 100644 --- a/pkg/querier/tripperware/query.go +++ b/pkg/querier/tripperware/query.go @@ -50,7 +50,7 @@ type Codec interface { // Merger is used by middlewares making multiple requests to merge back all responses into a single one. type Merger interface { // MergeResponse merges responses from multiple requests into a single Response - MergeResponse(context.Context, ...Response) (Response, error) + MergeResponse(context.Context, Request, ...Response) (Response, error) } // Response represents a query range response. diff --git a/pkg/querier/tripperware/queryrange/query_range.go b/pkg/querier/tripperware/queryrange/query_range.go index 56654f18be2..958e1ac5776 100644 --- a/pkg/querier/tripperware/queryrange/query_range.go +++ b/pkg/querier/tripperware/queryrange/query_range.go @@ -127,7 +127,7 @@ func NewEmptyPrometheusResponse() *PrometheusResponse { } } -func (c prometheusCodec) MergeResponse(ctx context.Context, responses ...tripperware.Response) (tripperware.Response, error) { +func (c prometheusCodec) MergeResponse(ctx context.Context, _ tripperware.Request, responses ...tripperware.Response) (tripperware.Response, error) { sp, _ := opentracing.StartSpanFromContext(ctx, "QueryRangeResponse.MergeResponse") sp.SetTag("response_count", len(responses)) defer sp.Finish() diff --git a/pkg/querier/tripperware/queryrange/query_range_test.go b/pkg/querier/tripperware/queryrange/query_range_test.go index bd61485d064..5c808cfe8bb 100644 --- a/pkg/querier/tripperware/queryrange/query_range_test.go +++ b/pkg/querier/tripperware/queryrange/query_range_test.go @@ -652,7 +652,7 @@ func TestMergeAPIResponses(t *testing.T) { }, }} { t.Run(tc.name, func(t *testing.T) { - output, err := PrometheusCodec.MergeResponse(context.Background(), tc.input...) + output, err := PrometheusCodec.MergeResponse(context.Background(), nil, tc.input...) require.NoError(t, err) require.Equal(t, tc.expected, output) }) diff --git a/pkg/querier/tripperware/queryrange/results_cache.go b/pkg/querier/tripperware/queryrange/results_cache.go index 5c0fc543759..565280337f8 100644 --- a/pkg/querier/tripperware/queryrange/results_cache.go +++ b/pkg/querier/tripperware/queryrange/results_cache.go @@ -407,7 +407,7 @@ func (s resultsCache) handleHit(ctx context.Context, r tripperware.Request, exte return nil, nil, err } if len(requests) == 0 { - response, err := s.merger.MergeResponse(context.Background(), responses...) + response, err := s.merger.MergeResponse(context.Background(), r, responses...) // No downstream requests so no need to write back to the cache. return response, nil, err } @@ -469,7 +469,7 @@ func (s resultsCache) handleHit(ctx context.Context, r tripperware.Request, exte if err != nil { return nil, nil, err } - merged, err := s.merger.MergeResponse(ctx, accumulator.Response, currentRes) + merged, err := s.merger.MergeResponse(ctx, r, accumulator.Response, currentRes) if err != nil { return nil, nil, err } @@ -481,7 +481,7 @@ func (s resultsCache) handleHit(ctx context.Context, r tripperware.Request, exte return nil, nil, err } - response, err := s.merger.MergeResponse(ctx, responses...) + response, err := s.merger.MergeResponse(ctx, r, responses...) return response, mergedExtents, err } diff --git a/pkg/querier/tripperware/queryrange/split_by_interval.go b/pkg/querier/tripperware/queryrange/split_by_interval.go index 4ad6e8f47f8..ab7fa2cfc47 100644 --- a/pkg/querier/tripperware/queryrange/split_by_interval.go +++ b/pkg/querier/tripperware/queryrange/split_by_interval.go @@ -61,7 +61,7 @@ func (s splitByInterval) Do(ctx context.Context, r tripperware.Request) (tripper resps = append(resps, reqResp.Response) } - response, err := s.merger.MergeResponse(ctx, resps...) + response, err := s.merger.MergeResponse(ctx, nil, resps...) if err != nil { return nil, err } diff --git a/pkg/querier/tripperware/queryrange/split_by_interval_test.go b/pkg/querier/tripperware/queryrange/split_by_interval_test.go index 22e9565bce1..6ac73d005cf 100644 --- a/pkg/querier/tripperware/queryrange/split_by_interval_test.go +++ b/pkg/querier/tripperware/queryrange/split_by_interval_test.go @@ -266,7 +266,7 @@ func TestSplitQuery(t *testing.T) { } func TestSplitByDay(t *testing.T) { - mergedResponse, err := PrometheusCodec.MergeResponse(context.Background(), parsedResponse, parsedResponse) + mergedResponse, err := PrometheusCodec.MergeResponse(context.Background(), nil, parsedResponse, parsedResponse) require.NoError(t, err) mergedHTTPResponse, err := PrometheusCodec.EncodeResponse(context.Background(), mergedResponse) diff --git a/pkg/querier/tripperware/shard_by.go b/pkg/querier/tripperware/shard_by.go index c596f40b637..39bef61ca90 100644 --- a/pkg/querier/tripperware/shard_by.go +++ b/pkg/querier/tripperware/shard_by.go @@ -79,7 +79,7 @@ func (s shardBy) Do(ctx context.Context, r Request) (Response, error) { resps = append(resps, reqResp.Response) } - return s.merger.MergeResponse(ctx, resps...) + return s.merger.MergeResponse(ctx, r, resps...) } func (s shardBy) shardQuery(l log.Logger, numShards int, r Request, analysis querysharding.QueryAnalysis) []Request { From cabb0f3fdb27ea315e37b3a2e4cc362ac78780fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=8C=B2=20Harry=20=F0=9F=8C=8A=20John=20=F0=9F=8F=94?= Date: Wed, 15 Feb 2023 09:17:30 -0800 Subject: [PATCH 2/4] Bug fix for sort and sort_desc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 🌲 Harry 🌊 John 🏔 --- .../tripperware/instantquery/instant_query.go | 75 ++++++++++++++++--- .../instantquery/instant_query_test.go | 37 ++++++++- 2 files changed, 100 insertions(+), 12 deletions(-) diff --git a/pkg/querier/tripperware/instantquery/instant_query.go b/pkg/querier/tripperware/instantquery/instant_query.go index 91b9d8f1bb2..bb0162d5120 100644 --- a/pkg/querier/tripperware/instantquery/instant_query.go +++ b/pkg/querier/tripperware/instantquery/instant_query.go @@ -27,6 +27,7 @@ import ( "github.com/cortexproject/cortex/pkg/querier/tripperware/queryrange" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/spanlogger" + promqlparser "github.com/prometheus/prometheus/promql/parser" ) var ( @@ -245,7 +246,7 @@ func (instantQueryCodec) EncodeResponse(ctx context.Context, res tripperware.Res return &resp, nil } -func (instantQueryCodec) MergeResponse(ctx context.Context, _ tripperware.Request, responses ...tripperware.Response) (tripperware.Response, error) { +func (instantQueryCodec) MergeResponse(ctx context.Context, req tripperware.Request, responses ...tripperware.Response) (tripperware.Response, error) { sp, _ := opentracing.StartSpanFromContext(ctx, "PrometheusInstantQueryResponse.MergeResponse") sp.SetTag("response_count", len(responses)) defer sp.Finish() @@ -265,11 +266,15 @@ func (instantQueryCodec) MergeResponse(ctx context.Context, _ tripperware.Reques // For now, we only shard queries that returns a vector. switch promResponses[0].Data.ResultType { case model.ValVector.String(): + v, err := vectorMerge(req, promResponses) + if err != nil { + return nil, err + } data = PrometheusInstantQueryData{ ResultType: model.ValVector.String(), Result: PrometheusInstantQueryResult{ Result: &PrometheusInstantQueryResult_Vector{ - Vector: vectorMerge(promResponses), + Vector: v, }, }, Stats: statsMerge(promResponses), @@ -297,8 +302,12 @@ func (instantQueryCodec) MergeResponse(ctx context.Context, _ tripperware.Reques return res, nil } -func vectorMerge(resps []*PrometheusInstantQueryResponse) *Vector { +func vectorMerge(req tripperware.Request, resps []*PrometheusInstantQueryResponse) (*Vector, error) { output := map[string]*Sample{} + sortAsc, sortDesc, err := parseQueryForSort(req.GetQuery()) + if err != nil { + return nil, err + } buf := make([]byte, 0, 1024) for _, resp := range resps { if resp == nil { @@ -327,22 +336,66 @@ func vectorMerge(resps []*PrometheusInstantQueryResponse) *Vector { if len(output) == 0 { return &Vector{ Samples: make([]*Sample, 0), - } + }, nil } - keys := make([]string, 0, len(output)) - for key := range output { - keys = append(keys, key) + type pair struct { + metric string + s *Sample + } + + samples := make([]*pair, 0, len(output)) + for k, v := range output { + samples = append(samples, &pair{ + metric: k, + s: v, + }) } - sort.Strings(keys) + sort.Slice(samples, func(i, j int) bool { + // Order is determined by the sortFn in the query. + if sortAsc { + return samples[i].s.Sample.Value < samples[j].s.Sample.Value + } else if sortDesc { + return samples[i].s.Sample.Value > samples[j].s.Sample.Value + } else { + // Fallback on sorting by labels. + return samples[i].metric < samples[j].metric + } + }) result := &Vector{ Samples: make([]*Sample, 0, len(output)), } - for _, key := range keys { - result.Samples = append(result.Samples, output[key]) + for _, p := range samples { + result.Samples = append(result.Samples, p.s) } - return result + return result, nil +} + +func parseQueryForSort(q string) (bool, bool, error) { + expr, err := promqlparser.ParseExpr(q) + if err != nil { + return false, false, err + } + var sortAsc bool = false + var sortDesc bool = false + done := errors.New("done") + promqlparser.Inspect(expr, func(n promqlparser.Node, _ []promqlparser.Node) error { + if n, ok := n.(*promqlparser.Call); ok { + if n.Func != nil { + if n.Func.Name == "sort" { + sortAsc = true + return done + } + if n.Func.Name == "sort_desc" { + sortDesc = true + return done + } + } + } + return nil + }) + return sortAsc, sortDesc, nil } func matrixMerge(resps []*PrometheusInstantQueryResponse) []tripperware.SampleStream { diff --git a/pkg/querier/tripperware/instantquery/instant_query_test.go b/pkg/querier/tripperware/instantquery/instant_query_test.go index 4ddb902b14b..ee6f3ae238b 100644 --- a/pkg/querier/tripperware/instantquery/instant_query_test.go +++ b/pkg/querier/tripperware/instantquery/instant_query_test.go @@ -198,34 +198,43 @@ func TestResponse(t *testing.T) { } func TestMergeResponse(t *testing.T) { + defaultReq := &PrometheusRequest{ + Query: "sum(up)", + } for _, tc := range []struct { name string + req tripperware.Request resps []string expectedResp string expectedErr error }{ { name: "empty response", + req: defaultReq, resps: []string{`{"status":"success","data":{"resultType":"vector","result":[]}}`}, expectedResp: `{"status":"success","data":{"resultType":"vector","result":[]}}`, }, { name: "empty response with stats", + req: defaultReq, resps: []string{`{"status":"success","data":{"resultType":"vector","result":[],"stats":{"samples":{"totalQueryableSamples":0,"totalQueryableSamplesPerStep":[]}}}}`}, expectedResp: `{"status":"success","data":{"resultType":"vector","result":[],"stats":{"samples":{"totalQueryableSamples":0,"totalQueryableSamplesPerStep":[]}}}}`, }, { name: "single response", + req: defaultReq, resps: []string{`{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up"},"value":[1,"1"]}]}}`}, expectedResp: `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up"},"value":[1,"1"]}]}}`, }, { name: "single response with stats", + req: defaultReq, resps: []string{`{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up"},"value":[1,"1"]}],"stats":{"samples":{"totalQueryableSamples":10,"totalQueryableSamplesPerStep":[[1,10]]}}}}`}, expectedResp: `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up"},"value":[1,"1"]}],"stats":{"samples":{"totalQueryableSamples":10,"totalQueryableSamplesPerStep":[[1,10]]}}}}`, }, { name: "duplicated response", + req: defaultReq, resps: []string{ `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up"},"value":[1,"1"]}]}}`, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up"},"value":[1,"1"]}]}}`, @@ -234,6 +243,7 @@ func TestMergeResponse(t *testing.T) { }, { name: "duplicated response with stats", + req: defaultReq, resps: []string{ `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up"},"value":[1,"1"]}],"stats":{"samples":{"totalQueryableSamples":10,"totalQueryableSamplesPerStep":[[1,10]]}}}}`, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up"},"value":[1,"1"]}],"stats":{"samples":{"totalQueryableSamples":10,"totalQueryableSamplesPerStep":[[1,10]]}}}}`, @@ -242,14 +252,34 @@ func TestMergeResponse(t *testing.T) { }, { name: "merge two responses", + req: defaultReq, resps: []string{ `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up","job":"foo"},"value":[1,"1"]}]}}`, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up","job":"bar"},"value":[2,"2"]}]}}`, }, expectedResp: `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up","job":"bar"},"value":[2,"2"]},{"metric":{"__name__":"up","job":"foo"},"value":[1,"1"]}]}}`, }, + { + name: "merge two responses with sort", + req: &PrometheusRequest{Query: "sort(up)"}, + resps: []string{ + `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up","job":"foo"},"value":[1,"1"]}]}}`, + `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up","job":"bar"},"value":[1,"2"]}]}}`, + }, + expectedResp: `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up","job":"foo"},"value":[1,"1"]},{"metric":{"__name__":"up","job":"bar"},"value":[1,"2"]}]}}`, + }, + { + name: "merge two responses with sort_desc", + req: &PrometheusRequest{Query: "sort_desc(up)"}, + resps: []string{ + `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up","job":"foo"},"value":[1,"1"]}]}}`, + `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up","job":"bar"},"value":[1,"2"]}]}}`, + }, + expectedResp: `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up","job":"bar"},"value":[1,"2"]},{"metric":{"__name__":"up","job":"foo"},"value":[1,"1"]}]}}`, + }, { name: "merge two responses with stats", + req: defaultReq, resps: []string{ `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up","job":"foo"},"value":[1,"1"]}],"stats":{"samples":{"totalQueryableSamples":10,"totalQueryableSamplesPerStep":[[1,10]]}}}}`, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up","job":"bar"},"value":[2,"2"]}],"stats":{"samples":{"totalQueryableSamples":10,"totalQueryableSamplesPerStep":[[1,10]]}}}}`, @@ -258,6 +288,7 @@ func TestMergeResponse(t *testing.T) { }, { name: "responses don't contain vector, should return an error", + req: defaultReq, resps: []string{ `{"status":"success","data":{"resultType":"string","result":[1662682521.409,"foo"]}}`, `{"status":"success","data":{"resultType":"string","result":[1662682521.409,"foo"]}}`, @@ -266,6 +297,7 @@ func TestMergeResponse(t *testing.T) { }, { name: "single matrix response", + req: defaultReq, resps: []string{ `{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"up"},"values":[[1,"1"],[2,"2"]]}]}}`, }, @@ -273,6 +305,7 @@ func TestMergeResponse(t *testing.T) { }, { name: "multiple matrix responses without duplicated series", + req: defaultReq, resps: []string{ `{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"bar"},"values":[[1,"1"],[2,"2"]]}]}}`, `{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"foo"},"values":[[3,"3"],[4,"4"]]}]}}`, @@ -281,6 +314,7 @@ func TestMergeResponse(t *testing.T) { }, { name: "multiple matrix responses with duplicated series, but not same samples", + req: defaultReq, resps: []string{ `{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"bar"},"values":[[1,"1"],[2,"2"]]}]}}`, `{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"bar"},"values":[[3,"3"]]}]}}`, @@ -289,6 +323,7 @@ func TestMergeResponse(t *testing.T) { }, { name: "multiple matrix responses with duplicated series and same samples", + req: defaultReq, resps: []string{ `{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"bar"},"values":[[1,"1"],[2,"2"]]}]}}`, `{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"bar"},"values":[[1,"1"],[2,"2"],[3,"3"]]}]}}`, @@ -308,7 +343,7 @@ func TestMergeResponse(t *testing.T) { require.NoError(t, err) resps = append(resps, dr) } - resp, err := InstantQueryCodec.MergeResponse(context.Background(), nil, resps...) + resp, err := InstantQueryCodec.MergeResponse(context.Background(), tc.req, resps...) assert.Equal(t, err, tc.expectedErr) if err != nil { return From 518ca3c16664f5894bfdf2c0a32c83c6dbec345d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=8C=B2=20Harry=20=F0=9F=8C=8A=20John=20=F0=9F=8F=94?= Date: Wed, 15 Feb 2023 09:21:08 -0800 Subject: [PATCH 3/4] Update CHANGELOG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 🌲 Harry 🌊 John 🏔 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2911d8ca82c..d51a4ef054b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ * [BUGFIX] Ingester: Ingesters returning empty response for metadata APIs. #5081 * [BUGFIX] Ingester: Fix panic when querying metadata from blocks that are being deleted. #5119 * [BUGFIX] Ring: Fix case when dynamodb kv reaches the limit of 25 actions per batch call. #5136 +* [BUGFIX] Query-frontend: Fix sorted queries do not produce sorted results for shardable queries. #5148 * [FEATURE] Alertmanager: Add support for time_intervals. #5102 ## 1.14.0 2022-12-02 From d3a8c59c1608536cf62f19eb330dbb8bae34f0f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=8C=B2=20Harry=20=F0=9F=8C=8A=20John=20=F0=9F=8F=94?= Date: Wed, 15 Feb 2023 09:34:09 -0800 Subject: [PATCH 4/4] Fix lint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 🌲 Harry 🌊 John 🏔 --- pkg/querier/tripperware/instantquery/instant_query.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/querier/tripperware/instantquery/instant_query.go b/pkg/querier/tripperware/instantquery/instant_query.go index bb0162d5120..880366f73f8 100644 --- a/pkg/querier/tripperware/instantquery/instant_query.go +++ b/pkg/querier/tripperware/instantquery/instant_query.go @@ -22,12 +22,13 @@ import ( "github.com/weaveworks/common/httpgrpc" "google.golang.org/grpc/status" + promqlparser "github.com/prometheus/prometheus/promql/parser" + "github.com/cortexproject/cortex/pkg/cortexpb" "github.com/cortexproject/cortex/pkg/querier/tripperware" "github.com/cortexproject/cortex/pkg/querier/tripperware/queryrange" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/spanlogger" - promqlparser "github.com/prometheus/prometheus/promql/parser" ) var (