From 0d10dcea818a566e5f3a4a9dd36fd9a92184b1a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Thu, 6 Feb 2020 09:12:45 +0100 Subject: [PATCH 01/29] =?UTF-8?q?Updated=20Prometheus=20to=2022a04239c937b?= =?UTF-8?q?e61df95fdb60f0661684693cf3b=20(v2.16.0-rc.0)=20Signed-off-by:?= =?UTF-8?q?=20Peter=20S=CC=8Ctibrany=CC=81=20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 6 +- go.sum | 8 + .../prometheus/common/route/route.go | 6 + .../prometheus/common/version/info.go | 17 +- .../prometheus/prometheus/config/config.go | 5 +- .../prometheus/discovery/manager.go | 4 +- .../prometheus/pkg/labels/labels.go | 14 + .../prometheus/pkg/logging/dedupe.go | 1 + .../prometheus/prometheus/pkg/logging/file.go | 62 + .../prometheus/pkg/logging/ratelimit.go | 1 + .../prometheus/pkg/rulefmt/rulefmt.go | 145 +- .../prometheus/prometheus/prompb/remote.pb.go | 2 +- .../prometheus/prometheus/prompb/remote.proto | 2 +- .../prometheus/prometheus/promql/ast.go | 115 +- .../prometheus/prometheus/promql/engine.go | 245 +- .../prometheus/prometheus/promql/functions.go | 7 +- .../prometheus/promql/generated_parser.y | 745 ++-- .../prometheus/promql/generated_parser.y.go | 1205 +++++-- .../prometheus/prometheus/promql/lex.go | 131 +- .../prometheus/prometheus/promql/parse.go | 895 ++--- .../prometheus/prometheus/promql/printer.go | 14 +- .../prometheus/promql/query_logger.go | 73 +- .../prometheus/prometheus/promql/test.go | 33 +- .../prometheus/prometheus/rules/manager.go | 71 +- .../prometheus/prometheus/scrape/manager.go | 72 +- .../prometheus/prometheus/scrape/scrape.go | 95 +- .../prometheus/prometheus/scrape/target.go | 24 + .../prometheus/prometheus/storage/fanout.go | 13 +- .../prometheus/storage/interface.go | 3 + .../prometheus/prometheus/storage/noop.go | 4 + .../prometheus/storage/remote/client.go | 3 +- .../prometheus/storage/remote/codec.go | 2 +- .../storage/remote/queue_manager.go | 34 +- .../prometheus/storage/remote/read.go | 7 + .../prometheus/storage/remote/storage.go | 2 +- .../prometheus/storage/tsdb/tsdb.go | 8 + .../prometheus/prometheus/tsdb/db.go | 34 +- .../prometheus/prometheus/tsdb/head.go | 149 +- .../prometheus/prometheus/tsdb/index/index.go | 95 +- .../prometheus/prometheus/tsdb/querier.go | 57 +- .../prometheus/tsdb/tombstones/tombstones.go | 18 +- .../prometheus/tsdb/tsdbblockutil.go | 7 +- .../prometheus/tsdb/wal/checkpoint.go | 4 + .../prometheus/prometheus/tsdb/wal/wal.go | 7 + .../prometheus/util/httputil/context.go | 52 + .../prometheus/prometheus/util/stats/timer.go | 6 +- .../prometheus/util/testutil/testing.go | 2 +- .../prometheus/prometheus/web/api/v1/api.go | 115 +- vendor/gopkg.in/yaml.v2/.travis.yml | 18 +- vendor/gopkg.in/yaml.v2/scannerc.go | 76 +- vendor/gopkg.in/yaml.v2/yaml.go | 2 +- vendor/gopkg.in/yaml.v3/.travis.yml | 16 + vendor/gopkg.in/yaml.v3/LICENSE | 50 + vendor/gopkg.in/yaml.v3/NOTICE | 13 + vendor/gopkg.in/yaml.v3/README.md | 150 + vendor/gopkg.in/yaml.v3/apic.go | 746 ++++ vendor/gopkg.in/yaml.v3/decode.go | 931 +++++ vendor/gopkg.in/yaml.v3/emitterc.go | 1992 +++++++++++ vendor/gopkg.in/yaml.v3/encode.go | 546 +++ vendor/gopkg.in/yaml.v3/go.mod | 5 + vendor/gopkg.in/yaml.v3/parserc.go | 1229 +++++++ vendor/gopkg.in/yaml.v3/readerc.go | 434 +++ vendor/gopkg.in/yaml.v3/resolve.go | 326 ++ vendor/gopkg.in/yaml.v3/scannerc.go | 3031 +++++++++++++++++ vendor/gopkg.in/yaml.v3/sorter.go | 134 + vendor/gopkg.in/yaml.v3/writerc.go | 48 + vendor/gopkg.in/yaml.v3/yaml.go | 662 ++++ vendor/gopkg.in/yaml.v3/yamlh.go | 804 +++++ vendor/gopkg.in/yaml.v3/yamlprivateh.go | 198 ++ vendor/modules.txt | 8 +- 70 files changed, 14273 insertions(+), 1766 deletions(-) create mode 100644 vendor/github.com/prometheus/prometheus/pkg/logging/file.go create mode 100644 vendor/github.com/prometheus/prometheus/util/httputil/context.go create mode 100644 vendor/gopkg.in/yaml.v3/.travis.yml create mode 100644 vendor/gopkg.in/yaml.v3/LICENSE create mode 100644 vendor/gopkg.in/yaml.v3/NOTICE create mode 100644 vendor/gopkg.in/yaml.v3/README.md create mode 100644 vendor/gopkg.in/yaml.v3/apic.go create mode 100644 vendor/gopkg.in/yaml.v3/decode.go create mode 100644 vendor/gopkg.in/yaml.v3/emitterc.go create mode 100644 vendor/gopkg.in/yaml.v3/encode.go create mode 100644 vendor/gopkg.in/yaml.v3/go.mod create mode 100644 vendor/gopkg.in/yaml.v3/parserc.go create mode 100644 vendor/gopkg.in/yaml.v3/readerc.go create mode 100644 vendor/gopkg.in/yaml.v3/resolve.go create mode 100644 vendor/gopkg.in/yaml.v3/scannerc.go create mode 100644 vendor/gopkg.in/yaml.v3/sorter.go create mode 100644 vendor/gopkg.in/yaml.v3/writerc.go create mode 100644 vendor/gopkg.in/yaml.v3/yaml.go create mode 100644 vendor/gopkg.in/yaml.v3/yamlh.go create mode 100644 vendor/gopkg.in/yaml.v3/yamlprivateh.go diff --git a/go.mod b/go.mod index b2b6e652785..656ce190349 100644 --- a/go.mod +++ b/go.mod @@ -52,8 +52,8 @@ require ( github.com/prometheus/alertmanager v0.19.0 github.com/prometheus/client_golang v1.2.1 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 - github.com/prometheus/common v0.7.0 - github.com/prometheus/prometheus v1.8.2-0.20200107122003-4708915ac6ef + github.com/prometheus/common v0.8.0 + github.com/prometheus/prometheus v1.8.2-0.20200131223856-22a04239c937 github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1 github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e github.com/spf13/afero v1.2.2 @@ -71,7 +71,7 @@ require ( golang.org/x/time v0.0.0-20191024005414-555d28b269f0 google.golang.org/api v0.14.0 google.golang.org/grpc v1.25.1 - gopkg.in/yaml.v2 v2.2.5 + gopkg.in/yaml.v2 v2.2.7 sigs.k8s.io/yaml v1.1.0 ) diff --git a/go.sum b/go.sum index f69ed698bd6..9cd2e4b22a6 100644 --- a/go.sum +++ b/go.sum @@ -671,6 +671,8 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.8.0 h1:bLkjvFe2ZRX1DpcgZcdf7j/+MnusEps5hktST/FHA34= +github.com/prometheus/common v0.8.0/go.mod h1:PC/OgXc+UN7B4ALwvn1yzVZmVwvhXp5JsbBv6wSv6i0= github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -686,6 +688,8 @@ github.com/prometheus/prometheus v0.0.0-20180315085919-58e2a31db8de/go.mod h1:oA github.com/prometheus/prometheus v0.0.0-20190818123050-43acd0e2e93f/go.mod h1:rMTlmxGCvukf2KMu3fClMDKLLoJ5hl61MhcJ7xKakf0= github.com/prometheus/prometheus v1.8.2-0.20200107122003-4708915ac6ef h1:pYYKXo/zGx25kyViw+Gdbxd0ItIg+vkVKpwgWUEyIc4= github.com/prometheus/prometheus v1.8.2-0.20200107122003-4708915ac6ef/go.mod h1:7U90zPoLkWjEIQcy/rweQla82OCTUzxVHE51G3OhJbI= +github.com/prometheus/prometheus v1.8.2-0.20200131223856-22a04239c937 h1:6i96jDUJjjPC/lmRv7082vCA8L7e4tlGF3gXce+Cqz0= +github.com/prometheus/prometheus v1.8.2-0.20200131223856-22a04239c937/go.mod h1:fkIPPkuZnkXyopYHmXPxf9rgiPkVgZCN8w9o8+UgBlY= github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1 h1:+kGqA4dNN5hn7WwvKdzHl0rdN5AEkbNZd0VjRltAiZg= github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1/go.mod h1:JaY6n2sDr+z2WTsXkOmNRUfDy6FN0L6Nk7x06ndm4tY= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -1064,6 +1068,10 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2 h1:XZx7nhd5GMaZpmDaEHFVafUZC7ya0fuo7cSJ3UCKYmM= +gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/vendor/github.com/prometheus/common/route/route.go b/vendor/github.com/prometheus/common/route/route.go index 1bd0a1edd3e..cc05516c801 100644 --- a/vendor/github.com/prometheus/common/route/route.go +++ b/vendor/github.com/prometheus/common/route/route.go @@ -53,6 +53,12 @@ func New() *Router { // WithInstrumentation returns a router with instrumentation support. func (r *Router) WithInstrumentation(instrh func(handlerName string, handler http.HandlerFunc) http.HandlerFunc) *Router { + if r.instrh != nil { + newInstrh := instrh + instrh = func(handlerName string, handler http.HandlerFunc) http.HandlerFunc { + return newInstrh(handlerName, r.instrh(handlerName, handler)) + } + } return &Router{rtr: r.rtr, prefix: r.prefix, instrh: instrh} } diff --git a/vendor/github.com/prometheus/common/version/info.go b/vendor/github.com/prometheus/common/version/info.go index 84489a51044..ac9af1febd9 100644 --- a/vendor/github.com/prometheus/common/version/info.go +++ b/vendor/github.com/prometheus/common/version/info.go @@ -33,9 +33,10 @@ var ( GoVersion = runtime.Version() ) -// NewCollector returns a collector which exports metrics about current version information. -func NewCollector(program string) *prometheus.GaugeVec { - buildInfo := prometheus.NewGaugeVec( +// NewCollector returns a collector that exports metrics about current version +// information. +func NewCollector(program string) prometheus.Collector { + return prometheus.NewGaugeFunc( prometheus.GaugeOpts{ Namespace: program, Name: "build_info", @@ -43,11 +44,15 @@ func NewCollector(program string) *prometheus.GaugeVec { "A metric with a constant '1' value labeled by version, revision, branch, and goversion from which %s was built.", program, ), + ConstLabels: prometheus.Labels{ + "version": Version, + "revision": Revision, + "branch": Branch, + "goversion": GoVersion, + }, }, - []string{"version", "revision", "branch", "goversion"}, + func() float64 { return 1 }, ) - buildInfo.WithLabelValues(Version, Revision, Branch, GoVersion).Set(1) - return buildInfo } // versionInfoTmpl contains the template used by Info. diff --git a/vendor/github.com/prometheus/prometheus/config/config.go b/vendor/github.com/prometheus/prometheus/config/config.go index 794ccc0ee68..11667a8268a 100644 --- a/vendor/github.com/prometheus/prometheus/config/config.go +++ b/vendor/github.com/prometheus/prometheus/config/config.go @@ -299,6 +299,8 @@ type GlobalConfig struct { ScrapeTimeout model.Duration `yaml:"scrape_timeout,omitempty"` // How frequently to evaluate rules by default. EvaluationInterval model.Duration `yaml:"evaluation_interval,omitempty"` + // File to which PromQL queries are logged. + QueryLogFile string `yaml:"query_log_file,omitempty"` // The labels to add to any timeseries that this Prometheus instance scrapes. ExternalLabels labels.Labels `yaml:"external_labels,omitempty"` } @@ -349,7 +351,8 @@ func (c *GlobalConfig) isZero() bool { return c.ExternalLabels == nil && c.ScrapeInterval == 0 && c.ScrapeTimeout == 0 && - c.EvaluationInterval == 0 + c.EvaluationInterval == 0 && + c.QueryLogFile == "" } // ScrapeConfig configures a scraping unit for Prometheus. diff --git a/vendor/github.com/prometheus/prometheus/discovery/manager.go b/vendor/github.com/prometheus/prometheus/discovery/manager.go index 5457bd9b2e7..d135cd54e70 100644 --- a/vendor/github.com/prometheus/prometheus/discovery/manager.go +++ b/vendor/github.com/prometheus/prometheus/discovery/manager.go @@ -304,8 +304,8 @@ func (m *Manager) updateGroup(poolKey poolKey, tgs []*targetgroup.Group) { } func (m *Manager) allGroups() map[string][]*targetgroup.Group { - m.mtx.Lock() - defer m.mtx.Unlock() + m.mtx.RLock() + defer m.mtx.RUnlock() tSets := map[string][]*targetgroup.Group{} for pkey, tsets := range m.targets { diff --git a/vendor/github.com/prometheus/prometheus/pkg/labels/labels.go b/vendor/github.com/prometheus/prometheus/pkg/labels/labels.go index 3b462e68d71..b919408ed53 100644 --- a/vendor/github.com/prometheus/prometheus/pkg/labels/labels.go +++ b/vendor/github.com/prometheus/prometheus/pkg/labels/labels.go @@ -200,6 +200,20 @@ func (ls Labels) Has(name string) bool { return false } +// HasDuplicateLabels returns whether ls has duplicate label names. +// It assumes that the labelset is sorted. +func (ls Labels) HasDuplicateLabelNames() (string, bool) { + for i, l := range ls { + if i == 0 { + continue + } + if l.Name == ls[i-1].Name { + return l.Name, true + } + } + return "", false +} + // WithoutEmpty returns the labelset without empty labels. // May return the same labelset. func (ls Labels) WithoutEmpty() Labels { diff --git a/vendor/github.com/prometheus/prometheus/pkg/logging/dedupe.go b/vendor/github.com/prometheus/prometheus/pkg/logging/dedupe.go index f040b2f23d6..1d911ca2f6e 100644 --- a/vendor/github.com/prometheus/prometheus/pkg/logging/dedupe.go +++ b/vendor/github.com/prometheus/prometheus/pkg/logging/dedupe.go @@ -10,6 +10,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + package logging import ( diff --git a/vendor/github.com/prometheus/prometheus/pkg/logging/file.go b/vendor/github.com/prometheus/prometheus/pkg/logging/file.go new file mode 100644 index 00000000000..be118fad051 --- /dev/null +++ b/vendor/github.com/prometheus/prometheus/pkg/logging/file.go @@ -0,0 +1,62 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logging + +import ( + "os" + "time" + + "github.com/go-kit/kit/log" + "github.com/pkg/errors" +) + +var ( + timestampFormat = log.TimestampFormat( + func() time.Time { return time.Now().UTC() }, + "2006-01-02T15:04:05.000Z07:00", + ) +) + +// JSONFileLogger represents a logger that writes JSON to a file. +type JSONFileLogger struct { + logger log.Logger + file *os.File +} + +// NewJSONFileLogger returns a new JSONFileLogger. +func NewJSONFileLogger(s string) (*JSONFileLogger, error) { + if s == "" { + return nil, nil + } + + f, err := os.OpenFile(s, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) + if err != nil { + return nil, errors.Wrap(err, "can't create json logger") + } + + return &JSONFileLogger{ + logger: log.With(log.NewJSONLogger(f), "ts", timestampFormat), + file: f, + }, nil +} + +// Close closes the underlying file. +func (l *JSONFileLogger) Close() error { + return l.file.Close() +} + +// Log calls the Log function of the underlying logger. +func (l *JSONFileLogger) Log(i ...interface{}) error { + return l.logger.Log(i...) +} diff --git a/vendor/github.com/prometheus/prometheus/pkg/logging/ratelimit.go b/vendor/github.com/prometheus/prometheus/pkg/logging/ratelimit.go index bf4d9dbda82..d3567eaa0e1 100644 --- a/vendor/github.com/prometheus/prometheus/pkg/logging/ratelimit.go +++ b/vendor/github.com/prometheus/prometheus/pkg/logging/ratelimit.go @@ -10,6 +10,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + package logging import ( diff --git a/vendor/github.com/prometheus/prometheus/pkg/rulefmt/rulefmt.go b/vendor/github.com/prometheus/prometheus/pkg/rulefmt/rulefmt.go index 501079862c7..6971657ba89 100644 --- a/vendor/github.com/prometheus/prometheus/pkg/rulefmt/rulefmt.go +++ b/vendor/github.com/prometheus/prometheus/pkg/rulefmt/rulefmt.go @@ -21,7 +21,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/common/model" - yaml "gopkg.in/yaml.v2" + yaml "gopkg.in/yaml.v3" "github.com/prometheus/prometheus/pkg/timestamp" "github.com/prometheus/prometheus/promql" @@ -33,11 +33,24 @@ type Error struct { Group string Rule int RuleName string - Err error + Err WrappedError +} + +// WrappedError wraps error with the yaml node which can be used to represent +// the line and column numbers of the error. +type WrappedError struct { + err error + node *yaml.Node + nodeAlt *yaml.Node } func (err *Error) Error() string { - return errors.Wrapf(err.Err, "group %q, rule %d, %q", err.Group, err.Rule, err.RuleName).Error() + if err.Err.nodeAlt != nil { + return errors.Wrapf(err.Err.err, "%d:%d: %d:%d: group %q, rule %d, %q", err.Err.node.Line, err.Err.node.Column, err.Err.nodeAlt.Line, err.Err.nodeAlt.Column, err.Group, err.Rule, err.RuleName).Error() + } else if err.Err.node != nil { + return errors.Wrapf(err.Err.err, "%d:%d: group %q, rule %d, %q", err.Err.node.Line, err.Err.node.Column, err.Group, err.Rule, err.RuleName).Error() + } + return errors.Wrapf(err.Err.err, "group %q, rule %d, %q", err.Group, err.Rule, err.RuleName).Error() } // RuleGroups is a set of rule groups that are typically exposed in a file. @@ -45,28 +58,32 @@ type RuleGroups struct { Groups []RuleGroup `yaml:"groups"` } +type ruleGroups struct { + Groups []yaml.Node `yaml:"groups"` +} + // Validate validates all rules in the rule groups. -func (g *RuleGroups) Validate() (errs []error) { +func (g *RuleGroups) Validate(node ruleGroups) (errs []error) { set := map[string]struct{}{} - for _, g := range g.Groups { + for j, g := range g.Groups { if g.Name == "" { - errs = append(errs, errors.Errorf("Groupname should not be empty")) + errs = append(errs, errors.Errorf("%d:%d: Groupname should not be empty", node.Groups[j].Line, node.Groups[j].Column)) } if _, ok := set[g.Name]; ok { errs = append( errs, - errors.Errorf("groupname: \"%s\" is repeated in the same file", g.Name), + errors.Errorf("%d:%d: groupname: \"%s\" is repeated in the same file", node.Groups[j].Line, node.Groups[j].Column, g.Name), ) } set[g.Name] = struct{}{} for i, r := range g.Rules { - for _, err := range r.Validate() { - var ruleName string - if r.Alert != "" { + for _, node := range r.Validate() { + var ruleName yaml.Node + if r.Alert.Value != "" { ruleName = r.Alert } else { ruleName = r.Record @@ -74,8 +91,8 @@ func (g *RuleGroups) Validate() (errs []error) { errs = append(errs, &Error{ Group: g.Name, Rule: i, - RuleName: ruleName, - Err: err, + RuleName: ruleName.Value, + Err: node, }) } } @@ -88,7 +105,7 @@ func (g *RuleGroups) Validate() (errs []error) { type RuleGroup struct { Name string `yaml:"name"` Interval model.Duration `yaml:"interval,omitempty"` - Rules []Rule `yaml:"rules"` + Rules []RuleNode `yaml:"rules"` } // Rule describes an alerting or recording rule. @@ -101,55 +118,104 @@ type Rule struct { Annotations map[string]string `yaml:"annotations,omitempty"` } +// RuleNode adds yaml.v3 layer to support line and column outputs for invalid rules. +type RuleNode struct { + Record yaml.Node `yaml:"record,omitempty"` + Alert yaml.Node `yaml:"alert,omitempty"` + Expr yaml.Node `yaml:"expr"` + For model.Duration `yaml:"for,omitempty"` + Labels map[string]string `yaml:"labels,omitempty"` + Annotations map[string]string `yaml:"annotations,omitempty"` +} + // Validate the rule and return a list of encountered errors. -func (r *Rule) Validate() (errs []error) { - if r.Record != "" && r.Alert != "" { - errs = append(errs, errors.Errorf("only one of 'record' and 'alert' must be set")) +func (r *RuleNode) Validate() (nodes []WrappedError) { + if r.Record.Value != "" && r.Alert.Value != "" { + nodes = append(nodes, WrappedError{ + err: errors.Errorf("only one of 'record' and 'alert' must be set"), + node: &r.Record, + nodeAlt: &r.Alert, + }) } - if r.Record == "" && r.Alert == "" { - errs = append(errs, errors.Errorf("one of 'record' or 'alert' must be set")) + if r.Record.Value == "" && r.Alert.Value == "" { + if r.Record.Value == "0" { + nodes = append(nodes, WrappedError{ + err: errors.Errorf("one of 'record' or 'alert' must be set"), + node: &r.Alert, + }) + } else { + nodes = append(nodes, WrappedError{ + err: errors.Errorf("one of 'record' or 'alert' must be set"), + node: &r.Record, + }) + } } - if r.Expr == "" { - errs = append(errs, errors.Errorf("field 'expr' must be set in rule")) - } else if _, err := promql.ParseExpr(r.Expr); err != nil { - errs = append(errs, errors.Wrap(err, "could not parse expression")) + if r.Expr.Value == "" { + nodes = append(nodes, WrappedError{ + err: errors.Errorf("field 'expr' must be set in rule"), + node: &r.Expr, + }) + } else if _, err := promql.ParseExpr(r.Expr.Value); err != nil { + nodes = append(nodes, WrappedError{ + err: errors.Wrapf(err, "could not parse expression"), + node: &r.Expr, + }) } - if r.Record != "" { + if r.Record.Value != "" { if len(r.Annotations) > 0 { - errs = append(errs, errors.Errorf("invalid field 'annotations' in recording rule")) + nodes = append(nodes, WrappedError{ + err: errors.Errorf("invalid field 'annotations' in recording rule"), + node: &r.Record, + }) } if r.For != 0 { - errs = append(errs, errors.Errorf("invalid field 'for' in recording rule")) + nodes = append(nodes, WrappedError{ + err: errors.Errorf("invalid field 'for' in recording rule"), + node: &r.Record, + }) } - if !model.IsValidMetricName(model.LabelValue(r.Record)) { - errs = append(errs, errors.Errorf("invalid recording rule name: %s", r.Record)) + if !model.IsValidMetricName(model.LabelValue(r.Record.Value)) { + nodes = append(nodes, WrappedError{ + err: errors.Errorf("invalid recording rule name: %s", r.Record.Value), + node: &r.Record, + }) } } for k, v := range r.Labels { if !model.LabelName(k).IsValid() { - errs = append(errs, errors.Errorf("invalid label name: %s", k)) + nodes = append(nodes, WrappedError{ + err: errors.Errorf("invalid label name: %s", k), + }) } if !model.LabelValue(v).IsValid() { - errs = append(errs, errors.Errorf("invalid label value: %s", v)) + nodes = append(nodes, WrappedError{ + err: errors.Errorf("invalid label value: %s", v), + }) } } for k := range r.Annotations { if !model.LabelName(k).IsValid() { - errs = append(errs, errors.Errorf("invalid annotation name: %s", k)) + nodes = append(nodes, WrappedError{ + err: errors.Errorf("invalid annotation name: %s", k), + }) } } - return append(errs, testTemplateParsing(r)...) + for _, err := range testTemplateParsing(r) { + nodes = append(nodes, WrappedError{err: err}) + } + + return } // testTemplateParsing checks if the templates used in labels and annotations // of the alerting rules are parsed correctly. -func testTemplateParsing(rl *Rule) (errs []error) { - if rl.Alert == "" { +func testTemplateParsing(rl *RuleNode) (errs []error) { + if rl.Alert.Value == "" { // Not an alerting rule. return errs } @@ -165,7 +231,7 @@ func testTemplateParsing(rl *Rule) (errs []error) { tmpl := template.NewTemplateExpander( context.TODO(), strings.Join(append(defs, text), ""), - "__alert_"+rl.Alert, + "__alert_"+rl.Alert.Value, tmplData, model.Time(timestamp.FromTime(time.Now())), nil, @@ -195,11 +261,16 @@ func testTemplateParsing(rl *Rule) (errs []error) { // Parse parses and validates a set of rules. func Parse(content []byte) (*RuleGroups, []error) { - var groups RuleGroups - if err := yaml.UnmarshalStrict(content, &groups); err != nil { + var ( + groups RuleGroups + node ruleGroups + ) + err := yaml.Unmarshal(content, &groups) + _err := yaml.Unmarshal(content, &node) + if err != nil || _err != nil { return nil, []error{err} } - return &groups, groups.Validate() + return &groups, groups.Validate(node) } // ParseFile reads and parses rules from a file. diff --git a/vendor/github.com/prometheus/prometheus/prompb/remote.pb.go b/vendor/github.com/prometheus/prometheus/prompb/remote.pb.go index 5dcb254f2c7..56fd03e9477 100644 --- a/vendor/github.com/prometheus/prometheus/prompb/remote.pb.go +++ b/vendor/github.com/prometheus/prometheus/prompb/remote.pb.go @@ -340,7 +340,7 @@ func (m *QueryResult) GetTimeseries() []*TimeSeries { // ChunkedReadResponse is a response when response_type equals STREAMED_XOR_CHUNKS. // We strictly stream full series after series, optionally split by time. This means that a single frame can contain // partition of the single series, but once a new series is started to be streamed it means that no more chunks will -// be sent for previous one. +// be sent for previous one. Series are returned sorted in the same way TSDB block are internally. type ChunkedReadResponse struct { ChunkedSeries []*ChunkedSeries `protobuf:"bytes,1,rep,name=chunked_series,json=chunkedSeries,proto3" json:"chunked_series,omitempty"` // query_index represents an index of the query from ReadRequest.queries these chunks relates to. diff --git a/vendor/github.com/prometheus/prometheus/prompb/remote.proto b/vendor/github.com/prometheus/prometheus/prompb/remote.proto index da2b06f2977..ecd8f0bb198 100644 --- a/vendor/github.com/prometheus/prometheus/prompb/remote.proto +++ b/vendor/github.com/prometheus/prometheus/prompb/remote.proto @@ -73,7 +73,7 @@ message QueryResult { // ChunkedReadResponse is a response when response_type equals STREAMED_XOR_CHUNKS. // We strictly stream full series after series, optionally split by time. This means that a single frame can contain // partition of the single series, but once a new series is started to be streamed it means that no more chunks will -// be sent for previous one. +// be sent for previous one. Series are returned sorted in the same way TSDB block are internally. message ChunkedReadResponse { repeated prometheus.ChunkedSeries chunked_series = 1; diff --git a/vendor/github.com/prometheus/prometheus/promql/ast.go b/vendor/github.com/prometheus/prometheus/promql/ast.go index 973971d2929..803c9892eb1 100644 --- a/vendor/github.com/prometheus/prometheus/promql/ast.go +++ b/vendor/github.com/prometheus/prometheus/promql/ast.go @@ -39,6 +39,9 @@ type Node interface { // String representation of the node that returns the given node when parsed // as part of a valid query. String() string + + // PositionRange returns the position of the AST Node in the query string. + PositionRange() PositionRange } // Statement is a generic interface for all statements. @@ -84,6 +87,7 @@ type AggregateExpr struct { Param Expr // Parameter used by some aggregators. Grouping []string // The labels by which to group the Vector. Without bool // Whether to drop the given labels rather than keep them. + PosRange PositionRange } // BinaryExpr represents a binary expression between two child expressions. @@ -103,18 +107,18 @@ type BinaryExpr struct { type Call struct { Func *Function // The function that was called. Args Expressions // Arguments used in the call. + + PosRange PositionRange } // MatrixSelector represents a Matrix selection. type MatrixSelector struct { - Name string - Range time.Duration - Offset time.Duration - LabelMatchers []*labels.Matcher + // It is safe to assume that this is an VectorSelector + // if the parser hasn't returned an error. + VectorSelector Expr + Range time.Duration - // The unexpanded seriesSet populated at query preparation time. - unexpandedSeriesSet storage.SeriesSet - series []storage.Series + EndPos Pos } // SubqueryExpr represents a subquery. @@ -123,22 +127,28 @@ type SubqueryExpr struct { Range time.Duration Offset time.Duration Step time.Duration + + EndPos Pos } // NumberLiteral represents a number. type NumberLiteral struct { Val float64 + + PosRange PositionRange } // ParenExpr wraps an expression so it cannot be disassembled as a consequence // of operator precedence. type ParenExpr struct { - Expr Expr + Expr Expr + PosRange PositionRange } // StringLiteral represents a string. type StringLiteral struct { - Val string + Val string + PosRange PositionRange } // UnaryExpr represents a unary operation on another expression. @@ -146,6 +156,8 @@ type StringLiteral struct { type UnaryExpr struct { Op ItemType Expr Expr + + StartPos Pos } // VectorSelector represents a Vector selection. @@ -157,6 +169,8 @@ type VectorSelector struct { // The unexpanded seriesSet populated at query preparation time. unexpandedSeriesSet storage.SeriesSet series []storage.Series + + PosRange PositionRange } func (e *AggregateExpr) Type() ValueType { return ValueTypeVector } @@ -316,10 +330,91 @@ func Children(node Node) []Node { return []Node{n.Expr} case *UnaryExpr: return []Node{n.Expr} - case *MatrixSelector, *NumberLiteral, *StringLiteral, *VectorSelector: + case *MatrixSelector: + return []Node{n.VectorSelector} + case *NumberLiteral, *StringLiteral, *VectorSelector: // nothing to do return []Node{} default: panic(errors.Errorf("promql.Children: unhandled node type %T", node)) } } + +// PositionRange describes a position in the input string of the parser. +type PositionRange struct { + Start Pos + End Pos +} + +// mergeRanges is a helper function to merge the PositionRanges of two Nodes. +// Note that the arguments must be in the same order as they +// occur in the input string. +func mergeRanges(first Node, last Node) PositionRange { + return PositionRange{ + Start: first.PositionRange().Start, + End: last.PositionRange().End, + } +} + +// Item implements the Node interface. +// This makes it possible to call mergeRanges on them. +func (i *Item) PositionRange() PositionRange { + return PositionRange{ + Start: i.Pos, + End: i.Pos + Pos(len(i.Val)), + } +} + +func (e *AggregateExpr) PositionRange() PositionRange { + return e.PosRange +} +func (e *BinaryExpr) PositionRange() PositionRange { + return mergeRanges(e.LHS, e.RHS) +} +func (e *Call) PositionRange() PositionRange { + return e.PosRange +} +func (e *EvalStmt) PositionRange() PositionRange { + return e.Expr.PositionRange() +} +func (e Expressions) PositionRange() PositionRange { + if len(e) == 0 { + // Position undefined. + return PositionRange{ + Start: -1, + End: -1, + } + } else { + return mergeRanges(e[0], e[len(e)-1]) + } +} +func (e *MatrixSelector) PositionRange() PositionRange { + return PositionRange{ + Start: e.VectorSelector.PositionRange().Start, + End: e.EndPos, + } +} +func (e *SubqueryExpr) PositionRange() PositionRange { + return PositionRange{ + Start: e.Expr.PositionRange().Start, + End: e.EndPos, + } +} +func (e *NumberLiteral) PositionRange() PositionRange { + return e.PosRange +} +func (e *ParenExpr) PositionRange() PositionRange { + return e.PosRange +} +func (e *StringLiteral) PositionRange() PositionRange { + return e.PosRange +} +func (e *UnaryExpr) PositionRange() PositionRange { + return PositionRange{ + Start: e.StartPos, + End: e.Expr.PositionRange().End, + } +} +func (e *VectorSelector) PositionRange() PositionRange { + return e.PosRange +} diff --git a/vendor/github.com/prometheus/prometheus/promql/engine.go b/vendor/github.com/prometheus/prometheus/promql/engine.go index 6afb3d4fbde..0add9e1473d 100644 --- a/vendor/github.com/prometheus/prometheus/promql/engine.go +++ b/vendor/github.com/prometheus/prometheus/promql/engine.go @@ -33,7 +33,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/pkg/gate" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/timestamp" "github.com/prometheus/prometheus/pkg/value" @@ -76,6 +75,8 @@ func GetDefaultEvaluationInterval() int64 { type engineMetrics struct { currentQueries prometheus.Gauge maxConcurrentQueries prometheus.Gauge + queryLogEnabled prometheus.Gauge + queryLogFailures prometheus.Counter queryQueueTime prometheus.Summary queryPrepareTime prometheus.Summary queryInnerEval prometheus.Summary @@ -112,6 +113,13 @@ func (e ErrStorage) Error() string { return e.Err.Error() } +// QueryLogger is an interface that can be used to log all the queries logged +// by the engine. +type QueryLogger interface { + Log(...interface{}) error + Close() error +} + // A Query is derived from an a raw query string and can be run against an engine // it is associated with. type Query interface { @@ -146,6 +154,10 @@ type query struct { ng *Engine } +type queryCtx int + +var queryOrigin queryCtx + // Statement implements the Query interface. func (q *query) Statement() Statement { return q.stmt @@ -176,19 +188,9 @@ func (q *query) Exec(ctx context.Context) *Result { span.SetTag(queryTag, q.stmt.String()) } - // Log query in active log. - var queryIndex int - if q.ng.activeQueryTracker != nil { - queryIndex = q.ng.activeQueryTracker.Insert(q.q) - } - // Exec query. res, warnings, err := q.ng.exec(ctx, q) - // Delete query from active log. - if q.ng.activeQueryTracker != nil { - q.ng.activeQueryTracker.Delete(queryIndex) - } return &Result{Err: err, Value: res, Warnings: warnings} } @@ -215,7 +217,6 @@ func contextErr(err error, env string) error { type EngineOpts struct { Logger log.Logger Reg prometheus.Registerer - MaxConcurrent int MaxSamples int Timeout time.Duration ActiveQueryTracker *ActiveQueryTracker @@ -227,9 +228,10 @@ type Engine struct { logger log.Logger metrics *engineMetrics timeout time.Duration - gate *gate.Gate maxSamplesPerQuery int activeQueryTracker *ActiveQueryTracker + queryLogger QueryLogger + queryLoggerLock sync.RWMutex } // NewEngine returns a new engine. @@ -245,6 +247,18 @@ func NewEngine(opts EngineOpts) *Engine { Name: "queries", Help: "The current number of queries being executed or waiting.", }), + queryLogEnabled: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "query_log_enabled", + Help: "State of the query log.", + }), + queryLogFailures: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "query_log_failures_total", + Help: "The number of query log failures.", + }), maxConcurrentQueries: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: subsystem, @@ -284,12 +298,19 @@ func NewEngine(opts EngineOpts) *Engine { Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }), } - metrics.maxConcurrentQueries.Set(float64(opts.MaxConcurrent)) + + if t := opts.ActiveQueryTracker; t != nil { + metrics.maxConcurrentQueries.Set(float64(t.GetMaxConcurrent())) + } else { + metrics.maxConcurrentQueries.Set(-1) + } if opts.Reg != nil { opts.Reg.MustRegister( metrics.currentQueries, metrics.maxConcurrentQueries, + metrics.queryLogEnabled, + metrics.queryLogFailures, metrics.queryQueueTime, metrics.queryPrepareTime, metrics.queryInnerEval, @@ -298,7 +319,6 @@ func NewEngine(opts EngineOpts) *Engine { } return &Engine{ - gate: gate.New(opts.MaxConcurrent), timeout: opts.Timeout, logger: opts.Logger, metrics: metrics, @@ -307,6 +327,29 @@ func NewEngine(opts EngineOpts) *Engine { } } +// SetQueryLogger sets the query logger. +func (ng *Engine) SetQueryLogger(l QueryLogger) { + ng.queryLoggerLock.Lock() + defer ng.queryLoggerLock.Unlock() + + if ng.queryLogger != nil { + // An error closing the old file descriptor should + // not make reload fail; only log a warning. + err := ng.queryLogger.Close() + if err != nil { + level.Warn(ng.logger).Log("msg", "error while closing the previous query log file", "err", err) + } + } + + ng.queryLogger = l + + if l != nil { + ng.metrics.queryLogEnabled.Set(1) + } else { + ng.metrics.queryLogEnabled.Set(0) + } +} + // NewInstantQuery returns an evaluation query for the given expression at the given time. func (ng *Engine) NewInstantQuery(q storage.Queryable, qs string, ts time.Time) (Query, error) { expr, err := ParseExpr(qs) @@ -358,6 +401,13 @@ type testStmt func(context.Context) error func (testStmt) String() string { return "test statement" } func (testStmt) stmt() {} +func (testStmt) PositionRange() PositionRange { + return PositionRange{ + Start: -1, + End: -1, + } +} + func (ng *Engine) newTestQuery(f func(context.Context) error) Query { qry := &query{ q: "test statement", @@ -372,23 +422,55 @@ func (ng *Engine) newTestQuery(f func(context.Context) error) Query { // // At this point per query only one EvalStmt is evaluated. Alert and record // statements are not handled by the Engine. -func (ng *Engine) exec(ctx context.Context, q *query) (Value, storage.Warnings, error) { +func (ng *Engine) exec(ctx context.Context, q *query) (v Value, w storage.Warnings, err error) { ng.metrics.currentQueries.Inc() defer ng.metrics.currentQueries.Dec() ctx, cancel := context.WithTimeout(ctx, ng.timeout) q.cancel = cancel + defer func() { + ng.queryLoggerLock.RLock() + if l := ng.queryLogger; l != nil { + params := make(map[string]interface{}, 4) + params["query"] = q.q + if eq, ok := q.Statement().(*EvalStmt); ok { + params["start"] = formatDate(eq.Start) + params["end"] = formatDate(eq.End) + params["step"] = eq.Interval + } + f := []interface{}{"params", params} + if err != nil { + f = append(f, "error", err) + } + f = append(f, "stats", stats.NewQueryStats(q.Stats())) + if origin := ctx.Value(queryOrigin); origin != nil { + for k, v := range origin.(map[string]interface{}) { + f = append(f, k, v) + } + } + if err := l.Log(f...); err != nil { + ng.metrics.queryLogFailures.Inc() + level.Error(ng.logger).Log("msg", "can't log query", "err", err) + } + } + ng.queryLoggerLock.RUnlock() + }() + execSpanTimer, ctx := q.stats.GetSpanTimer(ctx, stats.ExecTotalTime) defer execSpanTimer.Finish() queueSpanTimer, _ := q.stats.GetSpanTimer(ctx, stats.ExecQueueTime, ng.metrics.queryQueueTime) - - if err := ng.gate.Start(ctx); err != nil { - return nil, nil, contextErr(err, "query queue") + // Log query in active log. The active log guarantees that we don't run over + // MaxConcurrent queries. + if ng.activeQueryTracker != nil { + queryIndex, err := ng.activeQueryTracker.Insert(ctx, q.q) + if err != nil { + queueSpanTimer.Finish() + return nil, nil, contextErr(err, "query queue") + } + defer ng.activeQueryTracker.Delete(queryIndex) } - defer ng.gate.Done() - queueSpanTimer.Finish() // Cancel when execution is done or an error was raised. @@ -452,6 +534,7 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *EvalStmt) ( defaultEvalInterval: GetDefaultEvaluationInterval(), logger: ng.logger, } + val, err := evaluator.Eval(s.Expr) if err != nil { return nil, warnings, err @@ -459,10 +542,17 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *EvalStmt) ( evalSpanTimer.Finish() - mat, ok := val.(Matrix) - if !ok { + var mat Matrix + + switch result := val.(type) { + case Matrix: + mat = result + case String: + return result, warnings, nil + default: panic(errors.Errorf("promql.Engine.exec: invalid expression type %q", val.Type())) } + query.matrix = mat switch s.Expr.Type() { case ValueTypeVector: @@ -510,7 +600,6 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *EvalStmt) ( return nil, warnings, err } - // TODO(fabxc): order ensured by storage? // TODO(fabxc): where to ensure metric labels are a copy from the storage internals. sortSpanTimer, _ := query.stats.GetSpanTimer(ctx, stats.ResultSortTime, ng.metrics.queryResultSort) sort.Sort(mat) @@ -547,8 +636,8 @@ func (ng *Engine) populateSeries(ctx context.Context, q storage.Queryable, s *Ev if maxOffset < n.Range+subqOffset { maxOffset = n.Range + subqOffset } - if n.Offset+n.Range+subqOffset > maxOffset { - maxOffset = n.Offset + n.Range + subqOffset + if m := n.VectorSelector.(*VectorSelector).Offset + n.Range + subqOffset; m > maxOffset { + maxOffset = m } } return nil @@ -563,6 +652,11 @@ func (ng *Engine) populateSeries(ctx context.Context, q storage.Queryable, s *Ev var warnings storage.Warnings + // Whenever a MatrixSelector is evaluated this variable is set to the corresponding range. + // The evaluation of the VectorSelector inside then evaluates the given range and unsets + // the variable. + var evalRange time.Duration + Inspect(s.Expr, func(node Node, path []Node) error { var set storage.SeriesSet var wrn storage.Warnings @@ -582,7 +676,16 @@ func (ng *Engine) populateSeries(ctx context.Context, q storage.Queryable, s *Ev switch n := node.(type) { case *VectorSelector: - params.Start = params.Start - durationMilliseconds(LookbackDelta) + if evalRange == 0 { + params.Start = params.Start - durationMilliseconds(LookbackDelta) + } else { + params.Range = durationMilliseconds(evalRange) + // For all matrix queries we want to ensure that we have (end-start) + range selected + // this way we have `range` data before the start time + params.Start = params.Start - durationMilliseconds(evalRange) + evalRange = 0 + } + params.Func = extractFuncFromPath(path) params.By, params.Grouping = extractGroupsFromPath(path) if n.Offset > 0 { @@ -600,24 +703,7 @@ func (ng *Engine) populateSeries(ctx context.Context, q storage.Queryable, s *Ev n.unexpandedSeriesSet = set case *MatrixSelector: - params.Func = extractFuncFromPath(path) - params.Range = durationMilliseconds(n.Range) - // For all matrix queries we want to ensure that we have (end-start) + range selected - // this way we have `range` data before the start time - params.Start = params.Start - durationMilliseconds(n.Range) - if n.Offset > 0 { - offsetMilliseconds := durationMilliseconds(n.Offset) - params.Start = params.Start - offsetMilliseconds - params.End = params.End - offsetMilliseconds - } - - set, wrn, err = querier.Select(params, n.LabelMatchers...) - warnings = append(warnings, wrn...) - if err != nil { - level.Error(ng.logger).Log("msg", "error selecting series set", "err", err) - return err - } - n.unexpandedSeriesSet = set + evalRange = n.Range } return nil }) @@ -658,14 +744,7 @@ func extractGroupsFromPath(p []Node) (bool, []string) { func checkForSeriesSetExpansion(ctx context.Context, expr Expr) { switch e := expr.(type) { case *MatrixSelector: - if e.series == nil { - series, err := expandSeriesSet(ctx, e.unexpandedSeriesSet) - if err != nil { - panic(err) - } else { - e.series = series - } - } + checkForSeriesSetExpansion(ctx, e.VectorSelector) case *VectorSelector: if e.series == nil { series, err := expandSeriesSet(ctx, e.unexpandedSeriesSet) @@ -923,13 +1002,16 @@ func (ev *evaluator) rangeEval(f func([]Value, *EvalNodeHelper) Vector, exprs .. // evaluated MatrixSelector in its place. Note that the Name and LabelMatchers are not set. func (ev *evaluator) evalSubquery(subq *SubqueryExpr) *MatrixSelector { val := ev.eval(subq).(Matrix) - ms := &MatrixSelector{ - Range: subq.Range, + vs := &VectorSelector{ Offset: subq.Offset, series: make([]storage.Series, 0, len(val)), } + ms := &MatrixSelector{ + Range: subq.Range, + VectorSelector: vs, + } for _, s := range val { - ms.series = append(ms.series, NewStorageSeries(s)) + vs.series = append(vs.series, NewStorageSeries(s)) } return ms } @@ -945,6 +1027,7 @@ func (ev *evaluator) eval(expr Expr) Value { switch e := expr.(type) { case *AggregateExpr: + unwrapParenExpr(&e.Param) if s, ok := e.Param.(*StringLiteral); ok { return ev.rangeEval(func(v []Value, enh *EvalNodeHelper) Vector { return ev.aggregation(e.Op, e.Grouping, e.Without, s.Val, v[0].(Vector), enh) @@ -974,7 +1057,9 @@ func (ev *evaluator) eval(expr Expr) Value { // Check if the function has a matrix argument. var matrixArgIndex int var matrixArg bool - for i, a := range e.Args { + for i := range e.Args { + unwrapParenExpr(&e.Args[i]) + a := e.Args[i] if _, ok := a.(*MatrixSelector); ok { matrixArgIndex = i matrixArg = true @@ -1009,9 +1094,11 @@ func (ev *evaluator) eval(expr Expr) Value { } sel := e.Args[matrixArgIndex].(*MatrixSelector) + selVS := sel.VectorSelector.(*VectorSelector) + checkForSeriesSetExpansion(ev.ctx, sel) - mat := make(Matrix, 0, len(sel.series)) // Output matrix. - offset := durationMilliseconds(sel.Offset) + mat := make(Matrix, 0, len(selVS.series)) // Output matrix. + offset := durationMilliseconds(selVS.Offset) selRange := durationMilliseconds(sel.Range) stepRange := selRange if stepRange > ev.interval { @@ -1024,17 +1111,17 @@ func (ev *evaluator) eval(expr Expr) Value { enh := &EvalNodeHelper{out: make(Vector, 0, 1)} // Process all the calls for one time series at a time. it := storage.NewBuffer(selRange) - for i, s := range sel.series { + for i, s := range selVS.series { points = points[:0] it.Reset(s.Iterator()) ss := Series{ // For all range vector functions, the only change to the // output labels is dropping the metric name so just do // it once here. - Metric: dropMetricName(sel.series[i].Labels()), + Metric: dropMetricName(selVS.series[i].Labels()), Points: getPointSlice(numSteps), } - inMatrix[0].Metric = sel.series[i].Labels() + inMatrix[0].Metric = selVS.series[i].Labels() for ts, step := ev.startTimestamp, -1; ts <= ev.endTimestamp; ts += ev.interval { step++ // Set the non-matrix arguments. @@ -1247,6 +1334,8 @@ func (ev *evaluator) eval(expr Expr) Value { res := newEv.eval(e.Expr) ev.currentSamples = newEv.currentSamples return res + case *StringLiteral: + return String{V: e.Val, T: ev.startTimestamp} } panic(errors.Errorf("unhandled expression of type: %T", expr)) @@ -1332,21 +1421,25 @@ func putPointSlice(p []Point) { func (ev *evaluator) matrixSelector(node *MatrixSelector) Matrix { checkForSeriesSetExpansion(ev.ctx, node) + vs := node.VectorSelector.(*VectorSelector) + var ( - offset = durationMilliseconds(node.Offset) + offset = durationMilliseconds(vs.Offset) maxt = ev.startTimestamp - offset mint = maxt - durationMilliseconds(node.Range) - matrix = make(Matrix, 0, len(node.series)) + matrix = make(Matrix, 0, len(vs.series)) ) it := storage.NewBuffer(durationMilliseconds(node.Range)) - for i, s := range node.series { + series := vs.series + + for i, s := range series { if err := contextDone(ev.ctx, "expression evaluation"); err != nil { ev.error(err) } it.Reset(s.Iterator()) ss := Series{ - Metric: node.series[i].Labels(), + Metric: series[i].Labels(), } ss.Points = ev.matrixIterSlice(it, mint, maxt, getPointSlice(16)) @@ -2000,6 +2093,11 @@ func shouldDropMetricName(op ItemType) bool { } } +// NewOriginContext returns a new context with data about the origin attached. +func NewOriginContext(ctx context.Context, data map[string]interface{}) context.Context { + return context.WithValue(ctx, queryOrigin, data) +} + // documentedType returns the internal type to the equivalent // user facing terminology as defined in the documentation. func documentedType(t ValueType) string { @@ -2012,3 +2110,18 @@ func documentedType(t ValueType) string { return string(t) } } + +func formatDate(t time.Time) string { + return t.UTC().Format("2006-01-02T15:04:05.000Z07:00") +} + +// unwrapParenExpr does the AST equivalent of removing parentheses around a expression. +func unwrapParenExpr(e *Expr) { + for { + if p, ok := (*e).(*ParenExpr); ok { + *e = p.Expr + } else { + break + } + } +} diff --git a/vendor/github.com/prometheus/prometheus/promql/functions.go b/vendor/github.com/prometheus/prometheus/promql/functions.go index 1475bb73869..4765965ab3e 100644 --- a/vendor/github.com/prometheus/prometheus/promql/functions.go +++ b/vendor/github.com/prometheus/prometheus/promql/functions.go @@ -64,11 +64,12 @@ func funcTime(vals []Value, args Expressions, enh *EvalNodeHelper) Vector { // the result as either per-second (if isRate is true) or overall. func extrapolatedRate(vals []Value, args Expressions, enh *EvalNodeHelper, isCounter bool, isRate bool) Vector { ms := args[0].(*MatrixSelector) + vs := ms.VectorSelector.(*VectorSelector) var ( samples = vals[0].(Matrix)[0] - rangeStart = enh.ts - durationMilliseconds(ms.Range+ms.Offset) - rangeEnd = enh.ts - durationMilliseconds(ms.Offset) + rangeStart = enh.ts - durationMilliseconds(ms.Range+vs.Offset) + rangeEnd = enh.ts - durationMilliseconds(vs.Offset) ) // No sense in trying to compute a rate without at least two points. Drop @@ -1243,7 +1244,7 @@ func createLabelsForAbsentFunction(expr Expr) labels.Labels { case *VectorSelector: lm = n.LabelMatchers case *MatrixSelector: - lm = n.LabelMatchers + lm = n.VectorSelector.(*VectorSelector).LabelMatchers default: return m } diff --git a/vendor/github.com/prometheus/prometheus/promql/generated_parser.y b/vendor/github.com/prometheus/prometheus/promql/generated_parser.y index aae970d799b..0cb98df7553 100644 --- a/vendor/github.com/prometheus/prometheus/promql/generated_parser.y +++ b/vendor/github.com/prometheus/prometheus/promql/generated_parser.y @@ -18,6 +18,7 @@ import ( "math" "sort" "strconv" + "time" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/value" @@ -35,209 +36,265 @@ import ( series []sequenceValue uint uint64 float float64 + duration time.Duration } -%token ERROR -%token EOF -%token COMMENT -%token IDENTIFIER -%token METRIC_IDENTIFIER -%token LEFT_PAREN -%token RIGHT_PAREN -%token LEFT_BRACE -%token RIGHT_BRACE -%token LEFT_BRACKET -%token RIGHT_BRACKET -%token COMMA -%token ASSIGN -%token COLON -%token SEMICOLON -%token STRING -%token NUMBER -%token DURATION -%token BLANK -%token TIMES -%token SPACE +%token +ASSIGN +BLANK +COLON +COMMA +COMMENT +DURATION +EOF +ERROR +IDENTIFIER +LEFT_BRACE +LEFT_BRACKET +LEFT_PAREN +METRIC_IDENTIFIER +NUMBER +RIGHT_BRACE +RIGHT_BRACKET +RIGHT_PAREN +SEMICOLON +SPACE +STRING +TIMES -%token operatorsStart // Operators. -%token SUB -%token ADD -%token MUL -%token MOD -%token DIV -%token LAND -%token LOR -%token LUNLESS -%token EQL -%token NEQ -%token LTE -%token LSS -%token GTE -%token GTR -%token EQL_REGEX -%token NEQ_REGEX -%token POW +%token operatorsStart +%token +ADD +DIV +EQL +EQL_REGEX +GTE +GTR +LAND +LOR +LSS +LTE +LUNLESS +MOD +MUL +NEQ +NEQ_REGEX +POW +SUB %token operatorsEnd -%token aggregatorsStart // Aggregators. -%token AVG -%token COUNT -%token SUM -%token MIN -%token MAX -%token STDDEV -%token STDVAR -%token TOPK -%token BOTTOMK -%token COUNT_VALUES -%token QUANTILE +%token aggregatorsStart +%token +AVG +BOTTOMK +COUNT +COUNT_VALUES +MAX +MIN +QUANTILE +STDDEV +STDVAR +SUM +TOPK %token aggregatorsEnd -%token keywordsStart // Keywords. -%token OFFSET -%token BY -%token WITHOUT -%token ON -%token IGNORING -%token GROUP_LEFT -%token GROUP_RIGHT -%token BOOL - +%token keywordsStart +%token +BOOL +BY +GROUP_LEFT +GROUP_RIGHT +IGNORING +OFFSET +ON +WITHOUT %token keywordsEnd -%token startSymbolsStart // Start symbols for the generated parser. -%token START_LABELS -%token START_METRIC -%token START_GROUPING_LABELS -%token START_SERIES_DESCRIPTION +%token startSymbolsStart +%token +START_METRIC +START_SERIES_DESCRIPTION +START_EXPRESSION +START_METRIC_SELECTOR %token startSymbolsEnd -%type label_matchers label_match_list + +// Type definitions for grammar rules. +%type label_match_list %type label_matcher -%type match_op metric_identifier grouping_label maybe_label +%type aggregate_op grouping_label match_op maybe_label metric_identifier unary_op -%type label_set_list label_set metric -%type