From f38032466d5558ff6d0ea5b44595378aa9b09ee4 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 9 Apr 2020 10:57:29 -0400 Subject: [PATCH 01/15] Added query frontend to the single binary Signed-off-by: Joe Elliott --- pkg/cortex/modules.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index ee976fd607e..d79db475279 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -611,6 +611,6 @@ var modules = map[ModuleName]module{ }, All: { - deps: []ModuleName{Querier, Ingester, Distributor, TableManager, DataPurger, StoreGateway}, + deps: []ModuleName{QueryFrontend, Querier, Ingester, Distributor, TableManager, DataPurger, StoreGateway}, }, } From 23235c7453109e951ea5371a7560684334f39851 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 9 Apr 2020 13:36:19 -0400 Subject: [PATCH 02/15] First pass single binary Signed-off-by: Joe Elliott --- pkg/api/api.go | 62 +++++++++++++++++++++++++++---------------- pkg/cortex/modules.go | 4 +-- 2 files changed, 41 insertions(+), 25 deletions(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index e98b49e750f..c2057d2286b 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -29,6 +29,7 @@ import ( "github.com/cortexproject/cortex/pkg/ruler" "github.com/cortexproject/cortex/pkg/storegateway" "github.com/cortexproject/cortex/pkg/util/push" + "github.com/gorilla/mux" ) type Config struct { @@ -79,15 +80,19 @@ func New(cfg Config, s *server.Server, logger log.Logger) (*API, error) { } func (a *API) registerRoute(path string, handler http.Handler, auth bool, methods ...string) { + a.registerRouteWithRouter(a.server.HTTP, path, handler, auth, methods...) +} + +func (a *API) registerRouteWithRouter(router *mux.Router, path string, handler http.Handler, auth bool, methods ...string) { level.Debug(a.logger).Log("msg", "api: registering route", "methods", strings.Join(methods, ","), "path", path, "auth", auth) if auth { handler = a.authMiddleware.Wrap(handler) } if len(methods) == 0 { - a.server.HTTP.Path(path).Handler(handler) + router.Path(path).Handler(handler) return } - a.server.HTTP.Path(path).Methods(methods...).Handler(handler) + router.Path(path).Methods(methods...).Handler(handler) } func (a *API) registerRoutesWithPrefix(prefix string, handler http.Handler, auth bool, methods ...string) { @@ -236,7 +241,7 @@ func (a *API) RegisterCompactor(c *compactor.Compactor) { // RegisterQuerier registers the Prometheus routes supported by the // Cortex querier service. Currently this can not be registered simultaneously // with the QueryFrontend. -func (a *API) RegisterQuerier(queryable storage.Queryable, engine *promql.Engine, distributor *distributor.Distributor) { +func (a *API) RegisterQuerier(queryable storage.Queryable, engine *promql.Engine, distributor *distributor.Distributor, registerRoutesExternally bool) http.Handler { api := v1.NewAPI( engine, queryable, @@ -255,36 +260,47 @@ func (a *API) RegisterQuerier(queryable storage.Queryable, engine *promql.Engine &v1.PrometheusVersion{}, ) - promRouter := route.New().WithPrefix(a.cfg.ServerPrefix + a.cfg.PrometheusHTTPPrefix + "/api/v1") - api.Register(promRouter) - promHandler := fakeRemoteAddr(promRouter) - - a.registerRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/read", querier.RemoteReadHandler(queryable), true, "GET") - a.registerRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/query", promHandler, true, "GET", "POST") - a.registerRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/query_range", promHandler, true, "GET", "POST") - a.registerRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/labels", promHandler, true, "GET", "POST") - a.registerRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/label/{name}/values", promHandler, true, "GET") - a.registerRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/series", promHandler, true, "GET", "POST", "DELETE") - a.registerRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/metadata", promHandler, true, "GET") - + // these routes are always registered to the default server a.registerRoute("/api/v1/user_stats", http.HandlerFunc(distributor.UserStatsHandler), true) a.registerRoute("/api/v1/chunks", querier.ChunksHandler(queryable), true) - // Legacy Routes a.registerRoute(a.cfg.LegacyHTTPPrefix+"/user_stats", http.HandlerFunc(distributor.UserStatsHandler), true) a.registerRoute(a.cfg.LegacyHTTPPrefix+"/chunks", querier.ChunksHandler(queryable), true) + // these routes are either registered the default server OR to an internal mux. The internal mux is + // for use in a single binary mode when both the query frontend and the querier would attempt to claim these routes + auth := false + router := mux.NewRouter() + if registerRoutesExternally { + auth = true + router = a.server.HTTP + } + + promRouter := route.New().WithPrefix(a.cfg.ServerPrefix + a.cfg.PrometheusHTTPPrefix + "/api/v1") + api.Register(promRouter) + promHandler := fakeRemoteAddr(promRouter) + + a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/read", querier.RemoteReadHandler(queryable), auth, "GET") + a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/query", promHandler, auth, "GET", "POST") + a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/query_range", promHandler, auth, "GET", "POST") + a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/labels", promHandler, auth, "GET", "POST") + a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/label/{name}/values", promHandler, auth, "GET") + a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/series", promHandler, auth, "GET", "POST", "DELETE") + a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/metadata", promHandler, auth, "GET") + legacyPromRouter := route.New().WithPrefix(a.cfg.ServerPrefix + a.cfg.LegacyHTTPPrefix + "/api/v1") api.Register(legacyPromRouter) legacyPromHandler := fakeRemoteAddr(legacyPromRouter) - a.registerRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/read", querier.RemoteReadHandler(queryable), true, "GET") - a.registerRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/query", legacyPromHandler, true, "GET", "POST") - a.registerRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/query_range", legacyPromHandler, true, "GET", "POST") - a.registerRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/labels", legacyPromHandler, true, "GET", "POST") - a.registerRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/label/{name}/values", legacyPromHandler, true, "GET") - a.registerRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/series", legacyPromHandler, true, "GET", "POST", "DELETE") - a.registerRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/metadata", legacyPromHandler, true, "GET") + a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/read", querier.RemoteReadHandler(queryable), auth, "GET") + a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/query", legacyPromHandler, auth, "GET", "POST") + a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/query_range", legacyPromHandler, auth, "GET", "POST") + a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/labels", legacyPromHandler, auth, "GET", "POST") + a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/label/{name}/values", legacyPromHandler, auth, "GET") + a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/series", legacyPromHandler, auth, "GET", "POST", "DELETE") + a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/metadata", legacyPromHandler, auth, "GET") + + return router } // RegisterQueryFrontend registers the Prometheus routes supported by the diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index d79db475279..dbb512fa612 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -196,11 +196,11 @@ func (t *Cortex) initQuerier(cfg *Config) (serv services.Service, err error) { queryable, engine := querier.New(cfg.Querier, t.distributor, t.storeQueryable, tombstonesLoader, prometheus.DefaultRegisterer) - t.api.RegisterQuerier(queryable, engine, t.distributor) + handler := t.api.RegisterQuerier(queryable, engine, t.distributor) // Query frontend worker will only be started after all its dependencies are started, not here. // Worker may also be nil, if not configured, which is OK. - worker, err := frontend.NewWorker(cfg.Worker, httpgrpc_server.NewServer(t.server.HTTPServer.Handler), util.Logger) + worker, err := frontend.NewWorker(cfg.Worker, httpgrpc_server.NewServer(handler), util.Logger) if err != nil { return } From 349627622fea210bd560aa4f57e7d78c630c8215 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 9 Apr 2020 13:49:57 -0400 Subject: [PATCH 03/15] Pass parameter correctly Signed-off-by: Joe Elliott --- pkg/cortex/modules.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index dbb512fa612..d4ce67ef49a 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -196,7 +196,9 @@ func (t *Cortex) initQuerier(cfg *Config) (serv services.Service, err error) { queryable, engine := querier.New(cfg.Querier, t.distributor, t.storeQueryable, tombstonesLoader, prometheus.DefaultRegisterer) - handler := t.api.RegisterQuerier(queryable, engine, t.distributor) + // if we are not configured for single binary mode then the querier needs to register its paths externally + registerExternally := cfg.Target != All + handler := t.api.RegisterQuerier(queryable, engine, t.distributor, registerExternally) // Query frontend worker will only be started after all its dependencies are started, not here. // Worker may also be nil, if not configured, which is OK. From 90c7f79216ed5b25dfdbf452f2ce4d14b43030c7 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 9 Apr 2020 15:26:25 -0400 Subject: [PATCH 04/15] lint Signed-off-by: Joe Elliott --- pkg/api/api.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index c2057d2286b..85c1ef84e79 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -17,6 +17,8 @@ import ( "github.com/weaveworks/common/middleware" "github.com/weaveworks/common/server" + "github.com/gorilla/mux" + "github.com/cortexproject/cortex/pkg/alertmanager" "github.com/cortexproject/cortex/pkg/chunk/purger" "github.com/cortexproject/cortex/pkg/compactor" @@ -29,7 +31,6 @@ import ( "github.com/cortexproject/cortex/pkg/ruler" "github.com/cortexproject/cortex/pkg/storegateway" "github.com/cortexproject/cortex/pkg/util/push" - "github.com/gorilla/mux" ) type Config struct { From 50769a3134b16f82de209f058c7cc652b9779f8e Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 9 Apr 2020 16:00:19 -0400 Subject: [PATCH 05/15] Added label tests to single binary Signed-off-by: Joe Elliott --- integration/e2e/util.go | 13 ++++++++---- integration/e2ecortex/client.go | 12 +++++++++++ ...ting_started_single_process_config_test.go | 21 +++++++++++++++++-- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/integration/e2e/util.go b/integration/e2e/util.go index 23116216ef9..bf746130d54 100644 --- a/integration/e2e/util.go +++ b/integration/e2e/util.go @@ -68,15 +68,20 @@ func TimeToMilliseconds(t time.Time) int64 { return int64(math.Round(float64(t.UnixNano()) / 1000000)) } -func GenerateSeries(name string, ts time.Time) (series []prompb.TimeSeries, vector model.Vector) { +func GenerateSeries(name string, ts time.Time, additionalLabels ...prompb.Label) (series []prompb.TimeSeries, vector model.Vector) { tsMillis := TimeToMilliseconds(ts) value := rand.Float64() - // Generate the series - series = append(series, prompb.TimeSeries{ - Labels: []prompb.Label{ + lbls := append( + []prompb.Label{ {Name: labels.MetricName, Value: name}, }, + additionalLabels..., + ) + + // Generate the series + series = append(series, prompb.TimeSeries{ + Labels: lbls, Samples: []prompb.Sample{ {Value: value, Timestamp: tsMillis}, }, diff --git a/integration/e2ecortex/client.go b/integration/e2ecortex/client.go index b16c0aac591..bd45780516b 100644 --- a/integration/e2ecortex/client.go +++ b/integration/e2ecortex/client.go @@ -112,6 +112,18 @@ func (c *Client) Query(query string, ts time.Time) (model.Value, error) { return value, err } +// LabelValues gets label values +func (c *Client) LabelValues(label string) (model.LabelValues, error) { + value, _, err := c.querierClient.LabelValues(context.Background(), label) + return value, err +} + +// LabelNames gets label names +func (c *Client) LabelNames() ([]string, error) { + value, _, err := c.querierClient.LabelNames(context.Background()) + return value, err +} + type addOrgIDRoundTripper struct { orgID string next http.RoundTripper diff --git a/integration/getting_started_single_process_config_test.go b/integration/getting_started_single_process_config_test.go index 6455b6d8a87..833097dfb3b 100644 --- a/integration/getting_started_single_process_config_test.go +++ b/integration/getting_started_single_process_config_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/prompb" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -38,7 +39,7 @@ func TestGettingStartedSingleProcessConfigWithChunksStorage(t *testing.T) { // Push some series to Cortex. now := time.Now() - series, expectedVector := generateSeries("series_1", now) + series, expectedVector := generateSeries("series_1", now, prompb.Label{Name: "foo", Value: "bar"}) res, err := c.Push(series) require.NoError(t, err) @@ -49,6 +50,14 @@ func TestGettingStartedSingleProcessConfigWithChunksStorage(t *testing.T) { require.NoError(t, err) require.Equal(t, model.ValVector, result.Type()) assert.Equal(t, expectedVector, result.(model.Vector)) + + labelValues, err := c.LabelValues("foo") + require.NoError(t, err) + require.Equal(t, model.LabelValues{"bar"}, labelValues) + + labelNames, err := c.LabelNames() + require.NoError(t, err) + require.Equal(t, []string{"foo"}, labelNames) } func TestGettingStartedSingleProcessConfigWithBlocksStorage(t *testing.T) { @@ -82,7 +91,7 @@ func TestGettingStartedSingleProcessConfigWithBlocksStorage(t *testing.T) { // Push some series to Cortex. now := time.Now() - series, expectedVector := generateSeries("series_1", now) + series, expectedVector := generateSeries("series_1", now, prompb.Label{Name: "foo", Value: "bar"}) res, err := c.Push(series) require.NoError(t, err) @@ -93,4 +102,12 @@ func TestGettingStartedSingleProcessConfigWithBlocksStorage(t *testing.T) { require.NoError(t, err) require.Equal(t, model.ValVector, result.Type()) assert.Equal(t, expectedVector, result.(model.Vector)) + + labelValues, err := c.LabelValues("foo") + require.NoError(t, err) + require.Equal(t, model.LabelValues{"bar"}, labelValues) + + labelNames, err := c.LabelNames() + require.NoError(t, err) + require.Equal(t, []string{"foo"}, labelNames) } From 1fad54c2100c57d68eba1667cee0f98386d47ff8 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 9 Apr 2020 16:08:25 -0400 Subject: [PATCH 06/15] Added a warning for bad single process config Signed-off-by: Joe Elliott --- pkg/cortex/modules.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index d4ce67ef49a..6d7b4739e07 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -207,6 +207,10 @@ func (t *Cortex) initQuerier(cfg *Config) (serv services.Service, err error) { return } + if worker == nil && cfg.Target == All { + level.Error(util.Logger).Log("msg", "Worker is nil in single binary mode. This probably means that the query api will be unresponsive. Please configure the worker.") + } + return worker, nil } From 292a97c05ae31661524e9ab79fa3e297f1bc781e Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 9 Apr 2020 16:23:46 -0400 Subject: [PATCH 07/15] Added auto config of worker Signed-off-by: Joe Elliott --- pkg/cortex/modules.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index 6d7b4739e07..3e69af65138 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -207,8 +207,16 @@ func (t *Cortex) initQuerier(cfg *Config) (serv services.Service, err error) { return } + // single binary will be mysteriously unresponsive unless worker is working. warn and configure here if worker == nil && cfg.Target == All { - level.Error(util.Logger).Log("msg", "Worker is nil in single binary mode. This probably means that the query api will be unresponsive. Please configure the worker.") + address := fmt.Sprintf(":%d", cfg.Server.GRPCListenPort) + level.Warn(util.Logger).Log("msg", "Worker is nil in single binary mode. Attempting automatic worker configuration. If queries are unresponsive consider configuring the worker explicitly.", "address", address) + cfg.Worker.Address = address + + worker, err = frontend.NewWorker(cfg.Worker, httpgrpc_server.NewServer(handler), util.Logger) + if err != nil { + return + } } return worker, nil From 1034ff512dac3b8e58552abd20315ddd4179f763 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 9 Apr 2020 16:35:06 -0400 Subject: [PATCH 08/15] Always add auth middleware. Signed-off-by: Joe Elliott --- pkg/api/api.go | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index 85c1ef84e79..8fc42977e15 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -270,10 +270,8 @@ func (a *API) RegisterQuerier(queryable storage.Queryable, engine *promql.Engine // these routes are either registered the default server OR to an internal mux. The internal mux is // for use in a single binary mode when both the query frontend and the querier would attempt to claim these routes - auth := false router := mux.NewRouter() if registerRoutesExternally { - auth = true router = a.server.HTTP } @@ -281,25 +279,25 @@ func (a *API) RegisterQuerier(queryable storage.Queryable, engine *promql.Engine api.Register(promRouter) promHandler := fakeRemoteAddr(promRouter) - a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/read", querier.RemoteReadHandler(queryable), auth, "GET") - a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/query", promHandler, auth, "GET", "POST") - a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/query_range", promHandler, auth, "GET", "POST") - a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/labels", promHandler, auth, "GET", "POST") - a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/label/{name}/values", promHandler, auth, "GET") - a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/series", promHandler, auth, "GET", "POST", "DELETE") - a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/metadata", promHandler, auth, "GET") + a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/read", querier.RemoteReadHandler(queryable), true, "GET") + a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/query", promHandler, true, "GET", "POST") + a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/query_range", promHandler, true, "GET", "POST") + a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/labels", promHandler, true, "GET", "POST") + a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/label/{name}/values", promHandler, true, "GET") + a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/series", promHandler, true, "GET", "POST", "DELETE") + a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/metadata", promHandler, true, "GET") legacyPromRouter := route.New().WithPrefix(a.cfg.ServerPrefix + a.cfg.LegacyHTTPPrefix + "/api/v1") api.Register(legacyPromRouter) legacyPromHandler := fakeRemoteAddr(legacyPromRouter) - a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/read", querier.RemoteReadHandler(queryable), auth, "GET") - a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/query", legacyPromHandler, auth, "GET", "POST") - a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/query_range", legacyPromHandler, auth, "GET", "POST") - a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/labels", legacyPromHandler, auth, "GET", "POST") - a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/label/{name}/values", legacyPromHandler, auth, "GET") - a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/series", legacyPromHandler, auth, "GET", "POST", "DELETE") - a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/metadata", legacyPromHandler, auth, "GET") + a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/read", querier.RemoteReadHandler(queryable), true, "GET") + a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/query", legacyPromHandler, true, "GET", "POST") + a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/query_range", legacyPromHandler, true, "GET", "POST") + a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/labels", legacyPromHandler, true, "GET", "POST") + a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/label/{name}/values", legacyPromHandler, true, "GET") + a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/series", legacyPromHandler, true, "GET", "POST", "DELETE") + a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/metadata", legacyPromHandler, true, "GET") return router } From be758dcae01e941c0a96ad89802f9997a4d6dbdb Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 9 Apr 2020 16:44:04 -0400 Subject: [PATCH 09/15] Fixed label tests and used address that worked in e2e tests Signed-off-by: Joe Elliott --- integration/e2e/util.go | 3 +++ integration/getting_started_single_process_config_test.go | 4 ++-- pkg/cortex/modules.go | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/integration/e2e/util.go b/integration/e2e/util.go index bf746130d54..83d65e87ff5 100644 --- a/integration/e2e/util.go +++ b/integration/e2e/util.go @@ -90,6 +90,9 @@ func GenerateSeries(name string, ts time.Time, additionalLabels ...prompb.Label) // Generate the expected vector when querying it metric := model.Metric{} metric[labels.MetricName] = model.LabelValue(name) + for _, lbl := range additionalLabels { + metric[model.LabelName(lbl.Name)] = model.LabelValue(lbl.Value) + } vector = append(vector, &model.Sample{ Metric: metric, diff --git a/integration/getting_started_single_process_config_test.go b/integration/getting_started_single_process_config_test.go index 833097dfb3b..c5cd709178d 100644 --- a/integration/getting_started_single_process_config_test.go +++ b/integration/getting_started_single_process_config_test.go @@ -57,7 +57,7 @@ func TestGettingStartedSingleProcessConfigWithChunksStorage(t *testing.T) { labelNames, err := c.LabelNames() require.NoError(t, err) - require.Equal(t, []string{"foo"}, labelNames) + require.Equal(t, []string{"__name__", "foo"}, labelNames) } func TestGettingStartedSingleProcessConfigWithBlocksStorage(t *testing.T) { @@ -109,5 +109,5 @@ func TestGettingStartedSingleProcessConfigWithBlocksStorage(t *testing.T) { labelNames, err := c.LabelNames() require.NoError(t, err) - require.Equal(t, []string{"foo"}, labelNames) + require.Equal(t, []string{"__name__", "foo"}, labelNames) } diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index 3e69af65138..91ec189631a 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -209,7 +209,7 @@ func (t *Cortex) initQuerier(cfg *Config) (serv services.Service, err error) { // single binary will be mysteriously unresponsive unless worker is working. warn and configure here if worker == nil && cfg.Target == All { - address := fmt.Sprintf(":%d", cfg.Server.GRPCListenPort) + address := fmt.Sprintf("127.0.0.1:%d", cfg.Server.GRPCListenPort) level.Warn(util.Logger).Log("msg", "Worker is nil in single binary mode. Attempting automatic worker configuration. If queries are unresponsive consider configuring the worker explicitly.", "address", address) cfg.Worker.Address = address From 9747d43db723f44d70d5f79c6c5ca3c2594c76f5 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 9 Apr 2020 16:47:29 -0400 Subject: [PATCH 10/15] updated changelog Signed-off-by: Joe Elliott --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a19a313d1a..57909e54e35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ * [BUGFIX] Cassandra Storage: Fix endpoint TLS host verification. #2109 * [BUGFIX] Experimental TSDB: fixed response status code from `422` to `500` when an error occurs while iterating chunks with the experimental blocks storage. #2402 * [BUGFIX] Ring: Fixed a situation where upgrading from pre-1.0 cortex with a rolling strategy caused new 1.0 ingesters to lose their zone value in the ring until manually forced to re-register. #2404 +* [CHANGE] Single Binary: Added query-frontend. #2437 ## 1.0.0 / 2020-04-02 From de982b9b15324467d78088e5ac0756f37ab791f4 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Fri, 10 Apr 2020 13:04:35 +0200 Subject: [PATCH 11/15] Run unbounded queries to fetch label names/values in the blocks storage Signed-off-by: Marco Pracucci --- pkg/ingester/ingester_v2.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pkg/ingester/ingester_v2.go b/pkg/ingester/ingester_v2.go index aa622ac34a4..f3ea5dc68f4 100644 --- a/pkg/ingester/ingester_v2.go +++ b/pkg/ingester/ingester_v2.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "math" "net/http" "os" "path/filepath" @@ -431,9 +432,9 @@ func (i *Ingester) v2LabelValues(ctx context.Context, req *client.LabelValuesReq return &client.LabelValuesResponse{}, nil } - through := time.Now() - from := through.Add(-i.cfg.TSDBConfig.Retention) - q, err := db.Querier(from.Unix()*1000, through.Unix()*1000) + // Since we ingester runs with a very limited TSDB retention, we can (and should) query + // label values without any time range bound. + q, err := db.Querier(0, math.MaxInt64) if err != nil { return nil, err } @@ -460,9 +461,9 @@ func (i *Ingester) v2LabelNames(ctx context.Context, req *client.LabelNamesReque return &client.LabelNamesResponse{}, nil } - through := time.Now() - from := through.Add(-i.cfg.TSDBConfig.Retention) - q, err := db.Querier(from.Unix()*1000, through.Unix()*1000) + // Since we ingester runs with a very limited TSDB retention, we can (and should) query + // label names without any time range bound. + q, err := db.Querier(0, math.MaxInt64) if err != nil { return nil, err } From 69751387d6726b2ac28da8433d39ac4709290e36 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 10 Apr 2020 08:32:02 -0400 Subject: [PATCH 12/15] Moved automatic worker config attempt above worker Signed-off-by: Joe Elliott --- pkg/cortex/modules.go | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index 91ec189631a..2a0d8603583 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -200,6 +200,14 @@ func (t *Cortex) initQuerier(cfg *Config) (serv services.Service, err error) { registerExternally := cfg.Target != All handler := t.api.RegisterQuerier(queryable, engine, t.distributor, registerExternally) + // single binary mode requires a properly configured worker. if the operator did not attempt to configure the + // worker we will attempt an automatic configuration here + if cfg.Worker.Address == "" && cfg.Target == All { + address := fmt.Sprintf("127.0.0.1:%d", cfg.Server.GRPCListenPort) + level.Warn(util.Logger).Log("msg", "Worker address is empty in single binary mode. Attempting automatic worker configuration. If queries are unresponsive consider configuring the worker explicitly.", "address", address) + cfg.Worker.Address = address + } + // Query frontend worker will only be started after all its dependencies are started, not here. // Worker may also be nil, if not configured, which is OK. worker, err := frontend.NewWorker(cfg.Worker, httpgrpc_server.NewServer(handler), util.Logger) @@ -207,18 +215,6 @@ func (t *Cortex) initQuerier(cfg *Config) (serv services.Service, err error) { return } - // single binary will be mysteriously unresponsive unless worker is working. warn and configure here - if worker == nil && cfg.Target == All { - address := fmt.Sprintf("127.0.0.1:%d", cfg.Server.GRPCListenPort) - level.Warn(util.Logger).Log("msg", "Worker is nil in single binary mode. Attempting automatic worker configuration. If queries are unresponsive consider configuring the worker explicitly.", "address", address) - cfg.Worker.Address = address - - worker, err = frontend.NewWorker(cfg.Worker, httpgrpc_server.NewServer(handler), util.Logger) - if err != nil { - return - } - } - return worker, nil } From 17427491a0c6bdbb96f3930b81b099446091c154 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 10 Apr 2020 08:49:51 -0400 Subject: [PATCH 13/15] Moved changelog entry and expanded Signed-off-by: Joe Elliott --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57909e54e35..8e8732892fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ * `cortex_querier_bucket_store_blocks_meta_sync_consistency_delay_seconds` > `cortex_querier_blocks_meta_sync_consistency_delay_seconds` * [CHANGE] Experimental TSDB: Modified default values for `compactor.deletion-delay` option from 48h to 12h and `-experimental.tsdb.bucket-store.ignore-deletion-marks-delay` from 24h to 6h. #2414 * [CHANGE] Experimental WAL: Default value of `-ingester.checkpoint-enabled` changed to `true`. #2416 +* [CHANGE] Single Binary: Added query-frontend to the single binary. Single binary users will now benefit from various query-frontend features. Primarily: sharding, parallelization, load shedding, additional caching (if configured), and query retries. #2437 * [FEATURE] Ruler: The `-ruler.evaluation-delay` flag was added to allow users to configure a default evaluation delay for all rules in cortex. The default value is 0 which is the current behavior. #2423 * [ENHANCEMENT] Experimental TSDB: sample ingestion errors are now reported via existing `cortex_discarded_samples_total` metric. #2370 * [ENHANCEMENT] Failures on samples at distributors and ingesters return the first validation error as opposed to the last. #2383 @@ -26,7 +27,6 @@ * [BUGFIX] Cassandra Storage: Fix endpoint TLS host verification. #2109 * [BUGFIX] Experimental TSDB: fixed response status code from `422` to `500` when an error occurs while iterating chunks with the experimental blocks storage. #2402 * [BUGFIX] Ring: Fixed a situation where upgrading from pre-1.0 cortex with a rolling strategy caused new 1.0 ingesters to lose their zone value in the ring until manually forced to re-register. #2404 -* [CHANGE] Single Binary: Added query-frontend. #2437 ## 1.0.0 / 2020-04-02 From 8a394886eac2353ae025a541b0a6e21b859e94b8 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Tue, 14 Apr 2020 08:19:13 -0400 Subject: [PATCH 14/15] Change => Enhancement Signed-off-by: Joe Elliott --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e8732892fa..f0584d6b7c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,13 +15,13 @@ * `cortex_querier_bucket_store_blocks_meta_sync_consistency_delay_seconds` > `cortex_querier_blocks_meta_sync_consistency_delay_seconds` * [CHANGE] Experimental TSDB: Modified default values for `compactor.deletion-delay` option from 48h to 12h and `-experimental.tsdb.bucket-store.ignore-deletion-marks-delay` from 24h to 6h. #2414 * [CHANGE] Experimental WAL: Default value of `-ingester.checkpoint-enabled` changed to `true`. #2416 -* [CHANGE] Single Binary: Added query-frontend to the single binary. Single binary users will now benefit from various query-frontend features. Primarily: sharding, parallelization, load shedding, additional caching (if configured), and query retries. #2437 * [FEATURE] Ruler: The `-ruler.evaluation-delay` flag was added to allow users to configure a default evaluation delay for all rules in cortex. The default value is 0 which is the current behavior. #2423 * [ENHANCEMENT] Experimental TSDB: sample ingestion errors are now reported via existing `cortex_discarded_samples_total` metric. #2370 * [ENHANCEMENT] Failures on samples at distributors and ingesters return the first validation error as opposed to the last. #2383 * [ENHANCEMENT] Experimental TSDB: Added `cortex_querier_blocks_meta_synced`, which reflects current state of synced blocks over all tenants. #2392 * [ENHANCEMENT] Added `cortex_distributor_latest_seen_sample_timestamp_seconds` metric to see how far behind Prometheus servers are in sending data. #2371 * [ENHANCEMENT] FIFO cache to support eviction based on memory usage. The `-.fifocache.size` CLI flag has been renamed to `-.fifocache.max-size-items` as well as its YAML config option `size` renamed to `max_size_items`. Added `-.fifocache.max-size-bytes` CLI flag and YAML config option `max_size_bytes` to specify memory limit of the cache. #2319 +* [ENHANCEMENT] Single Binary: Added query-frontend to the single binary. Single binary users will now benefit from various query-frontend features. Primarily: sharding, parallelization, load shedding, additional caching (if configured), and query retries. #2437 * [BUGFIX] Fixes #2411, Ensure requests are properly routed to the prometheus api embedded in the query if `-server.path-prefix` is set. #2372 * [BUGFIX] Experimental TSDB: fixed chunk data corruption when querying back series using the experimental blocks storage. #2400 * [BUGFIX] Cassandra Storage: Fix endpoint TLS host verification. #2109 From 6aac5360115bc74e28741c0817c20ef43123775a Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Tue, 14 Apr 2020 11:25:47 -0400 Subject: [PATCH 15/15] Added todo Signed-off-by: Joe Elliott --- pkg/api/api.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/api/api.go b/pkg/api/api.go index 1f59e0b4449..ddcd035f86f 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -270,6 +270,7 @@ func (a *API) RegisterQuerier(queryable storage.Queryable, engine *promql.Engine // these routes are either registered the default server OR to an internal mux. The internal mux is // for use in a single binary mode when both the query frontend and the querier would attempt to claim these routes + // TODO: Add support to expose querier paths with a configurable prefix in single binary mode. router := mux.NewRouter() if registerRoutesExternally { router = a.server.HTTP