From 08c5d55f9c14fea1ee33bd14572cc18cb2bfdbb1 Mon Sep 17 00:00:00 2001 From: Dhruvan Tanna Date: Thu, 1 Aug 2024 21:17:33 +0530 Subject: [PATCH] Adding feature to export metrics for specific index --- README.md | 1 + collector/indices.go | 21 +++++++++++++++------ collector/indices_mappings.go | 22 ++++++++++++++-------- collector/indices_mappings_test.go | 2 +- collector/indices_settings.go | 14 +++++++++----- collector/indices_settings_test.go | 2 +- collector/indices_test.go | 4 ++-- main.go | 11 ++++++++--- 8 files changed, 51 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index d5304f5c..f161c8fb 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ elasticsearch_exporter --help | es.aliases | 1.0.4rc1 | If true, include informational aliases metrics. | true | | es.shards | 1.0.3rc1 | If true, query stats for all indices in the cluster, including shard-level stats (implies `es.indices=true`). | false | | es.snapshots | 1.0.4rc1 | If true, query stats for the cluster snapshots. | false | +| es.filter_index | | If set, filter index to export metrics for [settings, stats, mappings] based on the provided pattern. `example: index_prefix_1*,index2` | _all | | es.slm | | If true, query stats for SLM. | false | | es.timeout | 1.0.2 | Timeout for trying to get stats from Elasticsearch. (ex: 20s) | 5s | | es.ca | 1.0.2 | Path to PEM file that contains trusted Certificate Authorities for the Elasticsearch connection. | | diff --git a/collector/indices.go b/collector/indices.go index 85413125..865d4e03 100644 --- a/collector/indices.go +++ b/collector/indices.go @@ -11,21 +11,24 @@ // See the License for the specific language governing permissions and // limitations under the License. +// changes of exporting metrics for specific Index are inspired by https://github.com/prometheus-community/elasticsearch_exporter/pull/764 + package collector import ( "encoding/json" "fmt" - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/prometheus-community/elasticsearch_exporter/pkg/clusterinfo" - "github.com/prometheus/client_golang/prometheus" "io/ioutil" "net/http" "net/url" "path" "sort" "strconv" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/prometheus-community/elasticsearch_exporter/pkg/clusterinfo" + "github.com/prometheus/client_golang/prometheus" ) type labels struct { @@ -59,6 +62,7 @@ type Indices struct { logger log.Logger client *http.Client url *url.URL + filterIndex string shards bool aliases bool clusterInfoCh chan *clusterinfo.Response @@ -74,7 +78,7 @@ type Indices struct { } // NewIndices defines Indices Prometheus metrics -func NewIndices(logger log.Logger, client *http.Client, url *url.URL, shards bool, includeAliases bool) *Indices { +func NewIndices(logger log.Logger, client *http.Client, url *url.URL, shards bool, includeAliases bool, filterIndex string) *Indices { indexLabels := labels{ keys: func(...string) []string { @@ -122,6 +126,7 @@ func NewIndices(logger log.Logger, client *http.Client, url *url.URL, shards boo logger: logger, client: client, url: url, + filterIndex: filterIndex, shards: shards, aliases: includeAliases, clusterInfoCh: make(chan *clusterinfo.Response), @@ -1102,7 +1107,11 @@ func (i *Indices) fetchAndDecodeIndexStats() (indexStatsResponse, error) { var isr indexStatsResponse u := *i.url - u.Path = path.Join(u.Path, "/_all/_stats") + indexStatsRequest := fmt.Sprintf("/%s/_stats", i.filterIndex) + _ = level.Debug(i.logger).Log( + "msg", fmt.Sprintf("Query path for the index: %s", indexStatsRequest), + ) + u.Path = path.Join(u.Path, indexStatsRequest) if i.shards { u.RawQuery = "ignore_unavailable=true&level=shards" } else { diff --git a/collector/indices_mappings.go b/collector/indices_mappings.go index 689d12da..3ba7f453 100644 --- a/collector/indices_mappings.go +++ b/collector/indices_mappings.go @@ -11,6 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// changes of exporting metrics for specific Index are inspired by https://github.com/prometheus-community/elasticsearch_exporter/pull/764 + package collector import ( @@ -38,9 +40,10 @@ type indicesMappingsMetric struct { // IndicesMappings information struct type IndicesMappings struct { - logger log.Logger - client *http.Client - url *url.URL + logger log.Logger + client *http.Client + url *url.URL + filterIndex string up prometheus.Gauge totalScrapes, jsonParseFailures prometheus.Counter @@ -49,13 +52,14 @@ type IndicesMappings struct { } // NewIndicesMappings defines Indices IndexMappings Prometheus metrics -func NewIndicesMappings(logger log.Logger, client *http.Client, url *url.URL) *IndicesMappings { +func NewIndicesMappings(logger log.Logger, client *http.Client, url *url.URL, filterIndex string) *IndicesMappings { subsystem := "indices_mappings_stats" return &IndicesMappings{ - logger: logger, - client: client, - url: url, + logger: logger, + client: client, + url: url, + filterIndex: filterIndex, up: prometheus.NewGauge(prometheus.GaugeOpts{ Name: prometheus.BuildFQName(namespace, subsystem, "up"), @@ -155,7 +159,9 @@ func (im *IndicesMappings) getAndParseURL(u *url.URL) (*IndicesMappingsResponse, func (im *IndicesMappings) fetchAndDecodeIndicesMappings() (*IndicesMappingsResponse, error) { u := *im.url - u.Path = path.Join(u.Path, "/_all/_mappings") + // Inspired by https://github.com/prometheus-community/elasticsearch_exporter/pull/764 + indexMappingRequest := fmt.Sprintf("/%s/_mappings", im.filterIndex) + u.Path = path.Join(u.Path, indexMappingRequest) return im.getAndParseURL(&u) } diff --git a/collector/indices_mappings_test.go b/collector/indices_mappings_test.go index 87cb3ef5..9853879d 100644 --- a/collector/indices_mappings_test.go +++ b/collector/indices_mappings_test.go @@ -124,7 +124,7 @@ func TestMapping(t *testing.T) { if err != nil { t.Fatalf("Failed to parse URL: %s", err) } - c := NewIndicesMappings(log.NewNopLogger(), http.DefaultClient, u) + c := NewIndicesMappings(log.NewNopLogger(), http.DefaultClient, u, "_all") imr, err := c.fetchAndDecodeIndicesMappings() if err != nil { t.Fatalf("Failed to fetch or decode indices mappings: %s", err) diff --git a/collector/indices_settings.go b/collector/indices_settings.go index 5b6b55a1..2e31c016 100644 --- a/collector/indices_settings.go +++ b/collector/indices_settings.go @@ -11,6 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// changes of exporting metrics for specific Index are inspired by https://github.com/prometheus-community/elasticsearch_exporter/pull/764 + package collector import ( @@ -29,9 +31,10 @@ import ( // IndicesSettings information struct type IndicesSettings struct { - logger log.Logger - client *http.Client - url *url.URL + logger log.Logger + client *http.Client + url *url.URL + filterIndex string up prometheus.Gauge readOnlyIndices prometheus.Gauge @@ -52,7 +55,7 @@ type indicesSettingsMetric struct { } // NewIndicesSettings defines Indices Settings Prometheus metrics -func NewIndicesSettings(logger log.Logger, client *http.Client, url *url.URL) *IndicesSettings { +func NewIndicesSettings(logger log.Logger, client *http.Client, url *url.URL, filterIndex string) *IndicesSettings { return &IndicesSettings{ logger: logger, client: client, @@ -139,7 +142,8 @@ func (cs *IndicesSettings) getAndParseURL(u *url.URL, data interface{}) error { func (cs *IndicesSettings) fetchAndDecodeIndicesSettings() (IndicesSettingsResponse, error) { u := *cs.url - u.Path = path.Join(u.Path, "/_all/_settings") + indexSettingsRequest := fmt.Sprintf("/%s/_settings", cs.filterIndex) + u.Path = path.Join(u.Path, indexSettingsRequest) var asr IndicesSettingsResponse err := cs.getAndParseURL(&u, &asr) if err != nil { diff --git a/collector/indices_settings_test.go b/collector/indices_settings_test.go index 7a3d0fcd..a16f360f 100644 --- a/collector/indices_settings_test.go +++ b/collector/indices_settings_test.go @@ -70,7 +70,7 @@ func TestIndicesSettings(t *testing.T) { if err != nil { t.Fatalf("Failed to parse URL: %s", err) } - c := NewIndicesSettings(log.NewNopLogger(), http.DefaultClient, u) + c := NewIndicesSettings(log.NewNopLogger(), http.DefaultClient, u, "_all") nsr, err := c.fetchAndDecodeIndicesSettings() if err != nil { t.Fatalf("Failed to fetch or decode indices settings: %s", err) diff --git a/collector/indices_test.go b/collector/indices_test.go index 1d164b65..d325d645 100644 --- a/collector/indices_test.go +++ b/collector/indices_test.go @@ -49,7 +49,7 @@ func TestIndices(t *testing.T) { if err != nil { t.Fatalf("Failed to parse URL: %s", err) } - i := NewIndices(log.NewNopLogger(), http.DefaultClient, u, false, false) + i := NewIndices(log.NewNopLogger(), http.DefaultClient, u, false, false, "_all") stats, err := i.fetchAndDecodeIndexStats() if err != nil { t.Fatalf("Failed to fetch or decode indices stats: %s", err) @@ -123,7 +123,7 @@ func TestAliases(t *testing.T) { if err != nil { t.Fatalf("Failed to parse URL: %s", err) } - i := NewIndices(log.NewNopLogger(), http.DefaultClient, u, false, true) + i := NewIndices(log.NewNopLogger(), http.DefaultClient, u, false, true, "_all") stats, err := i.fetchAndDecodeIndexStats() if err != nil { t.Fatalf("Failed to fetch or decode indices stats: %s", err) diff --git a/main.go b/main.go index 66cacd31..146d320f 100644 --- a/main.go +++ b/main.go @@ -11,6 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// changes of exporting metrics for specific Index are inspired by https://github.com/prometheus-community/elasticsearch_exporter/pull/764 + package main import ( @@ -65,6 +67,9 @@ func main() { esNode = kingpin.Flag("es.node", "Node's name of which metrics should be exposed."). Default("_local").String() + esFilterIndex = kingpin.Flag("es.filter_index", + "Filter index to export the metrics for"). + Default("_all").String() esExportIndices = kingpin.Flag("es.indices", "Export stats for indices in the cluster."). Default("false").Bool() @@ -186,7 +191,7 @@ func main() { if *esExportIndices || *esExportShards { prometheus.MustRegister(collector.NewShards(logger, httpClient, esURL)) - iC := collector.NewIndices(logger, httpClient, esURL, *esExportShards, *esExportIndexAliases) + iC := collector.NewIndices(logger, httpClient, esURL, *esExportShards, *esExportIndexAliases, *esFilterIndex) prometheus.MustRegister(iC) if registerErr := clusterInfoRetriever.RegisterConsumer(iC); registerErr != nil { _ = level.Error(logger).Log("msg", "failed to register indices collector in cluster info") @@ -207,11 +212,11 @@ func main() { } if *esExportIndicesSettings { - prometheus.MustRegister(collector.NewIndicesSettings(logger, httpClient, esURL)) + prometheus.MustRegister(collector.NewIndicesSettings(logger, httpClient, esURL, *esFilterIndex)) } if *esExportIndicesMappings { - prometheus.MustRegister(collector.NewIndicesMappings(logger, httpClient, esURL)) + prometheus.MustRegister(collector.NewIndicesMappings(logger, httpClient, esURL, *esFilterIndex)) } // create a http server