Skip to content

Adding feature to export metrics for specific index #97

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: build
Choose a base branch
from
Open

Adding feature to export metrics for specific index #97

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ elasticsearch_exporter --help
| es.aliases | 1.0.4rc1 | If true, include informational aliases metrics. | true |
| es.shards | 1.0.3rc1 | If true, query stats for all indices in the cluster, including shard-level stats (implies `es.indices=true`). | false |
| es.snapshots | 1.0.4rc1 | If true, query stats for the cluster snapshots. | false |
| es.filter_index | | If set, filter index to export metrics for [settings, stats, mappings] based on the provided pattern. `example: index_prefix_1*,index2` | _all |
| es.slm | | If true, query stats for SLM. | false |
| es.timeout | 1.0.2 | Timeout for trying to get stats from Elasticsearch. (ex: 20s) | 5s |
| es.ca | 1.0.2 | Path to PEM file that contains trusted Certificate Authorities for the Elasticsearch connection. | |
Expand Down
21 changes: 15 additions & 6 deletions collector/indices.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,21 +11,24 @@
// See the License for the specific language governing permissions and
// limitations under the License.

// changes of exporting metrics for specific Index are inspired by https://github.com/prometheus-community/elasticsearch_exporter/pull/764

package collector

import (
"encoding/json"
"fmt"
"github.com/go-kit/log"
"github.com/go-kit/log/level"
"github.com/prometheus-community/elasticsearch_exporter/pkg/clusterinfo"
"github.com/prometheus/client_golang/prometheus"
"io/ioutil"
"net/http"
"net/url"
"path"
"sort"
"strconv"

"github.com/go-kit/log"
"github.com/go-kit/log/level"
"github.com/prometheus-community/elasticsearch_exporter/pkg/clusterinfo"
"github.com/prometheus/client_golang/prometheus"
)

type labels struct {
Expand Down Expand Up @@ -59,6 +62,7 @@ type Indices struct {
logger log.Logger
client *http.Client
url *url.URL
filterIndex string
shards bool
aliases bool
clusterInfoCh chan *clusterinfo.Response
Expand All @@ -74,7 +78,7 @@ type Indices struct {
}

// NewIndices defines Indices Prometheus metrics
func NewIndices(logger log.Logger, client *http.Client, url *url.URL, shards bool, includeAliases bool) *Indices {
func NewIndices(logger log.Logger, client *http.Client, url *url.URL, shards bool, includeAliases bool, filterIndex string) *Indices {

indexLabels := labels{
keys: func(...string) []string {
Expand Down Expand Up @@ -122,6 +126,7 @@ func NewIndices(logger log.Logger, client *http.Client, url *url.URL, shards boo
logger: logger,
client: client,
url: url,
filterIndex: filterIndex,
shards: shards,
aliases: includeAliases,
clusterInfoCh: make(chan *clusterinfo.Response),
Expand Down Expand Up @@ -1102,7 +1107,11 @@ func (i *Indices) fetchAndDecodeIndexStats() (indexStatsResponse, error) {
var isr indexStatsResponse

u := *i.url
u.Path = path.Join(u.Path, "/_all/_stats")
indexStatsRequest := fmt.Sprintf("/%s/_stats", i.filterIndex)
_ = level.Debug(i.logger).Log(
"msg", fmt.Sprintf("Query path for the index: %s", indexStatsRequest),
)
u.Path = path.Join(u.Path, indexStatsRequest)
if i.shards {
u.RawQuery = "ignore_unavailable=true&level=shards"
} else {
Expand Down
22 changes: 14 additions & 8 deletions collector/indices_mappings.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.

// changes of exporting metrics for specific Index are inspired by https://github.com/prometheus-community/elasticsearch_exporter/pull/764

package collector

import (
Expand Down Expand Up @@ -38,9 +40,10 @@ type indicesMappingsMetric struct {

// IndicesMappings information struct
type IndicesMappings struct {
logger log.Logger
client *http.Client
url *url.URL
logger log.Logger
client *http.Client
url *url.URL
filterIndex string

up prometheus.Gauge
totalScrapes, jsonParseFailures prometheus.Counter
Expand All @@ -49,13 +52,14 @@ type IndicesMappings struct {
}

// NewIndicesMappings defines Indices IndexMappings Prometheus metrics
func NewIndicesMappings(logger log.Logger, client *http.Client, url *url.URL) *IndicesMappings {
func NewIndicesMappings(logger log.Logger, client *http.Client, url *url.URL, filterIndex string) *IndicesMappings {
subsystem := "indices_mappings_stats"

return &IndicesMappings{
logger: logger,
client: client,
url: url,
logger: logger,
client: client,
url: url,
filterIndex: filterIndex,

up: prometheus.NewGauge(prometheus.GaugeOpts{
Name: prometheus.BuildFQName(namespace, subsystem, "up"),
Expand Down Expand Up @@ -155,7 +159,9 @@ func (im *IndicesMappings) getAndParseURL(u *url.URL) (*IndicesMappingsResponse,

func (im *IndicesMappings) fetchAndDecodeIndicesMappings() (*IndicesMappingsResponse, error) {
u := *im.url
u.Path = path.Join(u.Path, "/_all/_mappings")
// Inspired by https://github.com/prometheus-community/elasticsearch_exporter/pull/764
indexMappingRequest := fmt.Sprintf("/%s/_mappings", im.filterIndex)
u.Path = path.Join(u.Path, indexMappingRequest)
return im.getAndParseURL(&u)
}

Expand Down
2 changes: 1 addition & 1 deletion collector/indices_mappings_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ func TestMapping(t *testing.T) {
if err != nil {
t.Fatalf("Failed to parse URL: %s", err)
}
c := NewIndicesMappings(log.NewNopLogger(), http.DefaultClient, u)
c := NewIndicesMappings(log.NewNopLogger(), http.DefaultClient, u, "_all")
imr, err := c.fetchAndDecodeIndicesMappings()
if err != nil {
t.Fatalf("Failed to fetch or decode indices mappings: %s", err)
Expand Down
14 changes: 9 additions & 5 deletions collector/indices_settings.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.

// changes of exporting metrics for specific Index are inspired by https://github.com/prometheus-community/elasticsearch_exporter/pull/764

package collector

import (
Expand All @@ -29,9 +31,10 @@ import (

// IndicesSettings information struct
type IndicesSettings struct {
logger log.Logger
client *http.Client
url *url.URL
logger log.Logger
client *http.Client
url *url.URL
filterIndex string

up prometheus.Gauge
readOnlyIndices prometheus.Gauge
Expand All @@ -52,7 +55,7 @@ type indicesSettingsMetric struct {
}

// NewIndicesSettings defines Indices Settings Prometheus metrics
func NewIndicesSettings(logger log.Logger, client *http.Client, url *url.URL) *IndicesSettings {
func NewIndicesSettings(logger log.Logger, client *http.Client, url *url.URL, filterIndex string) *IndicesSettings {
return &IndicesSettings{
logger: logger,
client: client,
Expand Down Expand Up @@ -139,7 +142,8 @@ func (cs *IndicesSettings) getAndParseURL(u *url.URL, data interface{}) error {
func (cs *IndicesSettings) fetchAndDecodeIndicesSettings() (IndicesSettingsResponse, error) {

u := *cs.url
u.Path = path.Join(u.Path, "/_all/_settings")
indexSettingsRequest := fmt.Sprintf("/%s/_settings", cs.filterIndex)
u.Path = path.Join(u.Path, indexSettingsRequest)
var asr IndicesSettingsResponse
err := cs.getAndParseURL(&u, &asr)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion collector/indices_settings_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ func TestIndicesSettings(t *testing.T) {
if err != nil {
t.Fatalf("Failed to parse URL: %s", err)
}
c := NewIndicesSettings(log.NewNopLogger(), http.DefaultClient, u)
c := NewIndicesSettings(log.NewNopLogger(), http.DefaultClient, u, "_all")
nsr, err := c.fetchAndDecodeIndicesSettings()
if err != nil {
t.Fatalf("Failed to fetch or decode indices settings: %s", err)
Expand Down
4 changes: 2 additions & 2 deletions collector/indices_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func TestIndices(t *testing.T) {
if err != nil {
t.Fatalf("Failed to parse URL: %s", err)
}
i := NewIndices(log.NewNopLogger(), http.DefaultClient, u, false, false)
i := NewIndices(log.NewNopLogger(), http.DefaultClient, u, false, false, "_all")
stats, err := i.fetchAndDecodeIndexStats()
if err != nil {
t.Fatalf("Failed to fetch or decode indices stats: %s", err)
Expand Down Expand Up @@ -123,7 +123,7 @@ func TestAliases(t *testing.T) {
if err != nil {
t.Fatalf("Failed to parse URL: %s", err)
}
i := NewIndices(log.NewNopLogger(), http.DefaultClient, u, false, true)
i := NewIndices(log.NewNopLogger(), http.DefaultClient, u, false, true, "_all")
stats, err := i.fetchAndDecodeIndexStats()
if err != nil {
t.Fatalf("Failed to fetch or decode indices stats: %s", err)
Expand Down
11 changes: 8 additions & 3 deletions main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.

// changes of exporting metrics for specific Index are inspired by https://github.com/prometheus-community/elasticsearch_exporter/pull/764

package main

import (
Expand Down Expand Up @@ -65,6 +67,9 @@ func main() {
esNode = kingpin.Flag("es.node",
"Node's name of which metrics should be exposed.").
Default("_local").String()
esFilterIndex = kingpin.Flag("es.filter_index",
"Filter index to export the metrics for").
Default("_all").String()
esExportIndices = kingpin.Flag("es.indices",
"Export stats for indices in the cluster.").
Default("false").Bool()
Expand Down Expand Up @@ -186,7 +191,7 @@ func main() {

if *esExportIndices || *esExportShards {
prometheus.MustRegister(collector.NewShards(logger, httpClient, esURL))
iC := collector.NewIndices(logger, httpClient, esURL, *esExportShards, *esExportIndexAliases)
iC := collector.NewIndices(logger, httpClient, esURL, *esExportShards, *esExportIndexAliases, *esFilterIndex)
prometheus.MustRegister(iC)
if registerErr := clusterInfoRetriever.RegisterConsumer(iC); registerErr != nil {
_ = level.Error(logger).Log("msg", "failed to register indices collector in cluster info")
Expand All @@ -207,11 +212,11 @@ func main() {
}

if *esExportIndicesSettings {
prometheus.MustRegister(collector.NewIndicesSettings(logger, httpClient, esURL))
prometheus.MustRegister(collector.NewIndicesSettings(logger, httpClient, esURL, *esFilterIndex))
}

if *esExportIndicesMappings {
prometheus.MustRegister(collector.NewIndicesMappings(logger, httpClient, esURL))
prometheus.MustRegister(collector.NewIndicesMappings(logger, httpClient, esURL, *esFilterIndex))
}

// create a http server
Expand Down
Loading