Skip to content

Fix metrics to generate from DB and avoid early returns #618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ylembachar merged 8 commits into from
Aug 6, 2025
Merged

Fix metrics to generate from DB and avoid early returns #618

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
9b0d68a
update metrics flow to generate for keypers not part of the keyper set
ylembachar Aug 4, 2025
017883b
fix pre-commit issues
ylembachar Aug 4, 2025
ffa0fa0
set current eon from DB result
ylembachar Aug 4, 2025
2a8da05
populate other metrics from DB
ylembachar Aug 4, 2025
43276a4
fix pre-commit issue
ylembachar Aug 4, 2025
811f19a
remove early return to ensure all metrics populate
ylembachar Aug 5, 2025
b8afb7d
fix pre-commit issues
ylembachar Aug 5, 2025
21049ae
fix gofumpt issues
ylembachar Aug 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
155 changes: 101 additions & 54 deletions rolling-shutter/keyper/keypermetrics/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package keypermetrics
import (
"context"
"strconv"
"strings"

"github.com/jackc/pgx/v4/pgxpool"
"github.com/prometheus/client_golang/prometheus"
Expand Down Expand Up @@ -88,7 +89,7 @@ var MetricsKeyperBatchConfigInfo = prometheus.NewGaugeVec(
},
[]string{"batch_config_index", "keyper_addresses"})

var MetricsKeyperDKGstatus = prometheus.NewGaugeVec(
var MetricsKeyperDKGStatus = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: "shutter",
Subsystem: "keyper",
Expand All @@ -98,6 +99,23 @@ var MetricsKeyperDKGstatus = prometheus.NewGaugeVec(
[]string{"eon"},
)

var MetricsKeyperEthAddress = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: "shutter",
Subsystem: "keyper",
Name: "address",
Help: "Ethereum address of the Keyper",
}, []string{"address"})

var MetricsExecutionClientVersion = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: "shutter",
Subsystem: "keyper",
Name: "execution_client_version",
Help: "Version of the execution client",
},
[]string{"version"})

func InitMetrics(dbpool *pgxpool.Pool, config kprconfig.Config) {
prometheus.MustRegister(MetricsKeyperCurrentBlockL1)
prometheus.MustRegister(MetricsKeyperCurrentBlockShuttermint)
Expand All @@ -107,69 +125,98 @@ func InitMetrics(dbpool *pgxpool.Pool, config kprconfig.Config) {
prometheus.MustRegister(MetricsKeyperCurrentPhase)
prometheus.MustRegister(MetricsKeyperCurrentBatchConfigIndex)
prometheus.MustRegister(MetricsKeyperBatchConfigInfo)
prometheus.MustRegister(MetricsKeyperDKGstatus)
prometheus.MustRegister(MetricsKeyperDKGStatus)
prometheus.MustRegister(MetricsKeyperEthAddress)
prometheus.MustRegister(MetricsExecutionClientVersion)

ctx := context.Background()
queries := database.New(dbpool)
eons, err := queries.GetAllEons(context.Background())
if err != nil {
log.Error().Err(err).Msg("keypermetrics | Failed to get all eons")
return
}
keyperIndex, isKeyper, err := queries.GetKeyperIndex(context.Background(), eons[len(eons)-1].KeyperConfigIndex, config.GetAddress())
if err != nil {
log.Error().Err(err).Msg("keypermetrics | Failed to get keyper index")
return
}
if isKeyper {
MetricsKeyperIsKeyper.WithLabelValues(strconv.FormatInt(keyperIndex, 10)).Set(1)

MetricsKeyperEthAddress.WithLabelValues(config.GetAddress().Hex()).Set(1)

if version, err := chainsync.GetClientVersion(ctx, config.Ethereum.EthereumURL); err != nil {
log.Error().Err(err).Msg("keypermetrics | Failed to get execution client version")
} else {
MetricsKeyperIsKeyper.WithLabelValues(strconv.FormatInt(keyperIndex, 10)).Set(0)
MetricsExecutionClientVersion.WithLabelValues(version).Set(1)
}

dkgResult, err := queries.GetDKGResultForKeyperConfigIndex(context.Background(), eons[len(eons)-1].KeyperConfigIndex)
eons, err := queries.GetAllEons(ctx)
if err != nil {
MetricsKeyperDKGstatus.WithLabelValues(strconv.FormatInt(eons[len(eons)-1].Eon, 10)).Set(0)
log.Error().Err(err).Msg("keypermetrics | Failed to get dkg result")
return
log.Error().Err(err).Msg("keypermetrics | Failed to fetch eons")
} else if len(eons) == 0 {
log.Warn().Msg("keypermetrics | No eons found")
}
if dkgResult.Success {
MetricsKeyperDKGstatus.WithLabelValues(strconv.FormatInt(eons[len(eons)-1].Eon, 10)).Set(1)
} else {
MetricsKeyperDKGstatus.WithLabelValues(strconv.FormatInt(eons[len(eons)-1].Eon, 10)).Set(0)

if len(eons) > 0 {
currentEon := eons[len(eons)-1]

MetricsKeyperCurrentEon.Set(float64(currentEon.Eon))

MetricsKeyperCurrentBatchConfigIndex.Set(float64(currentEon.KeyperConfigIndex))

for _, eon := range eons {
eonStr := strconv.FormatInt(eon.Eon, 10)
MetricsKeyperEonStartBlock.WithLabelValues(eonStr).Set(float64(eon.ActivationBlockNumber))
}

// Populate MetricsKeyperDKGStatus
dkgResults, err := queries.GetAllDKGResults(ctx)
if err != nil {
log.Error().Err(err).Msg("keypermetrics | Failed to fetch DKG results")
} else {
dkgResultMap := make(map[int64]database.DkgResult)
for _, result := range dkgResults {
dkgResultMap[result.Eon] = result
}

// Set DKG status for all eons
for _, eon := range eons {
eonStr := strconv.FormatInt(eon.Eon, 10)

if dkgResult, exists := dkgResultMap[eon.Eon]; exists {
var dkgStatusValue float64
if dkgResult.Success {
dkgStatusValue = 1
}
MetricsKeyperDKGStatus.WithLabelValues(eonStr).Set(dkgStatusValue)
} else {
// No DKG result found for this eon, set to 0
MetricsKeyperDKGStatus.WithLabelValues(eonStr).Set(0)
}
}
}
}

version, err := chainsync.GetClientVersion(context.Background(), config.Ethereum.EthereumURL)
// Populate MetricsKeyperBatchConfigInfo && MetricsKeyperIsKeyper
batchConfigs, err := queries.GetBatchConfigs(ctx)
if err != nil {
log.Error().Err(err).Msg("execution_client_version metrics | Failed to get execution client version")
return
log.Error().Err(err).Msg("keypermetrics | Failed to fetch batch configs")
} else {
currentAddress := config.GetAddress().Hex()

for _, batchConfig := range batchConfigs {
batchConfigIndexStr := strconv.Itoa(int(batchConfig.KeyperConfigIndex))

// Join keyper addresses for the label
keyperAddresses := strings.Join(batchConfig.Keypers, ",")
MetricsKeyperBatchConfigInfo.WithLabelValues(batchConfigIndexStr, keyperAddresses).Set(1)

// Check if current node is a keyper in this batch config
isKeyper := false
for _, keyperAddr := range batchConfig.Keypers {
if strings.EqualFold(keyperAddr, currentAddress) {
isKeyper = true
break
}
}

var isKeyperValue float64
if isKeyper {
isKeyperValue = 1
}
MetricsKeyperIsKeyper.WithLabelValues(batchConfigIndexStr).Set(isKeyperValue)
}
}

executionClientVersion := prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: "shutter",
Subsystem: "keyper",
Name: "execution_client_version",
Help: "Version of the execution client",
ConstLabels: prometheus.Labels{
"version": version,
},
},
)
executionClientVersion.Set(1)

prometheus.MustRegister(executionClientVersion)
metricsKeyperEthAddress := prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: "shutter",
Subsystem: "keyper",
Name: "address",
Help: "Ethereum address of the Keyper",
ConstLabels: prometheus.Labels{
"address": config.GetAddress().Hex(),
},
},
)
metricsKeyperEthAddress.Set(1)

prometheus.MustRegister(metricsKeyperEthAddress)
log.Info().Msg("keypermetrics | Metrics population completed")
}
4 changes: 2 additions & 2 deletions rolling-shutter/keyper/smobserver/smstate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -489,7 +489,7 @@ func (st *ShuttermintState) finalizeDKG(
dkgresultmsg := shmsg.NewDKGResult(eon, err == nil)

if err != nil {
keypermetrics.MetricsKeyperDKGstatus.WithLabelValues(strconv.FormatUint(eon, 10)).Set(0)
keypermetrics.MetricsKeyperDKGStatus.WithLabelValues(strconv.FormatUint(eon, 10)).Set(0)
log.Error().Err(err).Uint64("eon", eon).Bool("success", false).
Msg("DKG process failed")
dkgerror = sql.NullString{String: err.Error(), Valid: true}
Expand All @@ -498,7 +498,7 @@ func (st *ShuttermintState) finalizeDKG(
return err
}
} else {
keypermetrics.MetricsKeyperDKGstatus.WithLabelValues(strconv.FormatUint(eon, 10)).Set(1)
keypermetrics.MetricsKeyperDKGStatus.WithLabelValues(strconv.FormatUint(eon, 10)).Set(1)
log.Info().Uint64("eon", eon).Bool("success", true).Msg("DKG process succeeded")
pureResult, err = shdb.EncodePureDKGResult(&dkgresult)
if err != nil {
Expand Down