Merge pull request SimplyStaking#380 from nivasan1/nikhil/add-peering-alert

Nikhil/add peering alert

Author: itsciccio

alerter/.coverage

@@ -43,3 +43,5 @@ class CosmosNodeAlertCode(AlertCode):
     TendermintRPCDataObtainedAlert = 'cosmos_node_alert_39'
     MetricNotFoundErrorAlert = 'cosmos_node_alert_40'
     MetricFoundAlert = 'cosmos_node_alert_41'
+    NodeIsNotPeeredWithSentinelAlert = 'cosmos_node_alert_42'
+    NodeIsPeeredWithSentinelAlert = 'cosmos_node_alert_43'

alerter/src/alerter/alert_code/node/cosmos_alert_code.py

@@ -46,6 +46,10 @@ def _equal_condition_function(current: Any, previous: Any) -> bool:
     def _is_true_condition_function(current: Any) -> bool:
         return current is True
 
+    @staticmethod
+    def _is_false_condition_function(current: Any) -> bool:
+        return current is False
+
     @staticmethod
     def _true_fn() -> bool:
         return True

alerter/src/alerter/alerters/alerter.py

@@ -422,7 +422,6 @@ def _process_tendermint_rpc_result(self, tendermint_data: Dict,
                 parent_id, node_id, configs, is_validator)
 
             # Check if some errors have been resolved
-
             self.alerting_factory.classify_error_alert(
                 InvalidUrlException.code,
                 cosmos_alerts.TendermintRPCInvalidUrlAlert,
@@ -451,11 +450,16 @@ def _process_tendermint_rpc_result(self, tendermint_data: Dict,
             )
 
             # Check if the alert rules are satisfied for the metrics
-
             is_syncing_configs = (
                 configs.validator_is_syncing if is_validator
                 else configs.node_is_syncing
             )
+
+            is_peered_with_sentinel_configs = (
+                configs.validator_is_peered_with_sentinel if is_validator
+                else configs.node_is_peered_with_sentinel
+            )
+
             classification_fn = (
                 self.alerting_factory
                     .classify_solvable_conditional_alert_no_repetition
@@ -473,6 +477,20 @@ def _process_tendermint_rpc_result(self, tendermint_data: Dict,
                         [node_name, Severity.INFO.value, last_monitored,
                          parent_id, node_id]
                     )
+            ## Only alert if the node is running mev_tendermint
+            if str_to_bool(is_peered_with_sentinel_configs['enabled']) and meta_data['is_mev_tendermint_node']:
+                current = data['is_peered_with_sentinel']['current']
+                if current is not None:
+                    classification_fn(
+                        parent_id, node_id, MetricCode.NodeIsNotPeeredWithSentinel.value,
+                        cosmos_alerts.NodeIsNotPeeredWithSentinelAlert,
+                        self._is_false_condition_function, [current],
+                        [node_name, is_peered_with_sentinel_configs['severity'],
+                         last_monitored, parent_id, node_id], data_for_alerting,
+                        cosmos_alerts.NodeIsPeeredWithSentinelAlert,
+                        [node_name, Severity.INFO.value, last_monitored,
+                         parent_id, node_id]
+                    )
 
             slashed_configs = configs.slashed
             if str_to_bool(slashed_configs['enabled']):

alerter/src/alerter/alerters/node/cosmos.py

@@ -80,6 +80,23 @@ def __init__(self, origin_name: str, severity: str, timestamp: float,
             timestamp, parent_id, origin_id,
             GroupedCosmosNodeAlertsMetricCode.NodeIsSyncing, [origin_id])
 
+class NodeIsPeeredWithSentinelAlert(Alert):
+    def __init__(self, origin_name: str, severity: str, timestamp: float,
+                 parent_id: str, origin_id: str) -> None:
+        super().__init__(
+            CosmosNodeAlertCode.NodeIsPeeredWithSentinelAlert,
+            "Node {} is peered with sentinel.".format(origin_name), severity,
+            timestamp, parent_id, origin_id,
+            GroupedCosmosNodeAlertsMetricCode.NodeIsNotPeeredWithSentinel, [origin_id])
+
+class NodeIsNotPeeredWithSentinelAlert(Alert):
+    def __init__(self, origin_name: str, severity: str, timestamp: float,
+                 parent_id: str, origin_id: str) -> None:
+        super().__init__(
+            CosmosNodeAlertCode.NodeIsNotPeeredWithSentinelAlert,
+            "Node {} is not peered with sentinel.".format(origin_name), severity,
+            timestamp, parent_id, origin_id,
+            GroupedCosmosNodeAlertsMetricCode.NodeIsNotPeeredWithSentinel, [origin_id])
 
 class ValidatorIsNotActiveAlert(Alert):
     def __init__(self, origin_name: str, severity: str, timestamp: float,

alerter/src/alerter/alerts/node/cosmos.py

@@ -107,6 +107,7 @@ def create_alerting_state(
                 AlertsMetricCode.MetricNotFound.value: False,
             }
             any_severity_sent = {
+                AlertsMetricCode.NodeIsNotPeeredWithSentinel.value: False,
                 AlertsMetricCode.NodeIsSyncing.value: False,
                 AlertsMetricCode.ValidatorIsNotActive.value: False,
                 AlertsMetricCode.ValidatorIsJailed.value: False,

alerter/src/alerter/factory/cosmos_node_alerting_factory.py

@@ -5,6 +5,7 @@ class GroupedCosmosNodeAlertsMetricCode(GroupedAlertsMetricCode):
     NodeIsDown = 'cosmos_node_is_down'
     ValidatorWasSlashed = 'cosmos_node_slashed'
     NodeIsSyncing = 'cosmos_node_syncing'
+    NodeIsNotPeeredWithSentinel = 'cosmos_node_is_not_peered_with_sentinel'
     ValidatorIsNotActive = 'cosmos_node_active'
     ValidatorIsJailed = 'cosmos_node_jailed'
     BlocksMissedThreshold = 'cosmos_node_blocks_missed'

alerter/src/alerter/grouped_alerts_metric_code/node/cosmos_node_metric_code.py

@@ -14,7 +14,8 @@ def __init__(
             cannot_access_tendermint_rpc_validator: Dict,
             cannot_access_tendermint_rpc_node: Dict, missed_blocks: Dict,
             slashed: Dict, node_is_syncing: Dict, validator_is_syncing: Dict,
-            validator_is_jailed: Dict) -> None:
+            validator_is_jailed: Dict,
+            node_is_peered_with_sentinel: Dict = None, validator_is_peered_with_sentinel: Dict = None) -> None:
         self._parent_id = parent_id
         self._cannot_access_validator = cannot_access_validator
         self._cannot_access_node = cannot_access_node
@@ -38,6 +39,8 @@ def __init__(
         self._node_is_syncing = node_is_syncing
         self._validator_is_syncing = validator_is_syncing
         self._validator_is_jailed = validator_is_jailed
+        self._node_is_peered_with_sentinel = node_is_peered_with_sentinel
+        self._validator_is_peered_with_sentinel = validator_is_peered_with_sentinel
 
     def __eq__(self, other: Any) -> bool:
         return self.__dict__ == other.__dict__
@@ -110,6 +113,14 @@ def node_is_syncing(self) -> Dict:
     def validator_is_syncing(self) -> Dict:
         return self._validator_is_syncing
 
+    @property
+    def node_is_peered_with_sentinel(self) -> Dict:
+        return self._node_is_peered_with_sentinel
+
+    @property
+    def validator_is_peered_with_sentinel(self) -> Dict:
+        return self._validator_is_peered_with_sentinel
+
     @property
     def validator_is_jailed(self) -> Dict:
         return self._validator_is_jailed

alerter/src/configs/alerts/node/cosmos.py

@@ -69,7 +69,6 @@ def get_chain_name(self, parent_id: str,
 
         return None
 
-
 class CosmosNodeAlertsConfigsFactory(CosmosAlertsConfigsFactory):
     """
     This class manages the node alerts configs. The configs are indexed by the
@@ -125,7 +124,9 @@ def add_new_config(self, chain_name: str, sent_configs: Dict) -> None:
             slashed=filtered['slashed'],
             node_is_syncing=filtered['node_is_syncing'],
             validator_is_syncing=filtered['validator_is_syncing'],
-            validator_is_jailed=filtered['validator_is_jailed']
+            validator_is_jailed=filtered['validator_is_jailed'],
+            node_is_peered_with_sentinel=filtered['node_is_peered_with_sentinel'],
+            validator_is_peered_with_sentinel=filtered['validator_is_peered_with_sentinel'],
         )
 
         self._configs[chain_name] = cosmos_node_alerts_config

alerter/src/configs/factory/alerts/cosmos_alerts.py

@@ -56,6 +56,7 @@
 _key_cosmos_node_last_monitored_prometheus = 'CosmosNode11'
 _key_cosmos_node_last_monitored_cosmos_rest = 'CosmosNode12'
 _key_cosmos_node_last_monitored_tendermint_rpc = 'CosmosNode13'
+_key_cosmos_node_is_peered = 'CosmosNode14'
 
 # CosmosNetworkX_<cosmos_network_id>
 _key_cosmos_network_proposals = 'CosmosNetwork1'
@@ -407,6 +408,10 @@ def get_cosmos_node_voting_power(cosmos_node_id: str) -> str:
     def get_cosmos_node_is_syncing(cosmos_node_id: str) -> str:
         return Keys._as_prefix(_key_cosmos_node_is_syncing) + cosmos_node_id
 
+    @staticmethod
+    def get_cosmos_node_is_peered(cosmos_node_id: str) -> str:
+        return Keys._as_prefix(_key_cosmos_node_is_peered) + cosmos_node_id
+
     @staticmethod
     def get_cosmos_node_bond_status(cosmos_node_id: str) -> str:
         return Keys._as_prefix(_key_cosmos_node_bond_status) + cosmos_node_id