fix: fetch used module operators keys and add sanity checks

src/modules/csm/distribution.py

@@ -111,7 +111,7 @@ def _get_ref_blockstamp_for_frame(
         )
 
     def _get_module_validators(self, blockstamp: ReferenceBlockStamp) -> ValidatorsByNodeOperator:
-        return self.w3.lido_validators.get_module_validators_by_node_operators(
+        return self.w3.lido_validators.get_used_module_validators_by_node_operators(
             StakingModuleAddress(self.w3.csm.module.address), blockstamp
         )
 

src/modules/submodules/oracle_module.py

@@ -15,7 +15,7 @@
 from src.modules.submodules.exceptions import IsNotMemberException, IncompatibleOracleVersion, ContractVersionMismatch
 from src.providers.http_provider import NotOkResponse
 from src.providers.ipfs import IPFSError
-from src.providers.keys.client import KeysOutdatedException
+from src.providers.keys.client import KAPIInconsistentData, KeysOutdatedException
 from src.utils.cache import clear_global_cache
 from src.web3py.extensions.lido_validators import CountOfKeysDiffersException
 from src.utils.blockstamp import build_blockstamp
@@ -102,6 +102,8 @@ def _cycle(self):
             logger.error({'msg': ''.join(traceback.format_exception(error))})
         except (NoSlotsAvailable, SlotNotFinalized, InconsistentData) as error:
             logger.error({'msg': 'Inconsistent response from consensus layer node.', 'error': str(error)})
+        except KAPIInconsistentData as error:
+            logger.error({'msg': 'Inconsistent response from Keys API service', 'error': str(error)})
         except KeysOutdatedException as error:
             logger.error({'msg': 'Keys API service returns outdated data.', 'error': str(error)})
         except CountOfKeysDiffersException as error:

src/providers/keys/client.py

@@ -16,9 +16,18 @@ class KAPIClientError(NotOkResponse):
     pass
 
 
+class KAPIInconsistentData(Exception):
+    pass
+
+
+class KAPIModule(TypedDict):
+    id: int
+    stakingModuleAddress: str
+
+
 class ModuleOperatorsKeys(TypedDict):
     keys: List[LidoKey]
-    module: dict
+    module: KAPIModule
     operators: list
 
 
@@ -36,7 +45,7 @@ class KeysAPIClient(HTTPProvider):
     PROMETHEUS_HISTOGRAM = KEYS_API_REQUESTS_DURATION
     PROVIDER_EXCEPTION = KAPIClientError
 
-    MODULE_OPERATORS_KEYS = 'v1/modules/{}/operators/keys'
+    USED_MODULE_OPERATORS_KEYS = 'v1/modules/{}/operators/keys?used=true'
     USED_KEYS = 'v1/keys?used=true'
     STATUS = 'v1/status'
 
@@ -61,17 +70,24 @@ def _get_with_blockstamp(self, url: str, blockstamp: BlockStamp, params: dict |
     @lru_cache(maxsize=1)
     def get_used_lido_keys(self, blockstamp: BlockStamp) -> list[LidoKey]:
         """Docs: https://keys-api.lido.fi/api/static/index.html#/keys/KeysController_get"""
-        return [LidoKey.from_response(**x) for x in self._get_with_blockstamp(self.USED_KEYS, blockstamp)]
+        data = [LidoKey.from_response(**x) for x in self._get_with_blockstamp(self.USED_KEYS, blockstamp)]
+        self._check_used_keys(data)
+        return data
 
     @lru_cache(maxsize=1)
-    def get_module_operators_keys(
+    def get_used_module_operators_keys(
         self, module_address: StakingModuleAddress, blockstamp: BlockStamp
     ) -> ModuleOperatorsKeys:
         """
         Docs: https://keys-api.lido.fi/api/static/index.html#/operators-keys/SRModulesOperatorsKeysController_getOperatorsKeys
         """
-        data = cast(dict, self._get_with_blockstamp(self.MODULE_OPERATORS_KEYS.format(module_address), blockstamp))
+        data = cast(dict, self._get_with_blockstamp(self.USED_MODULE_OPERATORS_KEYS.format(module_address), blockstamp))
+        if (kapi_module_address := data['module']['stakingModuleAddress']) != module_address:
+            raise KAPIInconsistentData(f"Module address mismatch: {kapi_module_address=} != {module_address=}")
+
         data['keys'] = [LidoKey.from_response(**k) for k in data['keys']]
+        self._check_used_keys(data['keys'])
+
         return cast(ModuleOperatorsKeys, data)
 
     def get_status(self) -> KeysApiStatus:
@@ -82,3 +98,12 @@ def get_status(self) -> KeysApiStatus:
     def _get_chain_id_with_provider(self, provider_index: int) -> int:
         data, _ = self._get_without_fallbacks(self.hosts[provider_index], self.STATUS, retval_validator=data_is_dict)
         return KeysApiStatus.from_response(**data).chainId
+
+    def _check_used_keys(self, keys: list[LidoKey]):
+        keys_seen: dict[str, LidoKey] = {}
+        for k in keys:
+            if not k.used:
+                raise KAPIInconsistentData(f"Got unused key={k}")
+            if k.key in keys_seen:
+                raise KAPIInconsistentData(f"Got duplicated key={k}, previously found={keys_seen[k.key]}")
+            keys_seen[k.key] = k

src/web3py/extensions/lido_validators.py

@@ -180,10 +180,10 @@ def get_lido_validators_by_node_operators(self, blockstamp: BlockStamp) -> Valid
         return no_validators
 
     @lru_cache(maxsize=1)
-    def get_module_validators_by_node_operators(
+    def get_used_module_validators_by_node_operators(
         self,
         module_address: StakingModuleAddress,
-        blockstamp: BlockStamp
+        blockstamp: BlockStamp,
     ) -> ValidatorsByNodeOperator:
         """
         Get module validators by querying the KeysAPI for the module keys.
@@ -195,21 +195,19 @@ def get_module_validators_by_node_operators(
         Returns:
             ValidatorsByNodeOperator: A mapping of node operator IDs to their corresponding validators.
         """
-        # Fetch module operator keys from the KeysAPI
-        kapi = self.w3.kac.get_module_operators_keys(module_address, blockstamp)
-        if (kapi_module_address := kapi['module']['stakingModuleAddress']) != module_address:
-            raise ValueError(f"Module address mismatch: {kapi_module_address=} != {module_address=}")
-        operators = kapi['operators']
-        keys = {k.key: k for k in kapi['keys']}
-        validators = self.w3.cc.get_validators(blockstamp)
-        module_id = StakingModuleId(int(kapi['module']['id']))
+
+        kapi = self.w3.kac.get_used_module_operators_keys(module_address, blockstamp)
+        module_id = StakingModuleId(kapi['module']['id'])
+
 
         # Make sure even empty NO will be presented in dict
         no_validators: ValidatorsByNodeOperator = {
-            (module_id, NodeOperatorId(int(operator['index']))): [] for operator in operators
+            (module_id, NodeOperatorId(int(operator['index']))): [] for operator in kapi['operators']
         }
 
         # Map validators to their corresponding node operators
+        validators = self.w3.cc.get_validators(blockstamp)
+        keys = {k.key: k for k in kapi['keys']}
         for validator in validators:
             lido_key = keys.get(HexStr(validator.validator.pubkey))
             if not lido_key: