Validate and Access all Parentchain Events in Enclave (#1272)

* Initial addition of retrieving event proof from parent-chain

* Adding in events retrieval

* removing old comments

* Passing in events and proofs per block to enclave

* refactoring a bit

* Fixing compiler errors

* slight refactor plus comment

* Clippy and fmt

* Addressing clippy concerns

* Taplo

* unpack events

* validated events

* adding in NodeMetaData extrinsic success and failed requests

* Refactor and implement groundwork

* passing events to correct place

* Adding in infra for events

* importing api client types and decoding events to log per block

* refactoring for passing events to triggered dispatcher

* Renaming module refactoring for triggered dispatcher refactor

* adding in queue methods for events queue

* cargo fmt

* fixing bug with chris on generic type

* Debugging decoding issue

* Was using storage key call incorrectly...

* small refactoring

* cargo fmt and taplo fmt

* adding in filtering of events for extrinsics

* cargo fmt

* taplo and cargo fmt in enclave

* refactoring for clippy

* cargo fmt

* changing log back

* minor comment removal

* refactoring to include new traits for testing

* cargo fmt

* refactoring for addressing comments

* cargo fmt

* clippy satisfaction

* cargo fmt

* fix for top pool tests

* cargo fmt

* refactor for comments

* making clippy happy

* adding back pallet system

* addressing comments

* refactor for comments

* adding some space

* add debug trait for logging

* cargo fmt

* addressing comments

* trace logs

* name changes

* couple log changes

* indentation fix

Author: coax1d

Cargo.lock

@@ -2770,6 +2770,7 @@ dependencies = [
  "itc-rest-client",
  "itc-rpc-client",
  "itc-rpc-server",
+ "itp-api-client-types",
  "itp-enclave-api",
  "itp-enclave-metrics",
  "itp-node-api",
@@ -3034,7 +3035,7 @@ name = "itc-parentchain-block-import-dispatcher"
 version = "0.9.0"
 dependencies = [
  "itc-parentchain-block-importer",
- "itp-block-import-queue",
+ "itp-import-queue",
  "itp-types",
  "log 0.4.17",
  "sgx_tstd",
@@ -3075,6 +3076,7 @@ dependencies = [
  "futures 0.3.8",
  "ita-stf",
  "itc-parentchain-test",
+ "itp-api-client-types",
  "itp-node-api",
  "itp-ocall-api",
  "itp-sgx-crypto",
@@ -3325,16 +3327,6 @@ dependencies = [
  "yasna 0.3.1",
 ]
 
-[[package]]
-name = "itp-block-import-queue"
-version = "0.8.0"
-dependencies = [
- "sgx_tstd",
- "sgx_types",
- "thiserror 1.0.40",
- "thiserror 1.0.9",
-]
-
 [[package]]
 name = "itp-component-container"
 version = "0.8.0"
@@ -3408,6 +3400,16 @@ dependencies = [
  "sp-core",
 ]
 
+[[package]]
+name = "itp-import-queue"
+version = "0.8.0"
+dependencies = [
+ "sgx_tstd",
+ "sgx_types",
+ "thiserror 1.0.40",
+ "thiserror 1.0.9",
+]
+
 [[package]]
 name = "itp-networking-utils"
 version = "0.9.0"
@@ -3895,8 +3897,8 @@ dependencies = [
  "fork-tree",
  "itc-parentchain-light-client",
  "itc-parentchain-test",
- "itp-block-import-queue",
  "itp-extrinsics-factory",
+ "itp-import-queue",
  "itp-node-api-metadata",
  "itp-node-api-metadata-provider",
  "itp-ocall-api",

Cargo.toml

@@ -17,7 +17,7 @@ members = [
     "core/rpc-server",
     "core/tls-websocket-server",
     "core-primitives/attestation-handler",
-    "core-primitives/block-import-queue",
+    "core-primitives/import-queue",
     "core-primitives/component-container",
     "core-primitives/enclave-api",
     "core-primitives/enclave-api/ffi",

core-primitives/block-import-queue/Cargo.toml renamed to core-primitives/import-queue/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "itp-block-import-queue"
+name = "itp-import-queue"
 version = "0.8.0"
 authors = ["Integritee AG <[email protected]>"]
 edition = "2021"

core-primitives/block-import-queue/src/error.rs renamed to core-primitives/import-queue/src/error.rs

@@ -15,7 +15,7 @@
 
 */
 
-//! Block import queue implementation
+//! Import queue implementation
 
 #[cfg(feature = "sgx")]
 use std::sync::SgxRwLock as RwLock;
@@ -25,64 +25,64 @@ use std::sync::RwLock;
 
 use crate::{
 	error::{Error, Result},
-	PeekBlockQueue, PopFromBlockQueue, PushToBlockQueue,
+	PeekQueue, PopFromQueue, PushToQueue,
 };
 use std::{collections::VecDeque, vec::Vec};
 
-/// Block import queue.
+/// Any import queue.
 ///
 /// Uses RwLock internally to guard against concurrent access and ensure all operations are atomic.
-pub struct BlockImportQueue<SignedBlock> {
-	queue: RwLock<VecDeque<SignedBlock>>,
+pub struct ImportQueue<Item> {
+	queue: RwLock<VecDeque<Item>>,
 }
 
-impl<SignedBlock> BlockImportQueue<SignedBlock> {
+impl<Item> ImportQueue<Item> {
 	pub fn is_empty(&self) -> Result<bool> {
 		let queue_lock = self.queue.read().map_err(|_| Error::PoisonedLock)?;
 		Ok(queue_lock.is_empty())
 	}
 }
 
-impl<SignedBlock> Default for BlockImportQueue<SignedBlock> {
+impl<Item> Default for ImportQueue<Item> {
 	fn default() -> Self {
-		BlockImportQueue { queue: Default::default() }
+		ImportQueue { queue: Default::default() }
 	}
 }
 
-impl<SignedBlock> PushToBlockQueue<SignedBlock> for BlockImportQueue<SignedBlock> {
-	fn push_multiple(&self, blocks: Vec<SignedBlock>) -> Result<()> {
+impl<Item> PushToQueue<Item> for ImportQueue<Item> {
+	fn push_multiple(&self, items: Vec<Item>) -> Result<()> {
 		let mut queue_lock = self.queue.write().map_err(|_| Error::PoisonedLock)?;
-		queue_lock.extend(blocks);
+		queue_lock.extend(items);
 		Ok(())
 	}
 
-	fn push_single(&self, block: SignedBlock) -> Result<()> {
+	fn push_single(&self, item: Item) -> Result<()> {
 		let mut queue_lock = self.queue.write().map_err(|_| Error::PoisonedLock)?;
-		queue_lock.push_back(block);
+		queue_lock.push_back(item);
 		Ok(())
 	}
 }
 
-impl<SignedBlock> PopFromBlockQueue for BlockImportQueue<SignedBlock> {
-	type BlockType = SignedBlock;
+impl<Item> PopFromQueue for ImportQueue<Item> {
+	type ItemType = Item;
 
-	fn pop_all_but_last(&self) -> Result<Vec<SignedBlock>> {
+	fn pop_all_but_last(&self) -> Result<Vec<Item>> {
 		let mut queue_lock = self.queue.write().map_err(|_| Error::PoisonedLock)?;
 		let queue_length = queue_lock.len();
 		if queue_length < 2 {
-			return Ok(Vec::<SignedBlock>::default())
+			return Ok(Vec::<Item>::default())
 		}
 		Ok(queue_lock.drain(..queue_length - 1).collect::<Vec<_>>())
 	}
 
-	fn pop_all(&self) -> Result<Vec<SignedBlock>> {
+	fn pop_all(&self) -> Result<Vec<Item>> {
 		let mut queue_lock = self.queue.write().map_err(|_| Error::PoisonedLock)?;
 		Ok(queue_lock.drain(..).collect::<Vec<_>>())
 	}
 
-	fn pop_until<Predicate>(&self, predicate: Predicate) -> Result<Vec<Self::BlockType>>
+	fn pop_until<Predicate>(&self, predicate: Predicate) -> Result<Vec<Self::ItemType>>
 	where
-		Predicate: FnMut(&Self::BlockType) -> bool,
+		Predicate: FnMut(&Self::ItemType) -> bool,
 	{
 		let mut queue_lock = self.queue.write().map_err(|_| Error::PoisonedLock)?;
 		match queue_lock.iter().position(predicate) {
@@ -91,28 +91,38 @@ impl<SignedBlock> PopFromBlockQueue for BlockImportQueue<SignedBlock> {
 		}
 	}
 
-	fn pop_front(&self) -> Result<Option<Self::BlockType>> {
+	fn pop_front(&self) -> Result<Option<Self::ItemType>> {
 		let mut queue_lock = self.queue.write().map_err(|_| Error::PoisonedLock)?;
 		Ok(queue_lock.pop_front())
 	}
+
+	fn pop_from_front_until(&self, amount: usize) -> Result<Vec<Self::ItemType>> {
+		let mut queue_lock = self.queue.write().map_err(|_| Error::PoisonedLock)?;
+		if amount > queue_lock.len() {
+			return Err(Error::Other(
+				"Cannot Pop more items from the queue than are available".into(),
+			))
+		}
+		Ok(queue_lock.drain(..amount).collect::<Vec<_>>())
+	}
 }
 
-impl<SignedBlock> PeekBlockQueue for BlockImportQueue<SignedBlock>
+impl<Item> PeekQueue for ImportQueue<Item>
 where
-	SignedBlock: Clone,
+	Item: Clone,
 {
-	type BlockType = SignedBlock;
+	type ItemType = Item;
 
-	fn peek_find<Predicate>(&self, predicate: Predicate) -> Result<Option<Self::BlockType>>
+	fn peek_find<Predicate>(&self, predicate: Predicate) -> Result<Option<Self::ItemType>>
 	where
-		Predicate: Fn(&Self::BlockType) -> bool,
+		Predicate: Fn(&Self::ItemType) -> bool,
 	{
 		let queue_lock = self.queue.read().map_err(|_| Error::PoisonedLock)?;
-		let maybe_block = queue_lock.iter().find(|&b| predicate(b));
-		Ok(maybe_block.cloned())
+		let maybe_item = queue_lock.iter().find(|&b| predicate(b));
+		Ok(maybe_item.cloned())
 	}
 
-	fn peek_last(&self) -> Result<Option<Self::BlockType>> {
+	fn peek_last(&self) -> Result<Option<Self::ItemType>> {
 		let queue_lock = self.queue.read().map_err(|_| Error::PoisonedLock)?;
 		Ok(queue_lock.back().cloned())
 	}
@@ -132,26 +142,26 @@ mod tests {
 
 	#[test]
 	fn default_queue_is_empty() {
-		let queue = BlockImportQueue::<TestBlock>::default();
+		let queue = ImportQueue::<TestBlock>::default();
 		assert!(queue.is_empty().unwrap());
 	}
 
 	#[test]
 	fn pop_all_on_default_returns_empty_vec() {
-		let queue = BlockImportQueue::<TestBlock>::default();
+		let queue = ImportQueue::<TestBlock>::default();
 		assert!(queue.pop_all().unwrap().is_empty());
 	}
 
 	#[test]
 	fn after_inserting_queue_is_not_empty() {
-		let queue = BlockImportQueue::<TestBlock>::default();
+		let queue = ImportQueue::<TestBlock>::default();
 		queue.push_single(TestBlock::default()).unwrap();
 		assert!(!queue.is_empty().unwrap());
 	}
 
 	#[test]
 	fn pop_all_after_inserting_leaves_empty_queue() {
-		let queue = BlockImportQueue::<TestBlock>::default();
+		let queue = ImportQueue::<TestBlock>::default();
 		queue
 			.push_multiple(vec![TestBlock::default(), TestBlock::default(), TestBlock::default()])
 			.unwrap();
@@ -163,20 +173,20 @@ mod tests {
 
 	#[test]
 	fn pop_all_except_last_on_default_returns_empty_vec() {
-		let queue = BlockImportQueue::<TestBlock>::default();
+		let queue = ImportQueue::<TestBlock>::default();
 		assert!(queue.pop_all_but_last().unwrap().is_empty());
 	}
 
 	#[test]
 	fn pop_all_except_last_with_single_element_returns_empty_vec() {
-		let queue = BlockImportQueue::<TestBlock>::default();
+		let queue = ImportQueue::<TestBlock>::default();
 		queue.push_single(TestBlock::default()).unwrap();
 		assert!(queue.pop_all_but_last().unwrap().is_empty());
 	}
 
 	#[test]
 	fn pop_all_except_last_with_multiple_elements_returns_all_but_last_inserted() {
-		let queue = BlockImportQueue::<TestBlock>::default();
+		let queue = ImportQueue::<TestBlock>::default();
 		queue.push_multiple(vec![1, 3, 5, 7]).unwrap();
 		assert_eq!(3, queue.pop_all_but_last().unwrap().len());
 		assert!(!queue.is_empty().unwrap());
@@ -185,7 +195,7 @@ mod tests {
 
 	#[test]
 	fn pop_until_returns_empty_vec_if_nothing_matches() {
-		let queue = BlockImportQueue::<TestBlock>::default();
+		let queue = ImportQueue::<TestBlock>::default();
 		queue.push_multiple(vec![1, 3, 5, 7]).unwrap();
 
 		let popped_elements = queue.pop_until(|i| i > &10u32).unwrap();
@@ -194,36 +204,36 @@ mod tests {
 
 	#[test]
 	fn pop_until_returns_elements_until_and_including_match() {
-		let queue = BlockImportQueue::<TestBlock>::default();
+		let queue = ImportQueue::<TestBlock>::default();
 		queue.push_multiple(vec![1, 2, 3, 10]).unwrap();
 
 		assert_eq!(queue.pop_until(|i| i == &3).unwrap(), vec![1, 2, 3]);
 	}
 
 	#[test]
 	fn pop_until_returns_all_elements_if_last_matches() {
-		let queue = BlockImportQueue::<TestBlock>::default();
+		let queue = ImportQueue::<TestBlock>::default();
 		queue.push_multiple(vec![1, 2, 3, 10]).unwrap();
 
 		assert_eq!(queue.pop_until(|i| i == &10).unwrap(), vec![1, 2, 3, 10]);
 	}
 
 	#[test]
 	fn pop_until_returns_first_element_if_it_matches() {
-		let queue = BlockImportQueue::<TestBlock>::default();
+		let queue = ImportQueue::<TestBlock>::default();
 		queue.push_single(4).unwrap();
 		assert_eq!(queue.pop_until(|i| i == &4).unwrap(), vec![4])
 	}
 
 	#[test]
 	fn pop_front_returns_none_if_queue_is_empty() {
-		let queue = BlockImportQueue::<TestBlock>::default();
+		let queue = ImportQueue::<TestBlock>::default();
 		assert_matches!(queue.pop_front().unwrap(), None);
 	}
 
 	#[test]
 	fn pop_front_works() {
-		let queue = BlockImportQueue::<TestBlock>::default();
+		let queue = ImportQueue::<TestBlock>::default();
 		queue.push_multiple(vec![1, 2, 3, 5]).unwrap();
 		assert_eq!(queue.pop_front().unwrap(), Some(1));
 		assert_eq!(queue.pop_front().unwrap(), Some(2));
@@ -234,7 +244,7 @@ mod tests {
 
 	#[test]
 	fn peek_find_works() {
-		let queue = BlockImportQueue::<TestBlock>::default();
+		let queue = ImportQueue::<TestBlock>::default();
 		queue.push_multiple(vec![1, 2, 3, 5]).unwrap();
 
 		assert_eq!(None, queue.peek_find(|i| i == &4).unwrap());
@@ -244,20 +254,20 @@ mod tests {
 
 	#[test]
 	fn peek_find_on_empty_queue_returns_none() {
-		let queue = BlockImportQueue::<TestBlock>::default();
+		let queue = ImportQueue::<TestBlock>::default();
 		assert_eq!(None, queue.peek_find(|i| i == &1).unwrap());
 	}
 
 	#[test]
 	fn peek_last_works() {
-		let queue = BlockImportQueue::<TestBlock>::default();
+		let queue = ImportQueue::<TestBlock>::default();
 		queue.push_multiple(vec![1, 2, 3, 5, 6, 9, 10]).unwrap();
 		assert_eq!(queue.peek_last().unwrap(), Some(10));
 	}
 
 	#[test]
 	fn peek_last_on_empty_queue_returns_none() {
-		let queue = BlockImportQueue::<TestBlock>::default();
+		let queue = ImportQueue::<TestBlock>::default();
 		assert_eq!(None, queue.peek_last().unwrap());
 	}
 }