Subgraph Composition: Cache blocks processed by firehose block stream

chain/arweave/src/chain.rs

@@ -144,6 +144,7 @@ impl Blockchain for Chain {
         });
 
         Ok(Box::new(FirehoseBlockStream::new(
+            self.chain_store(),
             deployment.hash,
             self.chain_client(),
             store.block_ptr(),
@@ -269,7 +270,7 @@ impl TriggersAdapterTrait<Chain> for TriggersAdapter {
         }))
     }
 
-    async fn load_blocks_by_numbers(
+    async fn load_block_ptrs_by_numbers(
         &self,
         _logger: Logger,
         _block_numbers: HashSet<BlockNumber>,

chain/cosmos/src/chain.rs

@@ -137,6 +137,7 @@ impl Blockchain for Chain {
         });
 
         Ok(Box::new(FirehoseBlockStream::new(
+            self.chain_store(),
             deployment.hash,
             self.chain_client(),
             store.block_ptr(),
@@ -197,15 +198,13 @@ impl TriggersAdapterTrait<Chain> for TriggersAdapter {
     ) -> Result<Option<codec::Block>, Error> {
         panic!("Should never be called since not used by FirehoseBlockStream")
     }
-
-    async fn load_blocks_by_numbers(
+    async fn load_block_ptrs_by_numbers(
         &self,
         _logger: Logger,
         _block_numbers: HashSet<BlockNumber>,
     ) -> Result<Vec<Block>, Error> {
-        unimplemented!()
+        todo!()
     }
-
     async fn chain_head_ptr(&self) -> Result<Option<BlockPtr>, Error> {
         unimplemented!()
     }

chain/ethereum/src/adapter.rs

@@ -1,5 +1,6 @@
 use anyhow::Error;
 use ethabi::{Error as ABIError, Function, ParamType, Token};
+use graph::blockchain::BlockPtrExt;
 use graph::blockchain::ChainIdentifier;
 use graph::components::subgraph::MappingError;
 use graph::data::store::ethereum::call;
@@ -1109,12 +1110,12 @@ pub trait EthereumAdapter: Send + Sync + 'static {
         block_hash: H256,
     ) -> Box<dyn Future<Item = LightEthereumBlock, Error = Error> + Send>;
 
-    async fn load_blocks_by_numbers(
+    async fn load_block_ptrs_by_numbers(
         &self,
         _logger: Logger,
         _chain_store: Arc<dyn ChainStore>,
         _block_numbers: HashSet<BlockNumber>,
-    ) -> Box<dyn Stream<Item = Arc<LightEthereumBlock>, Error = Error> + Send>;
+    ) -> Box<dyn Stream<Item = Arc<BlockPtrExt>, Error = Error> + Send>;
 
     /// Load Ethereum blocks in bulk, returning results as they come back as a Stream.
     /// May use the `chain_store` as a cache.

chain/ethereum/src/chain.rs

@@ -3,7 +3,7 @@ use anyhow::{Context, Error};
 use graph::blockchain::client::ChainClient;
 use graph::blockchain::firehose_block_ingestor::{FirehoseBlockIngestor, Transforms};
 use graph::blockchain::{
-    BlockIngestor, BlockTime, BlockchainKind, ChainIdentifier, TriggerFilterWrapper,
+    BlockIngestor, BlockPtrExt, BlockTime, BlockchainKind, ChainIdentifier, TriggerFilterWrapper,
     TriggersAdapterSelector,
 };
 use graph::components::adapter::ChainId;
@@ -100,6 +100,7 @@ impl BlockStreamBuilder<Chain> for EthereumStreamBuilder {
         let firehose_mapper = Arc::new(FirehoseMapper { adapter, filter });
 
         Ok(Box::new(FirehoseBlockStream::new(
+            chain.chain_store(),
             deployment.hash,
             chain.chain_client(),
             subgraph_current_block,
@@ -156,6 +157,7 @@ impl BlockStreamBuilder<Chain> for EthereumStreamBuilder {
         unified_api_version: UnifiedMappingApiVersion,
     ) -> Result<Box<dyn BlockStream<Chain>>> {
         let requirements = filter.chain_filter.node_capabilities();
+        let is_using_subgraph_composition = !source_subgraph_stores.is_empty();
         let adapter = TriggersAdapterWrapper::new(
             chain
                 .triggers_adapter(&deployment, &requirements, unified_api_version.clone())
@@ -181,20 +183,32 @@ impl BlockStreamBuilder<Chain> for EthereumStreamBuilder {
         // This is ok because Celo blocks are always final. And we _need_ to do this because
         // some events appear only in eth_getLogs but not in transaction receipts.
         // See also ca0edc58-0ec5-4c89-a7dd-2241797f5e50.
-        let chain_id = match chain.chain_client().as_ref() {
+        let reorg_threshold = match chain.chain_client().as_ref() {
             ChainClient::Rpc(adapter) => {
-                adapter
+                let chain_id = adapter
                     .cheapest()
                     .await
                     .ok_or(anyhow!("unable to get eth adapter for chan_id call"))?
                     .chain_id()
-                    .await?
+                    .await?;
+
+                if CELO_CHAIN_IDS.contains(&chain_id) {
+                    0
+                } else {
+                    chain.reorg_threshold
+                }
             }
-            _ => panic!("expected rpc when using polling blockstream"),
+            _ if is_using_subgraph_composition => chain.reorg_threshold,
+            _ => panic!(
+                "expected rpc when using polling blockstream : {}",
+                is_using_subgraph_composition
+            ),
         };
-        let reorg_threshold = match CELO_CHAIN_IDS.contains(&chain_id) {
-            false => chain.reorg_threshold,
-            true => 0,
+
+        let max_block_range_size = if is_using_subgraph_composition {
+            ENV_VARS.max_block_range_size * 10
+        } else {
+            ENV_VARS.max_block_range_size
         };
 
         Ok(Box::new(PollingBlockStream::new(
@@ -207,7 +221,7 @@ impl BlockStreamBuilder<Chain> for EthereumStreamBuilder {
             start_blocks,
             reorg_threshold,
             logger,
-            ENV_VARS.max_block_range_size,
+            max_block_range_size,
             ENV_VARS.target_triggers_per_block_range,
             unified_api_version,
             subgraph_current_block,
@@ -617,6 +631,8 @@ pub enum BlockFinality {
 
     // If a block may still be reorged, we need to work with more local data.
     NonFinal(EthereumBlockWithCalls),
+
+    Ptr(Arc<BlockPtrExt>),
 }
 
 impl Default for BlockFinality {
@@ -630,6 +646,7 @@ impl BlockFinality {
         match self {
             BlockFinality::Final(block) => block,
             BlockFinality::NonFinal(block) => &block.ethereum_block.block,
+            BlockFinality::Ptr(_) => unreachable!("light_block called on HeaderOnly"),
         }
     }
 }
@@ -639,6 +656,7 @@ impl<'a> From<&'a BlockFinality> for BlockPtr {
         match block {
             BlockFinality::Final(b) => BlockPtr::from(&**b),
             BlockFinality::NonFinal(b) => BlockPtr::from(&b.ethereum_block),
+            BlockFinality::Ptr(b) => BlockPtr::new(b.hash.clone(), b.number),
         }
     }
 }
@@ -648,13 +666,17 @@ impl Block for BlockFinality {
         match self {
             BlockFinality::Final(block) => block.block_ptr(),
             BlockFinality::NonFinal(block) => block.ethereum_block.block.block_ptr(),
+            BlockFinality::Ptr(block) => BlockPtr::new(block.hash.clone(), block.number),
         }
     }
 
     fn parent_ptr(&self) -> Option<BlockPtr> {
         match self {
             BlockFinality::Final(block) => block.parent_ptr(),
             BlockFinality::NonFinal(block) => block.ethereum_block.block.parent_ptr(),
+            BlockFinality::Ptr(block) => {
+                Some(BlockPtr::new(block.parent_hash.clone(), block.number - 1))
+            }
         }
     }
 
@@ -687,13 +709,15 @@ impl Block for BlockFinality {
                 json::to_value(eth_block)
             }
             BlockFinality::NonFinal(block) => json::to_value(&block.ethereum_block),
+            BlockFinality::Ptr(_) => Ok(json::Value::Null),
         }
     }
 
     fn timestamp(&self) -> BlockTime {
         let ts = match self {
             BlockFinality::Final(block) => block.timestamp,
             BlockFinality::NonFinal(block) => block.ethereum_block.block.timestamp,
+            BlockFinality::Ptr(block) => block.timestamp,
         };
         let ts = i64::try_from(ts.as_u64()).unwrap();
         BlockTime::since_epoch(ts, 0)
@@ -735,7 +759,7 @@ impl TriggersAdapterTrait<Chain> for TriggersAdapter {
         .await
     }
 
-    async fn load_blocks_by_numbers(
+    async fn load_block_ptrs_by_numbers(
         &self,
         logger: Logger,
         block_numbers: HashSet<BlockNumber>,
@@ -749,9 +773,9 @@ impl TriggersAdapterTrait<Chain> for TriggersAdapter {
             .await?;
 
         let blocks = adapter
-            .load_blocks_by_numbers(logger, self.chain_store.clone(), block_numbers)
+            .load_block_ptrs_by_numbers(logger, self.chain_store.clone(), block_numbers)
             .await
-            .map(|block| BlockFinality::Final(block))
+            .map(|block| BlockFinality::Ptr(block))
             .collect()
             .compat()
             .await?;
@@ -812,6 +836,7 @@ impl TriggersAdapterTrait<Chain> for TriggersAdapter {
                 triggers.append(&mut parse_block_triggers(&filter.block, full_block));
                 Ok(BlockWithTriggers::new(block, triggers, logger))
             }
+            BlockFinality::Ptr(_) => unreachable!("triggers_in_block called on HeaderOnly"),
         }
     }
 

chain/ethereum/src/env.rs

@@ -33,6 +33,9 @@ pub struct EnvVars {
     /// Set by the environment variable `ETHEREUM_BLOCK_BATCH_SIZE`. The
     /// default value is 10 blocks.
     pub block_batch_size: usize,
+    /// Set by the environment variable `ETHEREUM_BLOCK_PTR_BATCH_SIZE`. The
+    /// default value is 10 blocks.
+    pub block_ptr_batch_size: usize,
     /// Maximum number of blocks to request in each chunk.
     ///
     /// Set by the environment variable `GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE`.
@@ -116,6 +119,7 @@ impl From<Inner> for EnvVars {
             trace_stream_step_size: x.trace_stream_step_size,
             max_event_only_range: x.max_event_only_range,
             block_batch_size: x.block_batch_size,
+            block_ptr_batch_size: x.block_ptr_batch_size,
             max_block_range_size: x.max_block_range_size,
             json_rpc_timeout: Duration::from_secs(x.json_rpc_timeout_in_secs),
             block_receipts_check_timeout: Duration::from_secs(
@@ -160,6 +164,8 @@ struct Inner {
     max_event_only_range: BlockNumber,
     #[envconfig(from = "ETHEREUM_BLOCK_BATCH_SIZE", default = "10")]
     block_batch_size: usize,
+    #[envconfig(from = "ETHEREUM_BLOCK_PTR_BATCH_SIZE", default = "100")]
+    block_ptr_batch_size: usize,
     #[envconfig(from = "GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE", default = "2000")]
     max_block_range_size: BlockNumber,
     #[envconfig(from = "GRAPH_ETHEREUM_JSON_RPC_TIMEOUT", default = "180")]