diff --git a/src/cli.rs b/src/cli.rs index e8959f2..0d2518b 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,5 +1,7 @@ +use std::{fmt::Display, time::Instant}; + use anyhow::Result; -use clap::{Parser, Subcommand}; +use clap::{Parser, Subcommand, ValueEnum}; use indicatif::{ProgressBar, ProgressStyle}; use revm::InMemoryDB; use tokio::sync::mpsc; @@ -7,7 +9,7 @@ use tokio::sync::mpsc; use crate::{ evm_map::erc20_contract_to_system_address, fs::{download_blocks, read_abci_state, read_blocks, read_evm_state}, - run::{run_blocks, MAINNET_CHAIN_ID}, + run::run_blocks, state::State, types::PreprocessedBlock, }; @@ -17,6 +19,7 @@ use anyhow::anyhow; const CHUNK_SIZE: u64 = 1000; // only store this many blocks in memory const READ_LIMIT: u64 = 100000; +const TESTNET_BLOCK_THRESHOLD: u64 = 26800000; #[derive(Parser)] #[command(name = "hyper-evm-sync")] @@ -25,9 +28,30 @@ pub struct Cli { commands: Commands, } +#[derive(Debug, Clone, Copy, ValueEnum)] +pub enum Chain { + Mainnet, + Testnet, +} + +impl Display for Chain { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + match self { + Chain::Mainnet => "Mainnet", + Chain::Testnet => "Testnet", + } + ) + } +} + #[derive(Subcommand)] enum Commands { DownloadBlocks { + #[arg(long)] + chain: Chain, #[arg(short, long)] dir: String, #[arg(short, long, default_value_t = 1)] @@ -36,6 +60,8 @@ enum Commands { end_block: u64, }, SyncFromState { + #[arg(long)] + chain: Chain, #[arg(long)] is_abci: bool, #[arg(short, long)] @@ -60,12 +86,12 @@ enum Commands { impl Cli { pub async fn execute(self) -> Result<()> { match self.commands { - Commands::DownloadBlocks { start_block, end_block, dir } => { - download_blocks(&dir, start_block, end_block).await?; - println!("Downloaded {start_block} -> {end_block}."); + Commands::DownloadBlocks { chain, start_block, end_block, dir } => { + download_blocks(chain, &dir, start_block, end_block).await?; + println!("Downloaded {start_block} -> {end_block} from {chain}."); } - Commands::SyncFromState { fln, is_abci, snapshot_dir, chunk_size, blocks_dir, end_block } => { - run_from_state(blocks_dir, fln, is_abci, snapshot_dir, chunk_size, end_block).await? + Commands::SyncFromState { chain, fln, is_abci, snapshot_dir, chunk_size, blocks_dir, end_block } => { + run_from_state(chain, blocks_dir, fln, is_abci, snapshot_dir, chunk_size, end_block).await? } Commands::NextBlockNumber { abci_state_fln, evm_state_fln } => { if let Some(fln) = abci_state_fln { @@ -82,6 +108,7 @@ impl Cli { } async fn run_from_state( + chain: Chain, blocks_dir: String, state_fln: Option, is_abci: bool, @@ -89,7 +116,7 @@ async fn run_from_state( chunk_size: u64, end_block: u64, ) -> Result<()> { - let erc20_contract_to_system_address = erc20_contract_to_system_address(MAINNET_CHAIN_ID).await?; + let erc20_contract_to_system_address = erc20_contract_to_system_address(chain).await?; let (start_block, mut state) = if let Some(state_fln) = state_fln { if is_abci { read_abci_state(state_fln)? @@ -97,9 +124,18 @@ async fn run_from_state( read_evm_state(state_fln)? } } else { + if let Chain::Testnet = chain { + return Err(anyhow!("Testnet must start from a snapshot")); + } (1, InMemoryDB::genesis()) }; - println!("{start_block} -> {end_block}"); + if let Chain::Testnet = chain { + if start_block < TESTNET_BLOCK_THRESHOLD { + return Err(anyhow!("Testnet must be run after {TESTNET_BLOCK_THRESHOLD}")); + } + } + + println!("{start_block} -> {end_block} on {chain}"); let pb = ProgressBar::new(end_block - start_block + 1); pb.set_style( ProgressStyle::default_bar() @@ -110,10 +146,13 @@ async fn run_from_state( let (tx, mut rx) = mpsc::channel::)>>(1); let processor = tokio::spawn(async move { + let start = Instant::now(); + let hash = state.blake3_hash_slow(); + println!("Computed state hash after block={start_block}: {hash:?} in {:?}", start.elapsed()); while let Some(blocks) = rx.recv().await { run_blocks( Some(pb.clone()), - MAINNET_CHAIN_ID, + chain, &mut state, blocks, &erc20_contract_to_system_address, @@ -135,10 +174,10 @@ async fn run_from_state( let (processor_res, reader_res) = tokio::join!(processor, reader); if let Err(e) = processor_res { - eprintln!("Processor failed: {}", e); + eprintln!("Processor failed: {e}"); } if let Err(e) = reader_res { - eprintln!("Reader failed: {}", e); + eprintln!("Reader failed: {e}"); } Ok(()) } diff --git a/src/evm_map.rs b/src/evm_map.rs index 86695ac..68c2236 100644 --- a/src/evm_map.rs +++ b/src/evm_map.rs @@ -1,6 +1,6 @@ -use crate::run::{MAINNET_CHAIN_ID, TESTNET_CHAIN_ID}; +use crate::cli::Chain; use alloy::primitives::Address; -use anyhow::{Error, Result}; +use anyhow::Result; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; @@ -21,19 +21,22 @@ struct SpotMeta { tokens: Vec, } -async fn fetch_spot_meta(chain_id: u64) -> Result { - let url = match chain_id { - MAINNET_CHAIN_ID => "https://api.hyperliquid.xyz/info", - TESTNET_CHAIN_ID => "https://api.hyperliquid-testnet.xyz/info", - _ => return Err(Error::msg("unknown chain id")), - }; +fn info_url(chain: Chain) -> &'static str { + match chain { + Chain::Mainnet => "https://api.hyperliquid.xyz/info", + Chain::Testnet => "https://api.hyperliquid-testnet.xyz/info", + } +} + +async fn fetch_spot_meta(chain: Chain) -> Result { + let url = info_url(chain); let client = reqwest::Client::new(); let response = client.post(url).json(&serde_json::json!({"type": "spotMeta"})).send().await?; Ok(response.json().await?) } -pub async fn erc20_contract_to_system_address(chain_id: u64) -> Result> { - let meta = fetch_spot_meta(chain_id).await?; +pub async fn erc20_contract_to_system_address(chain: Chain) -> Result> { + let meta = fetch_spot_meta(chain).await?; let mut map = BTreeMap::new(); for token in &meta.tokens { if let Some(evm_contract) = &token.evm_contract { diff --git a/src/fs.rs b/src/fs.rs index d34b6d5..1e985e0 100644 --- a/src/fs.rs +++ b/src/fs.rs @@ -1,6 +1,9 @@ -use crate::types::{AbciState, BlockAndReceipts, EvmBlock, EvmState, PreprocessedBlock}; +use crate::{ + cli::Chain, + types::{AbciState, BlockAndReceipts, EvmBlock, EvmState, PreprocessedBlock}, +}; use anyhow::Result; -use aws_config::{meta::region::RegionProviderChain, BehaviorVersion}; +use aws_config::{BehaviorVersion, Region}; use aws_sdk_s3::{types::RequestPayer, Client}; use futures::{stream, StreamExt, TryStreamExt}; use indicatif::{ProgressBar, ProgressStyle}; @@ -17,7 +20,7 @@ use std::{ }; const DOWNLOAD_CHUNK_SIZE: u64 = 10000; -const CONCURRENCY_LIMIT: usize = 500; +const CONCURRENCY_LIMIT: usize = 1000; fn decompress(data: &[u8]) -> Result, lz4_flex::frame::Error> { let mut decoder = lz4_flex::frame::FrameDecoder::new(data); @@ -26,6 +29,13 @@ fn decompress(data: &[u8]) -> Result, lz4_flex::frame::Error> { Ok(decompressed) } +fn bucket(chain: Chain) -> &'static str { + match chain { + Chain::Mainnet => "hl-mainnet-evm-blocks", + Chain::Testnet => "hl-testnet-evm-blocks", + } +} + fn read_block_and_receipts(file_path: &Path) -> Result { let mut file = File::open(file_path)?; let mut buffer = Vec::new(); @@ -66,7 +76,6 @@ pub fn read_blocks(dir: &str, start_block: u64, end_block: u64, chunk_size: u64) PreprocessedBlock { block_num, block_and_receipts, signers } }) .collect(); - // let blocks = stream::iter(futures).buffered(CONCURRENCY_LIMIT).collect().await; println!("Deserialized blocks {}-{} in {:?}", start_block, end_block, start.elapsed()); all_blocks.push((chunk, blocks)); } @@ -113,11 +122,6 @@ fn block_key(block_num: u64) -> String { async fn fetch_block(block_num: u64, dir: PathBuf, s3: Arc, pb: ProgressBar, bucket: &str) -> Result<()> { let key = block_key(block_num); let local_path: PathBuf = dir.join(&key); - - if let Some(parent) = local_path.parent() { - create_dir_all(parent)?; - } - if local_path.is_file() { pb.inc(1); return Ok(()); @@ -126,6 +130,9 @@ async fn fetch_block(block_num: u64, dir: PathBuf, s3: Arc, pb: Progress let obj = s3.get_object().bucket(bucket).key(key).request_payer(RequestPayer::Requester).send().await?; let mut body = obj.body.into_async_read(); + if let Some(parent) = local_path.parent() { + create_dir_all(parent)?; + } let mut file = tokio::fs::File::create(&local_path).await?; tokio::io::copy(&mut body, &mut file).await?; @@ -133,7 +140,7 @@ async fn fetch_block(block_num: u64, dir: PathBuf, s3: Arc, pb: Progress Ok(()) } -pub async fn download_blocks(dir: &str, start_block: u64, end_block: u64) -> Result<()> { +pub async fn download_blocks(chain: Chain, dir: &str, start_block: u64, end_block: u64) -> Result<()> { let pb = ProgressBar::new(end_block - start_block + 1); pb.set_style( ProgressStyle::default_bar() @@ -141,11 +148,12 @@ pub async fn download_blocks(dir: &str, start_block: u64, end_block: u64) -> Res .unwrap() .progress_chars("##-"), ); - let region = RegionProviderChain::default_provider().or_else("us-east-1"); + let region = Region::new("ap-northeast-1".to_string()); let config = aws_config::defaults(BehaviorVersion::latest()).region(region).load().await; let s3 = Arc::new(Client::new(&config)); - let bucket = "hl-mainnet-evm-blocks"; + let bucket = bucket(chain); + let mut cur_block = start_block; while cur_block <= end_block { let next_block = (end_block + 1).min(cur_block + DOWNLOAD_CHUNK_SIZE); @@ -165,6 +173,7 @@ pub async fn download_blocks(dir: &str, start_block: u64, end_block: u64) -> Res #[cfg(test)] mod tests { use crate::{ + cli::Chain, fs::{download_blocks, read_abci_state, read_evm_state, snapshot_evm_state}, state::State, }; @@ -174,7 +183,7 @@ mod tests { #[tokio::test] async fn test_block_download() -> Result<()> { let time = Instant::now(); - download_blocks("hl-mainnet-evm-blocks", 4000000, 4001000).await?; + download_blocks(Chain::Mainnet, "hl-mainnet-evm-blocks", 4000000, 4001000).await?; println!("downloaded in {:?}", time.elapsed()); Ok(()) } diff --git a/src/run.rs b/src/run.rs index 9d2b94c..2db7d45 100644 --- a/src/run.rs +++ b/src/run.rs @@ -1,4 +1,5 @@ use crate::{ + cli::Chain, fs::snapshot_evm_state, precompile::set_replay_precompiles, state::{State, StateHash}, @@ -35,26 +36,31 @@ fn deploy_system_contract(state: &mut S, contract_address: Address, de } } -fn deploy_system_contracts(state: &mut S, block_number: u64) { - if block_number == 1 { - deploy_system_contract( +fn deploy_system_contracts(state: &mut S, chain: Chain, block_number: u64) { + match chain { + Chain::Mainnet => { + if block_number == 1 { + deploy_system_contract( state, NATIVE_TOKEN_SYSTEM_ADDRESS, bytes!( "0x608060405236603f5760405134815233907f88a5966d370b9919b20f3e2c13ff65706f196a4e32cc2c12bf57088f885258749060200160405180910390a2005b600080fdfea2646970667358221220ca425db50898ac19f9e4676e86e8ebed9853baa048942f6306fe8a86b8d4abb964736f6c63430008090033" ), ); - deploy_system_contract( + deploy_system_contract( state, WHYPE_CONTRACT_ADDRESS, bytes!( "0x6080604052600436106100bc5760003560e01c8063313ce56711610074578063a9059cbb1161004e578063a9059cbb146102cb578063d0e30db0146100bc578063dd62ed3e14610311576100bc565b8063313ce5671461024b57806370a082311461027657806395d89b41146102b6576100bc565b806318160ddd116100a557806318160ddd146101aa57806323b872dd146101d15780632e1a7d4d14610221576100bc565b806306fdde03146100c6578063095ea7b314610150575b6100c4610359565b005b3480156100d257600080fd5b506100db6103a8565b6040805160208082528351818301528351919283929083019185019080838360005b838110156101155781810151838201526020016100fd565b50505050905090810190601f1680156101425780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b34801561015c57600080fd5b506101966004803603604081101561017357600080fd5b5073ffffffffffffffffffffffffffffffffffffffff8135169060200135610454565b604080519115158252519081900360200190f35b3480156101b657600080fd5b506101bf6104c7565b60408051918252519081900360200190f35b3480156101dd57600080fd5b50610196600480360360608110156101f457600080fd5b5073ffffffffffffffffffffffffffffffffffffffff8135811691602081013590911690604001356104cb565b34801561022d57600080fd5b506100c46004803603602081101561024457600080fd5b503561066b565b34801561025757600080fd5b50610260610700565b6040805160ff9092168252519081900360200190f35b34801561028257600080fd5b506101bf6004803603602081101561029957600080fd5b503573ffffffffffffffffffffffffffffffffffffffff16610709565b3480156102c257600080fd5b506100db61071b565b3480156102d757600080fd5b50610196600480360360408110156102ee57600080fd5b5073ffffffffffffffffffffffffffffffffffffffff8135169060200135610793565b34801561031d57600080fd5b506101bf6004803603604081101561033457600080fd5b5073ffffffffffffffffffffffffffffffffffffffff813581169160200135166107a7565b33600081815260036020908152604091829020805434908101909155825190815291517fe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c9281900390910190a2565b6000805460408051602060026001851615610100027fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0190941693909304601f8101849004840282018401909252818152929183018282801561044c5780601f106104215761010080835404028352916020019161044c565b820191906000526020600020905b81548152906001019060200180831161042f57829003601f168201915b505050505081565b33600081815260046020908152604080832073ffffffffffffffffffffffffffffffffffffffff8716808552908352818420869055815186815291519394909390927f8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925928290030190a350600192915050565b4790565b73ffffffffffffffffffffffffffffffffffffffff83166000908152600360205260408120548211156104fd57600080fd5b73ffffffffffffffffffffffffffffffffffffffff84163314801590610573575073ffffffffffffffffffffffffffffffffffffffff841660009081526004602090815260408083203384529091529020547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff14155b156105ed5773ffffffffffffffffffffffffffffffffffffffff841660009081526004602090815260408083203384529091529020548211156105b557600080fd5b73ffffffffffffffffffffffffffffffffffffffff841660009081526004602090815260408083203384529091529020805483900390555b73ffffffffffffffffffffffffffffffffffffffff808516600081815260036020908152604080832080548890039055938716808352918490208054870190558351868152935191937fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef929081900390910190a35060019392505050565b3360009081526003602052604090205481111561068757600080fd5b33600081815260036020526040808220805485900390555183156108fc0291849190818181858888f193505050501580156106c6573d6000803e3d6000fd5b5060408051828152905133917f7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65919081900360200190a250565b60025460ff1681565b60036020526000908152604090205481565b60018054604080516020600284861615610100027fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0190941693909304601f8101849004840282018401909252818152929183018282801561044c5780601f106104215761010080835404028352916020019161044c565b60006107a03384846104cb565b9392505050565b60046020908152600092835260408084209091529082529020548156fea265627a7a72315820e87684b404839c5657b1e7820bfa5ac4539ac8c83c21e28ec1086123db902cfe64736f6c63430005110032" ), ); - } + } - if block_number == CORE_WRITER_DEPLOY_BLOCK_NUMBER { - deploy_system_contract(state, CORE_WRITER_ADDRESS, bytes!("0x608060405234801561000f575f5ffd5b5060043610610029575f3560e01c806317938e131461002d575b5f5ffd5b61004760048036038101906100429190610123565b610049565b005b5f5f90505b61019081101561006557808060010191505061004e565b503373ffffffffffffffffffffffffffffffffffffffff167f8c7f585fb295f7eb1e6aeb8fba61b23a4fe60beda405f0045073b185c74412e383836040516100ae9291906101c8565b60405180910390a25050565b5f5ffd5b5f5ffd5b5f5ffd5b5f5ffd5b5f5ffd5b5f5f83601f8401126100e3576100e26100c2565b5b8235905067ffffffffffffffff811115610100576100ff6100c6565b5b60208301915083600182028301111561011c5761011b6100ca565b5b9250929050565b5f5f60208385031215610139576101386100ba565b5b5f83013567ffffffffffffffff811115610156576101556100be565b5b610162858286016100ce565b92509250509250929050565b5f82825260208201905092915050565b828183375f83830152505050565b5f601f19601f8301169050919050565b5f6101a7838561016e565b93506101b483858461017e565b6101bd8361018c565b840190509392505050565b5f6020820190508181035f8301526101e181848661019c565b9050939250505056fea2646970667358221220f01517e1fbaff8af4bd72cb063cccecbacbb00b07354eea7dd52265d355474fb64736f6c634300081c0033")); + if block_number == CORE_WRITER_DEPLOY_BLOCK_NUMBER { + deploy_system_contract(state, CORE_WRITER_ADDRESS, bytes!("0x608060405234801561000f575f5ffd5b5060043610610029575f3560e01c806317938e131461002d575b5f5ffd5b61004760048036038101906100429190610123565b610049565b005b5f5f90505b61019081101561006557808060010191505061004e565b503373ffffffffffffffffffffffffffffffffffffffff167f8c7f585fb295f7eb1e6aeb8fba61b23a4fe60beda405f0045073b185c74412e383836040516100ae9291906101c8565b60405180910390a25050565b5f5ffd5b5f5ffd5b5f5ffd5b5f5ffd5b5f5ffd5b5f5f83601f8401126100e3576100e26100c2565b5b8235905067ffffffffffffffff811115610100576100ff6100c6565b5b60208301915083600182028301111561011c5761011b6100ca565b5b9250929050565b5f5f60208385031215610139576101386100ba565b5b5f83013567ffffffffffffffff811115610156576101556100be565b5b610162858286016100ce565b92509250509250929050565b5f82825260208201905092915050565b828183375f83830152505050565b5f601f19601f8301169050919050565b5f6101a7838561016e565b93506101b483858461017e565b6101bd8361018c565b840190509392505050565b5f6020820190508181035f8301526101e181848661019c565b9050939250505056fea2646970667358221220f01517e1fbaff8af4bd72cb063cccecbacbb00b07354eea7dd52265d355474fb64736f6c634300081c0033")); + } + } + Chain::Testnet => {} } } @@ -178,7 +184,7 @@ where } fn process_block( - chain_id: u64, + chain: Chain, state: &mut S, erc20_contract_to_system_address: &BTreeMap, block_and_receipts: BlockAndReceipts, @@ -211,13 +217,13 @@ fn process_block( Arc::new(res) }; - deploy_system_contracts(state, block.number); + deploy_system_contracts(state, chain, block.number); let mut cumulative_gas_used = 0; for (tx_index, system_tx) in system_txs.into_iter().enumerate() { let SystemTx { tx, receipt } = system_tx; let computed_receipt = apply_tx(ApplyTxArgs { - chain_id, + chain_id: chain_id(chain), block: &block, precompile_results: &precompile_results, sender: if tx.input().is_empty() { @@ -243,7 +249,7 @@ fn process_block( for (tx_index, (tx_signed, signer)) in txs { let transaction = &tx_signed.transaction; let receipt = apply_tx(ApplyTxArgs { - chain_id, + chain_id: chain_id(chain), block: &block, precompile_results: &precompile_results, sender: signer, @@ -267,7 +273,7 @@ fn process_block( #[allow(clippy::type_complexity)] pub fn run_blocks( pb: Option, - chain_id: u64, + chain: Chain, state: &mut S, blocks: Vec<(u64, Vec)>, erc20_contract_to_system_address: &BTreeMap, @@ -292,7 +298,7 @@ where } let BlockAndReceipts { block: EvmBlock::Reth115(block), .. } = &block_and_receipts; assert_eq!(block_num, block.number); - process_block(chain_id, state, erc20_contract_to_system_address, block_and_receipts, signers); + process_block(chain, state, erc20_contract_to_system_address, block_and_receipts, signers); if block_num % chunk_size == 0 || block_num == end_block { let start = Instant::now(); let hash = state.blake3_hash_slow(); @@ -319,6 +325,13 @@ where pub const MAINNET_CHAIN_ID: u64 = 999; pub const TESTNET_CHAIN_ID: u64 = 998; +pub const fn chain_id(chain: Chain) -> u64 { + match chain { + Chain::Mainnet => MAINNET_CHAIN_ID, + Chain::Testnet => TESTNET_CHAIN_ID, + } +} + const NATIVE_TOKEN_SYSTEM_ADDRESS: Address = address!("0x2222222222222222222222222222222222222222"); const CORE_WRITER_ADDRESS: Address = address!("0x3333333333333333333333333333333333333333"); const WHYPE_CONTRACT_ADDRESS: Address = address!("0x5555555555555555555555555555555555555555");