diff --git a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs index f68cc8920d..ce2258bfb3 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs @@ -19,7 +19,7 @@ use crate::storage::storage_api::{ block_aux_data::{BlockAuxData, BlockWithExtraData}, AmountWithDecimals, ApiServerStorageError, BlockInfo, CoinOrTokenStatistic, Delegation, FungibleTokenData, LockedUtxo, NftWithOwner, Order, PoolBlockStats, PoolDataWithExtraInfo, - TransactionInfo, Utxo, UtxoLock, UtxoWithExtraInfo, + TransactionInfo, TransactionWithBlockInfo, Utxo, UtxoLock, UtxoWithExtraInfo, }; use common::{ address::Address, @@ -29,7 +29,7 @@ use common::{ Block, ChainConfig, DelegationId, Destination, Genesis, OrderId, PoolId, Transaction, UtxoOutPoint, }, - primitives::{id::WithId, Amount, BlockHeight, CoinOrTokenId, Id}, + primitives::{id::WithId, Amount, BlockHeight, CoinOrTokenId, Id, Idable}, }; use std::{ cmp::Reverse, @@ -51,6 +51,7 @@ struct ApiServerInMemoryStorage { main_chain_blocks_table: BTreeMap>, pool_data_table: BTreeMap>, transaction_table: BTreeMap, (Id, TransactionInfo)>, + ordered_transaction_table: BTreeMap>, utxo_table: BTreeMap>, address_utxos: BTreeMap>, locked_utxo_table: BTreeMap>, @@ -78,6 +79,7 @@ impl ApiServerInMemoryStorage { main_chain_blocks_table: BTreeMap::new(), pool_data_table: BTreeMap::new(), transaction_table: BTreeMap::new(), + ordered_transaction_table: BTreeMap::new(), utxo_table: BTreeMap::new(), address_utxos: BTreeMap::new(), locked_utxo_table: BTreeMap::new(), @@ -194,11 +196,11 @@ impl ApiServerInMemoryStorage { })) } - fn get_transactions_with_block( + fn get_transactions_with_block_info( &self, len: u32, - offset: u32, - ) -> Result, ApiServerStorageError> { + offset: u64, + ) -> Result, ApiServerStorageError> { Ok(self .main_chain_blocks_table .values() @@ -208,13 +210,21 @@ impl ApiServerInMemoryStorage { let block = self.block_table.get(block_id).expect("must exist"); block.block.transactions().iter().zip(block.tx_additional_infos.iter()).map( |(tx, additinal_data)| { - ( - *block_aux, - TransactionInfo { + let tx_global_index = self + .ordered_transaction_table + .iter() + .find(|(_, tx_id)| **tx_id == tx.transaction().get_id()) + .expect("must exist") + .0; + + TransactionWithBlockInfo { + tx_info: TransactionInfo { tx: tx.clone(), additional_info: additinal_data.clone(), }, - ) + block_aux: *block_aux, + global_tx_index: *tx_global_index, + } }, ) }) @@ -223,6 +233,32 @@ impl ApiServerInMemoryStorage { .collect()) } + fn get_transactions_with_block_before_tx_global_index( + &self, + len: u32, + tx_global_index: u64, + ) -> Result, ApiServerStorageError> { + Ok(self + .ordered_transaction_table + .range(..tx_global_index) + .rev() + .take(len as usize) + .map(|(tx_global_index, tx_id)| { + let (block_id, tx_info) = self.transaction_table.get(tx_id).expect("must exist"); + let block_aux = self.block_aux_data_table.get(block_id).expect("must exist"); + TransactionWithBlockInfo { + tx_info: tx_info.clone(), + block_aux: *block_aux, + global_tx_index: *tx_global_index, + } + }) + .collect()) + } + + fn get_last_transaction_global_indeex(&self) -> Result, ApiServerStorageError> { + Ok(self.ordered_transaction_table.keys().last().copied()) + } + #[allow(clippy::type_complexity)] fn get_transaction_with_block( &self, @@ -380,7 +416,7 @@ impl ApiServerInMemoryStorage { fn get_orders_by_height( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { let len = len as usize; let offset = offset as usize; @@ -413,7 +449,7 @@ impl ApiServerInMemoryStorage { &self, pair: (CoinOrTokenId, CoinOrTokenId), len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { let len = len as usize; let offset = offset as usize; @@ -447,7 +483,7 @@ impl ApiServerInMemoryStorage { fn get_latest_pool_ids( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { let len = len as usize; let offset = offset as usize; @@ -478,7 +514,7 @@ impl ApiServerInMemoryStorage { fn get_pool_data_with_largest_staker_balance( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { let len = len as usize; let offset = offset as usize; @@ -671,7 +707,7 @@ impl ApiServerInMemoryStorage { .or_else(|| self.nft_token_issuances.get(&token_id).map(|_| 0))) } - fn get_token_ids(&self, len: u32, offset: u32) -> Result, ApiServerStorageError> { + fn get_token_ids(&self, len: u32, offset: u64) -> Result, ApiServerStorageError> { Ok(self .fungible_token_data .keys() @@ -685,7 +721,7 @@ impl ApiServerInMemoryStorage { fn get_token_ids_by_ticker( &self, len: u32, - offset: u32, + offset: u64, ticker: &[u8], ) -> Result, ApiServerStorageError> { Ok(self @@ -971,6 +1007,7 @@ impl ApiServerInMemoryStorage { fn set_transaction( &mut self, transaction_id: Id, + tx_global_index: u64, owning_block: Id, transaction: &TransactionInfo, ) -> Result<(), ApiServerStorageError> { @@ -983,6 +1020,7 @@ impl ApiServerInMemoryStorage { self.transaction_table .insert(transaction_id, (owning_block, transaction.clone())); + self.ordered_transaction_table.insert(tx_global_index, transaction_id); Ok(()) } diff --git a/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs b/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs index cff9e7259d..7d0b1729d6 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs @@ -26,7 +26,8 @@ use common::{ use crate::storage::storage_api::{ block_aux_data::BlockAuxData, AmountWithDecimals, ApiServerStorageError, ApiServerStorageRead, BlockInfo, CoinOrTokenStatistic, Delegation, FungibleTokenData, NftWithOwner, Order, - PoolBlockStats, PoolDataWithExtraInfo, TransactionInfo, Utxo, UtxoWithExtraInfo, + PoolBlockStats, PoolDataWithExtraInfo, TransactionInfo, TransactionWithBlockInfo, Utxo, + UtxoWithExtraInfo, }; use super::ApiServerInMemoryStorageTransactionalRo; @@ -88,12 +89,27 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRo<'_> { self.transaction.get_transaction_with_block(transaction_id) } - async fn get_transactions_with_block( + async fn get_transactions_with_block_info( &self, len: u32, - offset: u32, - ) -> Result, ApiServerStorageError> { - self.transaction.get_transactions_with_block(len, offset) + offset: u64, + ) -> Result, ApiServerStorageError> { + self.transaction.get_transactions_with_block_info(len, offset) + } + + async fn get_transactions_with_block_info_before_tx_global_index( + &self, + len: u32, + tx_global_index: u64, + ) -> Result, ApiServerStorageError> { + self.transaction + .get_transactions_with_block_before_tx_global_index(len, tx_global_index) + } + + async fn get_last_transaction_global_index( + &self, + ) -> Result, ApiServerStorageError> { + self.transaction.get_last_transaction_global_indeex() } async fn get_delegation( @@ -121,7 +137,7 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRo<'_> { async fn get_latest_pool_data( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { self.transaction.get_latest_pool_ids(len, offset) } @@ -129,7 +145,7 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRo<'_> { async fn get_pool_data_with_largest_staker_balance( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { self.transaction.get_pool_data_with_largest_staker_balance(len, offset) } @@ -243,7 +259,7 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRo<'_> { async fn get_token_ids( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { self.transaction.get_token_ids(len, offset) } @@ -251,7 +267,7 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRo<'_> { async fn get_token_ids_by_ticker( &self, len: u32, - offset: u32, + offset: u64, ticker: &[u8], ) -> Result, ApiServerStorageError> { self.transaction.get_token_ids_by_ticker(len, offset, ticker) @@ -279,7 +295,7 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRo<'_> { async fn get_all_orders( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { self.transaction.get_orders_by_height(len, offset) } @@ -288,7 +304,7 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRo<'_> { &self, pair: (CoinOrTokenId, CoinOrTokenId), len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { self.transaction.get_orders_for_trading_pair(pair, len, offset) } diff --git a/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs b/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs index fcb086c1a1..b8dc530c3f 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs @@ -19,7 +19,8 @@ use crate::storage::storage_api::{ block_aux_data::{BlockAuxData, BlockWithExtraData}, AmountWithDecimals, ApiServerStorageError, ApiServerStorageRead, ApiServerStorageWrite, BlockInfo, CoinOrTokenStatistic, Delegation, FungibleTokenData, LockedUtxo, NftWithOwner, - Order, PoolBlockStats, PoolDataWithExtraInfo, TransactionInfo, Utxo, UtxoWithExtraInfo, + Order, PoolBlockStats, PoolDataWithExtraInfo, TransactionInfo, TransactionWithBlockInfo, Utxo, + UtxoWithExtraInfo, }; use common::{ address::Address, @@ -125,10 +126,12 @@ impl ApiServerStorageWrite for ApiServerInMemoryStorageTransactionalRw<'_> { async fn set_transaction( &mut self, transaction_id: Id, + order_number: u64, owning_block: Id, transaction: &TransactionInfo, ) -> Result<(), ApiServerStorageError> { - self.transaction.set_transaction(transaction_id, owning_block, transaction) + self.transaction + .set_transaction(transaction_id, order_number, owning_block, transaction) } async fn set_block_aux_data( @@ -395,12 +398,27 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRw<'_> { self.transaction.get_transaction_with_block(transaction_id) } - async fn get_transactions_with_block( + async fn get_transactions_with_block_info( + &self, + len: u32, + offset: u64, + ) -> Result, ApiServerStorageError> { + self.transaction.get_transactions_with_block_info(len, offset) + } + + async fn get_transactions_with_block_info_before_tx_global_index( &self, len: u32, - offset: u32, - ) -> Result, ApiServerStorageError> { - self.transaction.get_transactions_with_block(len, offset) + tx_global_index: u64, + ) -> Result, ApiServerStorageError> { + self.transaction + .get_transactions_with_block_before_tx_global_index(len, tx_global_index) + } + + async fn get_last_transaction_global_index( + &self, + ) -> Result, ApiServerStorageError> { + self.transaction.get_last_transaction_global_indeex() } async fn get_pool_data( @@ -413,7 +431,7 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRw<'_> { async fn get_latest_pool_data( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { self.transaction.get_latest_pool_ids(len, offset) } @@ -421,7 +439,7 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRw<'_> { async fn get_pool_data_with_largest_staker_balance( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { self.transaction.get_pool_data_with_largest_staker_balance(len, offset) } @@ -500,7 +518,7 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRw<'_> { async fn get_token_ids( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { self.transaction.get_token_ids(len, offset) } @@ -508,7 +526,7 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRw<'_> { async fn get_token_ids_by_ticker( &self, len: u32, - offset: u32, + offset: u64, ticker: &[u8], ) -> Result, ApiServerStorageError> { self.transaction.get_token_ids_by_ticker(len, offset, ticker) @@ -536,7 +554,7 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRw<'_> { async fn get_all_orders( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { self.transaction.get_orders_by_height(len, offset) } @@ -545,7 +563,7 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRw<'_> { &self, pair: (CoinOrTokenId, CoinOrTokenId), len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { self.transaction.get_orders_for_trading_pair(pair, len, offset) } diff --git a/api-server/api-server-common/src/storage/impls/mod.rs b/api-server/api-server-common/src/storage/impls/mod.rs index da84c75fcb..4e3c9c5041 100644 --- a/api-server/api-server-common/src/storage/impls/mod.rs +++ b/api-server/api-server-common/src/storage/impls/mod.rs @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub const CURRENT_STORAGE_VERSION: u32 = 21; +pub const CURRENT_STORAGE_VERSION: u32 = 22; pub mod in_memory; pub mod postgres; diff --git a/api-server/api-server-common/src/storage/impls/postgres/queries.rs b/api-server/api-server-common/src/storage/impls/postgres/queries.rs index 260cb37b09..dc75a12007 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/queries.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/queries.rs @@ -39,7 +39,7 @@ use crate::storage::{ block_aux_data::{BlockAuxData, BlockWithExtraData}, AmountWithDecimals, ApiServerStorageError, BlockInfo, CoinOrTokenStatistic, Delegation, FungibleTokenData, LockedUtxo, NftWithOwner, Order, PoolBlockStats, PoolDataWithExtraInfo, - TransactionInfo, Utxo, UtxoWithExtraInfo, + TransactionInfo, TransactionWithBlockInfo, Utxo, UtxoWithExtraInfo, }, }; @@ -83,6 +83,13 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .map_err(|_| ApiServerStorageError::TimestampTooHigh(block_timestamp)) } + fn tx_global_index_to_postgres_friendly(tx_global_index: u64) -> i64 { + // Postgres doesn't like u64, so we have to convert it to i64 + tx_global_index + .try_into() + .unwrap_or_else(|e| panic!("Invalid tx global index: {e}")) + } + pub async fn is_initialized(&mut self) -> Result { let query_str = Self::get_table_exists_query("misc_data"); let row_count = self @@ -670,6 +677,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { self.just_execute( "CREATE TABLE ml.transactions ( transaction_id bytea PRIMARY KEY, + global_tx_index bigint NOT NULL, owning_block_id bytea NOT NULL REFERENCES ml.blocks(block_id), transaction_data bytea NOT NULL );", // block_id can be null if the transaction is not in the main chain @@ -1560,7 +1568,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { pub async fn get_latest_pool_data( &self, len: u32, - offset: u32, + offset: u64, chain_config: &ChainConfig, ) -> Result, ApiServerStorageError> { let len = len as i64; @@ -1604,7 +1612,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { pub async fn get_pool_data_with_largest_staker_balance( &self, len: u32, - offset: u32, + offset: u64, chain_config: &ChainConfig, ) -> Result, ApiServerStorageError> { let len = len as i64; @@ -1776,8 +1784,8 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { pub async fn get_transactions_with_block( &self, len: u32, - offset: u32, - ) -> Result, ApiServerStorageError> { + offset: u64, + ) -> Result, ApiServerStorageError> { let len = len as i64; let offset = offset as i64; let rows = self @@ -1786,15 +1794,13 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { r#" SELECT t.transaction_data, - b.aux_data + b.aux_data, + t.global_tx_index FROM - ml.blocks mb - INNER JOIN - ml.transactions t ON t.owning_block_id = mb.block_id + ml.transactions t INNER JOIN ml.block_aux_data b ON t.owning_block_id = b.block_id - WHERE mb.block_height IS NOT NULL - ORDER BY mb.block_height DESC + ORDER BY t.global_tx_index DESC OFFSET $1 LIMIT $2; "#, @@ -1807,8 +1813,9 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .map(|data| { let transaction_data: Vec = data.get(0); let block_data: Vec = data.get(1); + let global_tx_index: i64 = data.get(2); - let block_data = + let block_aux = BlockAuxData::decode_all(&mut block_data.as_slice()).map_err(|e| { ApiServerStorageError::DeserializationError(format!( "Block deserialization failed: {}", @@ -1816,21 +1823,107 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { )) })?; - let transaction = TransactionInfo::decode_all(&mut transaction_data.as_slice()) + let tx_info = TransactionInfo::decode_all(&mut transaction_data.as_slice()) .map_err(|e| { ApiServerStorageError::DeserializationError(format!( "Transaction deserialization failed: {e}" )) })?; - Ok((block_data, transaction)) + Ok(TransactionWithBlockInfo { + tx_info, + block_aux, + global_tx_index: global_tx_index as u64, + }) }) .collect() } + pub async fn get_transactions_with_block_before_tx_global_index( + &self, + len: u32, + global_tx_index: u64, + ) -> Result, ApiServerStorageError> { + let len = len as i64; + let tx_global_index = Self::tx_global_index_to_postgres_friendly(global_tx_index); + let rows = self + .tx + .query( + r#" + SELECT + t.transaction_data, + b.aux_data, + t.global_tx_index + FROM + ml.transactions t + INNER JOIN + ml.block_aux_data b ON t.owning_block_id = b.block_id + WHERE t.global_tx_index < $1 + ORDER BY t.global_tx_index DESC + LIMIT $2; + "#, + &[&tx_global_index, &len], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + + rows.into_iter() + .map(|data| { + let transaction_data: Vec = data.get(0); + let block_data: Vec = data.get(1); + let global_tx_index: i64 = data.get(2); + + let block_aux = + BlockAuxData::decode_all(&mut block_data.as_slice()).map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "Block deserialization failed: {}", + e + )) + })?; + + let tx_info = TransactionInfo::decode_all(&mut transaction_data.as_slice()) + .map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "Transaction deserialization failed: {e}" + )) + })?; + + Ok(TransactionWithBlockInfo { + tx_info, + block_aux, + global_tx_index: global_tx_index as u64, + }) + }) + .collect() + } + + pub async fn get_last_transaction_global_index( + &self, + ) -> Result, ApiServerStorageError> { + let row = self + .tx + .query_opt( + r#" + SELECT + MAX(t.global_tx_index) + FROM + ml.transactions t; + "#, + &[], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + + Ok(row.map(|row| { + let last_tx_global_index: i64 = row.get(0); + last_tx_global_index as u64 + })) + } + pub async fn set_transaction( &mut self, transaction_id: Id, + global_tx_index: u64, owning_block: Id, transaction: &TransactionInfo, ) -> Result<(), ApiServerStorageError> { @@ -1839,11 +1932,13 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { transaction_id, owning_block ); + let global_tx_index = Self::tx_global_index_to_postgres_friendly(global_tx_index); self.tx.execute( - "INSERT INTO ml.transactions (transaction_id, owning_block_id, transaction_data) VALUES ($1, $2, $3) + "INSERT INTO ml.transactions (transaction_id, owning_block_id, transaction_data, global_tx_index) VALUES ($1, $2, $3, $4) ON CONFLICT (transaction_id) DO UPDATE - SET owning_block_id = $2, transaction_data = $3;", &[&transaction_id.encode(), &owning_block.encode(), &transaction.encode()] + SET owning_block_id = $2, transaction_data = $3, global_tx_index = $4;", + &[&transaction_id.encode(), &owning_block.encode(), &transaction.encode(), &global_tx_index] ).await .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; @@ -2250,7 +2345,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { pub async fn get_token_ids( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { let len = len as i64; let offset = offset as i64; @@ -2292,7 +2387,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { pub async fn get_token_ids_by_ticker( &self, len: u32, - offset: u32, + offset: u64, ticker: &[u8], ) -> Result, ApiServerStorageError> { let len = len as i64; @@ -2716,7 +2811,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { pub async fn get_orders_by_height( &self, len: u32, - offset: u32, + offset: u64, chain_config: &ChainConfig, ) -> Result, ApiServerStorageError> { let len = len as i64; @@ -2749,7 +2844,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { &self, pair: (CoinOrTokenId, CoinOrTokenId), len: u32, - offset: u32, + offset: u64, chain_config: &ChainConfig, ) -> Result, ApiServerStorageError> { let len = len as i64; diff --git a/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs b/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs index 3a37ce9e5f..3a17e183c9 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs @@ -26,8 +26,8 @@ use crate::storage::{ storage_api::{ block_aux_data::BlockAuxData, AmountWithDecimals, ApiServerStorageError, ApiServerStorageRead, BlockInfo, CoinOrTokenStatistic, Delegation, FungibleTokenData, - NftWithOwner, Order, PoolBlockStats, PoolDataWithExtraInfo, TransactionInfo, Utxo, - UtxoWithExtraInfo, + NftWithOwner, Order, PoolBlockStats, PoolDataWithExtraInfo, TransactionInfo, + TransactionWithBlockInfo, Utxo, UtxoWithExtraInfo, }, }; use std::collections::BTreeMap; @@ -194,17 +194,39 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRo<'_> { Ok(res) } - async fn get_transactions_with_block( + async fn get_transactions_with_block_info( &self, len: u32, - offset: u32, - ) -> Result, ApiServerStorageError> { + offset: u64, + ) -> Result, ApiServerStorageError> { let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); let res = conn.get_transactions_with_block(len, offset).await?; Ok(res) } + async fn get_transactions_with_block_info_before_tx_global_index( + &self, + len: u32, + order_number: u64, + ) -> Result, ApiServerStorageError> { + let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); + let res = conn + .get_transactions_with_block_before_tx_global_index(len, order_number) + .await?; + + Ok(res) + } + + async fn get_last_transaction_global_index( + &self, + ) -> Result, ApiServerStorageError> { + let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); + let res = conn.get_last_transaction_global_index().await?; + + Ok(res) + } + async fn get_pool_data( &self, pool_id: PoolId, @@ -219,7 +241,7 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRo<'_> { async fn get_latest_pool_data( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); let res = conn.get_latest_pool_data(len, offset, &self.chain_config).await?; @@ -230,7 +252,7 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRo<'_> { async fn get_pool_data_with_largest_staker_balance( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); let res = conn @@ -344,7 +366,7 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRo<'_> { async fn get_token_ids( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); let res = conn.get_token_ids(len, offset).await?; @@ -355,7 +377,7 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRo<'_> { async fn get_token_ids_by_ticker( &self, len: u32, - offset: u32, + offset: u64, ticker: &[u8], ) -> Result, ApiServerStorageError> { let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); @@ -395,7 +417,7 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRo<'_> { async fn get_all_orders( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); let res = conn.get_orders_by_height(len, offset, &self.chain_config).await?; @@ -407,7 +429,7 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRo<'_> { &self, pair: (CoinOrTokenId, CoinOrTokenId), len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); let res = conn.get_orders_for_trading_pair(pair, len, offset, &self.chain_config).await?; diff --git a/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs b/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs index cbd7640237..7f5549ef9f 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs @@ -31,7 +31,8 @@ use crate::storage::{ block_aux_data::{BlockAuxData, BlockWithExtraData}, AmountWithDecimals, ApiServerStorageError, ApiServerStorageRead, ApiServerStorageWrite, BlockInfo, CoinOrTokenStatistic, Delegation, FungibleTokenData, LockedUtxo, NftWithOwner, - Order, PoolBlockStats, PoolDataWithExtraInfo, TransactionInfo, Utxo, UtxoWithExtraInfo, + Order, PoolBlockStats, PoolDataWithExtraInfo, TransactionInfo, TransactionWithBlockInfo, + Utxo, UtxoWithExtraInfo, }, }; @@ -148,11 +149,13 @@ impl ApiServerStorageWrite for ApiServerPostgresTransactionalRw<'_> { async fn set_transaction( &mut self, transaction_id: Id, + order_number: u64, owning_block: Id, transaction: &TransactionInfo, ) -> Result<(), ApiServerStorageError> { let mut conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); - conn.set_transaction(transaction_id, owning_block, transaction).await?; + conn.set_transaction(transaction_id, order_number, owning_block, transaction) + .await?; Ok(()) } @@ -507,17 +510,39 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRw<'_> { Ok(res) } - async fn get_transactions_with_block( + async fn get_transactions_with_block_info( &self, len: u32, - offset: u32, - ) -> Result, ApiServerStorageError> { + offset: u64, + ) -> Result, ApiServerStorageError> { let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); let res = conn.get_transactions_with_block(len, offset).await?; Ok(res) } + async fn get_transactions_with_block_info_before_tx_global_index( + &self, + len: u32, + tx_global_index: u64, + ) -> Result, ApiServerStorageError> { + let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); + let res = conn + .get_transactions_with_block_before_tx_global_index(len, tx_global_index) + .await?; + + Ok(res) + } + + async fn get_last_transaction_global_index( + &self, + ) -> Result, ApiServerStorageError> { + let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); + let res = conn.get_last_transaction_global_index().await?; + + Ok(res) + } + async fn get_pool_data( &self, pool_id: PoolId, @@ -531,7 +556,7 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRw<'_> { async fn get_latest_pool_data( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); let res = conn.get_latest_pool_data(len, offset, &self.chain_config).await?; @@ -542,7 +567,7 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRw<'_> { async fn get_pool_data_with_largest_staker_balance( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); let res = conn @@ -677,7 +702,7 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRw<'_> { async fn get_token_ids( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); let res = conn.get_token_ids(len, offset).await?; @@ -688,7 +713,7 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRw<'_> { async fn get_token_ids_by_ticker( &self, len: u32, - offset: u32, + offset: u64, ticker: &[u8], ) -> Result, ApiServerStorageError> { let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); @@ -728,7 +753,7 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRw<'_> { async fn get_all_orders( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); let res = conn.get_orders_by_height(len, offset, &self.chain_config).await?; @@ -740,7 +765,7 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRw<'_> { &self, pair: (CoinOrTokenId, CoinOrTokenId), len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError> { let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); let res = conn.get_orders_for_trading_pair(pair, len, offset, &self.chain_config).await?; diff --git a/api-server/api-server-common/src/storage/storage_api/mod.rs b/api-server/api-server-common/src/storage/storage_api/mod.rs index 0401cc7ada..d93afa9a81 100644 --- a/api-server/api-server-common/src/storage/storage_api/mod.rs +++ b/api-server/api-server-common/src/storage/storage_api/mod.rs @@ -544,6 +544,13 @@ pub struct TransactionInfo { pub additional_info: TxAdditionalInfo, } +#[derive(Debug, Clone)] +pub struct TransactionWithBlockInfo { + pub tx_info: TransactionInfo, + pub block_aux: BlockAuxData, + pub global_tx_index: u64, +} + pub struct PoolBlockStats { pub block_count: u64, } @@ -638,13 +645,13 @@ pub trait ApiServerStorageRead: Sync { async fn get_latest_pool_data( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError>; async fn get_pool_data_with_largest_staker_balance( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError>; #[allow(clippy::type_complexity)] @@ -659,11 +666,20 @@ pub trait ApiServerStorageRead: Sync { transaction_id: Id, ) -> Result, TransactionInfo)>, ApiServerStorageError>; - async fn get_transactions_with_block( + async fn get_transactions_with_block_info( &self, len: u32, - offset: u32, - ) -> Result, ApiServerStorageError>; + offset: u64, + ) -> Result, ApiServerStorageError>; + + async fn get_transactions_with_block_info_before_tx_global_index( + &self, + len: u32, + tx_global_index: u64, + ) -> Result, ApiServerStorageError>; + + async fn get_last_transaction_global_index(&self) + -> Result, ApiServerStorageError>; async fn get_utxo(&self, outpoint: UtxoOutPoint) -> Result, ApiServerStorageError>; @@ -712,13 +728,13 @@ pub trait ApiServerStorageRead: Sync { async fn get_token_ids( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError>; async fn get_token_ids_by_ticker( &self, len: u32, - offset: u32, + offset: u64, ticker: &[u8], ) -> Result, ApiServerStorageError>; @@ -738,14 +754,14 @@ pub trait ApiServerStorageRead: Sync { async fn get_all_orders( &self, len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError>; async fn get_orders_for_trading_pair( &self, pair: (CoinOrTokenId, CoinOrTokenId), len: u32, - offset: u32, + offset: u64, ) -> Result, ApiServerStorageError>; } @@ -811,6 +827,7 @@ pub trait ApiServerStorageWrite: ApiServerStorageRead { async fn set_transaction( &mut self, transaction_id: Id, + tx_global_index: u64, owning_block: Id, transaction: &TransactionInfo, ) -> Result<(), ApiServerStorageError>; diff --git a/api-server/scanner-lib/src/blockchain_state/mod.rs b/api-server/scanner-lib/src/blockchain_state/mod.rs index d9a4e10b48..6f60525bd8 100644 --- a/api-server/scanner-lib/src/blockchain_state/mod.rs +++ b/api-server/scanner-lib/src/blockchain_state/mod.rs @@ -117,6 +117,8 @@ impl LocalBlockchainState for BlockchainState disconnect_tables_above_height(&mut db_tx, common_block_height) .await .expect("Unable to disconnect tables"); + let mut next_order_number = + db_tx.get_last_transaction_global_index().await?.map_or(0, |idx| idx + 1); // Connect the new blocks in the new chain for (index, block) in blocks.into_iter().map(WithId::new).enumerate() { @@ -190,12 +192,18 @@ impl LocalBlockchainState for BlockchainState .await .expect("Unable to set block"); - for tx_info in transactions { + for (i, tx_info) in transactions.iter().enumerate() { db_tx - .set_transaction(tx_info.tx.transaction().get_id(), block_id, &tx_info) + .set_transaction( + tx_info.tx.transaction().get_id(), + next_order_number + i as u64, + block_id, + tx_info, + ) .await .expect("Unable to set transaction"); } + next_order_number += transactions.len() as u64; db_tx .set_block_aux_data( diff --git a/api-server/stack-test-suite/tests/v2/block.rs b/api-server/stack-test-suite/tests/v2/block.rs index 51097e4106..1c529836dc 100644 --- a/api-server/stack-test-suite/tests/v2/block.rs +++ b/api-server/stack-test-suite/tests/v2/block.rs @@ -157,7 +157,7 @@ async fn ok(#[case] seed: Seed) { "transactions": block.transactions() .iter() .zip(tx_additional_data.iter()) - .map(|(tx, additional_data)| tx_to_json(tx.transaction(), additional_data, tf.chain_config())) + .map(|(tx, additional_data)| tx_to_json(tx, additional_data, tf.chain_config())) .collect::>(), }, }); @@ -209,7 +209,7 @@ async fn ok(#[case] seed: Seed) { "transactions": block.transactions() .iter() .zip(tx_additional_data.iter()) - .map(|(tx, additinal_data)| tx_to_json(tx.transaction(), additinal_data, tf.chain_config())) + .map(|(tx, additinal_data)| tx_to_json(tx, additinal_data, tf.chain_config())) .collect::>(), }, }); diff --git a/api-server/stack-test-suite/tests/v2/htlc.rs b/api-server/stack-test-suite/tests/v2/htlc.rs index 92cc416ce8..a1d426d42f 100644 --- a/api-server/stack-test-suite/tests/v2/htlc.rs +++ b/api-server/stack-test-suite/tests/v2/htlc.rs @@ -62,6 +62,8 @@ fn create_htlc( #[case(Seed::from_entropy())] #[tokio::test] async fn spend(#[case] seed: Seed) { + use api_web_server::api::json_helpers::to_json_string; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); @@ -131,7 +133,7 @@ async fn spend(#[case] seed: Seed) { &tx2, &[Some(&tx_1.transaction().outputs()[0])], 0, - secret, + secret.clone(), &mut rng, ) .unwrap(); @@ -150,6 +152,7 @@ async fn spend(#[case] seed: Seed) { tx_1_id.to_hash().encode_hex::(), block2.get_id().to_hash().encode_hex::(), tx_2_id.to_hash().encode_hex::(), + secret, )); vec![ @@ -189,7 +192,7 @@ async fn spend(#[case] seed: Seed) { web_server(listener, web_server_state, true).await }); - let (block1_id, tx_1_id, block2_id, tx_2_id) = rx.await.unwrap(); + let (block1_id, tx_1_id, block2_id, tx_2_id, secret) = rx.await.unwrap(); let url = format!("/api/v2/block/{block1_id}"); let response = reqwest::get(format!("http://{}:{}{url}", addr.ip(), addr.port())) @@ -215,6 +218,9 @@ async fn spend(#[case] seed: Seed) { .unwrap(); assert_eq!(response.status(), 200); + let body = response.text().await.unwrap(); + assert!(body.contains(&format!("\"secret\":{}", to_json_string(secret.secret())))); + task.abort(); } @@ -373,5 +379,8 @@ async fn refund(#[case] seed: Seed) { assert_eq!(response.status(), 200); + let body = response.text().await.unwrap(); + assert!(body.contains("\"secret\":null")); + task.abort(); } diff --git a/api-server/stack-test-suite/tests/v2/transactions.rs b/api-server/stack-test-suite/tests/v2/transactions.rs index 5c44191b23..4bd4eeed6b 100644 --- a/api-server/stack-test-suite/tests/v2/transactions.rs +++ b/api-server/stack-test-suite/tests/v2/transactions.rs @@ -17,6 +17,7 @@ use api_server_common::storage::storage_api::{ block_aux_data::BlockAuxData, TransactionInfo, TxAdditionalInfo, }; use api_web_server::api::json_helpers::to_tx_json_with_block_info; +use serde_json::Value; use super::*; @@ -34,6 +35,20 @@ async fn invalid_offset() { task.abort(); } +#[tokio::test] +async fn invalid_before_tx_global_index() { + let (task, response) = spawn_webserver("/api/v2/transaction?offset_mode=asd").await; + + assert_eq!(response.status(), 400); + + let body = response.text().await.unwrap(); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert_eq!(body["error"].as_str().unwrap(), "Invalid offset mode"); + + task.abort(); +} + #[tokio::test] async fn invalid_num_items() { let (task, response) = spawn_webserver("/api/v2/transaction?items=asd").await; @@ -81,8 +96,8 @@ async fn ok(#[case] seed: Seed) { let task = tokio::spawn(async move { let web_server_state = { let mut rng = make_seedable_rng(seed); - let n_blocks = rng.gen_range(2..100); - let num_tx = rng.gen_range(1..n_blocks); + let n_blocks = rng.gen_range(3..100); + let num_tx = rng.gen_range(2..n_blocks); let chain_config = create_unit_test_config(); @@ -95,6 +110,15 @@ async fn ok(#[case] seed: Seed) { .create_chain_return_ids(&tf.genesis().get_id().into(), n_blocks, &mut rng) .unwrap(); + let mut num_txs: usize = chainstate_block_ids + .iter() + .map(|id| { + let block_id = tf.to_chain_block_id(id); + let block = tf.block(block_id); + block.transactions().len() + }) + .sum(); + let txs: Vec = chainstate_block_ids .windows(2) .rev() @@ -122,6 +146,9 @@ async fn ok(#[case] seed: Seed) { }) .collect(); + let tx_global_index = + num_txs - block.transactions().len() + transaction_index; + num_txs -= block.transactions().len(); to_tx_json_with_block_info( &TransactionInfo { tx: signed_transaction.clone(), @@ -138,6 +165,7 @@ async fn ok(#[case] seed: Seed) { BlockHeight::new((n_blocks - idx) as u64), block.timestamp(), ), + tx_global_index as u64, ) }) .take(num_tx) @@ -182,11 +210,9 @@ async fn ok(#[case] seed: Seed) { let expected_transactions = rx.await.unwrap(); let num_tx = expected_transactions.len(); + let url = format!("/api/v2/transaction?offset=0&items={num_tx}"); - // Given that the listener port is open, this will block until a - // response is made (by the web server, which takes the listener - // over) let response = reqwest::get(format!("http://{}:{}{url}", addr.ip(), addr.port())) .await .unwrap(); @@ -194,37 +220,91 @@ async fn ok(#[case] seed: Seed) { assert_eq!(response.status(), 200); let body = response.text().await.unwrap(); + eprintln!("body: {}", body); let body: serde_json::Value = serde_json::from_str(&body).unwrap(); let arr_body = body.as_array().unwrap(); + assert_eq!(arr_body.len(), num_tx); for (expected_transaction, body) in expected_transactions.iter().zip(arr_body) { - assert_eq!( - body.get("block_id").unwrap(), - &expected_transaction["block_id"] - ); - assert_eq!( - body.get("version_byte").unwrap(), - &expected_transaction["version_byte"] - ); - assert_eq!( - body.get("is_replaceable").unwrap(), - &expected_transaction["is_replaceable"] - ); - assert_eq!(body.get("flags").unwrap(), &expected_transaction["flags"]); - assert_eq!(body.get("inputs").unwrap(), &expected_transaction["inputs"]); - assert_eq!( - body.get("outputs").unwrap(), - &expected_transaction["outputs"] - ); - assert_eq!( - body.get("timestamp").unwrap(), - &expected_transaction["timestamp"] - ); - assert_eq!( - body.get("confirmations").unwrap(), - &expected_transaction["confirmations"] - ); + compare_body(body, expected_transaction); + } + + let mut rng = make_seedable_rng(seed); + let offset = rng.gen_range(1..num_tx); + let items = num_tx - offset; + let url = format!("/api/v2/transaction?offset={offset}&items={items}"); + + let response = reqwest::get(format!("http://{}:{}{url}", addr.ip(), addr.port())) + .await + .unwrap(); + + assert_eq!(response.status(), 200); + + let body = response.text().await.unwrap(); + eprintln!("body: {}", body); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + let arr_body = body.as_array().unwrap(); + + assert_eq!(arr_body.len(), num_tx - offset); + for (expected_transaction, body) in expected_transactions[offset..].iter().zip(arr_body) { + compare_body(body, expected_transaction); + } + + // test before_tx_global_index instead of offset + let tx_global_index = &expected_transactions[offset - 1]["tx_global_index"].as_str().unwrap(); + eprintln!("tx_global_index: '{tx_global_index}'"); + let url = + format!("/api/v2/transaction?offset={tx_global_index}&items={items}&offset_mode=absolute"); + + let response = reqwest::get(format!("http://{}:{}{url}", addr.ip(), addr.port())) + .await + .unwrap(); + + assert_eq!(response.status(), 200); + + let body = response.text().await.unwrap(); + eprintln!("body: {}", body); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + let arr_body = body.as_array().unwrap(); + + assert_eq!(arr_body.len(), num_tx - offset); + for (expected_transaction, body) in expected_transactions[offset..].iter().zip(arr_body) { + compare_body(body, expected_transaction); } task.abort(); } + +#[track_caller] +fn compare_body(body: &Value, expected_transaction: &Value) { + assert_eq!( + body.get("block_id").unwrap(), + &expected_transaction["block_id"] + ); + assert_eq!( + body.get("version_byte").unwrap(), + &expected_transaction["version_byte"] + ); + assert_eq!( + body.get("is_replaceable").unwrap(), + &expected_transaction["is_replaceable"] + ); + assert_eq!(body.get("flags").unwrap(), &expected_transaction["flags"]); + assert_eq!(body.get("inputs").unwrap(), &expected_transaction["inputs"]); + assert_eq!( + body.get("outputs").unwrap(), + &expected_transaction["outputs"] + ); + assert_eq!( + body.get("timestamp").unwrap(), + &expected_transaction["timestamp"] + ); + assert_eq!( + body.get("confirmations").unwrap(), + &expected_transaction["confirmations"] + ); + assert_eq!( + body.get("tx_global_index").unwrap(), + &expected_transaction["tx_global_index"] + ); +} diff --git a/api-server/storage-test-suite/src/basic.rs b/api-server/storage-test-suite/src/basic.rs index 832593e528..2166bddfb1 100644 --- a/api-server/storage-test-suite/src/basic.rs +++ b/api-server/storage-test-suite/src/basic.rs @@ -327,7 +327,7 @@ where }; let random_owning_block = Id::::new(H256::random_using(&mut rng)); let result = db_tx - .set_transaction(tx1.transaction().get_id(), random_owning_block, &tx_info) + .set_transaction(tx1.transaction().get_id(), 1, random_owning_block, &tx_info) .await .unwrap_err(); @@ -347,7 +347,7 @@ where }, }; db_tx - .set_transaction(tx1.transaction().get_id(), owning_block1, &tx_info) + .set_transaction(tx1.transaction().get_id(), 2, owning_block1, &tx_info) .await .unwrap(); @@ -360,6 +360,50 @@ where } db_tx.commit().await.unwrap(); + + let mut db_tx = storage.transaction_rw().await.unwrap(); + { + let height1_u64 = rng.gen_range::(1..i64::MAX as u64); + let height1 = height1_u64.into(); + let random_block_timestamp = BlockTimestamp::from_int_seconds(rng.gen::()); + let aux_data1 = + BlockAuxData::new(owning_block1.into(), height1, random_block_timestamp); + db_tx.set_block_aux_data(owning_block1, &aux_data1).await.unwrap(); + + let tx_info = TransactionInfo { + tx: tx1.clone(), + additional_info: TxAdditionalInfo { + fee: Amount::from_atoms(rng.gen_range(0..100)), + input_utxos: tx1_input_utxos.clone(), + token_decimals: BTreeMap::new(), + }, + }; + + let expected_last_tx_global_index = 20; + let start_tx_global_index = 10; + for i in start_tx_global_index..=expected_last_tx_global_index { + let tx_id = H256::random_using(&mut rng); + db_tx.set_transaction(tx_id.into(), i, owning_block1, &tx_info).await.unwrap(); + } + + let last_num = db_tx.get_last_transaction_global_index().await.unwrap(); + assert_eq!(last_num, Some(expected_last_tx_global_index)); + + let take_txs = rng.gen_range(1..expected_last_tx_global_index - start_tx_global_index); + let txs = db_tx + .get_transactions_with_block_info_before_tx_global_index( + take_txs as u32, + last_num.unwrap() + 1, + ) + .await + .unwrap(); + assert_eq!(txs.len() as u64, take_txs); + for (i, tx) in txs.iter().enumerate() { + assert_eq!(tx.global_tx_index, expected_last_tx_global_index - i as u64); + } + } + + db_tx.rollback().await.unwrap(); } // Test setting/getting block aux data diff --git a/api-server/web-server/src/api/json_helpers.rs b/api-server/web-server/src/api/json_helpers.rs index da28fd0d4c..8b028b3223 100644 --- a/api-server/web-server/src/api/json_helpers.rs +++ b/api-server/web-server/src/api/json_helpers.rs @@ -23,15 +23,20 @@ use common::{ chain::{ block::ConsensusData, output_value::OutputValue, + signature::inputsig::{ + authorize_hashed_timelock_contract_spend::AuthorizedHashedTimelockContractSpend, + InputWitness, + }, tokens::{IsTokenUnfreezable, NftIssuance, TokenId, TokenTotalSupply}, AccountCommand, AccountSpending, Block, ChainConfig, Destination, OrderAccountCommand, - OrderId, OutPointSourceId, PoolId, Transaction, TxInput, TxOutput, UtxoOutPoint, + OrderId, OutPointSourceId, PoolId, SignedTransaction, TxInput, TxOutput, UtxoOutPoint, }, primitives::{Amount, BlockHeight, CoinOrTokenId, Idable}, Uint256, }; use hex::ToHex; use serde_json::json; +use serialization::DecodeAll; pub enum TokenDecimals<'a> { Map(&'a BTreeMap), @@ -107,6 +112,17 @@ pub fn txoutput_to_json( out: &TxOutput, chain_config: &ChainConfig, token_decimals: &TokenDecimals, +) -> serde_json::Value { + opt_spent_utxo_to_json(out, chain_config, token_decimals, None) +} + +/// This is reused for both Tx outputs and UTXOs. +/// In the UTXOs case we have a signature which we use to extract the HTLC secret if any. +fn opt_spent_utxo_to_json( + out: &TxOutput, + chain_config: &ChainConfig, + token_decimals: &TokenDecimals, + signature: Option<&InputWitness>, ) -> serde_json::Value { match out { TxOutput::Transfer(value, dest) => { @@ -202,10 +218,26 @@ pub fn txoutput_to_json( }) } TxOutput::Htlc(value, htlc) => { + let secret = if let Some(InputWitness::Standard(sig)) = signature { + let htlc_sig = + AuthorizedHashedTimelockContractSpend::decode_all(&mut sig.raw_signature()) + .expect("proper signature"); + + match htlc_sig { + AuthorizedHashedTimelockContractSpend::Secret(secret, _) => { + Some(to_json_string(secret.secret())) + } + AuthorizedHashedTimelockContractSpend::Multisig(_) => None, + } + } else { + None + }; + json!({ "type": "Htlc", "value": outputvalue_to_json(value, chain_config, token_decimals), "htlc": { + "secret": secret, "secret_hash": to_json_string(htlc.secret_hash.as_bytes()), "spend_key": Address::new(chain_config, htlc.spend_key.clone()).expect("no error").as_str(), "refund_timelock": htlc.refund_timelock, @@ -432,21 +464,30 @@ pub fn tx_input_to_json( } pub fn tx_to_json( - tx: &Transaction, + signed_tx: &SignedTransaction, additional_info: &TxAdditionalInfo, chain_config: &ChainConfig, ) -> serde_json::Value { let token_decimals = &(&additional_info.token_decimals).into(); + let tx = signed_tx.transaction(); json!({ "id": tx.get_id().to_hash().encode_hex::(), "version_byte": tx.version_byte(), "is_replaceable": tx.is_replaceable(), "flags": tx.flags(), "fee": amount_to_json(additional_info.fee, chain_config.coin_decimals()), - "inputs": tx.inputs().iter().zip(additional_info.input_utxos.iter()).map(|(inp, utxo)| json!({ - "input": tx_input_to_json(inp, token_decimals, chain_config), - "utxo": utxo.as_ref().map(|txo| txoutput_to_json(txo, chain_config, token_decimals)), - })).collect::>(), + "inputs": tx.inputs() + .iter() + .zip(additional_info.input_utxos.iter()) + .zip(signed_tx.signatures()) + .map(|((inp, utxo), sig)| + json!({ + "input": tx_input_to_json(inp, token_decimals, chain_config), + "utxo": utxo + .as_ref() + .map(|txo| opt_spent_utxo_to_json(txo, chain_config, token_decimals, Some(sig))), + })) + .collect::>(), "outputs": tx.outputs() .iter() .map(|out| txoutput_to_json(out, chain_config, token_decimals)) @@ -459,8 +500,9 @@ pub fn to_tx_json_with_block_info( chain_config: &ChainConfig, tip_height: BlockHeight, block: BlockAuxData, + tx_global_index: u64, ) -> serde_json::Value { - let mut json = tx_to_json(tx.tx.transaction(), &tx.additional_info, chain_config); + let mut json = tx_to_json(&tx.tx, &tx.additional_info, chain_config); let obj = json.as_object_mut().expect("object"); let confirmations = tip_height.sub(block.block_height()); @@ -477,6 +519,7 @@ pub fn to_tx_json_with_block_info( "confirmations".into(), confirmations.map_or("".to_string(), |c| c.to_string()).into(), ); + obj.insert("tx_global_index".into(), tx_global_index.to_string().into()); json } diff --git a/api-server/web-server/src/api/v2.rs b/api-server/web-server/src/api/v2.rs index b3bf0c7672..0cde5cc75c 100644 --- a/api-server/web-server/src/api/v2.rs +++ b/api-server/web-server/src/api/v2.rs @@ -194,7 +194,7 @@ pub async fn block( .iter() .zip(block.tx_additional_infos.iter()) .map(|(tx, additinal_info)| { - tx_to_json(tx.transaction(), additinal_info, &state.chain_config) + tx_to_json(tx, additinal_info, &state.chain_config) }) .collect::>(), }, @@ -444,31 +444,72 @@ pub async fn submit_transaction( )) } +enum OffsetMode { + Legacy, + Absolute, +} + +impl FromStr for OffsetMode { + type Err = ApiServerWebServerClientError; + + fn from_str(input: &str) -> Result { + match input { + "legacy" => Ok(Self::Legacy), + "absolute" => Ok(Self::Absolute), + _ => Err(ApiServerWebServerClientError::InvalidOffsetMode), + } + } +} + pub async fn transactions( Query(params): Query>, State(state): State, Arc>>, ) -> Result { + const OFFSET_MODE: &str = "offset_mode"; + let offset_mode = params + .get(OFFSET_MODE) + .map(|mode| OffsetMode::from_str(mode)) + .transpose()? + .unwrap_or(OffsetMode::Legacy); let offset_and_items = get_offset_and_items(¶ms)?; - let txs = state - .db - .transaction_ro() - .await - .map_err(|e| { - logging::log::error!("internal error: {e}"); - ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) - })? - .get_transactions_with_block(offset_and_items.items, offset_and_items.offset) - .await - .map_err(|e| { - logging::log::error!("internal error: {e}"); - ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) - })?; + let db_tx = state.db.transaction_ro().await.map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + })?; + + let txs = match offset_mode { + OffsetMode::Absolute => { + db_tx + .get_transactions_with_block_info_before_tx_global_index( + offset_and_items.items, + offset_and_items.offset, + ) + .await + } + OffsetMode::Legacy => { + db_tx + .get_transactions_with_block_info(offset_and_items.items, offset_and_items.offset) + .await + } + } + .map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + })?; let tip_height = best_block(&state).await?.block_height(); let txs = txs .into_iter() - .map(|(block, tx)| to_tx_json_with_block_info(&tx, &state.chain_config, tip_height, block)) + .map(|tx| { + to_tx_json_with_block_info( + &tx.tx_info, + &state.chain_config, + tip_height, + tx.block_aux, + tx.global_tx_index, + ) + }) .collect(); Ok(Json(serde_json::Value::Array(txs))) @@ -492,7 +533,7 @@ pub async fn transaction( } else { None }; - let mut json = tx_to_json(tx.transaction(), &additional_info, &state.chain_config); + let mut json = tx_to_json(&tx, &additional_info, &state.chain_config); let obj = json.as_object_mut().expect("object"); obj.insert( @@ -848,7 +889,7 @@ pub async fn pools( let sort = params .get(SORT) - .map(|offset| PoolSorting::from_str(offset)) + .map(|sort| PoolSorting::from_str(sort)) .transpose()? .unwrap_or(PoolSorting::ByHeight); @@ -1506,7 +1547,7 @@ pub async fn order_pair( } struct OffsetAndItems { - offset: u32, + offset: u64, items: u32, } @@ -1520,7 +1561,7 @@ fn get_offset_and_items( let offset = params .get(OFFSET) - .map(|offset| u32::from_str(offset)) + .map(|offset| u64::from_str(offset)) .transpose() .map_err(|_| { ApiServerWebServerError::ClientError(ApiServerWebServerClientError::InvalidOffset) diff --git a/api-server/web-server/src/error.rs b/api-server/web-server/src/error.rs index 59da940a70..cccb970dde 100644 --- a/api-server/web-server/src/error.rs +++ b/api-server/web-server/src/error.rs @@ -91,6 +91,8 @@ pub enum ApiServerWebServerClientError { InvalidPoolId, #[error("Invalid offset")] InvalidOffset, + #[error("Invalid offset mode")] + InvalidOffsetMode, #[error("Invalid number of items")] InvalidNumItems, #[error("Invalid pools sort order")]