feat(cardano-services): improve transaction from blockfrost using cbor to reduce the nr of calls

iccicci · iccicci · commit 78232f8752ab · 2024-10-22T11:24:24.000+02:00
diff --git a/packages/cardano-services/src/ChainHistory/BlockrostChainHistoryProvider/BlockfrostChainHistoryProvider.ts b/packages/cardano-services/src/ChainHistory/BlockrostChainHistoryProvider/BlockfrostChainHistoryProvider.ts
diff --git a/packages/cardano-services/src/NetworkInfo/BlockfrostNetworkInfoProvider/BlockfrostNetworkInfoProvider.ts b/packages/cardano-services/src/NetworkInfo/BlockfrostNetworkInfoProvider/BlockfrostNetworkInfoProvider.ts
@@ -1,5 +1,5 @@
 import { BlockfrostProvider } from '../../util/BlockfrostProvider/BlockfrostProvider';
-import { BlockfrostToCore, blockfrostToProviderError, networkMagicToIdMap } from '../../util';
+import { BlockfrostToCore, blockfrostToProviderError } from '../../util';
 import {
   Cardano,
   EraSummary,
@@ -63,7 +63,10 @@ export class BlockfrostNetworkInfoProvider extends BlockfrostProvider implements
         epochLength: response.epoch_length,
         maxKesEvolutions: response.max_kes_evolutions,
         maxLovelaceSupply: BigInt(response.max_lovelace_supply),
-        networkId: networkMagicToIdMap[response.network_magic],
+        networkId:
+          response.network_magic === Cardano.NetworkMagics.Mainnet
+            ? Cardano.NetworkId.Mainnet
+            : Cardano.NetworkId.Testnet,
         networkMagic: response.network_magic,
         securityParameter: response.security_param,
         slotLength: Seconds(response.slot_length),
@@ -80,7 +83,7 @@ export class BlockfrostNetworkInfoProvider extends BlockfrostProvider implements
     try {
       // Although Blockfrost have the endpoint, the blockfrost-js library don't have a call for it
       // https://github.com/blockfrost/blockfrost-js/issues/294
-      const response = await this.blockfrost.instance<Schemas['network-eras']>('network-eras');
+      const response = await this.blockfrost.instance<Schemas['network-eras']>('network/eras');
       return response.body;
     } catch (error) {
       throw handleError(error);
diff --git a/packages/cardano-services/src/Program/programs/providerServer.ts b/packages/cardano-services/src/Program/programs/providerServer.ts
@@ -6,6 +6,7 @@ import {
   CardanoNode,
   ChainHistoryProvider,
   HandleProvider,
+  NetworkInfoProvider,
   Provider,
   RewardsProvider,
   Seconds,
@@ -330,8 +331,17 @@ const serviceMapFactory = (options: ServiceMapFactoryOptions) => {
     });
   }, ServiceNames.NetworkInfo);
 
-  const getBlockfrostChainHistoryProvider = () =>
-    new BlockfrostChainHistoryProvider({ blockfrost: getBlockfrostApi(), logger });
+  let networkInfoProvider: NetworkInfoProvider;
+  const getNetworkInfoProvider = () => {
+    if (!networkInfoProvider)
+      networkInfoProvider =
+        args.networkInfoProvider === ProviderImplementation.BLOCKFROST
+          ? getBlockfrostNetworkInfoProvider()
+          : getDbSyncNetworkInfoProvider();
+    return networkInfoProvider;
+  };
+  const getBlockfrostChainHistoryProvider = (nInfoProvider: NetworkInfoProvider | DbSyncNetworkInfoProvider) =>
+    new BlockfrostChainHistoryProvider({ blockfrost: getBlockfrostApi(), logger, networkInfoProvider: nInfoProvider });
 
   const getBlockfrostRewardsProvider = () => new BlockfrostRewardsProvider({ blockfrost: getBlockfrostApi(), logger });
 
@@ -395,7 +405,10 @@ const serviceMapFactory = (options: ServiceMapFactoryOptions) => {
       new ChainHistoryHttpService({
         chainHistoryProvider: selectProviderImplementation<ChainHistoryProvider>(
           args.chainHistoryProvider ?? ProviderImplementation.DBSYNC,
-          { blockfrost: getBlockfrostChainHistoryProvider, dbsync: getDbSyncChainHistoryProvider },
+          {
+            blockfrost: () => getBlockfrostChainHistoryProvider(getNetworkInfoProvider()),
+            dbsync: getDbSyncChainHistoryProvider
+          },
           logger,
           ServiceNames.ChainHistory
         ),
@@ -412,16 +425,11 @@ const serviceMapFactory = (options: ServiceMapFactoryOptions) => {
           ServiceNames.Rewards
         )
       }),
-    [ServiceNames.NetworkInfo]: async () => {
-      const networkInfoProvider =
-        args.networkInfoProvider === ProviderImplementation.BLOCKFROST
-          ? getBlockfrostNetworkInfoProvider()
-          : getDbSyncNetworkInfoProvider();
-      return new NetworkInfoHttpService({
+    [ServiceNames.NetworkInfo]: async () =>
+      new NetworkInfoHttpService({
         logger,
-        networkInfoProvider
-      });
-    },
+        networkInfoProvider: getNetworkInfoProvider()
+      }),
     [ServiceNames.TxSubmit]: async () => {
       const txSubmitProvider = args.useSubmitApi
         ? getSubmitApiProvider()
diff --git a/packages/cardano-services/src/Utxo/BlockfrostUtxoProvider/BlockfrostUtxoProvider.ts b/packages/cardano-services/src/Utxo/BlockfrostUtxoProvider/BlockfrostUtxoProvider.ts
@@ -1,19 +1,56 @@
 import { BlockfrostProvider } from '../../util/BlockfrostProvider/BlockfrostProvider';
-import { BlockfrostToCore, BlockfrostUtxo, blockfrostToProviderError, fetchByAddressSequentially } from '../../util';
-import { Cardano, UtxoByAddressesArgs, UtxoProvider } from '@cardano-sdk/core';
+import { BlockfrostToCore, blockfrostToProviderError, fetchByAddressSequentially } from '../../util';
+import { Cardano, Serialization, UtxoByAddressesArgs, UtxoProvider } from '@cardano-sdk/core';
+import { PaginationOptions } from '@blockfrost/blockfrost-js/lib/types';
 import { Responses } from '@blockfrost/blockfrost-js';
+import { Schemas } from '@blockfrost/blockfrost-js/lib/types/open-api';
 
 export class BlockfrostUtxoProvider extends BlockfrostProvider implements UtxoProvider {
+  protected async fetchUtxos(addr: Cardano.PaymentAddress, pagination: PaginationOptions): Promise<Cardano.Utxo[]> {
+    const utxos: Responses['address_utxo_content'] = (await this.blockfrost.addressesUtxos(
+      addr.toString(),
+      pagination
+    )) as Responses['address_utxo_content'];
+
+    const utxoPromises = utxos.map((utxo) =>
+      this.fetchDetailsFromCBOR(utxo.tx_hash).then((tx) => {
+        const txOut = tx ? tx.body.outputs.find((output) => output.address === utxo.address) : undefined;
+        return BlockfrostToCore.addressUtxoContent(addr.toString(), utxo, txOut);
+      })
+    );
+    return Promise.all(utxoPromises);
+  }
+
+  async fetchCBOR(hash: string): Promise<string> {
+    return this.blockfrost
+      .instance<Schemas['script_cbor']>(`txs/${hash}/cbor`)
+      .then((response) => {
+        if (response.body.cbor) return response.body.cbor;
+        throw new Error('CBOR is null');
+      })
+      .catch((_error) => {
+        throw new Error('CBOR fetch failed');
+      });
+  }
+  protected async fetchDetailsFromCBOR(hash: string) {
+    return this.fetchCBOR(hash)
+      .then((cbor) => {
+        const tx = Serialization.Transaction.fromCbor(Serialization.TxCBOR(cbor)).toCore();
+        this.logger.info('Fetched details from CBOR for tx', hash);
+        return tx;
+      })
+      .catch((error) => {
+        this.logger.warn('Failed to fetch details from CBOR for tx', hash, error);
+        return null;
+      });
+  }
   public async utxoByAddresses({ addresses }: UtxoByAddressesArgs): Promise<Cardano.Utxo[]> {
     try {
       const utxoResults = await Promise.all(
         addresses.map(async (address) =>
-          fetchByAddressSequentially<Cardano.Utxo, BlockfrostUtxo>({
+          fetchByAddressSequentially<Cardano.Utxo, Cardano.Utxo>({
             address,
-            request: (addr: Cardano.PaymentAddress, pagination) =>
-              this.blockfrost.addressesUtxos(addr.toString(), pagination),
-            responseTranslator: (addr: Cardano.PaymentAddress, response: Responses['address_utxo_content']) =>
-              BlockfrostToCore.addressUtxoContent(addr.toString(), response)
+            request: async (addr: Cardano.PaymentAddress, pagination) => await this.fetchUtxos(addr, pagination)
           })
         )
       );
diff --git a/packages/cardano-services/src/util/BlockfrostProvider/BlockfrostClientFactory.ts b/packages/cardano-services/src/util/BlockfrostProvider/BlockfrostClientFactory.ts
@@ -14,7 +14,8 @@ export const getBlockfrostApi = () => {
   // custom hosted instance
   if (process.env.BLOCKFROST_CUSTOM_BACKEND_URL && process.env.BLOCKFROST_CUSTOM_BACKEND_URL !== '') {
     blockfrostApi = new BlockFrostAPI({
-      customBackend: process.env.BLOCKFROST_CUSTOM_BACKEND_URL
+      customBackend: process.env.BLOCKFROST_CUSTOM_BACKEND_URL,
+      rateLimiter: false
     });
 
     return blockfrostApi;
@@ -29,7 +30,8 @@ export const getBlockfrostApi = () => {
   // network is not mandatory, we keep it for safety.
   blockfrostApi = new BlockFrostAPI({
     network: process.env.NETWORK as AvailableNetworks,
-    projectId: process.env.BLOCKFROST_API_KEY
+    projectId: process.env.BLOCKFROST_API_KEY,
+    rateLimiter: false
   });
 
   return blockfrostApi;
diff --git a/packages/cardano-services/src/util/BlockfrostProvider/BlockfrostProvider.ts b/packages/cardano-services/src/util/BlockfrostProvider/BlockfrostProvider.ts
@@ -3,7 +3,7 @@ import { HealthCheckResponse, Provider, ProviderDependencies } from '@cardano-sd
 import { blockfrostToProviderError } from './blockfrostUtil';
 import type { Logger } from 'ts-log';
 
-/** Properties that are need to create a BlockfrostProvider */
+/** Properties needed to create a BlockfrostProvider */
 export interface BlockfrostProviderDependencies extends ProviderDependencies {
   blockfrost: BlockFrostAPI;
   logger: Logger;
diff --git a/packages/cardano-services/src/util/BlockfrostProvider/BlockfrostToCore.ts b/packages/cardano-services/src/util/BlockfrostProvider/BlockfrostToCore.ts
@@ -1,4 +1,6 @@
-import { Cardano } from '@cardano-sdk/core';
+import { Cardano, Serialization } from '@cardano-sdk/core';
+import { Hash32ByteBase16 } from '@cardano-sdk/crypto';
+import { HexBlob } from '@cardano-sdk/util';
 import { Responses } from '@blockfrost/blockfrost-js';
 
 type Unpacked<T> = T extends (infer U)[] ? U : T;
@@ -11,11 +13,15 @@ export type BlockfrostTransactionContent = Unpacked<Responses['address_transacti
 export type BlockfrostUtxo = Unpacked<BlockfrostAddressUtxoContent>;
 
 export const BlockfrostToCore = {
-  addressUtxoContent: (address: string, blockfrost: Responses['address_utxo_content']): Cardano.Utxo[] =>
-    blockfrost.map((utxo) => [
+  addressUtxoContent: (
+    address: string,
+    utxo: Responses['address_utxo_content'][0],
+    txOutFromCbor?: Cardano.TxOut
+  ): Cardano.Utxo =>
+    [
       BlockfrostToCore.hydratedTxIn(BlockfrostToCore.inputFromUtxo(address, utxo)),
-      BlockfrostToCore.txOut(BlockfrostToCore.outputFromUtxo(address, utxo))
-    ]) as Cardano.Utxo[],
+      BlockfrostToCore.txOut(BlockfrostToCore.outputFromUtxo(address, utxo), txOutFromCbor)
+    ] as Cardano.Utxo,
 
   blockToTip: (block: Responses['block_content']): Cardano.Tip => ({
     blockNo: Cardano.BlockNo(block.height!),
@@ -92,30 +98,30 @@ export const BlockfrostToCore = {
     treasuryExpansion: blockfrost.tau.toString()
   }),
 
-  transactionUtxos: (utxoResponse: Responses['tx_content_utxo']) => ({
-    collaterals: utxoResponse.inputs.filter((input) => input.collateral).map(BlockfrostToCore.hydratedTxIn),
-    inputs: utxoResponse.inputs.filter((input) => !input.collateral).map(BlockfrostToCore.hydratedTxIn),
-    outputs: utxoResponse.outputs.map(BlockfrostToCore.txOut)
-  }),
-
-  txContentUtxo: (blockfrost: Responses['tx_content_utxo']) => ({
-    hash: blockfrost.hash,
-    inputs: BlockfrostToCore.inputs(blockfrost.inputs),
-    outputs: BlockfrostToCore.outputs(blockfrost.outputs)
-  }),
+  txOut: (blockfrost: BlockfrostOutput, txOutFromCbor?: Cardano.TxOut): Cardano.TxOut => {
+    const value: Cardano.Value = {
+      coins: BigInt(blockfrost.amount.find(({ unit }) => unit === 'lovelace')!.quantity)
+    };
 
-  txOut: (blockfrost: BlockfrostOutput): Cardano.TxOut => {
     const assets: Cardano.TokenMap = new Map();
     for (const { quantity, unit } of blockfrost.amount) {
       if (unit === 'lovelace') continue;
       assets.set(Cardano.AssetId(unit), BigInt(quantity));
     }
-    return {
+
+    if (assets.size > 0) value.assets = assets;
+
+    const txOut: Cardano.TxOut = {
       address: Cardano.PaymentAddress(blockfrost.address),
-      value: {
-        assets,
-        coins: BigInt(blockfrost.amount.find(({ unit }) => unit === 'lovelace')!.quantity)
-      }
+      value
     };
+
+    if (blockfrost.inline_datum)
+      txOut.datum = Serialization.PlutusData.fromCbor(HexBlob(blockfrost.inline_datum)).toCore();
+    if (blockfrost.data_hash) txOut.datumHash = Hash32ByteBase16(blockfrost.data_hash);
+
+    if (txOutFromCbor?.scriptReference) txOut.scriptReference = txOutFromCbor.scriptReference;
+
+    return txOut;
   }
 };
diff --git a/packages/cardano-services/src/util/BlockfrostProvider/blockfrostUtil.ts b/packages/cardano-services/src/util/BlockfrostProvider/blockfrostUtil.ts
@@ -39,7 +39,7 @@ export const fetchSequentially = async <Item, Arg, Response>(
     request: (arg: Arg, pagination: PaginationOptions) => Promise<Response[]>;
     responseTranslator?: (response: Response[], arg: Arg) => Item[];
     /**
-     * @returns true to indicatate that current result set should be returned
+     * @returns true to indicate that current result set should be returned
      */
     haveEnoughItems?: (items: Item[]) => boolean;
     paginationOptions?: PaginationOptions;
@@ -88,7 +88,7 @@ export const fetchByAddressSequentially = async <Item, Response>(props: {
   request: (address: Cardano.PaymentAddress, pagination: PaginationOptions) => Promise<Response[]>;
   responseTranslator?: (address: Cardano.PaymentAddress, response: Response[]) => Item[];
   /**
-   * @returns true to indicatate that current result set should be returned
+   * @returns true to indicate that current result set should be returned
    */
   haveEnoughItems?: (items: Item[]) => boolean;
   paginationOptions?: PaginationOptions;
@@ -103,11 +103,6 @@ export const fetchByAddressSequentially = async <Item, Response>(props: {
       : undefined
   });
 
-export const networkMagicToIdMap: { [key in number]: Cardano.NetworkId } = {
-  [Cardano.NetworkMagics.Mainnet]: Cardano.NetworkId.Mainnet,
-  [Cardano.NetworkMagics.Preprod]: Cardano.NetworkId.Testnet
-};
-
 // copied from util-dev
 export const testnetEraSummaries: EraSummary[] = [
   {
diff --git a/packages/cardano-services/test/ChainHistory/BlockfrostChainHistoryProvider/BlockfrostChainHistoryProvider.test.ts b/packages/cardano-services/test/ChainHistory/BlockfrostChainHistoryProvider/BlockfrostChainHistoryProvider.test.ts
