LW-11470 Blockfrost based backend

.github/workflows/continuous-integration-blockfrost-e2e.yaml

@@ -0,0 +1,93 @@
+name: Continuous Integration - E2E BF
+
+env:
+  TL_DEPTH: ${{ github.event.pull_request.head.repo.fork && '0' || fromJson(vars.TL_DEPTH) }}
+  TL_LEVEL: ${{ github.event.pull_request.head.repo.fork && 'info' || vars.TL_LEVEL }}
+  # -----------------------------------------------------------------------------------------
+  DB_SYNC_CONNECTION_STRING: 'postgresql://postgres:doNoUseThisSecret!@localhost:5435/cexplorer'
+  KEY_MANAGEMENT_PROVIDER: 'inMemory'
+  KEY_MANAGEMENT_PARAMS: '{"bip32Ed25519": "Sodium", "accountIndex": 0, "chainId":{"networkId": 0, "networkMagic": 888}, "passphrase":"some_passphrase","mnemonic":"vacant violin soft weird deliver render brief always monitor general maid smart jelly core drastic erode echo there clump dizzy card filter option defense"}'
+  OGMIOS_URL: 'ws://localhost:1340/'
+  STAKE_POOL_CONNECTION_STRING: 'postgresql://postgres:doNoUseThisSecret!@localhost:5435/stake_pool'
+  STAKE_POOL_TEST_CONNECTION_STRING: 'postgresql://postgres:doNoUseThisSecret!@localhost:5435/stake_pool_test'
+  TEST_CLIENT_ASSET_PROVIDER: 'http'
+  TEST_CLIENT_ASSET_PROVIDER_PARAMS: '{"baseUrl":"http://localhost:4014/"}'
+  TEST_CLIENT_CHAIN_HISTORY_PROVIDER: 'http'
+  TEST_CLIENT_CHAIN_HISTORY_PROVIDER_PARAMS: '{"baseUrl":"http://localhost:4001/"}'
+  TEST_CLIENT_HANDLE_PROVIDER: 'http'
+  TEST_CLIENT_HANDLE_PROVIDER_PARAMS: '{"baseUrl":"http://localhost:4011/"}'
+  TEST_CLIENT_NETWORK_INFO_PROVIDER: 'ws'
+  TEST_CLIENT_NETWORK_INFO_PROVIDER_PARAMS: '{"baseUrl":"http://localhost:4000/"}'
+  TEST_CLIENT_REWARDS_PROVIDER: 'http'
+  TEST_CLIENT_REWARDS_PROVIDER_PARAMS: '{"baseUrl":"http://localhost:4001/"}'
+  TEST_CLIENT_TX_SUBMIT_PROVIDER: 'http'
+  TEST_CLIENT_TX_SUBMIT_PROVIDER_PARAMS: '{"baseUrl":"http://localhost:4000/"}'
+  TEST_CLIENT_UTXO_PROVIDER: 'http'
+  TEST_CLIENT_UTXO_PROVIDER_PARAMS: '{"baseUrl":"http://localhost:4001/"}'
+  TEST_CLIENT_STAKE_POOL_PROVIDER: 'http'
+  TEST_CLIENT_STAKE_POOL_PROVIDER_PARAMS: '{"baseUrl":"http://localhost:4000/"}'
+  WS_PROVIDER_URL: 'http://localhost:4100/ws'
+
+on:
+  pull_request:
+  push:
+    branches: ['master']
+    tags: ['*.*.*']
+
+jobs:
+  build_and_test:
+    strategy:
+      matrix:
+        os: [ubuntu-20.04]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: 📥 Checkout repository
+        uses: actions/checkout@v3
+
+      - name: 🧰 Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18.12.0
+
+      - name: 🔨 Build
+        run: |
+          yarn install --immutable --inline-builds --mode=skip-build
+          yarn workspace @cardano-sdk/cardano-services-client build:cjs
+          yarn workspace @cardano-sdk/cardano-services build:cjs
+          yarn workspace @cardano-sdk/e2e build:cjs
+          yarn workspace @cardano-sdk/util-dev build:cjs
+          docker build --no-cache .
+        env:
+          NODE_OPTIONS: '--max_old_space_size=8192'
+
+      - name: 🌐 Setup local test network
+        working-directory: packages/e2e
+        run: |
+          yarn local-network:up -d
+        env:
+          CARDANO_NODE_CHAINDB_LOG_LEVEL: 'Warning'
+          CARDANO_NODE_LOG_LEVEL: 'Warning'
+          OGMIOS_PORT: '1340'
+          OGMIOS_URL: 'ws://ogmios:1340'
+          POSTGRES_PORT: '5435'
+
+      - name: Wait for network init
+        run: |
+          yarn workspace @cardano-sdk/e2e wait-for-network-init
+
+      - name: 🔬 Test - e2e - wallet at epoch 0
+        run: |
+          yarn workspace @cardano-sdk/e2e test:wallet:epoch0
+
+      - name: Wait for epoch 3
+        run: |
+          yarn workspace @cardano-sdk/e2e wait-for-network-epoch-3
+
+      - name: 🔬 Test - e2e - wallet at epoch 3
+        run: |
+          yarn workspace @cardano-sdk/e2e test:wallet:epoch3
+          yarn workspace @cardano-sdk/e2e test:blockfrost:providers
+
+      - name: Dump docker logs
+        if: ${{ cancelled() || failure() }}
+        uses: jwalton/gh-docker-logs@v2

.github/workflows/continuous-integration-e2e.yaml

@@ -4,20 +4,20 @@ env:
   TL_DEPTH: ${{ github.event.pull_request.head.repo.fork && '0' || fromJson(vars.TL_DEPTH) }}
   TL_LEVEL: ${{ github.event.pull_request.head.repo.fork && 'info' || vars.TL_LEVEL }}
   # -----------------------------------------------------------------------------------------
+  DB_SYNC_CONNECTION_STRING: 'postgresql://postgres:doNoUseThisSecret!@localhost:5435/cexplorer'
   KEY_MANAGEMENT_PROVIDER: 'inMemory'
   KEY_MANAGEMENT_PARAMS: '{"bip32Ed25519": "Sodium", "accountIndex": 0, "chainId":{"networkId": 0, "networkMagic": 888}, "passphrase":"some_passphrase","mnemonic":"vacant violin soft weird deliver render brief always monitor general maid smart jelly core drastic erode echo there clump dizzy card filter option defense"}'
+  OGMIOS_URL: 'ws://localhost:1340/'
+  STAKE_POOL_CONNECTION_STRING: 'postgresql://postgres:doNoUseThisSecret!@localhost:5435/stake_pool'
+  STAKE_POOL_TEST_CONNECTION_STRING: 'postgresql://postgres:doNoUseThisSecret!@localhost:5435/stake_pool_test'
   TEST_CLIENT_ASSET_PROVIDER: 'http'
   TEST_CLIENT_ASSET_PROVIDER_PARAMS: '{"baseUrl":"http://localhost:4014/"}'
   TEST_CLIENT_CHAIN_HISTORY_PROVIDER: 'ws'
   TEST_CLIENT_CHAIN_HISTORY_PROVIDER_PARAMS: '{"baseUrl":"http://localhost:4000/"}'
-  DB_SYNC_CONNECTION_STRING: 'postgresql://postgres:doNoUseThisSecret!@localhost:5435/cexplorer'
   TEST_CLIENT_HANDLE_PROVIDER: 'http'
   TEST_CLIENT_HANDLE_PROVIDER_PARAMS: '{"baseUrl":"http://localhost:4011/"}'
-  STAKE_POOL_CONNECTION_STRING: 'postgresql://postgres:doNoUseThisSecret!@localhost:5435/stake_pool'
-  STAKE_POOL_TEST_CONNECTION_STRING: 'postgresql://postgres:doNoUseThisSecret!@localhost:5435/stake_pool_test'
   TEST_CLIENT_NETWORK_INFO_PROVIDER: 'ws'
   TEST_CLIENT_NETWORK_INFO_PROVIDER_PARAMS: '{"baseUrl":"http://localhost:4000/"}'
-  OGMIOS_URL: 'ws://localhost:1340/'
   TEST_CLIENT_REWARDS_PROVIDER: 'http'
   TEST_CLIENT_REWARDS_PROVIDER_PARAMS: '{"baseUrl":"http://localhost:4000/"}'
   TEST_CLIENT_TX_SUBMIT_PROVIDER: 'http'

compose/common.yml

@@ -79,8 +79,6 @@ x-provider-server-environment: &provider-server-environment
   TX_SUBMIT_PROVIDER: ${TX_SUBMIT_PROVIDER:-submit-node}
   STAKE_POOL_PROVIDER: ${STAKE_POOL_PROVIDER:-dbsync}
   NETWORK: ${NETWORK:-mainnet}
-  BLOCKFROST_API_KEY: ${BLOCKFROST_API_KEY}
-  BLOCKFROST_CUSTOM_BACKEND_URL: ${BLOCKFROST_CUSTOM_BACKEND_URL}
 
 x-sdk-environment: &sdk-environment
   LOGGER_MIN_SEVERITY: ${LOGGER_MIN_SEVERITY:-info}
@@ -118,21 +116,23 @@ x-sdk-environment: &sdk-environment
   POSTGRES_USER_FILE_STAKE_POOL: /run/secrets/postgres_user
   POSTGRES_USER_FILE_WALLET_API: /run/secrets/postgres_user
   TOKEN_METADATA_SERVER_URL: https://metadata.world.dev.cardano.org
-  USE_WEB_SOCKET_API: true
   WEB_SOCKET_API_URL: ws://ws-server:3000/ws
 
 services:
   blockfrost-ryo:
     build:
-      context: "https://github.com/ginnun/blockfrost-backend-ryo.git#feat/custom-network-support"
+      context: 'https://github.com/ginnun/blockfrost-backend-ryo.git#feat/custom-network-support'
       dockerfile: Dockerfile
     environment:
       BLOCKFROST_CONFIG_SERVER_LISTEN_ADDRESS: 0.0.0.0
     depends_on:
       cardano-db-sync:
         condition: service_started
+    healthcheck:
+      test: ['CMD-SHELL', 'curl -s --fail http://localhost:3000/health']
     ports:
-      - "3015:3000"
+      - 3015:3000
+    restart: always
 
   cardano-db-sync:
     <<:
@@ -350,9 +350,34 @@ services:
       <<:
         - *sdk-environment
         - *provider-server-environment
+      USE_WEB_SOCKET_API: true
     ports:
       - ${API_PORT:-4000}:3000
 
+  blockfrost-provider-server:
+    <<:
+      - *from-sdk
+      - *logging
+      - *provider-server
+      - *with-postgres
+    depends_on:
+      blockfrost-ryo:
+        condition: service_healthy
+    environment:
+      <<:
+        - *sdk-environment
+        - *provider-server-environment
+      # ATM we don't have BlockfrostHandleProvider and BlockfrostStakePoolProvider
+      ASSET_PROVIDER: blockfrost
+      BLOCKFROST_CUSTOM_BACKEND_URL: 'http://blockfrost-ryo:3000'
+      CHAIN_HISTORY_PROVIDER: blockfrost
+      NETWORK_INFO_PROVIDER: blockfrost
+      REWARDS_PROVIDER: blockfrost
+      UTXO_PROVIDER: blockfrost
+    ports:
+      - ${API_PORT:-4001}:3000
+      - 9229:9229
+
   stake-pool-provider-server:
     <<:
       - *from-sdk

packages/cardano-services/package.json

@@ -29,7 +29,7 @@
     "cleanup": "rm -rf dist node_modules",
     "cli": "ts-node --transpile-only src/cli.ts",
     "compose:single:up": "yarn compose:up cardano-node ogmios postgres",
-    "compose:up": "__FIX_UMASK__=$(chmod -R a+r ../../compose/placeholder-secrets) docker compose --env-file environments/.env.$NETWORK -p cardano-services-$NETWORK -f docker-compose.yml -f ../../compose/common.yml -f ../../compose/pg-agent.yml -f ../../compose/$(uname -m).yml ${FILES:-}  --profile ${DOCKER_COMPOSE_PROFILE:-none} up",
+    "compose:up": "__FIX_UMASK__=$(chmod -R a+r ../../compose/placeholder-secrets) docker compose --env-file environments/.env.$NETWORK -p cardano-services-$NETWORK -f docker-compose.yml -f ../../compose/common.yml -f ../../compose/pg-agent.yml -f ../../compose/$(uname -m).yml ${FILES:-} up",
     "compose:down": "docker compose -p cardano-services-$NETWORK -f docker-compose.yml -f ../../compose/common.yml -f ../../compose/pg-agent.yml -f ../../compose/$(uname -m).yml down -t 120",
     "coverage": "yarn test --coverage || true",
     "circular-deps:check": "madge --circular dist/cjs",
@@ -38,21 +38,17 @@
     "generate-migration": "typeorm-ts-node-commonjs migration:generate src/Projection/migrations/migrations -d src/migrationDataSource.ts",
     "mainnet:single:up": "NETWORK=mainnet yarn compose:single:up",
     "mainnet:up": "NETWORK=mainnet SUBMIT_API_ARGS=--mainnet yarn compose:up",
-    "mainnet:blockfrost:up": "DOCKER_COMPOSE_PROFILE='blockfrost-ryo' yarn mainnet:up",
     "mainnet:down": "NETWORK=mainnet yarn compose:down",
     "prepack": "yarn build",
     "preprod:single:up": "NETWORK=preprod yarn compose:single:up",
     "preprod:up": "NETWORK=preprod SUBMIT_API_ARGS='--testnet-magic 1' yarn compose:up",
-    "preprod:blockfrost:up": "DOCKER_COMPOSE_PROFILE='blockfrost-ryo' yarn preprod:up",
     "preprod:down": "NETWORK=preprod yarn compose:down",
     "pretest": "yarn build",
     "preview:single:up": "NETWORK=preview yarn compose:single:up",
     "preview:up": "NETWORK=preview SUBMIT_API_ARGS='--testnet-magic 2' yarn compose:up",
-    "preview:blockfrost:up": "DOCKER_COMPOSE_PROFILE='blockfrost-ryo' yarn preview:up",
     "preview:down": "NETWORK=preview yarn compose:down",
     "sanchonet:single:up": "NETWORK=sanchonet yarn compose:single:up",
     "sanchonet:up": "NETWORK=sanchonet SUBMIT_API_ARGS='--testnet-magic 4' yarn compose:up",
-    "sanchonet:blockfrost:up": "DOCKER_COMPOSE_PROFILE='blockfrost-ryo' yarn sanchonet:up",
     "sanchonet:down": "NETWORK=sanchonet yarn compose:down",
     "test": "jest --runInBand -c ./jest.config.js --selectProjects unit",
     "test:build:verify": "tsc --build ./test",