Upgrade tracker

Author: opcatdev

packages/tracker/.env.example

@@ -12,4 +12,5 @@ RPC_PASSWORD=opcatAwesome
 
 NETWORK=mainnet
 API_PORT=3000
+WORKER_PORT=3001
 GENESIS_BLOCK_HEIGHT=0 # the height of block that CAT protocol launches

packages/tracker/.gitignore

@@ -55,7 +55,5 @@ pids
 # Diagnostic reports (https://nodejs.org/api/report.html)
 report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
 
-/docker/data/*
+/docker/*
 !/docker/data/bitcoin.conf
-/docker/pgdata/*
-!/docker/pgdata/.gitkeep

packages/tracker/README.md

@@ -1,77 +1,68 @@
 # CAT Tracker
 
-The tracker reads CAT token transactions from the blockchain, stores them in a database (`Postgres`) in a structured way, which can be quickly retrieved via RESTful APIs. The Swagger documentation for all the APIs can be found at http://127.0.0.1:3000 after running.
+The tracker reads CAT token transactions from the blockchain, stores them in Postgres in a structured way, which can be quickly retrieved via RESTful APIs. The Swagger documentation for all the APIs can be found at http://127.0.0.1:3000 after running.
 
 ## Installation
 
 ```bash
-yarn install
+yarn install && yarn build
 ```
 
-## Build
-
-```sh
-yarn build
-```
-
-## Before Run
+## Prerequisite
 
 The tracker needs a full node and Postgres. We use Fractal node as an example here.
 
-Make sure you have `docker` installed, you can follow this [guide](https://docs.docker.com/engine/install/) to install it.
+Make sure you have `docker` installed, or follow this [guide](https://docs.docker.com/engine/install/) to install it.
 
 1. Update `.env` file with your own configuration.
-2. Update directory permission
+
+2. Update directory permission to avoid `docker` doesn't have rights to write data.
 
 ```bash
-sudo chmod 777 docker/data
-sudo chmod 777 docker/pgdata
+sudo chmod o+w docker/data
 ```
 
-3. Run `postgresql` and `bitcoind`:
+3. Run `postgresql` and `bitcoind`.
 
 ```bash
 docker compose up -d
 ```
 
-## Run the tracker service
-
-### Use `Docker` (Recommended)
+## Migration
 
-1. Build docker image under the project root directory
+Run the following command to initialize database tables for the first-time run, or migrate data when upgrading from a previous version.
 
 ```bash
-cd ../../ && docker build -t tracker:latest .
+yarn migration:run
 ```
 
-2. Run the container
+**Note: If upgrading the tracker from a previous version, migrating the table data may take a few hours. Make a [database backup](https://www.postgresql.org/docs/current/app-pgdump.html) before this.**
 
-```bash
-docker run -d \
-    --name tracker \
-    --add-host="host.docker.internal:host-gateway" \
-    -e DATABASE_HOST="host.docker.internal" \
-    -e RPC_HOST="host.docker.internal" \
-    -p 3000:3000 \
-    tracker:latest
-```
+## Run
 
-3. Check tracker logs
+The tracker consists of two modules:
 
-```bash
-docker logs -f tracker
-```
+- `worker`, reads CAT token transactions from the blockchain and stores them in a database.
+- `api`, reads data from the database and serves the RESTful APIs.
 
-### Use `yarn`
+To run the `worker` module:
 
-* development mode
 ```bash
-yarn run start
+# development
+yarn start:worker
+
+# production mode
+yarn start:worker:prod
 ```
 
-* production mode
+To run the `api` module:
+
 ```bash
-yarn run start:prod
+# development
+yarn start:api
+
+# production mode
+yarn start:api:prod
 ```
 
-> **Note:** Make sure the tracker syncs to the latest block before you run CLI over it. The sync-up progress can be found at http://127.0.0.1:3000/api after running.
+**Note:** Make sure the tracker syncs to the latest block before you run CLI over it. The sync-up progress can be found at http://127.0.0.1:3000/api after running.

packages/tracker/docker-compose.yml

@@ -15,26 +15,11 @@ services:
   bitcoind:
     image: fractalbitcoin/fractal:v0.2.1
     restart: always
-    entrypoint: ["bitcoind", "-datadir=/data/", "-maxtipage=504576000"]
-    command: ""
+    entrypoint: ["bitcoind", "-datadir=/data/"]
     healthcheck:
       test: ["CMD", "bitcoin-cli", "-datadir=/data/", "getblockchaininfo"]
     ports:
-      - "8330:8330"
-      - "8331:8331"
       - "8332:8332"
       - "8333:8333"
-    deploy:
-      resources:
-        limits:
-          memory: 40G
-    memswap_limit: 60G
-    mem_swappiness: 100
     volumes:
       - ./docker/data:/data
-    logging:
-      driver: "json-file"
-      options:
-        labels: "env,filebeat,name"
-        max-size: "1g"
-        max-file: "3"

packages/tracker/docker/data/bitcoin.conf

@@ -13,10 +13,4 @@ rpcworkqueue=2048
 rpcthreads=32
 rpcservertimeout=120
 
-zmqpubhashblock=tcp://0.0.0.0:8330
-zmqpubrawtx=tcp://0.0.0.0:8331
-
 maxconnections=50
-
-debug=bench
-debug=netmsg

packages/tracker/nest-cli.json renamed to packages/tracker/nest-cli-api.json

@@ -2,6 +2,7 @@
   "$schema": "https://json.schemastore.org/nest-cli",
   "collection": "@nestjs/schematics",
   "sourceRoot": "src",
+  "entryFile": "main-api",
   "compilerOptions": {
     "deleteOutDir": true
   }

packages/tracker/nest-cli-worker.json

@@ -0,0 +1,9 @@
+{
+  "$schema": "https://json.schemastore.org/nest-cli",
+  "collection": "@nestjs/schematics",
+  "sourceRoot": "src",
+  "entryFile": "main-worker",
+  "compilerOptions": {
+    "deleteOutDir": true
+  }
+}

packages/tracker/package.json

@@ -8,10 +8,22 @@
   "scripts": {
     "build": "nest build",
     "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
-    "start": "nest start",
-    "start:dev": "nest start --watch",
-    "start:debug": "nest start --debug --watch",
-    "start:prod": "node dist/main",
+
+    "start:worker": "nest start -c nest-cli-worker.json",
+    "start:worker:dev": "nest start --watch -c nest-cli-worker.json",
+    "start:worker:debug": "nest start --debug --watch -c nest-cli-worker.json",
+    "start:worker:prod": "yarn build && node dist/main-worker",
+
+    "start:api": "nest start -c nest-cli-api.json",
+    "start:api:dev": "nest start --watch -c nest-cli-api.json",
+    "start:api:debug": "nest start --debug --watch -c nest-cli-api.json",
+    "start:api:prod": "yarn build && node dist/main-api",
+
+    "typeorm": "npx typeorm-ts-node-commonjs -d src/config/db.config.ts",
+    "migration:generate": "npm run typeorm -- migration:generate",
+    "migration:run": "npm run typeorm -- migration:run",
+    "migration:revert": "npm run typeorm -- migration:revert",
+
     "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
     "test": "jest",
     "test:watch": "jest --watch",

packages/tracker/src/app-api.module.ts

@@ -0,0 +1,35 @@
+import { Module } from '@nestjs/common';
+import { TypeOrmModule } from '@nestjs/typeorm';
+import { ConfigModule } from '@nestjs/config';
+// config
+import { appConfig } from './config/app.config';
+import { ormConfig } from './config/db.config';
+// routes
+import { HealthCheckModule } from './routes/healthCheck/healthCheck.module';
+import { TokenModule } from './routes/token/token.module';
+import { MinterModule } from './routes/minter/minter.module';
+import { AddressModule } from './routes/address/address.module';
+// serivces
+import { CommonModule } from './services/common/common.module';
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+require('dotenv').config();
+
+@Module({
+  imports: [
+    ConfigModule.forRoot({
+      load: [appConfig],
+    }),
+    TypeOrmModule.forRoot(ormConfig),
+
+    HealthCheckModule,
+    TokenModule,
+    MinterModule,
+    AddressModule,
+
+    CommonModule,
+  ],
+  controllers: [],
+  providers: [],
+})
+export class AppApiModule {}

packages/tracker/src/app-worker.module.ts

@@ -0,0 +1,31 @@
+import { Module } from '@nestjs/common';
+import { TypeOrmModule } from '@nestjs/typeorm';
+import { ConfigModule } from '@nestjs/config';
+// config
+import { appConfig } from './config/app.config';
+import { ormConfig } from './config/db.config';
+// services
+import { RpcModule } from './services/rpc/rpc.module';
+import { BlockModule } from './services/block/block.module';
+import { TxModule } from './services/tx/tx.module';
+import { CommonModule } from './services/common/common.module';
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+require('dotenv').config();
+
+@Module({
+  imports: [
+    ConfigModule.forRoot({
+      load: [appConfig],
+    }),
+    TypeOrmModule.forRoot(ormConfig),
+
+    RpcModule,
+    BlockModule,
+    TxModule,
+    CommonModule,
+  ],
+  controllers: [],
+  providers: [],
+})
+export class AppWorkerModule {}