refactor: introduce monorepo-like layout (#323)

Introduce the following repository layout:

- `bin` contains scripts shared by all components, currently `bin/migrate.js`
- `migrations` contains SQL migrations scripts and `index.js` for running them
- each component has its own directory with its own fly.toml
  - logshipper
  - api
  - spark-db
  - publish
- there is a single Dockerfile shared by all workspaces
- `publish` and `api` are npm workspaces with their own `package.json` file
- the top-level `package.json` file is a npm workspace root


Signed-off-by: Miroslav Bajtoš <[email protected]>

Author: bajtos

.dockerignore

@@ -3,3 +3,4 @@ Dockerfile
 .dockerignore
 node_modules
 .git
+.env

.github/dependabot.yml

@@ -3,11 +3,11 @@ updates:
   - package-ecosystem: "npm"
     directories:
       - "/"
-      - "/spark-publish"
     schedule:
       interval: "daily"
       time: "09:00"
       timezone: "Europe/Berlin"
+    versioning-strategy: increase
     commit-message:
       prefix: "deps"
       prefix-development: "deps(dev)"
@@ -28,7 +28,6 @@ updates:
   - package-ecosystem: "docker"
     directories:
       - "/"
-      - "/spark-publish"
     schedule:
       interval: "daily"
       time: "15:00"

.github/workflows/ci.yml

@@ -24,6 +24,7 @@ jobs:
     env:
       DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres
       GLIF_TOKEN: ${{ secrets.GLIF_TOKEN }}
+      NPM_CONFIG_WORKSPACE: api
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -50,40 +51,27 @@ jobs:
           --health-retries 5
     env:
       DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres
+      NPM_CONFIG_WORKSPACE: publish
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
           node-version: 20
       - run: npm ci
-      - run: npm run migrate
-      - run: cd spark-publish && npm ci
-      - run: cd spark-publish && npm test
+      - run: node bin/migrate.js
+      - run: npm test
 
-  docker-build-api:
+  lint-all:
     runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: read
-    env:
-      REGISTRY: ghcr.io
     steps:
       - uses: actions/checkout@v4
-
-      - uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build Docker image
-        uses: docker/build-push-action@v5
+      - uses: actions/setup-node@v4
         with:
-          context: .
-          cache-from: type=registry,ref=ghcr.io/filecoin-station/core
-          cache-to: type=inline
+          node-version: 20
+      - run: npm ci
+      - run: npm run lint
 
-  docker-build-publish:
+  docker-build:
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -99,21 +87,21 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build Docker image
+      - name: Build monorepo Docker image
         uses: docker/build-push-action@v5
         with:
-          context: spark-publish
+          context: .
           cache-from: type=registry,ref=ghcr.io/filecoin-station/core
           cache-to: type=inline
 
   deploy-api:
     if: github.ref == 'refs/heads/main'
-    needs: [build-api, build-publish, docker-build-api, docker-build-publish]
+    needs: [build-api, build-publish, docker-build]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: superfly/flyctl-actions/setup-flyctl@master
-      - run: flyctl deploy --remote-only
+      - run: flyctl deploy --remote-only -c api/fly.toml
         env:
           FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
       - if: failure()
@@ -137,12 +125,12 @@ jobs:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
   deploy-publish:
     if: github.ref == 'refs/heads/main'
-    needs: [build-api, build-publish, docker-build-api, docker-build-publish]
+    needs: [build-api, build-publish, docker-build]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: superfly/flyctl-actions/setup-flyctl@master
-      - run: cd spark-publish && flyctl deploy --remote-only
+      - run: flyctl deploy --remote-only -c publish/fly.toml
         env:
           FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN_PUBLISH }}
       - if: failure()

Dockerfile

@@ -12,23 +12,27 @@ WORKDIR /app
 # Set production environment
 ENV NODE_ENV production
 ENV SENTRY_ENVIRONMENT production
-ENV DOMAIN api.filspark.com
-ENV REQUEST_LOGGING false
 
 #######################################################################
 # Throw-away build stage to reduce size of final image
 FROM base as build
 
 # Install packages needed to build node modules
 RUN apt-get update -qq && \
-  apt-get install -y build-essential pkg-config python-is-python3
+    apt-get install -y build-essential pkg-config python-is-python3
 
 # Install node modules
 # NPM will not install any package listed in "devDependencies" when NODE_ENV is set to "production",
 # to install all modules: "npm install --production=false".
 # Ref: https://docs.npmjs.com/cli/v9/commands/npm-install#description
 COPY --link package-lock.json package.json ./
-RUN npm ci
+
+# We cannot use a wildcard until `COPY --parents` is stabilised
+# See https://docs.docker.com/reference/dockerfile/#copy---parents
+COPY --link api/package.json ./api/
+COPY --link publish/package.json ./publish/
+
+RUN npm ci --workspaces
 
 # Copy application code
 COPY --link . .
@@ -40,5 +44,8 @@ FROM base
 # Copy built application
 COPY --from=build /app /app
 
-# Start the server by default, this can be overwritten at runtime
-CMD [ "npm", "run", "start" ]
+# Set to `publish` or `api`
+# This argument controls the value passed to npm start --workspace parameter
+ENV SERVICE=""
+
+CMD npm start --workspace ${SERVICE}

README.md

@@ -84,18 +84,19 @@ docker run -d --name spark-db \
   postgres
 ```
 
-### `spark-api`
+### `api`
 
 Start the API service:
 
 ```bash
-npm start
+npm start --workspace api
 ```
 
 Run tests and linters:
 
 ```bash
-npm test
+npm test --workspace api
+npm run lint --workspace api
 ```
 
 ## Deployment
@@ -105,5 +106,5 @@ Pushes to `main` will be deployed automatically.
 Perform manual devops using [Fly.io](https://fly.io):
 
 ```bash
-$ fly deploy
+$ fly deploy api
 ```

bin/spark.js renamed to api/bin/spark.js

@@ -4,7 +4,7 @@ import { once } from 'node:events'
 import { createHandler } from '../index.js'
 import pg from 'pg'
 import { startRoundTracker } from '../lib/round-tracker.js'
-import { migrate } from '../lib/migrate.js'
+import { migrate } from '../../migrations/index.js'
 
 const {
   PORT = 8080,

fly.toml renamed to api/fly.toml

@@ -10,6 +10,9 @@ processes = []
   PORT = "8080"
   HOST = "0.0.0.0"
   SENTRY_ENVIRONMENT = "production"
+  DOMAIN = "api.filspark.com"
+  REQUEST_LOGGING = "false"
+  SERVICE="api"
 
 [experimental]
   auto_rollback = true

index.js renamed to api/index.js

@@ -1,5 +1,5 @@
 import { ethers } from 'ethers'
-import { RPC_URL, GLIF_TOKEN } from '../spark-publish/ie-contract-config.js'
+import { RPC_URL, GLIF_TOKEN } from '../../common/ie-contract-config.js'
 import * as SparkImpactEvaluator from '@filecoin-station/spark-impact-evaluator'
 
 const fetchRequest = new ethers.FetchRequest(RPC_URL)

lib/ie-contract.js renamed to api/lib/ie-contract.js

@@ -10,6 +10,7 @@ const pkg = JSON.parse(
     join(
       dirname(fileURLToPath(import.meta.url)),
       '..',
+      '..',
       'package.json'
     ),
     'utf8'

lib/inet-grouping.js renamed to api/lib/inet-grouping.js

@@ -0,0 +1,38 @@
+{
+  "name": "@filecoin-station/spark-api",
+  "private": true,
+  "version": "0.0.0",
+  "license": "MIT",
+  "repository": "filecoin-station/spark-api",
+  "type": "module",
+  "description": "API for SPARK",
+  "scripts": {
+    "start": "node bin/spark.js",
+    "lint": "standard",
+    "test": "mocha"
+  },
+  "devDependencies": {
+    "light-my-request": "^5.13.0",
+    "mocha": "^10.4.0",
+    "standard": "^17.1.0",
+    "varint": "^6.0.0"
+  },
+  "dependencies": {
+    "@filecoin-station/spark-impact-evaluator": "^1.1.1",
+    "@glif/filecoin-address": "^3.0.5",
+    "@sentry/node": "^8.7.0",
+    "compare-versions": "^6.1.0",
+    "ethers": "^6.12.1",
+    "http-assert": "^1.5.0",
+    "http-responders": "^2.0.2",
+    "multiformats": "^13.1.0",
+    "pg": "^8.11.5",
+    "postgrator": "^7.2.0",
+    "raw-body": "^2.5.2"
+  },
+  "standard": {
+    "env": [
+      "mocha"
+    ]
+  }
+}

lib/instrument.js renamed to api/lib/instrument.js

@@ -1,5 +1,5 @@
 import pg from 'pg'
-import { migrate } from '../lib/migrate.js'
+import { migrate } from '../../migrations/index.js'
 
 const { DATABASE_URL } = process.env
 

lib/round-tracker.js renamed to api/lib/round-tracker.js

@@ -3,7 +3,7 @@ import { setTimeout } from 'node:timers/promises'
 import pg from 'pg'
 import { mapRequestToInetGroup, mapRequestToSubnet, mapSubnetToInetGroup } from '../lib/inet-grouping.js'
 import { Request as FakeRequest } from 'light-my-request/lib/request.js'
-import { migrate } from '../lib/migrate.js'
+import { migrate } from '../../migrations/index.js'
 
 const { DATABASE_URL } = process.env
 

lib/validate.js renamed to api/lib/validate.js

@@ -6,8 +6,8 @@ import {
   mapCurrentMeridianRoundToSparkRound,
   startRoundTracker
 } from '../lib/round-tracker.js'
-import { migrate } from '../lib/migrate.js'
-import { assertApproximately } from './test-helpers.js'
+import { migrate } from '../../migrations/index.js'
+import { assertApproximately } from '../../test-helpers/assert.js'
 import { createMeridianContract } from '../lib/ie-contract.js'
 import { afterEach, beforeEach } from 'mocha'
 

api/package.json

@@ -1,4 +1,4 @@
-import { migrate } from '../lib/migrate.js'
+import { migrate } from '../migrations/index.js'
 import pg from 'pg'
 
 const { DATABASE_URL } = process.env