shutter-network · ylembachar · Sep 4, 2025 · Sep 4, 2025 · Sep 4, 2025 · Sep 4, 2025
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,5 @@
 .env
 data/
 config/
+.idea
+.env.backup*
diff --git a/README.md b/README.md
@@ -145,7 +145,17 @@ docker compose -f docker-compose.yml -f docker-compose.loki.yml up -d
 
 ## Backups
 
-Once your Keyper is up and running, you should regularly back up the following:
+### New Backup and Restore Guide
+
+We have introduced a new backup and restore process with dedicated scripts to make it easier and safer to preserve your Keyper’s data.
+
+**All keypers should follow the new process** described in the [Backup and Restore Guide](scripts/BACKUP_RESTORE.md) to ensure backups are complete and restores work correctly.
+
+### Previous Manual Backup
+
+(Deprecated – use only for reference.)
+
+If you are still using the manual process, regularly back up the following:
 
 - `.env`
 - `./config`
@@ -174,6 +184,14 @@ docker compose -f docker-compose.yml -f docker-compose.loki.yml up -d
 ```
 
 ## Version History
+### `gnosis/2025.09.01` – `2025-09-04`
+- Upgrade to [rolling-shutter v1.3.12](https://github.com/shutter-network/rolling-shutter/releases/tag/v1.3.12)
+- Add logic to allow for DB migrations
+- Fix issue with context cancellation
+- Improve keyper metrics to include DKG results, ETH address and EL/CL clients
+- Introduced a new backup and restore process with dedicated scripts.
+  The previous manual backup method is now deprecated — operators should follow
+  the new [Backup and Restore Guide](scripts/BACKUP_RESTORE.md).
 
 ### `gnosis/2025.06.01` – `2025-06-03`
 - Upgrade to **Gnosis Keyper v1.3.10**
@@ -235,4 +253,4 @@ Initial public release
   ValidatorRegistry: 0xefCC23E71f6bA9B22C4D28F7588141d44496A6D6
   keyperSetManager: 0x7C2337f9bFce19d8970661DA50dE8DD7d3D34abb
   keyBroadcastContract: 0x626dB87f9a9aC47070016A50e802dd5974341301
-```
+```
diff --git a/_container_scripts/keyper-db-init.sh b/_container_scripts/keyper-db-init.sh
@@ -2,4 +2,13 @@
 
 set -e
 
-createdb -U postgres keyper
+echo "Checking for backup dump file..."
+if [ -f "/var/lib/postgresql/dump/keyper.dump" ]; then
+    echo "Backup dump found, restoring database with full schema and data..."
+    pg_restore -U postgres -d postgres --create --clean -v /var/lib/postgresql/dump/keyper.dump
+    rm -f /var/lib/postgresql/dump/keyper.dump
+    echo "Database restore completed."
+else
+    echo "No backup dump file found, creating fresh database..."
+    createdb -U postgres keyper
+fi
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -4,7 +4,7 @@ x-logging: &logging
     max-file: 10
 
 x-image-main: &image-main
-  image: ${SHUTTER_IMAGE_ROLLING_SHUTTER:-ghcr.io/shutter-network/keyper:v1.3.10}
+  image: ${SHUTTER_IMAGE_ROLLING_SHUTTER:-ghcr.io/shutter-network/keyper:v1.3.12}
 
 x-image-assets: &image-assets
   image: ${SHUTTER_IMAGE_ASSETS:-ghcr.io/shutter-network/assets:shutter-gnosis-1000-set1.3}
@@ -51,6 +51,8 @@ services:
     volumes:
       - ./data/db:/var/lib/postgresql/data
       - ./_container_scripts/keyper-db-init.sh:/docker-entrypoint-initdb.d/keyper-db-init.sh:ro
+      - ./data/db-dump:/var/lib/postgresql/dump
+
     healthcheck:
       test: pg_isready -U postgres
       start_period: "30s"

diff --git a/scripts/BACKUP_RESTORE.md b/scripts/BACKUP_RESTORE.md
@@ -0,0 +1,110 @@
+# Backup and Restore Guide
+
+## Backup Process
+
+### Creating a Backup
+
+1. **Ensure services are running** - The backup process requires the database to be accessible
+2. **Run the backup script**:
+   ```bash
+   # Use default backup directory (data/backups/)
+   ./scripts/backup.sh
+
+   # Use custom backup directory
+   ./scripts/backup.sh /path/to/backups
+
+   # Show help
+   ./scripts/backup.sh -h
+   ```
+3. **Backup location** - Backups are stored in the specified directory (default: `data/backups/`)
+4. **Backup naming** - Files are named with timestamp: `gnosis-keyper-YYYY-MM-DDTHH-MM-SS.tar.xz`
+5. **Sanity check** - After the backup completes, verify in keyper logs that it resyncs events
+
+### What Gets Backed Up
+
+- Database dump (`keyper.dump`) - Contains full schema and data from the `keyper` database
+- Chain data (`data/chain/`) - Blockchain data and configuration
+- Keyper configuration (`config/`) - Application configuration files
+- Environment variables - Except Signing Key
+
+### What's NOT Backed Up
+
+- **Signing Key** - The `SIGNING_KEY` environment variable is intentionally excluded from backups for security reasons. You must manually preserve this value separately.
+
+### Security Considerations
+
+⚠️ **IMPORTANT**: Backup files contain sensitive information including:
+- Database contents with potentially sensitive data
+- Configuration files that may contain API keys, passwords, or other secrets
+- Chain data that could be used to reconstruct transaction history
+
+**Security Best Practices:**
+- Store backups on a different machine or secure cloud storage
+- Limit access to backup files to authorized personnel only
+- Consider using backup encryption tools for additional security
+
+## Restore Process
+
+### Prerequisites
+
+- **Empty keyper instance** - The restore *must* be performed on a fresh, empty deployment
+- **No running services** - Ensure all Docker containers are stopped before restore
+- **Backup file available** - The backup archive should be present in the specified backup directory
+- **Signing key available** - You must have the original `SIGNING_KEY` value from your deployment
+
+### Restore Steps
+
+1. **Run restore script**:
+   ```bash
+   # Use default backup directory (data/backups/)
+   ./scripts/restore.sh
+
+   # Use custom backup directory
+   ./scripts/restore.sh /path/to/backups
+
+   # Show help
+   ./scripts/restore.sh -h
+   ```
+    - This will automatically find the latest backup in the specified directory
+    - Prompts for confirmation before proceeding
+    - Restores all data to appropriate locations
+
+2. **Set the Signing Key**:
+    - After restoring, update the `.env` file by setting the `SIGNING_KEY` environment variable to the same value used in your original deployment.
+    - **CRITICAL**: Without the correct signing key, the restored deployment will not function properly and may not be able to process transactions.
+
+3. **Start services**:
+   If using loki log collection, please follow instructions given in README.md.
+
+   For basic restart:
+   ```bash
+   docker compose up -d
+   ```
+
+4. **Environment file preservation**:
+   The restore script automatically preserves any existing `.env` file by creating a timestamped backup (`.env.backup.YYYY-MM-DDTHH-MM-SS`) before overwriting it with restored configuration. This ensures no environment variables are lost during the restore process.
+
+### Restore Locations
+
+- **Database**: `data/db-dump/keyper.dump` - Automatically restored to PostgreSQL
+- **Chain data**: `data/chain/` - Keyper chain data and configuration
+- **Configuration**: `config/` - Application configuration files
+- **Environment**: `.env` - Updated with restored metrics settings
+
+### Important Notes
+
+- **Database restoration** - The database is automatically restored on first startup via the initialization script
+- **Service order** - Restore must be completed before starting any services
+- **Data integrity** - The restore process overwrites existing data; ensure you have a clean instance
+- **Configuration review** - Review restored configuration files before starting services
+- **Backup directory** - Both scripts accept an optional backup directory parameter, defaulting to `data/backups/`
+- **Incomplete recovery** - Backups do not contain the signing key; manual intervention is required to complete the restore
+- **Security** - Restored data may contain sensitive information; ensure proper access controls are in place
+
+### Troubleshooting
+
+- **No backup found** - Ensure backup files exist in the specified backup directory
+- **Permission errors** - Ensure proper file permissions on backup files and directories
+- **Configuration issues** - Verify that restored configuration files are valid
+- **Custom backup locations** - When using custom backup directories, ensure the path is accessible and writable
+- **Missing signing key** - You should be able to have original signing key at the time of restore.
diff --git a/scripts/backup.sh b/scripts/backup.sh
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+R='\033[0;31m'
+G='\033[0;32m'
+Y='\033[0;33m'
+B='\033[0;34m'
+DEF='\033[0m'
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+# Default backup directory
+DEFAULT_BACKUPS_DIR="${SCRIPT_DIR}/../data/backups"
+
+# Show usage if help is requested
+if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
+    echo "Usage: $0 [BACKUP_DIRECTORY]"
+    echo ""
+    echo "Creates a backup archive of the shutter-keyper deployment."
+    echo ""
+    echo "Arguments:"
+    echo "  BACKUP_DIRECTORY    Directory to store the backup (default: $DEFAULT_BACKUPS_DIR)"
+    echo ""
+    echo "Examples:"
+    echo "  $0                           # Use default backup directory"
+    echo "  $0 /path/to/backups         # Use custom backup directory"
+    echo "  $0 -h                       # Show this help message"
+    exit 0
+fi
+
+# Parse command line arguments
+BACKUPS_DIR="${1:-$DEFAULT_BACKUPS_DIR}"
+
+ARCHIVE_NAME="gnosis-keyper-$(date +%Y-%m-%dT%H-%M-%S).tar.xz"
+
+source "${SCRIPT_DIR}/../.env"
+
+mkdir -p "$BACKUPS_DIR"
+WORKDIR=$(mktemp -d -p "${BACKUPS_DIR}")
+
+cleanup_and_restart() {
+  rv=$?
+  set +e
+
+  rm -rf "$WORKDIR"
+  docker compose start
+
+  if [ $rv -ne 0 ]; then
+    echo -e "${R}Unexpected error, exit code: $rv${DEF}"
+  fi
+
+  exit $rv
+}
+
+trap cleanup_and_restart EXIT
+
+echo -e "${G}Creating backup archive${DEF}"
+echo -e "${B}Backup directory: ${Y}$BACKUPS_DIR${DEF}"
+
+echo -e "${B}[1/6] Stopping all services except database...${DEF}"
+docker compose stop keyper
+docker compose stop chain
+
+echo -e "${B}[2/6] Creating database dump...${DEF}"
+docker compose exec db pg_dump -U postgres -d keyper -Fc --create --clean -f /var/lib/postgresql/data/keyper.dump
+
+echo -e "${B}[3/6] Stopping database...${DEF}"
+docker compose stop db
+
+echo -e "${B}[4/6] Copying data...${DEF}"
+cp -a "${SCRIPT_DIR}/../data/chain/" "${WORKDIR}/chain"
+cp -a "${SCRIPT_DIR}/../data/db/keyper.dump" "${WORKDIR}/keyper.dump"
+cp -a "${SCRIPT_DIR}/../config" "${WORKDIR}/keyper-config"
+
+mkdir -p "${WORKDIR}/env-config"
+if [ -f "${SCRIPT_DIR}/../.env" ]; then
+    sed 's/^SIGNING_KEY=.*/SIGNING_KEY=PLACEHOLDER_REPLACE_WITH_YOUR_PRIVATE_KEY/' "${SCRIPT_DIR}/../.env" > "${WORKDIR}/env-config/.env"
+    echo -e "${G}✓ Environment configuration backed up (private key replaced with placeholder)${DEF}"
+else
+    echo -e "${Y}⚠ .env file not found, skipping environment backup${DEF}"
+fi
+
+echo -e "${B}[5/6] Compressing archive...${DEF}"
+docker run --rm -it -v "${WORKDIR}:/workdir" -v "$BACKUPS_DIR:/data" alpine:3.20.1 ash -c "apk -q --no-progress --no-cache add xz pv && tar -cf - -C /workdir . | pv -petabs \$(du -sb /workdir | cut -f 1) | xz -zq > /data/${ARCHIVE_NAME}"
+
+echo -e "${B}[6/6] Cleaning up...${DEF}"
+rm "${SCRIPT_DIR}/../data/db/keyper.dump" || true
+
+echo -e "${G}Done, backup archive created at ${B}$BACKUPS_DIR/${ARCHIVE_NAME}${DEF}"
+
+echo -e "\n\n${R}WARNING, IMPORTANT!${DEF}"
+echo -e "${Y}If you import this backup, make sure to stop this deployment first!${DEF}"