feat: Add bendsave which can backup and restore databend data

.github/actions/build_linux/action.yml

@@ -68,7 +68,7 @@ runs:
       if: inputs.artifacts == 'all'
       shell: bash
       run: |
-        artifacts="meta,metactl,metabench,query,sqllogictests"
+        artifacts="meta,metactl,metabench,query,sqllogictests,bendsave"
         binaries=""
         for artifact in ${artifacts//,/ }; do
           binaries="${binaries} --bin databend-$artifact"

.github/actions/test_sqllogic_standalone_linux_minio_with_bendsave/action.yml

@@ -0,0 +1,93 @@
+name: "Test sqllogic Standalone(minio) with bendsave"
+description: "Running sqllogic tests in standalone mode"
+
+runs:
+  using: "composite"
+  steps:
+    - uses: ./.github/actions/setup_test
+      with:
+        artifacts: sqllogictests,meta,query,bendsave
+
+    - name: Minio Setup for (ubuntu-latest only)
+      shell: bash
+      run: |
+        docker run -d --network host --name minio \
+                -e "MINIO_ACCESS_KEY=minioadmin" \
+                -e "MINIO_SECRET_KEY=minioadmin" \
+                -e "MINIO_ADDRESS=:9900" \
+                -v /tmp/data:/data \
+                -v /tmp/config:/root/.minio \
+                minio/minio server /data
+
+        export AWS_ACCESS_KEY_ID=minioadmin
+        export AWS_SECRET_ACCESS_KEY=minioadmin
+        export AWS_EC2_METADATA_DISABLED=true
+
+        aws --endpoint-url http://127.0.0.1:9900/ s3 mb s3://testbucket
+        aws --endpoint-url http://127.0.0.1:9900/ s3 mb s3://backupbucket
+
+    - name: Run sqllogic Tests with Standalone mode
+      shell: bash
+      env:
+        TEST_HANDLERS: "http"
+      run: bash ./scripts/ci/ci-run-sqllogic-tests-minio-with-bendsave.sh tpch
+
+    - name: Run bendsave backup
+      shell: bash
+      env:
+        AWS_ACCESS_KEY_ID: minioadmin
+        AWS_SECRET_ACCESS_KEY: minioadmin
+        AWS_EC2_METADATA_DISABLED: true
+        AWS_REGION: us-west-2
+      run: |
+        export STORAGE_TYPE=s3
+        export STORAGE_S3_BUCKET=testbucket
+        export STORAGE_S3_ROOT=admin
+        export STORAGE_S3_ENDPOINT_URL=http://127.0.0.1:9900
+        export STORAGE_S3_ACCESS_KEY_ID=minioadmin
+        export STORAGE_S3_SECRET_ACCESS_KEY=minioadmin
+        export STORAGE_ALLOW_INSECURE=true
+
+        ./target/${{ env.BUILD_PROFILE }}/databend-bendsave backup --from ./scripts/ci/deploy/config/databend-query-node-1.toml --to s3://backupbucket?endpoint=http://127.0.0.1:9900/
+
+        aws --endpoint-url http://127.0.0.1:9900/ s3 ls s3://backupbucket --recursive
+
+    - name: Destroy the existing services.
+      shell: bash
+      env:
+        AWS_ACCESS_KEY_ID: minioadmin
+        AWS_SECRET_ACCESS_KEY: minioadmin
+        AWS_EC2_METADATA_DISABLED: true
+        AWS_REGION: us-west-2
+      run: |
+        # kill all services
+        pkill databend-query
+        pkill databend-meta
+        # destory databend query
+        aws --endpoint-url http://127.0.0.1:9900/ s3 rm s3://testbucket --recursive
+        # destory databend meta
+        rm -rf /tmp/databend/meta1
+
+    - name: Run bendsave restore
+      shell: bash
+      env:
+        AWS_ACCESS_KEY_ID: minioadmin
+        AWS_SECRET_ACCESS_KEY: minioadmin
+        AWS_EC2_METADATA_DISABLED: true
+        AWS_REGION: us-west-2
+      run: |
+        export STORAGE_TYPE=s3
+        export STORAGE_S3_BUCKET=testbucket
+        export STORAGE_S3_ROOT=admin
+        export STORAGE_S3_ENDPOINT_URL=http://127.0.0.1:9900
+        export STORAGE_S3_ACCESS_KEY_ID=minioadmin
+        export STORAGE_S3_SECRET_ACCESS_KEY=minioadmin
+        export STORAGE_ALLOW_INSECURE=true
+
+        ./target/${{ env.BUILD_PROFILE }}/databend-bendsave restore --from s3://backupbucket?endpoint=http://127.0.0.1:9900/ --to-query ./scripts/ci/deploy/config/databend-query-node-1.toml --to-meta ./scripts/ci/deploy/config/databend-meta-node-for-bendsave.toml --confirm
+
+    - name: Run sqllogic Tests with Standalone mode again for testing
+      shell: bash
+      env:
+        TEST_HANDLERS: "http"
+      run: bash ./scripts/ci/ci-run-sqllogic-tests-minio-with-bendsave.sh tpch

.github/workflows/reuse.sqllogic.yml

@@ -25,7 +25,7 @@ env:
 
 jobs:
   management_mode:
-    runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
+    runs-on: [ self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}" ]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_sqllogic_management_mode_linux
@@ -72,7 +72,7 @@ jobs:
           name: test-sqllogic-standalone-${{ matrix.tests.dirs }}-${{ matrix.handler }}
 
   standalone_udf_server:
-    runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
+    runs-on: [ self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}" ]
     steps:
       - uses: actions/checkout@v4
       - name: Start UDF Server
@@ -94,7 +94,7 @@ jobs:
           name: test-sqllogic-standalone-udf-server
 
   standalone_cloud:
-    runs-on: [self-hosted, X64, Linux, 4c16g, "${{ inputs.runner_provider }}"]
+    runs-on: [ self-hosted, X64, Linux, 4c16g, "${{ inputs.runner_provider }}" ]
     steps:
       - uses: actions/checkout@v4
       - name: Start Cloud Control Server
@@ -115,7 +115,7 @@ jobs:
           name: test-sqllogic-standalone-cloud
 
   standalone_minio:
-    runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
+    runs-on: [ self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}" ]
     strategy:
       fail-fast: false
       matrix:
@@ -141,8 +141,24 @@ jobs:
         with:
           name: test-sqllogic-standalone-minio-${{ matrix.dirs }}-${{ matrix.handler }}-${{ matrix.format }}
 
+  standalone_minio_with_bendsave:
+    runs-on: [ self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}" ]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup_license
+        with:
+          runner_provider: ${{ inputs.runner_provider }}
+          type: ${{ inputs.license_type }}
+      - uses: ./.github/actions/test_sqllogic_standalone_linux_minio_with_bendsave
+        timeout-minutes: 30
+      - name: Upload failure
+        if: failure()
+        uses: ./.github/actions/artifact_failure
+        with:
+          name: test-sqllogic-standalone-minio-tpch-http-with-bendsave
+
   standalone_iceberg_tpch:
-    runs-on: [self-hosted, X64, Linux, 4c16g, "${{ inputs.runner_provider }}"]
+    runs-on: [ self-hosted, X64, Linux, 4c16g, "${{ inputs.runner_provider }}" ]
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-java@v4
@@ -201,7 +217,7 @@ jobs:
           name: test-sqllogic-cluster-${{ matrix.tests.dirs }}-${{ matrix.handler }}
 
   stage:
-    runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
+    runs-on: [ self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}" ]
     strategy:
       fail-fast: false
       matrix:
@@ -230,7 +246,7 @@ jobs:
           name: test-sqllogic-stage-${{ matrix.storage }}-${{ matrix.handler }}-${{ matrix.dedup }}
 
   standalone_no_table_meta_cache:
-    runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
+    runs-on: [ self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}" ]
     strategy:
       fail-fast: false
       matrix:
@@ -254,7 +270,7 @@ jobs:
           name: test-sqllogic-standalone-no-table-meta-cache-${{ matrix.dirs }}-${{ matrix.handler }}
 
   ee:
-    runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
+    runs-on: [ self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}" ]
     strategy:
       fail-fast: false
       matrix: