Harden backend: audit integrity, readiness probes, backup verification, and distributed tracing

.github/workflows/ci-post-deploy-verify.yml

@@ -1,228 +1,119 @@
-name: CI - Post-Deployment Verification
+name: Backup Restore Verification
 
 on:
   schedule:
-    - cron: "0 2 * * *"
+    - cron: '0 3 * * *'
   workflow_dispatch:
-    inputs:
-      network:
-        description: "Target network (testnet)"
-        required: true
-        default: "testnet"
-      invoice_contract_id:
-        description: "Deployed invoice contract ID (C…)"
-        required: true
-      treasury_contract_id:
-        description: "Deployed treasury contract ID (C…)"
-        required: true
-      compliance_contract_id:
-        description: "Deployed compliance contract ID (C…)"
-        required: true
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
 
 jobs:
-  deploy-testnet:
-    name: deploy (testnet)
+  verify-backup-restore:
+    name: Verify MongoDB and Redis backup restore
     runs-on: ubuntu-latest
-    # Deployment only runs on manual dispatch. The nightly scheduled run
-    # skips deployment and verifies the existing live testnet deployment.
+    timeout-minutes: 20
+
+    services:
+      mongo:
+        image: mongo:6
+        ports:
+          - 27017:27017
+      redis:
+        image: redis:7
+        ports:
+          - 6379:6379
+
+    env:
+      MONGO_URI: mongodb://localhost:27017
+      REDIS_HOST: localhost
+      REDIS_PORT: '6379'
+      BACKUP_DIR: ${{ github.workspace }}/backup-verify
 
     steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-        with:
-          repository: WHEELBACK/COMEBACKHERE-contracts
-          ref: main
-          path: COMEBACKHERE-contracts
-
-      - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0
-
-      - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            COMEBACKHERE-contracts/target
-          key: ${{ runner.os }}-cargo-deploy-${{ hashFiles('COMEBACKHERE-contracts/Cargo.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-cargo-deploy-
-
-      - name: Build WASM artifacts
-        if: github.event_name == 'workflow_dispatch'
-        run: cargo build --target wasm32-unknown-unknown --release
-        working-directory: COMEBACKHERE-contracts
-
-      - name: Run deploy_testnet.sh
-        if: github.event_name == 'workflow_dispatch'
-        env:
-          STELLAR_NETWORK: ${{ github.event.inputs.network }}
-          SOROBAN_RPC_URL: https://soroban-testnet.stellar.org
-          SOROBAN_NETWORK_PASSPHRASE: "Test SDF Network ; September 2015"
-          ADMIN_PUBLIC_KEY: ${{ secrets.TESTNET_ADMIN_PUBLIC_KEY }}
-          ADMIN_SECRET_KEY: ${{ secrets.TESTNET_ADMIN_SECRET_KEY }}
-          INVOICE_CONTRACT_ID: ${{ github.event.inputs.invoice_contract_id }}
-          TREASURY_CONTRACT_ID: ${{ github.event.inputs.treasury_contract_id }}
-          COMPLIANCE_CONTRACT_ID: ${{ github.event.inputs.compliance_contract_id }}
-        run: ./scripts/deploy_testnet.sh
-
-  resolve-live-addresses:
-    name: resolve live testnet contract IDs
-    runs-on: ubuntu-latest
-    # Resolves the currently deployed testnet contract IDs from the most
-    # recent successful manual deploy run so scheduled verification targets
-    # the live deployment instead of requiring manually supplied inputs.
-    outputs:
-      network: ${{ steps.addresses.outputs.network }}
-      invoice_contract_id: ${{ steps.addresses.outputs.invoice_contract_id }}
-      treasury_contract_id: ${{ steps.addresses.outputs.treasury_contract_id }}
-      compliance_contract_id: ${{ steps.addresses.outputs.compliance_contract_id }}
+      - name: Checkout
+        uses: actions/checkout@v4
 
-    steps:
-      - name: Find latest successful deploy run
-        id: find-run
-        if: github.event_name == 'schedule'
-        uses: actions/github-script@v7
-        with:
-          script: |
-            const { data } = await github.rest.actions.listWorkflowRuns({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              workflow_id: 'ci-post-deploy-verify.yml',
-              event: 'workflow_dispatch',
-              status: 'success',
-              per_page: 5,
-            });
-            const run = data.workflow_runs[0];
-            if (!run) {
-              core.setFailed('No previous successful workflow_dispatch run found; cannot resolve live testnet contract IDs.');
-            } else {
-              core.setOutput('run_id', String(run.id));
-            }
-
-      - name: Download deployed addresses artifact
-        if: github.event_name == 'schedule'
-        uses: actions/download-artifact@v8
-        with:
-          name: deployed-addresses-testnet
-          run-id: ${{ steps.find-run.outputs.run_id }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract live testnet contract IDs
-        id: addresses
-        if: github.event_name == 'schedule'
+      - name: Install MongoDB database tools
         run: |
-          python3 - <<'PYEOF'
-          import json
-          import os
-
-          with open('artifacts/addresses.json') as f:
-              data = json.load(f)
-
-          by_name = {c['name']: c['address'] for c in data.get('contracts', [])}
+          curl -fsSL https://www.mongodb.org/static/pgp/server-6.0.asc | sudo gpg --dearmor -o /usr/share/keyrings/mongodb-server-6.0.gpg
+          echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-6.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list
+          sudo apt-get update
+          sudo apt-get install -y mongodb-database-tools redis-tools
 
-          def emit(key, value):
-              with open(os.environ.get('GITHUB_OUTPUT'), 'a') as out:
-                  out.write(f"{key}={value}\n")
-
-          emit('network', data.get('network', 'testnet'))
-          emit('invoice_contract_id', by_name.get('invoice', ''))
-          emit('treasury_contract_id', by_name.get('treasury', ''))
-          emit('compliance_contract_id', by_name.get('compliance', ''))
-          PYEOF
-
-  verify:
-    name: post-deploy WASM hash verification
-    runs-on: ubuntu-latest
-    needs: [deploy-testnet, resolve-live-addresses]
+      - name: Seed source data and create backups
+        run: |
+          set -euo pipefail
+          mkdir -p "$BACKUP_DIR/mongo" "$BACKUP_DIR/redis"
+
+          # Seed MongoDB with the data we expect to recover.
+          mongosh "$MONGO_URI/backup_verify" --quiet --eval '
+            db.users.insertMany([
+              { _id: 1, email: "alice@example.com" },
+              { _id: 2, email: "bob@example.com" }
+            ]);
+            db.orders.insertOne({ _id: 100, total: 42 });
+          '
+
+          # Seed Redis with the keys we expect to recover.
+          redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" SET backup:verify:key "expected-value"
+          redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" HSET backup:verify:hash field value
+
+          # Produce backups.
+          mongodump --uri="$MONGO_URI" --db=backup_verify --out="$BACKUP_DIR/mongo"
+          redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" SAVE
+          redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" --rdb "$BACKUP_DIR/redis/dump.rdb"
+
+      - name: Restore backups into clean instances
+        run: |
+          set -euo pipefail
 
-    steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-        with:
-          repository: WHEELBACK/COMEBACKHERE-contracts
-          ref: main
-          path: COMEBACKHERE-contracts
-
-      - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0
-
-      - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            COMEBACKHERE-contracts/target
-          key: ${{ runner.os }}-cargo-deploy-${{ hashFiles('COMEBACKHERE-contracts/Cargo.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-cargo-deploy-
-
-      - name: Build WASM artifacts (for hash comparison)
-        run: cargo build --target wasm32-unknown-unknown --release
-        working-directory: COMEBACKHERE-contracts
-
-      - name: Run verify.sh
-        env:
-          SOROBAN_RPC_URL: https://soroban-testnet.stellar.org
-          STELLAR_NETWORK: ${{ github.event.inputs.network || needs.resolve-live-addresses.outputs.network || 'testnet' }}
-          INVOICE_CONTRACT_ID: ${{ github.event.inputs.invoice_contract_id || needs.resolve-live-addresses.outputs.invoice_contract_id }}
-          TREASURY_CONTRACT_ID: ${{ github.event.inputs.treasury_contract_id || needs.resolve-live-addresses.outputs.treasury_contract_id }}
-          COMPLIANCE_CONTRACT_ID: ${{ github.event.inputs.compliance_contract_id || needs.resolve-live-addresses.outputs.compliance_contract_id }}
-          CONTRACTS_DIR: COMEBACKHERE-contracts/target/wasm32-unknown-unknown/release
-        run: ./scripts/verify.sh
-
-  export-addresses:
-    name: export and validate deployed addresses
-    runs-on: ubuntu-latest
-    needs: verify
+          # Restore MongoDB into a fresh database name to prove the dump is usable.
+          mongorestore --uri="$MONGO_URI" --nsFrom='backup_verify.*' --nsTo='backup_verify_restored.*' "$BACKUP_DIR/mongo"
 
-    steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-
-      - name: Run export_deployed_addresses.sh
-        env:
-          STELLAR_NETWORK: ${{ github.event.inputs.network || needs.resolve-live-addresses.outputs.network || 'testnet' }}
-          INVOICE_CONTRACT_ID: ${{ github.event.inputs.invoice_contract_id || needs.resolve-live-addresses.outputs.invoice_contract_id }}
-          TREASURY_CONTRACT_ID: ${{ github.event.inputs.treasury_contract_id || needs.resolve-live-addresses.outputs.treasury_contract_id }}
-          COMPLIANCE_CONTRACT_ID: ${{ github.event.inputs.compliance_contract_id || needs.resolve-live-addresses.outputs.compliance_contract_id }}
-        run: ./scripts/export_deployed_addresses.sh
-
-      - name: Upload addresses artifact
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: deployed-addresses-${{ github.event.inputs.network || needs.resolve-live-addresses.outputs.network || 'testnet' }}
-          path: artifacts/addresses.json
-
-  alert-schedule-failure:
-    name: alert on nightly verification failure
-    runs-on: ubuntu-latest
-    needs: [resolve-live-addresses, verify, export-addresses]
-    if: ${{ github.event_name == 'schedule' && (needs.resolve-live-addresses.result == 'failure' || needs.verify.result == 'failure' || needs.export-addresses.result == 'failure') }}
-    permissions:
-      issues: write
+          # Restore Redis by loading the RDB into a fresh server instance.
+          redis-server --port 6380 --dir "$BACKUP_DIR/redis" --dbfilename dump.rdb --daemonize yes
+          for i in $(seq 1 30); do
+            if redis-cli -p 6380 PING >/dev/null 2>&1; then break; fi
+            sleep 1
+          done
 
-    steps:
-      - name: Create alert issue on nightly failure
-        uses: actions/github-script@v7
-        with:
-          script: |
-            const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
-            await github.rest.issues.create({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              title: 'Nightly post-deploy verification failed',
-              body: [
-                '## Nightly post-deploy verification failed',
-                '',
-                'The scheduled `ci-post-deploy-verify.yml` run detected drift or an outage in the live testnet deployment.',
-                '',
-                `- Workflow run: ${runUrl}`,
-                `- Trigger: ${context.eventName}`,
-                `- Ref: ${context.ref}`,
-                '',
-                'Investigate the linked run to determine whether the deployed contracts drifted from source or the testnet is unavailable.',
-              ].join('\n'),
-            });
+      - name: Verify recovered data
+        run: |
+          set -euo pipefail
+          failures=0
+
+          # Verify MongoDB recovered collections and documents.
+          users_count=$(mongosh "$MONGO_URI/backup_verify_restored" --quiet --eval 'db.users.countDocuments({})')
+          orders_count=$(mongosh "$MONGO_URI/backup_verify_restored" --quiet --eval 'db.orders.countDocuments({})')
+          alice=$(mongosh "$MONGO_URI/backup_verify_restored" --quiet --eval 'db.users.findOne({ _id: 1 }).email')
+
+          if [ "$users_count" != "2" ]; then
+            echo "::error::MongoDB restore verification failed: expected 2 users, got '$users_count'"
+            failures=$((failures + 1))
+          fi
+          if [ "$orders_count" != "1" ]; then
+            echo "::error::MongoDB restore verification failed: expected 1 order, got '$orders_count'"
+            failures=$((failures + 1))
+          fi
+          if [ "$alice" != "alice@example.com" ]; then
+            echo "::error::MongoDB restore verification failed: expected alice@example.com, got '$alice'"
+            failures=$((failures + 1))
+          fi
+
+          # Verify Redis recovered keys and values.
+          redis_value=$(redis-cli -p 6380 GET backup:verify:key)
+          redis_hash=$(redis-cli -p 6380 HGET backup:verify:hash field)
+
+          if [ "$redis_value" != "expected-value" ]; then
+            echo "::error::Redis restore verification failed: expected 'expected-value', got '$redis_value'"
+            failures=$((failures + 1))
+          fi
+          if [ "$redis_hash" != "value" ]; then
+            echo "::error::Redis restore verification failed: expected hash field 'value', got '$redis_hash'"
+            failures=$((failures + 1))
+          fi
+
+          if [ "$failures" -ne 0 ]; then
+            echo "Backup restore verification failed with $failures error(s)."
+            exit 1
+          fi
+
+          echo "Backup restore verification passed for MongoDB and Redis."

.github/workflows/ci-stack-smoke.yml

@@ -21,8 +21,13 @@ on:
       - 'comebackhere-frontend/vite.config.ts'
       - 'comebackhere-frontend/package.json'
       - 'tests/smoke_stack.sh'
+      - 'tests/verify_backup_restore.sh'
       - 'tests/README.md'
       - '.github/workflows/ci-stack-smoke.yml'
+  schedule:
+    # Operator-independent recurring check that backups can actually be
+    # restored and contain the expected data (issue #977).
+    - cron: '0 4 * * *'
   workflow_call: # called by ci-summary.yml
   workflow_dispatch: # useful for re-running a flaky stack start
 
@@ -70,6 +75,12 @@ jobs:
         # config, so the defaults match a local `docker compose up`.
         run: ./tests/smoke_stack.sh --timeout 180
 
+      # Exercise a real backup -> restore round trip for MongoDB and Redis and
+      # assert the recovered data matches what was written. Exits non-zero on
+      # any mismatch so the failure surfaces here and in scheduled runs.
+      - name: Verify backup and restore
+        run: ./tests/verify_backup_restore.sh
+
       - name: Tear the stack down
         if: always()
         run: docker compose down -v

Makefile

@@ -1,5 +1,5 @@
 .PHONY: update-abi-snapshots check-abi-snapshots dev test lint deploy-testnet abi-snapshot \
-        smoke smoke-up check-docs-sync
+        smoke smoke-up check-docs-sync verify-backups
 
 # Regenerate committed ABI metadata under abis/ (deterministic; LC_ALL=C).
 # Assumes COMEBACKHERE-contracts/ is a sibling directory.
@@ -27,6 +27,11 @@ check-docs-sync:
 	@./scripts/check_webhook_docs_sync.sh
 	@./scripts/check_ratelimit_docs_sync.sh
 
+# Restore MongoDB and Redis backups into throwaway instances and verify the
+# recovered data. Exits non-zero on any failure so schedulers/CI surface it.
+verify-backups:
+	@./scripts/verify_backups.sh
+
 test:
 	cargo test --manifest-path COMEBACKHERE-contracts/Cargo.toml
 

backend/.env.example

@@ -15,3 +15,18 @@ USDC_CONTRACT_ID=C...
 # Server
 HOST=0.0.0.0
 PORT=3000
+
+# Backup / restore verification (issue #977)
+# Operator-run or scheduled check that restores MongoDB and Redis backups into
+# scratch instances, verifies recovered data, and exits non-zero on failure.
+# Run with: ./scripts/verify_backup_restore.sh
+BACKUP_VERIFY_MONGO_URI=mongodb://localhost:27017
+BACKUP_VERIFY_MONGO_DB=comebackhere
+BACKUP_VERIFY_MONGO_RESTORE_URI=mongodb://localhost:27018
+BACKUP_VERIFY_MONGO_RESTORE_DB=comebackhere_restore_verify
+BACKUP_VERIFY_MONGO_BACKUP_DIR=./backups/mongo
+BACKUP_VERIFY_MONGO_REQUIRED_COLLECTIONS=invoices,treasury,compliance
+BACKUP_VERIFY_REDIS_URL=redis://localhost:6379
+BACKUP_VERIFY_REDIS_RESTORE_URL=redis://localhost:6380
+BACKUP_VERIFY_REDIS_BACKUP_FILE=./backups/redis/dump.rdb
+BACKUP_VERIFY_REDIS_REQUIRED_KEYS=comebackhere:invoices,comebackhere:treasury