Restore automatic DigitalOcean backend deployments from release

.github/workflows/deploy-digitalocean.yml

@@ -0,0 +1,72 @@
+name: Deploy workspace backend to DigitalOcean
+
+on:
+  push:
+    branches: [release]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: workspace-backend-production
+  cancel-in-progress: false
+
+jobs:
+  build:
+    if: github.ref == 'refs/heads/release'
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          persist-credentials: false
+      - name: Test deployment guards
+        run: python3 -m unittest discover -s workspace/operations/digitalocean/cd/tests -v
+      - name: Build exact release without deployment credentials
+        env:
+          RELEASE_SHA: ${{ github.sha }}
+          DOCKER_BUILDKIT: "1"
+        run: |
+          set -euo pipefail
+          # Canary files must be excluded even inside copied application paths.
+          mkdir -p workspace/backend/app/secrets
+          printf "not-a-real-secret" > workspace/backend/app/.env.deploy-canary
+          printf "not-a-real-secret" > workspace/backend/app/secrets/deploy-canary
+          docker build --file workspace/backend/production.Dockerfile \
+            --build-arg RELEASE_SHA="$RELEASE_SHA" \
+            --tag "openagents-backend:release-$RELEASE_SHA" workspace/backend
+          docker run --rm --network none --entrypoint python \
+            -e DATABASE_URL=sqlite:// -e BACKGROUND_JOBS_ENABLED=false \
+            "openagents-backend:release-$RELEASE_SHA" \
+            -c 'import app.main; import app.cache; import yaml; from pathlib import Path; assert not Path("/app/app/.env.deploy-canary").exists(); assert not Path("/app/app/secrets/deploy-canary").exists()'
+          docker save "openagents-backend:release-$RELEASE_SHA" | gzip -1 > backend-image.tar.gz
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: backend-image-${{ github.sha }}
+          path: backend-image.tar.gz
+          retention-days: 3
+          compression-level: 0
+          if-no-files-found: error
+
+  deploy:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: digitalocean-production
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          persist-credentials: false
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          name: backend-image-${{ github.sha }}
+      - name: Stage both nodes and request a rolling deployment
+        env:
+          RELEASE_SHA: ${{ github.sha }}
+          DO_DEPLOY_SSH_KEY: ${{ secrets.DO_DEPLOY_SSH_KEY }}
+          DO_DEPLOY_KNOWN_HOSTS: ${{ secrets.DO_DEPLOY_KNOWN_HOSTS }}
+          DO_PRIMARY_HOST: ${{ vars.DO_PRIMARY_HOST }}
+          DO_SECONDARY_HOST: ${{ vars.DO_SECONDARY_HOST }}
+          GH_TOKEN: ${{ github.token }}
+        run: python3 workspace/operations/digitalocean/cd/ci.py

.github/workflows/promote-release.yml

@@ -2,7 +2,7 @@ name: Promote develop to release
 
 # The only sanctioned way to move `release`.
 #
-# `release` is what Railway deploys. It must always be a fast-forward of
+# `release` triggers the DigitalOcean backend deployment and Vercel frontend deployment. It must always be a fast-forward of
 # `develop`: never a merge commit, never a commit that isn't on develop. A
 # ruleset on `release` blocks direct pushes from everyone; this workflow pushes
 # with a dedicated deploy key (RELEASE_PROMOTE_SSH_KEY) that the ruleset lets

.gitignore

@@ -280,3 +280,6 @@ workspace/backend/secrets/
 
 # End-to-end test credentials (tests/end_to_end/agents.example.json is the template)
 tests/end_to_end/agents.config.json
+
+# Reviewed production dependency constraints
+!workspace/backend/requirements.production.lock

workspace/backend/production.Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.12-slim
+WORKDIR /app
+RUN apt-get update && apt-get install -y --no-install-recommends libpq5 \
+    && rm -rf /var/lib/apt/lists/*
+COPY requirements.txt requirements.production.lock ./
+RUN pip install --no-cache-dir -r requirements.txt -c requirements.production.lock 'PyYAML>=6.0'
+# Explicit runtime allowlist: no credentials, tests, Git history or operations files.
+COPY app/ ./app/
+COPY alembic/ ./alembic/
+COPY openagents/ ./openagents/
+COPY cloud_providers/ ./cloud_providers/
+COPY registry/ ./registry/
+COPY alembic.ini entrypoint.sh ./
+RUN chmod +x entrypoint.sh
+ARG RELEASE_SHA
+LABEL org.opencontainers.image.revision=$RELEASE_SHA
+ENV PYTHONPATH=/app PYTHONUNBUFFERED=1
+EXPOSE 8000
+ENTRYPOINT ["./entrypoint.sh"]

workspace/backend/production.Dockerfile.dockerignore

@@ -0,0 +1,23 @@
+**
+!production.Dockerfile
+!requirements.txt
+!requirements.production.lock
+!entrypoint.sh
+!alembic.ini
+!app/
+!app/**
+!alembic/
+!alembic/**
+!openagents/
+!openagents/**
+!cloud_providers/
+!cloud_providers/**
+!registry/
+!registry/**
+**/__pycache__/**
+**/*.pyc
+**/.env*
+**/secrets/**
+**/*.pem
+**/*.key
+**/*.p8

workspace/backend/requirements.production.lock

@@ -0,0 +1,75 @@
+alembic==1.20.0
+annotated-doc==0.0.5
+annotated-types==0.8.0
+anthropic==1.13.0
+anyio==4.15.1
+boto3==1.43.111
+botocore==1.43.111
+CacheControl==0.14.4
+certifi==2026.7.22
+cffi==2.1.1
+charset-normalizer==3.5.2
+click==8.5.0
+cryptography==50.0.2
+docstring_parser==0.18.0
+fastapi==0.143.0
+firebase_admin==7.7.0
+google-api-core==2.42.0
+google-auth==2.61.0
+google-cloud-core==2.8.0
+google-cloud-firestore==2.34.1
+google-cloud-storage==3.17.0
+google-crc32c==1.9.0
+google-resumable-media==2.11.0
+googleapis-common-protos==1.75.5
+greenlet==3.5.6
+grpcio==1.84.0
+grpcio-status==1.84.0
+h11==0.16.0
+h2==4.4.1
+hpack==4.2.0
+httpcore==1.0.9
+httpcore2==2.13.1
+httptools==0.9.0
+httpx==0.28.1
+httpx2==2.13.1
+hyperframe==6.1.0
+idna==3.20
+jiter==0.17.0
+jmespath==1.1.0
+Mako==1.4.3
+MarkupSafe==3.0.4
+msgpack==1.2.3
+openai==3.28.0
+opentelemetry-api==1.45.1
+pillow==12.3.0
+playwright==1.63.0
+proto-plus==1.29.0
+protobuf==7.36.2
+psycopg2-binary==2.9.13
+pyasn1==0.6.4
+pyasn1_modules==0.4.2
+pycparser==3.11
+pydantic==2.14.0
+pydantic_core==2.50.0
+pyee==13.0.1
+PyJWT==2.15.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.2.4
+python-multipart==0.0.32
+PyYAML==6.0.3
+redis==8.1.0
+requests==2.34.2
+s3transfer==0.19.2
+six==1.17.0
+sniffio==1.3.1
+SQLAlchemy==2.0.54
+starlette==1.7.0
+truststore==0.10.4
+typing-inspection==0.4.4
+typing_extensions==4.16.0
+urllib3==2.8.0
+uvicorn==0.54.0
+uvloop==0.23.0
+watchfiles==1.3.0
+websockets==17.2

workspace/operations/digitalocean/cd/README.md

@@ -0,0 +1,70 @@
+# DigitalOcean automatic backend deployments
+
+A push to `release` builds that exact commit and rolls it out to the secondary,
+then the primary backend. Use the existing **Promote develop to release** workflow
+to ship a merged develop commit. The frontend continues using its existing deployment.
+The deployment workflow can also be run manually against `release` to retry it.
+
+The build job receives no deployment secrets. Its explicit Docker context allowlist
+excludes environment files, operations files and credentials. The deployment job uses
+the `digitalocean-production` GitHub environment, restricted to `release`, with:
+
+- Secrets `DO_DEPLOY_SSH_KEY`, `DO_DEPLOY_KNOWN_HOSTS`.
+- Variables `DO_PRIMARY_HOST`, `DO_SECONDARY_HOST`.
+
+The dedicated SSH key is restricted by root-owned forced commands on both servers.
+It can stage an image, start a rollout, and inspect deployment status, but cannot
+open an interactive shell or change routing directly. A separate restricted key
+allows the coordinator to update the secondary. Host keys are pinned. Anyone able
+to publish release code can execute application code in production; branch protections
+remain essential. Do not copy developer SSH keys into GitHub.
+
+## Server installation
+
+Install `deploy.py`, `rollout.py`, `migration_plan.py`, and `backup.py` under the
+root-owned `/opt/openagents/cd` directory (mode 0700). Python dependencies are PyYAML,
+boto3 and botocore; PostgreSQL client tools must support the server version.
+Root-owned `config.json` supplies coordinator flag, node IDs, peer host, local/peer
+Nginx upstream addresses, upstream file, public URL, load balancer ID, DO token file
+and migration Docker network. Node configuration and credentials are not tracked.
+
+Keep all credentials in `cd/secrets` (0700 directory, 0600 files): `migration.env`
+(admin DATABASE_URL), `smoke.json` (workspace and token), `backup.json` (libpq env and
+Spaces settings), and the coordinator's `peer-key`. The application's existing
+`/opt/openagents/secrets/production.env` stays on each server. Pin peer SSH host keys
+in `cd/peer-known-hosts`. Configure authorized_keys with `restrict` and a forced
+`python3 /opt/openagents/cd/deploy.py ci` or `peer` command; restrict the peer key's
+source address to the coordinator's private IP.
+
+## Rollout and migrations
+
+Both nodes must be ready, contain the candidate image, and be the exact members of
+the load balancer. The coordinator compares database revision to the candidate's
+Alembic head. Every pending migration must explicitly declare
+`deployment_backward_compatible = True`; review this declaration carefully. Use
+expand/contract changes compatible with the currently running image. Unmarked
+migrations stop deployment before any traffic changes. The declaration is a review
+contract, not a proof of compatibility.
+
+Pending migrations trigger a consistent PostgreSQL custom-format backup in the
+private Spaces bucket. The archive is checked with pg_restore, downloaded and
+SHA-256 verified, and anonymous access must be denied. Migration runs once, before
+replacing either node. Database schema/data are never automatically downgraded.
+
+For each node, remove it from the load balancer and temporarily forward new requests
+to its peer, update its image, check readiness/cache/authenticated reads, restore local
+routing and re-add it. Long-lived connections on the replaced process may reconnect.
+The primary retains background jobs; the secondary has them disabled. This does not
+provide automatic scheduler failover. Public verification follows the rollout.
+Failures restore previous images; failed rollback nodes remain out of rotation.
+
+The coordinator runs as `openagents-auto-deploy.service`, independently of Actions
+connectivity. `cd/result-<sha>.json` and `cd/last-result.json` record status. Detailed
+logs stay in root-only `cd/operations.log`; they may contain sensitive application
+errors and must not be pasted into GitHub. If status is `interrupted` or
+`rollback_failed`, inspect node readiness, routing and load balancer membership
+before retrying. Do not automatically restart a partially completed rollout.
+
+The forced-command scripts are host administration code, not automatically replaced
+by application releases. Changes to them require testing and explicit installation.
+Run tests with `python3 -m unittest discover -s workspace/operations/digitalocean/cd/tests -v`.

workspace/operations/digitalocean/cd/backup.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+"""Consistent pre-migration PostgreSQL snapshot, privately verified in Spaces."""
+import datetime
+import hashlib
+import json
+import os
+from pathlib import Path
+import subprocess
+import urllib.request
+import urllib.error
+import boto3
+from botocore.config import Config
+
+ROOT = Path('/opt/openagents/cd')
+
+
+def backup():
+    cfg = json.loads((ROOT / 'secrets/backup.json').read_text())
+    stamp = datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%dT%H%M%SZ')
+    directory = ROOT / 'backups'; directory.mkdir(mode=0o700, exist_ok=True)
+    path = directory / ('pre-migration-' + stamp + '.dump')
+    env = {**os.environ, **cfg['pg_env']}
+    # libpq credentials are inherited privately, never command-line arguments.
+    subprocess.run(['pg_dump', '--format=custom', '--no-owner', '--no-acl',
+                    '--file', str(path)], env=env, check=True, timeout=1200)
+    subprocess.run(['pg_restore', '--list', str(path)], stdout=subprocess.DEVNULL, check=True)
+    with path.open('rb') as f: digest = hashlib.file_digest(f, 'sha256').hexdigest()
+    s3 = boto3.client('s3', endpoint_url=cfg['endpoint'], region_name=cfg['region'],
+        aws_access_key_id=cfg['key_id'], aws_secret_access_key=cfg['secret'],
+        config=Config(signature_version='s3v4'))
+    key = 'database-backups/automatic-deploy/' + path.name
+    s3.upload_file(str(path), cfg['bucket'], key,
+                  ExtraArgs={'ACL': 'private', 'Metadata': {'sha256': digest}})
+    head = s3.head_object(Bucket=cfg['bucket'], Key=key)
+    if head['ContentLength'] != path.stat().st_size or head['Metadata'].get('sha256') != digest:
+        raise RuntimeError('Backup object metadata mismatch')
+    body = s3.get_object(Bucket=cfg['bucket'], Key=key)['Body']
+    verify = hashlib.sha256()
+    with body:
+        while chunk := body.read(8 * 1024 * 1024): verify.update(chunk)
+    if verify.hexdigest() != digest: raise RuntimeError('Backup checksum mismatch')
+    try:
+        with urllib.request.urlopen(urllib.request.Request(
+            cfg['public_bucket_url'] + '/' + key, method='HEAD'), timeout=15):
+            raise RuntimeError('Backup is publicly accessible')
+    except urllib.error.HTTPError as exc:
+        if exc.code not in (403, 404): raise
+    receipt = {'key': key, 'bytes': path.stat().st_size, 'sha256': digest,
+               'download_verified': True, 'anonymous_access_denied': True}
+    path.with_suffix('.json').write_text(json.dumps(receipt, indent=2))
+    print(json.dumps(receipt))
+
+
+if __name__ == '__main__':
+    os.umask(0o077)
+    backup()

workspace/operations/digitalocean/cd/ci.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+"""Actions client. SSH keys never enter the checkout, image or build job."""
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import tempfile
+import time
+
+
+def main():
+    sha = os.environ['RELEASE_SHA']
+    if not re.fullmatch('[0-9a-f]{40}', sha): raise ValueError('Invalid release')
+    hosts = [os.environ['DO_PRIMARY_HOST'], os.environ['DO_SECONDARY_HOST']]
+    if any(not re.fullmatch('[0-9.]+', host) for host in hosts):
+        raise ValueError('Invalid deployment host')
+    current = subprocess.check_output(['gh', 'api',
+        'repos/' + os.environ['GITHUB_REPOSITORY'] + '/git/ref/heads/release',
+        '--jq', '.object.sha'], text=True).strip()
+    if current != sha:
+        print('Skipping superseded release'); return
+    os.umask(0o077)
+    with tempfile.TemporaryDirectory(prefix='backend-deploy-') as directory:
+        key, known = Path(directory) / 'key', Path(directory) / 'known_hosts'
+        key.write_text(os.environ.pop('DO_DEPLOY_SSH_KEY').strip() + '\n')
+        known.write_text(os.environ.pop('DO_DEPLOY_KNOWN_HOSTS').strip() + '\n')
+        options = ['ssh', '-i', str(key), '-o', 'IdentitiesOnly=yes', '-o', 'BatchMode=yes',
+                   '-o', 'StrictHostKeyChecking=yes', '-o', 'UserKnownHostsFile=' + str(known),
+                   '-o', 'ConnectTimeout=15', '-o', 'ServerAliveInterval=15',
+                   '-o', 'ServerAliveCountMax=4']
+        def rpc(host, command, source=None, timeout=900):
+            output = subprocess.check_output(options + ['root@' + host, command],
+                                              stdin=source, timeout=timeout)
+            return json.loads(output)
+        for host in hosts:
+            with open('backend-image.tar.gz', 'rb') as archive:
+                rpc(host, 'stage ' + sha, archive)
+            print('Image staged on a backend', flush=True)
+        rpc(hosts[0], 'deploy ' + sha)
+        for _ in range(160):
+            time.sleep(15)
+            try:
+                result = rpc(hosts[0], 'result ' + sha, timeout=45)
+            except (subprocess.SubprocessError, json.JSONDecodeError):
+                continue  # server rollout survives a transient SSH interruption
+            if result['status'] == 'running': continue
+            print('Deployment result: ' + result['status'], flush=True)
+            if result['status'] != 'deployed':
+                raise RuntimeError('Deployment did not succeed; inspect private server logs')
+            return
+        raise RuntimeError('Timed out waiting; check server state before retrying')
+
+
+if __name__ == '__main__':
+    main()