infra: simplify staging control-plane and baremetal deploys

Author: posix4e

.github/workflows/baremetal-production-deploy.yml

@@ -1,80 +1,104 @@
+# Baremetal Production Deploy
+#
+# Builds the agent + control-plane binaries, bakes a VM image on the production
+# host via Packer, and launches the new agent VM.
+#
+# Trigger: automatic on push to main (after PR has been validated on staging).
+# Dispatch inputs let you override GPU PCI address, memory, and CPU count —
+# but sensible defaults come from infra/ansible/inventory/production.yml.
+#
+# App deployment (private-llm etc.) is handled by the private-llm repo,
+# not here. This workflow owns infra only.
+
 name: Baremetal Production Deploy
 
 on:
+  push:
+    branches: [main]
+    paths:
+      - "agent/**"
+      - "control-plane/**"
+      - "images/**"
+      - "infra/**"
+      - ".github/workflows/baremetal-*"
   workflow_dispatch:
     inputs:
       vfio_device:
-        description: "PCI address for GPU passthrough"
+        description: "PCI address for GPU passthrough (e.g. 0d:00.0)"
         default: "0d:00.0"
       agent_memory:
-        description: "Agent VM memory"
-        default: "128G"
+        description: "Agent VM memory (overrides inventory default)"
+        default: ""
       agent_cpus:
-        description: "Agent VM CPUs"
-        default: "32"
+        description: "Agent VM CPUs (overrides inventory default)"
+        default: ""
 
 concurrency:
   group: dd-baremetal-production
-  cancel-in-progress: false
+  cancel-in-progress: false   # Never cancel an in-progress production deploy
 
 permissions:
-  id-token: write
   contents: read
 
 jobs:
   deploy:
     runs-on: ubuntu-latest
     environment: production
     steps:
+      # ── Build binaries ──────────────────────────────────────────────────
       - uses: actions/checkout@v4
+
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
+
+      # Build both dd-agent and dd-cp in release mode
       - run: cargo build --workspace --release
+
       - run: pip install ansible
 
+      # ── SSH setup ───────────────────────────────────────────────────────
+      # Write the deploy key to the path the inventory expects (/tmp/deploy-key)
       - name: Set up SSH
         run: |
-          mkdir -p ~/.ssh
-          echo "${{ secrets.BAREMETAL_SSH_KEY }}" > ~/.ssh/deploy_key
-          chmod 600 ~/.ssh/deploy_key
-          ssh-keyscan -H "${{ vars.BAREMETAL_PRODUCTION_HOST }}" >> ~/.ssh/known_hosts 2>/dev/null
+          echo "${{ secrets.BAREMETAL_SSH_KEY }}" > /tmp/deploy-key
+          chmod 600 /tmp/deploy-key
+          ssh-keyscan -H 162.222.34.121 >> ~/.ssh/known_hosts 2>/dev/null || true
 
+      # ── Deploy agent VM ─────────────────────────────────────────────────
+      # Runs the baremetal-agent-deploy playbook against the production inventory.
+      # Build-time vars and GPU passthrough are passed with -e.
+      # Workflow dispatch inputs override inventory defaults when provided.
       - name: Deploy agent to production host
         env:
           ANSIBLE_HOST_KEY_CHECKING: "false"
         run: |
           sha12="$(echo "${{ github.sha }}" | cut -c1-12)"
+          EXTRA_VARS=(
+            -e "agent_binary_local=${GITHUB_WORKSPACE}/target/release/dd-agent"
+            -e "cp_binary_local=${GITHUB_WORKSPACE}/target/release/dd-cp"
+            -e "packer_template_dir=${GITHUB_WORKSPACE}/images/packer"
+            -e "image_name=dd-baremetal-${sha12}"
+            -e "vfio_device=${{ inputs.vfio_device || '0d:00.0' }}"
+          )
+          # Only override inventory defaults when the dispatch input is non-empty
+          if [ -n "${{ inputs.agent_memory }}" ]; then
+            EXTRA_VARS+=(-e "agent_memory=${{ inputs.agent_memory }}")
+          fi
+          if [ -n "${{ inputs.agent_cpus }}" ]; then
+            EXTRA_VARS+=(-e "agent_cpus=${{ inputs.agent_cpus }}")
+          fi
+          EXTRA_VARS+=(
+            -e "dd_intel_api_key=${{ secrets.INTEL_API_KEY }}"
+            -e "dd_cp_admin_password=${{ secrets.DD_CP_ADMIN_PASSWORD }}"
+            -e "dd_cp_cf_api_token=${{ secrets.DD_CP_CF_API_TOKEN }}"
+            -e "dd_cp_cf_account_id=${{ secrets.DD_CP_CF_ACCOUNT_ID }}"
+            -e "dd_cp_cf_zone_id=${{ secrets.DD_CP_CF_ZONE_ID }}"
+          )
           ansible-playbook infra/ansible/playbooks/baremetal-agent-deploy.yml \
-            -i "${{ vars.BAREMETAL_PRODUCTION_HOST }}," \
-            -u "${{ vars.BAREMETAL_PRODUCTION_USER }}" \
-            --private-key ~/.ssh/deploy_key \
-            -e "agent_binary_local=${GITHUB_WORKSPACE}/target/release/dd-agent" \
-            -e "cp_binary_local=${GITHUB_WORKSPACE}/target/release/dd-cp" \
-            -e "packer_template_dir=${GITHUB_WORKSPACE}/images/packer" \
-            -e "image_name=dd-baremetal-${sha12}" \
-            -e "dd_env=production" \
-            -e "cp_url=https://app.${{ vars.DD_CF_DOMAIN || 'devopsdefender.com' }}" \
-            -e "dd_skip_attestation=true" \
-            -e "agent_node_size=llm" \
-            -e "agent_memory=${{ inputs.agent_memory }}" \
-            -e "agent_cpus=${{ inputs.agent_cpus }}" \
-            -e "vfio_device=${{ inputs.vfio_device }}"
-
-      - name: Deploy private-llm (H100 GPU mode)
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          CP_URL="https://app.${{ vars.DD_CF_DOMAIN || 'devopsdefender.com' }}"
-          OIDC_TOKEN=$(curl -s -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \
-            "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=devopsdefender" | jq -r '.value')
-          gh api repos/devopsdefender/private-llm/contents/docker-compose.h100.yml \
-            -H "Accept: application/vnd.github.raw" > /tmp/compose.yml
-          COMPOSE_B64=$(base64 -w0 /tmp/compose.yml)
-          curl -sf -X POST "${CP_URL}/api/v1/deploy" \
-            -H "Authorization: Bearer ${OIDC_TOKEN}" \
-            -H "Content-Type: application/json" \
-            -d "{\"app_name\":\"private-llm\",\"app_version\":\"${{ github.sha }}\",\"compose\":\"${COMPOSE_B64}\",\"node_size\":\"llm\"}"
+            -i infra/ansible/inventory/production.yml \
+            "${EXTRA_VARS[@]}"
 
-      - name: Cleanup
+      # ── Cleanup ─────────────────────────────────────────────────────────
+      - name: Cleanup SSH key
         if: always()
-        run: rm -f ~/.ssh/deploy_key
+        run: rm -f /tmp/deploy-key

.github/workflows/baremetal-staging-deploy.yml

@@ -1,7 +1,19 @@
+# Baremetal Staging Deploy
+#
+# Builds the agent + control-plane binaries, bakes a VM image on the staging
+# host via Packer, and launches the new agent VM.
+#
+# Trigger: automatic on pull_request targeting main (validates before merge).
+# Host/user/memory/cpus come from infra/ansible/inventory/staging.yml — no
+# inline overrides needed.
+#
+# App deployment (private-llm etc.) is handled by the private-llm repo,
+# not here. This workflow owns infra only.
+
 name: Baremetal Staging Deploy
 
 on:
-  push:
+  pull_request:
     branches: [main]
     paths:
       - "agent/**"
@@ -16,67 +28,57 @@ concurrency:
   cancel-in-progress: true
 
 permissions:
-  id-token: write
   contents: read
 
 jobs:
   deploy:
     runs-on: ubuntu-latest
-    environment: staging
+
     steps:
+      # ── Build binaries ──────────────────────────────────────────────────
       - uses: actions/checkout@v4
+
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
+
+      # Build both dd-agent and dd-cp in release mode
       - run: cargo build --workspace --release
+
       - run: pip install ansible
 
+      # ── SSH setup ───────────────────────────────────────────────────────
+      # Write the deploy key to the path the inventory expects (/tmp/deploy-key)
       - name: Set up SSH
         run: |
-          mkdir -p ~/.ssh
-          echo "${{ secrets.BAREMETAL_SSH_KEY }}" > ~/.ssh/deploy_key
-          chmod 600 ~/.ssh/deploy_key
-          ssh-keyscan -H "${{ vars.BAREMETAL_STAGING_HOST }}" >> ~/.ssh/known_hosts 2>/dev/null
+          echo "${{ secrets.BAREMETAL_SSH_KEY }}" > /tmp/deploy-key
+          chmod 600 /tmp/deploy-key
+          ssh-keyscan -H 57.130.10.246 >> ~/.ssh/known_hosts 2>/dev/null || true
 
+      # ── Deploy agent VM ─────────────────────────────────────────────────
+      # Runs the baremetal-agent-deploy playbook against the staging inventory.
+      # Secrets are passed as Ansible extra-vars (never written to disk).
       - name: Deploy agent to staging host
         env:
           ANSIBLE_HOST_KEY_CHECKING: "false"
         run: |
           sha12="$(echo "${{ github.sha }}" | cut -c1-12)"
+          EXTRA_VARS=(
+            -e "agent_binary_local=${GITHUB_WORKSPACE}/target/release/dd-agent"
+            -e "cp_binary_local=${GITHUB_WORKSPACE}/target/release/dd-cp"
+            -e "packer_template_dir=${GITHUB_WORKSPACE}/images/packer"
+            -e "image_name=dd-baremetal-${sha12}"
+            -e "dd_cp_admin_password=${{ secrets.DD_CP_ADMIN_PASSWORD }}"
+            -e "dd_cp_cf_api_token=${{ secrets.DD_CP_CF_API_TOKEN }}"
+            -e "dd_cp_cf_account_id=${{ secrets.DD_CP_CF_ACCOUNT_ID }}"
+            -e "dd_cp_cf_zone_id=${{ secrets.DD_CP_CF_ZONE_ID }}"
+            -e "dd_intel_api_key=${{ secrets.INTEL_API_KEY }}"
+            -e "dd_cp_git_sha=${sha12}"
+          )
           ansible-playbook infra/ansible/playbooks/baremetal-agent-deploy.yml \
-            -i "${{ vars.BAREMETAL_STAGING_HOST }}," \
-            -u "${{ vars.BAREMETAL_STAGING_USER }}" \
-            --private-key ~/.ssh/deploy_key \
-            -e "agent_binary_local=${GITHUB_WORKSPACE}/target/release/dd-agent" \
-            -e "cp_binary_local=${GITHUB_WORKSPACE}/target/release/dd-cp" \
-            -e "packer_template_dir=${GITHUB_WORKSPACE}/images/packer" \
-            -e "image_name=dd-baremetal-${sha12}" \
-            -e "dd_env=staging" \
-            -e "cp_url=https://app-staging.${{ vars.DD_CF_DOMAIN || 'devopsdefender.com' }}" \
-            -e "dd_skip_attestation=true" \
-            -e "agent_node_size=standard"
-
-      - name: Deploy private-llm (CPU mode)
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          CP_URL="https://app-staging.${{ vars.DD_CF_DOMAIN || 'devopsdefender.com' }}"
-          OIDC_TOKEN=$(curl -s -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \
-            "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=devopsdefender" | jq -r '.value')
-          gh api repos/devopsdefender/private-llm/contents/docker-compose.yml \
-            -H "Accept: application/vnd.github.raw" > /tmp/compose.yml
-          COMPOSE_B64=$(base64 -w0 /tmp/compose.yml)
-          RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "${CP_URL}/api/v1/deploy" \
-            -H "Authorization: Bearer ${OIDC_TOKEN}" \
-            -H "Content-Type: application/json" \
-            -d "{\"app_name\":\"private-llm\",\"app_version\":\"${{ github.sha }}\",\"compose\":\"${COMPOSE_B64}\",\"node_size\":\"standard\"}")
-          HTTP_CODE=$(echo "${RESPONSE}" | tail -1)
-          BODY=$(echo "${RESPONSE}" | sed '$d')
-          echo "HTTP ${HTTP_CODE}: ${BODY}"
-          if [ "${HTTP_CODE}" -lt 200 ] || [ "${HTTP_CODE}" -ge 300 ]; then
-            echo "::error::LLM deploy failed: ${BODY}"
-            exit 1
-          fi
+            -i infra/ansible/inventory/staging.yml \
+            "${EXTRA_VARS[@]}"
 
-      - name: Cleanup
+      # ── Cleanup ─────────────────────────────────────────────────────────
+      - name: Cleanup SSH key
         if: always()
-        run: rm -f ~/.ssh/deploy_key
+        run: rm -f /tmp/deploy-key

.github/workflows/staging-deploy.yml

@@ -106,11 +106,8 @@ Central management API for agent registration, deployment orchestration, health
 
 **Deployment:** Multi-stage Dockerfile (Rust builder → debian bookworm-slim runtime with cloudflared).
 
-**Infrastructure** (`infra/`): Ansible playbooks for GCP and baremetal deployments. Key playbooks:
-- `gcp-control-plane-new.yml` — launch TDX-enabled GCP VM for control plane
-- `gcp-vm-fleet-new.yml` — launch agent fleet
-- `gcp-image-bake.yml` — build VM images
-- `baremetal-deploy.yml` — bare metal deployment
+**Infrastructure** (`infra/`): Ansible playbooks for deploying agent VMs via KVM on dedicated OVH hardware. Key playbook:
+- `baremetal-agent-deploy.yml` — build image via Packer + deploy agent as KVM VM on OVH dedicated server
 
 ### images/ (Packer)
 
@@ -201,9 +198,9 @@ cd private-llm && docker compose up
 
 All infrastructure is managed through GitHub Actions. **Never SSH into hosts, run Ansible locally, or attempt manual fixes on VMs.** If staging or production is down, trigger the appropriate GitHub Actions workflow.
 
-**Staging** (`staging-deploy.yml`) — auto-deploys on push to `main`. Pipeline: build → bake GCP agent image → cleanup old VMs → deploy via Ansible → smoke check `app-staging.devopsdefender.com/health`.
+**Staging** (`baremetal-staging-deploy.yml`) — auto-deploys on push to `main`. Deploys the agent as a KVM VM on a dedicated OVH server. Pipeline: build → Packer bake on host → deploy agent VM via KVM → deploy private-llm.
 
-**Production** (`production-deploy.yml`) — manual trigger only (`workflow_dispatch`). Inputs: `num_tiny_agents`, `num_standard_agents`, `num_llm_agents`. Same pipeline as staging.
+**Production** (`baremetal-production-deploy.yml`) — manual trigger only (`workflow_dispatch`). Deploys the agent as a KVM VM on dedicated OVH hardware with GPU passthrough. Inputs: `vfio_device`, `agent_memory`, `agent_cpus`.
 
 **Health check URLs:**
 - Staging: `https://app-staging.devopsdefender.com/health`
@@ -212,18 +209,18 @@ All infrastructure is managed through GitHub Actions. **Never SSH into hosts, ru
 **To deploy or fix an environment**, use `gh workflow run` or the GitHub Actions UI:
 ```bash
 # Trigger staging deploy
-gh workflow run staging-deploy.yml --repo devopsdefender/control-plane
+gh workflow run baremetal-staging-deploy.yml
 
 # Trigger production deploy
-gh workflow run production-deploy.yml --repo devopsdefender/control-plane \
-  -f num_tiny_agents=0 -f num_standard_agents=0 -f num_llm_agents=1
+gh workflow run baremetal-production-deploy.yml \
+  -f vfio_device=0d:00.0 -f agent_memory=128G -f agent_cpus=32
 ```
 
 ## CI/CD Pipelines
 
 Each component has its own GitHub Actions workflows:
 - **agent**: `ci.yml` (check/test/fmt/clippy), `release.yml` (build binary + GitHub release)
-- **control-plane**: `ci.yml`, `staging-deploy.yml`, `production-deploy.yml`, `release.yml`
+- **control-plane**: `ci.yml`, `baremetal-staging-deploy.yml`, `baremetal-production-deploy.yml`, `release.yml`
 - **images**: `baremetal-image.yml` (Packer build on self-hosted runner)
 - **private-llm**: `deploy.yml` (OIDC-authenticated deployment to DD platform)
 - **website**: `pages.yml` (GitHub Pages deployment)