Move staging to OVH baremetal with TDX attestation

.github/workflows/baremetal-production-deploy.yml

@@ -1,24 +1,39 @@
+# Baremetal Production Deploy
+#
+# Builds the agent + control-plane binaries, bakes a VM image on the production
+# host via Packer, and launches the new libvirt-managed OVH VM.
+#
+# Production keeps its external control plane. This workflow only replaces the
+# application VM on the OVH host.
+
 name: Baremetal Production Deploy
 
 on:
+  push:
+    branches: [main]
+    paths:
+      - "agent/**"
+      - "control-plane/**"
+      - "images/**"
+      - "infra/**"
+      - ".github/workflows/baremetal-*"
   workflow_dispatch:
     inputs:
       vfio_device:
-        description: "PCI address for GPU passthrough"
+        description: "PCI address for GPU passthrough (e.g. 0d:00.0)"
         default: "0d:00.0"
       agent_memory:
-        description: "Agent VM memory"
-        default: "128G"
+        description: "Agent VM memory (overrides inventory default)"
+        default: ""
       agent_cpus:
-        description: "Agent VM CPUs"
-        default: "32"
+        description: "Agent VM CPUs (overrides inventory default)"
+        default: ""
 
 concurrency:
   group: dd-baremetal-production
   cancel-in-progress: false
 
 permissions:
-  id-token: write
   contents: read
 
 jobs:
@@ -27,54 +42,43 @@ jobs:
     environment: production
     steps:
       - uses: actions/checkout@v4
+
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
-      - run: cargo build --workspace --release
+
+      - name: Build release binaries
+        run: cargo build --workspace --release
+
       - run: pip install ansible
 
       - name: Set up SSH
         run: |
-          mkdir -p ~/.ssh
-          echo "${{ secrets.BAREMETAL_SSH_KEY }}" > ~/.ssh/deploy_key
-          chmod 600 ~/.ssh/deploy_key
-          ssh-keyscan -H "${{ vars.BAREMETAL_PRODUCTION_HOST }}" >> ~/.ssh/known_hosts 2>/dev/null
+          echo "${{ secrets.BAREMETAL_SSH_KEY }}" > /tmp/deploy-key
+          chmod 600 /tmp/deploy-key
+          ssh-keyscan -H 162.222.34.121 >> ~/.ssh/known_hosts 2>/dev/null || true
 
       - name: Deploy agent to production host
         env:
           ANSIBLE_HOST_KEY_CHECKING: "false"
         run: |
           sha12="$(echo "${{ github.sha }}" | cut -c1-12)"
+          EXTRA_VARS=(
+            -e "agent_binary_local=${GITHUB_WORKSPACE}/target/release/dd-agent"
+            -e "cp_binary_local=${GITHUB_WORKSPACE}/target/release/dd-cp"
+            -e "packer_template_dir=${GITHUB_WORKSPACE}/images/packer"
+            -e "image_name=dd-baremetal-${sha12}"
+            -e "vfio_device=${{ inputs.vfio_device || '0d:00.0' }}"
+          )
+          if [ -n "${{ inputs.agent_memory }}" ]; then
+            EXTRA_VARS+=(-e "agent_memory=${{ inputs.agent_memory }}")
+          fi
+          if [ -n "${{ inputs.agent_cpus }}" ]; then
+            EXTRA_VARS+=(-e "agent_cpus=${{ inputs.agent_cpus }}")
+          fi
           ansible-playbook infra/ansible/playbooks/baremetal-agent-deploy.yml \
-            -i "${{ vars.BAREMETAL_PRODUCTION_HOST }}," \
-            -u "${{ vars.BAREMETAL_PRODUCTION_USER }}" \
-            --private-key ~/.ssh/deploy_key \
-            -e "agent_binary_local=${GITHUB_WORKSPACE}/target/release/dd-agent" \
-            -e "cp_binary_local=${GITHUB_WORKSPACE}/target/release/dd-cp" \
-            -e "packer_template_dir=${GITHUB_WORKSPACE}/images/packer" \
-            -e "image_name=dd-baremetal-${sha12}" \
-            -e "dd_env=production" \
-            -e "cp_url=https://app.${{ vars.DD_CF_DOMAIN || 'devopsdefender.com' }}" \
-            -e "dd_skip_attestation=true" \
-            -e "agent_node_size=llm" \
-            -e "agent_memory=${{ inputs.agent_memory }}" \
-            -e "agent_cpus=${{ inputs.agent_cpus }}" \
-            -e "vfio_device=${{ inputs.vfio_device }}"
-
-      - name: Deploy private-llm (H100 GPU mode)
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          CP_URL="https://app.${{ vars.DD_CF_DOMAIN || 'devopsdefender.com' }}"
-          OIDC_TOKEN=$(curl -s -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \
-            "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=devopsdefender" | jq -r '.value')
-          gh api repos/devopsdefender/private-llm/contents/docker-compose.h100.yml \
-            -H "Accept: application/vnd.github.raw" > /tmp/compose.yml
-          COMPOSE_B64=$(base64 -w0 /tmp/compose.yml)
-          curl -sf -X POST "${CP_URL}/api/v1/deploy" \
-            -H "Authorization: Bearer ${OIDC_TOKEN}" \
-            -H "Content-Type: application/json" \
-            -d "{\"app_name\":\"private-llm\",\"app_version\":\"${{ github.sha }}\",\"compose\":\"${COMPOSE_B64}\",\"node_size\":\"llm\"}"
+            -i infra/ansible/inventory/production.yml \
+            "${EXTRA_VARS[@]}"
 
-      - name: Cleanup
+      - name: Cleanup SSH key
         if: always()
-        run: rm -f ~/.ssh/deploy_key
+        run: rm -f /tmp/deploy-key

.github/workflows/baremetal-staging-deploy.yml

@@ -1,7 +1,15 @@
-name: Baremetal Staging Deploy
+# OVH Staging VM Deploy
+#
+# Builds the agent image, bakes a qcow2 on the OVH staging host, and launches
+# the example-app VM there via libvirt so it is visible in `virsh list`.
+#
+# The control plane and first/bootstrap validation agent stay on GCP.
+# This workflow only maintains the OVH VM that the example app targets.
+
+name: OVH Staging VM Deploy
 
 on:
-  push:
+  pull_request:
     branches: [main]
     paths:
       - "agent/**"
@@ -16,67 +24,44 @@ concurrency:
   cancel-in-progress: true
 
 permissions:
-  id-token: write
   contents: read
 
 jobs:
   deploy:
     runs-on: ubuntu-latest
-    environment: staging
+
     steps:
       - uses: actions/checkout@v4
+
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
-      - run: cargo build --workspace --release
+
+      - name: Build release binaries
+        run: cargo build --workspace --release
+
       - run: pip install ansible
 
       - name: Set up SSH
         run: |
-          mkdir -p ~/.ssh
-          echo "${{ secrets.BAREMETAL_SSH_KEY }}" > ~/.ssh/deploy_key
-          chmod 600 ~/.ssh/deploy_key
-          ssh-keyscan -H "${{ vars.BAREMETAL_STAGING_HOST }}" >> ~/.ssh/known_hosts 2>/dev/null
+          echo "${{ secrets.BAREMETAL_SSH_KEY }}" > /tmp/deploy-key
+          chmod 600 /tmp/deploy-key
+          ssh-keyscan -H 57.130.10.246 >> ~/.ssh/known_hosts 2>/dev/null || true
 
-      - name: Deploy agent to staging host
+      - name: Deploy OVH staging VM
         env:
           ANSIBLE_HOST_KEY_CHECKING: "false"
         run: |
           sha12="$(echo "${{ github.sha }}" | cut -c1-12)"
+          EXTRA_VARS=(
+            -e "agent_binary_local=${GITHUB_WORKSPACE}/target/release/dd-agent"
+            -e "cp_binary_local=${GITHUB_WORKSPACE}/target/release/dd-cp"
+            -e "packer_template_dir=${GITHUB_WORKSPACE}/images/packer"
+            -e "image_name=dd-baremetal-${sha12}"
+          )
           ansible-playbook infra/ansible/playbooks/baremetal-agent-deploy.yml \
-            -i "${{ vars.BAREMETAL_STAGING_HOST }}," \
-            -u "${{ vars.BAREMETAL_STAGING_USER }}" \
-            --private-key ~/.ssh/deploy_key \
-            -e "agent_binary_local=${GITHUB_WORKSPACE}/target/release/dd-agent" \
-            -e "cp_binary_local=${GITHUB_WORKSPACE}/target/release/dd-cp" \
-            -e "packer_template_dir=${GITHUB_WORKSPACE}/images/packer" \
-            -e "image_name=dd-baremetal-${sha12}" \
-            -e "dd_env=staging" \
-            -e "cp_url=https://app-staging.${{ vars.DD_CF_DOMAIN || 'devopsdefender.com' }}" \
-            -e "dd_skip_attestation=true" \
-            -e "agent_node_size=standard"
-
-      - name: Deploy private-llm (CPU mode)
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          CP_URL="https://app-staging.${{ vars.DD_CF_DOMAIN || 'devopsdefender.com' }}"
-          OIDC_TOKEN=$(curl -s -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \
-            "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=devopsdefender" | jq -r '.value')
-          gh api repos/devopsdefender/private-llm/contents/docker-compose.yml \
-            -H "Accept: application/vnd.github.raw" > /tmp/compose.yml
-          COMPOSE_B64=$(base64 -w0 /tmp/compose.yml)
-          RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "${CP_URL}/api/v1/deploy" \
-            -H "Authorization: Bearer ${OIDC_TOKEN}" \
-            -H "Content-Type: application/json" \
-            -d "{\"app_name\":\"private-llm\",\"app_version\":\"${{ github.sha }}\",\"compose\":\"${COMPOSE_B64}\",\"node_size\":\"standard\"}")
-          HTTP_CODE=$(echo "${RESPONSE}" | tail -1)
-          BODY=$(echo "${RESPONSE}" | sed '$d')
-          echo "HTTP ${HTTP_CODE}: ${BODY}"
-          if [ "${HTTP_CODE}" -lt 200 ] || [ "${HTTP_CODE}" -ge 300 ]; then
-            echo "::error::LLM deploy failed: ${BODY}"
-            exit 1
-          fi
+            -i infra/ansible/inventory/staging.yml \
+            "${EXTRA_VARS[@]}"
 
-      - name: Cleanup
+      - name: Cleanup SSH key
         if: always()
-        run: rm -f ~/.ssh/deploy_key
+        run: rm -f /tmp/deploy-key

.github/workflows/staging-deploy.yml

@@ -54,16 +54,23 @@ jobs:
             -e "packer_template_path=${GITHUB_WORKSPACE}/images/packer/gcp-agent-image.pkr.hcl" \
             -e "source_sha=${{ github.sha }}"
 
-      - name: Deploy
+      - name: Deploy GCP staging control plane and bootstrap agent
         env:
           GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+          DD_ENV: staging
           DD_CP_ADMIN_PASSWORD: ${{ secrets.DD_CP_ADMIN_PASSWORD }}
+          CP_ADMIN_PASSWORD: ${{ secrets.DD_CP_ADMIN_PASSWORD }}
           CLOUDFLARE_API_TOKEN: ${{ secrets.DD_CP_CF_API_TOKEN }}
           CLOUDFLARE_ACCOUNT_ID: ${{ secrets.DD_CP_CF_ACCOUNT_ID }}
           CLOUDFLARE_ZONE_ID: ${{ secrets.DD_CP_CF_ZONE_ID }}
           DD_DOMAIN: ${{ vars.DD_CF_DOMAIN || 'devopsdefender.com' }}
           DD_GIT_SHA: ${{ github.sha }}
+          INTEL_API_KEY: ${{ secrets.INTEL_API_KEY }}
         run: |
+          # Canonical staging lives on GCP: control plane plus one disposable
+          # bootstrap/test agent. The OVH example-app VM is handled separately.
+          sha12="$(echo "${{ github.sha }}" | cut -c1-12)"
+          export DD_GCP_IMAGE_NAME="dd-agent-main-${sha12}"
           ansible-playbook infra/ansible/playbooks/gcp-deploy.yml \
             -e num_tiny_agents=1 -e num_standard_agents=0 -e num_llm_agents=0
 

CLAUDE.md

@@ -106,11 +106,8 @@ Central management API for agent registration, deployment orchestration, health
 
 **Deployment:** Multi-stage Dockerfile (Rust builder → debian bookworm-slim runtime with cloudflared).
 
-**Infrastructure** (`infra/`): Ansible playbooks for GCP and baremetal deployments. Key playbooks:
-- `gcp-control-plane-new.yml` — launch TDX-enabled GCP VM for control plane
-- `gcp-vm-fleet-new.yml` — launch agent fleet
-- `gcp-image-bake.yml` — build VM images
-- `baremetal-deploy.yml` — bare metal deployment
+**Infrastructure** (`infra/`): Ansible playbooks for deploying agent VMs via KVM on dedicated OVH hardware. Key playbook:
+- `baremetal-agent-deploy.yml` — build image via Packer + deploy agent as KVM VM on OVH dedicated server
 
 ### images/ (Packer)
 
@@ -201,9 +198,9 @@ cd private-llm && docker compose up
 
 All infrastructure is managed through GitHub Actions. **Never SSH into hosts, run Ansible locally, or attempt manual fixes on VMs.** If staging or production is down, trigger the appropriate GitHub Actions workflow.
 
-**Staging** (`staging-deploy.yml`) — auto-deploys on push to `main`. Pipeline: build → bake GCP agent image → cleanup old VMs → deploy via Ansible → smoke check `app-staging.devopsdefender.com/health`.
+**Staging** (`baremetal-staging-deploy.yml`) — auto-deploys on push to `main`. Deploys the agent as a KVM VM on a dedicated OVH server. Pipeline: build → Packer bake on host → deploy agent VM via KVM → deploy private-llm.
 
-**Production** (`production-deploy.yml`) — manual trigger only (`workflow_dispatch`). Inputs: `num_tiny_agents`, `num_standard_agents`, `num_llm_agents`. Same pipeline as staging.
+**Production** (`baremetal-production-deploy.yml`) — manual trigger only (`workflow_dispatch`). Deploys the agent as a KVM VM on dedicated OVH hardware with GPU passthrough. Inputs: `vfio_device`, `agent_memory`, `agent_cpus`.
 
 **Health check URLs:**
 - Staging: `https://app-staging.devopsdefender.com/health`
@@ -212,18 +209,18 @@ All infrastructure is managed through GitHub Actions. **Never SSH into hosts, ru
 **To deploy or fix an environment**, use `gh workflow run` or the GitHub Actions UI:
 ```bash
 # Trigger staging deploy
-gh workflow run staging-deploy.yml --repo devopsdefender/control-plane
+gh workflow run baremetal-staging-deploy.yml
 
 # Trigger production deploy
-gh workflow run production-deploy.yml --repo devopsdefender/control-plane \
-  -f num_tiny_agents=0 -f num_standard_agents=0 -f num_llm_agents=1
+gh workflow run baremetal-production-deploy.yml \
+  -f vfio_device=0d:00.0 -f agent_memory=128G -f agent_cpus=32
 ```
 
 ## CI/CD Pipelines
 
 Each component has its own GitHub Actions workflows:
 - **agent**: `ci.yml` (check/test/fmt/clippy), `release.yml` (build binary + GitHub release)
-- **control-plane**: `ci.yml`, `staging-deploy.yml`, `production-deploy.yml`, `release.yml`
+- **control-plane**: `ci.yml`, `baremetal-staging-deploy.yml`, `baremetal-production-deploy.yml`, `release.yml`
 - **images**: `baremetal-image.yml` (Packer build on self-hosted runner)
 - **private-llm**: `deploy.yml` (OIDC-authenticated deployment to DD platform)
 - **website**: `pages.yml` (GitHub Pages deployment)