diff --git a/.github/workflows/baremetal-production-deploy.yml b/.github/workflows/baremetal-production-deploy.yml index 20936d0..18bd0b2 100644 --- a/.github/workflows/baremetal-production-deploy.yml +++ b/.github/workflows/baremetal-production-deploy.yml @@ -1,24 +1,39 @@ +# Baremetal Production Deploy +# +# Builds the agent + control-plane binaries, bakes a VM image on the production +# host via Packer, and launches the new libvirt-managed OVH VM. +# +# Production keeps its external control plane. This workflow only replaces the +# application VM on the OVH host. + name: Baremetal Production Deploy on: + push: + branches: [main] + paths: + - "agent/**" + - "control-plane/**" + - "images/**" + - "infra/**" + - ".github/workflows/baremetal-*" workflow_dispatch: inputs: vfio_device: - description: "PCI address for GPU passthrough" + description: "PCI address for GPU passthrough (e.g. 0d:00.0)" default: "0d:00.0" agent_memory: - description: "Agent VM memory" - default: "128G" + description: "Agent VM memory (overrides inventory default)" + default: "" agent_cpus: - description: "Agent VM CPUs" - default: "32" + description: "Agent VM CPUs (overrides inventory default)" + default: "" concurrency: group: dd-baremetal-production cancel-in-progress: false permissions: - id-token: write contents: read jobs: @@ -27,54 +42,43 @@ jobs: environment: production steps: - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - - run: cargo build --workspace --release + + - name: Build release binaries + run: cargo build --workspace --release + - run: pip install ansible - name: Set up SSH run: | - mkdir -p ~/.ssh - echo "${{ secrets.BAREMETAL_SSH_KEY }}" > ~/.ssh/deploy_key - chmod 600 ~/.ssh/deploy_key - ssh-keyscan -H "${{ vars.BAREMETAL_PRODUCTION_HOST }}" >> ~/.ssh/known_hosts 2>/dev/null + echo "${{ secrets.BAREMETAL_SSH_KEY }}" > /tmp/deploy-key + chmod 600 /tmp/deploy-key + ssh-keyscan -H 162.222.34.121 >> ~/.ssh/known_hosts 2>/dev/null || true - name: Deploy agent to production host env: ANSIBLE_HOST_KEY_CHECKING: "false" run: | sha12="$(echo "${{ github.sha }}" | cut -c1-12)" + EXTRA_VARS=( + -e "agent_binary_local=${GITHUB_WORKSPACE}/target/release/dd-agent" + -e "cp_binary_local=${GITHUB_WORKSPACE}/target/release/dd-cp" + -e "packer_template_dir=${GITHUB_WORKSPACE}/images/packer" + -e "image_name=dd-baremetal-${sha12}" + -e "vfio_device=${{ inputs.vfio_device || '0d:00.0' }}" + ) + if [ -n "${{ inputs.agent_memory }}" ]; then + EXTRA_VARS+=(-e "agent_memory=${{ inputs.agent_memory }}") + fi + if [ -n "${{ inputs.agent_cpus }}" ]; then + EXTRA_VARS+=(-e "agent_cpus=${{ inputs.agent_cpus }}") + fi ansible-playbook infra/ansible/playbooks/baremetal-agent-deploy.yml \ - -i "${{ vars.BAREMETAL_PRODUCTION_HOST }}," \ - -u "${{ vars.BAREMETAL_PRODUCTION_USER }}" \ - --private-key ~/.ssh/deploy_key \ - -e "agent_binary_local=${GITHUB_WORKSPACE}/target/release/dd-agent" \ - -e "cp_binary_local=${GITHUB_WORKSPACE}/target/release/dd-cp" \ - -e "packer_template_dir=${GITHUB_WORKSPACE}/images/packer" \ - -e "image_name=dd-baremetal-${sha12}" \ - -e "dd_env=production" \ - -e "cp_url=https://app.${{ vars.DD_CF_DOMAIN || 'devopsdefender.com' }}" \ - -e "dd_skip_attestation=true" \ - -e "agent_node_size=llm" \ - -e "agent_memory=${{ inputs.agent_memory }}" \ - -e "agent_cpus=${{ inputs.agent_cpus }}" \ - -e "vfio_device=${{ inputs.vfio_device }}" - - - name: Deploy private-llm (H100 GPU mode) - env: - GH_TOKEN: ${{ github.token }} - run: | - CP_URL="https://app.${{ vars.DD_CF_DOMAIN || 'devopsdefender.com' }}" - OIDC_TOKEN=$(curl -s -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \ - "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=devopsdefender" | jq -r '.value') - gh api repos/devopsdefender/private-llm/contents/docker-compose.h100.yml \ - -H "Accept: application/vnd.github.raw" > /tmp/compose.yml - COMPOSE_B64=$(base64 -w0 /tmp/compose.yml) - curl -sf -X POST "${CP_URL}/api/v1/deploy" \ - -H "Authorization: Bearer ${OIDC_TOKEN}" \ - -H "Content-Type: application/json" \ - -d "{\"app_name\":\"private-llm\",\"app_version\":\"${{ github.sha }}\",\"compose\":\"${COMPOSE_B64}\",\"node_size\":\"llm\"}" + -i infra/ansible/inventory/production.yml \ + "${EXTRA_VARS[@]}" - - name: Cleanup + - name: Cleanup SSH key if: always() - run: rm -f ~/.ssh/deploy_key + run: rm -f /tmp/deploy-key diff --git a/.github/workflows/baremetal-staging-deploy.yml b/.github/workflows/baremetal-staging-deploy.yml index 58fa119..cea0123 100644 --- a/.github/workflows/baremetal-staging-deploy.yml +++ b/.github/workflows/baremetal-staging-deploy.yml @@ -1,7 +1,15 @@ -name: Baremetal Staging Deploy +# OVH Staging VM Deploy +# +# Builds the agent image, bakes a qcow2 on the OVH staging host, and launches +# the example-app VM there via libvirt so it is visible in `virsh list`. +# +# The control plane and first/bootstrap validation agent stay on GCP. +# This workflow only maintains the OVH VM that the example app targets. + +name: OVH Staging VM Deploy on: - push: + pull_request: branches: [main] paths: - "agent/**" @@ -16,67 +24,44 @@ concurrency: cancel-in-progress: true permissions: - id-token: write contents: read jobs: deploy: runs-on: ubuntu-latest - environment: staging + steps: - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - - run: cargo build --workspace --release + + - name: Build release binaries + run: cargo build --workspace --release + - run: pip install ansible - name: Set up SSH run: | - mkdir -p ~/.ssh - echo "${{ secrets.BAREMETAL_SSH_KEY }}" > ~/.ssh/deploy_key - chmod 600 ~/.ssh/deploy_key - ssh-keyscan -H "${{ vars.BAREMETAL_STAGING_HOST }}" >> ~/.ssh/known_hosts 2>/dev/null + echo "${{ secrets.BAREMETAL_SSH_KEY }}" > /tmp/deploy-key + chmod 600 /tmp/deploy-key + ssh-keyscan -H 57.130.10.246 >> ~/.ssh/known_hosts 2>/dev/null || true - - name: Deploy agent to staging host + - name: Deploy OVH staging VM env: ANSIBLE_HOST_KEY_CHECKING: "false" run: | sha12="$(echo "${{ github.sha }}" | cut -c1-12)" + EXTRA_VARS=( + -e "agent_binary_local=${GITHUB_WORKSPACE}/target/release/dd-agent" + -e "cp_binary_local=${GITHUB_WORKSPACE}/target/release/dd-cp" + -e "packer_template_dir=${GITHUB_WORKSPACE}/images/packer" + -e "image_name=dd-baremetal-${sha12}" + ) ansible-playbook infra/ansible/playbooks/baremetal-agent-deploy.yml \ - -i "${{ vars.BAREMETAL_STAGING_HOST }}," \ - -u "${{ vars.BAREMETAL_STAGING_USER }}" \ - --private-key ~/.ssh/deploy_key \ - -e "agent_binary_local=${GITHUB_WORKSPACE}/target/release/dd-agent" \ - -e "cp_binary_local=${GITHUB_WORKSPACE}/target/release/dd-cp" \ - -e "packer_template_dir=${GITHUB_WORKSPACE}/images/packer" \ - -e "image_name=dd-baremetal-${sha12}" \ - -e "dd_env=staging" \ - -e "cp_url=https://app-staging.${{ vars.DD_CF_DOMAIN || 'devopsdefender.com' }}" \ - -e "dd_skip_attestation=true" \ - -e "agent_node_size=standard" - - - name: Deploy private-llm (CPU mode) - env: - GH_TOKEN: ${{ github.token }} - run: | - CP_URL="https://app-staging.${{ vars.DD_CF_DOMAIN || 'devopsdefender.com' }}" - OIDC_TOKEN=$(curl -s -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \ - "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=devopsdefender" | jq -r '.value') - gh api repos/devopsdefender/private-llm/contents/docker-compose.yml \ - -H "Accept: application/vnd.github.raw" > /tmp/compose.yml - COMPOSE_B64=$(base64 -w0 /tmp/compose.yml) - RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "${CP_URL}/api/v1/deploy" \ - -H "Authorization: Bearer ${OIDC_TOKEN}" \ - -H "Content-Type: application/json" \ - -d "{\"app_name\":\"private-llm\",\"app_version\":\"${{ github.sha }}\",\"compose\":\"${COMPOSE_B64}\",\"node_size\":\"standard\"}") - HTTP_CODE=$(echo "${RESPONSE}" | tail -1) - BODY=$(echo "${RESPONSE}" | sed '$d') - echo "HTTP ${HTTP_CODE}: ${BODY}" - if [ "${HTTP_CODE}" -lt 200 ] || [ "${HTTP_CODE}" -ge 300 ]; then - echo "::error::LLM deploy failed: ${BODY}" - exit 1 - fi + -i infra/ansible/inventory/staging.yml \ + "${EXTRA_VARS[@]}" - - name: Cleanup + - name: Cleanup SSH key if: always() - run: rm -f ~/.ssh/deploy_key + run: rm -f /tmp/deploy-key diff --git a/.github/workflows/staging-deploy.yml b/.github/workflows/staging-deploy.yml index 8fe5b56..ad90fc4 100644 --- a/.github/workflows/staging-deploy.yml +++ b/.github/workflows/staging-deploy.yml @@ -54,16 +54,23 @@ jobs: -e "packer_template_path=${GITHUB_WORKSPACE}/images/packer/gcp-agent-image.pkr.hcl" \ -e "source_sha=${{ github.sha }}" - - name: Deploy + - name: Deploy GCP staging control plane and bootstrap agent env: GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + DD_ENV: staging DD_CP_ADMIN_PASSWORD: ${{ secrets.DD_CP_ADMIN_PASSWORD }} + CP_ADMIN_PASSWORD: ${{ secrets.DD_CP_ADMIN_PASSWORD }} CLOUDFLARE_API_TOKEN: ${{ secrets.DD_CP_CF_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.DD_CP_CF_ACCOUNT_ID }} CLOUDFLARE_ZONE_ID: ${{ secrets.DD_CP_CF_ZONE_ID }} DD_DOMAIN: ${{ vars.DD_CF_DOMAIN || 'devopsdefender.com' }} DD_GIT_SHA: ${{ github.sha }} + INTEL_API_KEY: ${{ secrets.INTEL_API_KEY }} run: | + # Canonical staging lives on GCP: control plane plus one disposable + # bootstrap/test agent. The OVH example-app VM is handled separately. + sha12="$(echo "${{ github.sha }}" | cut -c1-12)" + export DD_GCP_IMAGE_NAME="dd-agent-main-${sha12}" ansible-playbook infra/ansible/playbooks/gcp-deploy.yml \ -e num_tiny_agents=1 -e num_standard_agents=0 -e num_llm_agents=0 diff --git a/CLAUDE.md b/CLAUDE.md index 3e7e210..db3e93a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -106,11 +106,8 @@ Central management API for agent registration, deployment orchestration, health **Deployment:** Multi-stage Dockerfile (Rust builder → debian bookworm-slim runtime with cloudflared). -**Infrastructure** (`infra/`): Ansible playbooks for GCP and baremetal deployments. Key playbooks: -- `gcp-control-plane-new.yml` — launch TDX-enabled GCP VM for control plane -- `gcp-vm-fleet-new.yml` — launch agent fleet -- `gcp-image-bake.yml` — build VM images -- `baremetal-deploy.yml` — bare metal deployment +**Infrastructure** (`infra/`): Ansible playbooks for deploying agent VMs via KVM on dedicated OVH hardware. Key playbook: +- `baremetal-agent-deploy.yml` — build image via Packer + deploy agent as KVM VM on OVH dedicated server ### images/ (Packer) @@ -201,9 +198,9 @@ cd private-llm && docker compose up All infrastructure is managed through GitHub Actions. **Never SSH into hosts, run Ansible locally, or attempt manual fixes on VMs.** If staging or production is down, trigger the appropriate GitHub Actions workflow. -**Staging** (`staging-deploy.yml`) — auto-deploys on push to `main`. Pipeline: build → bake GCP agent image → cleanup old VMs → deploy via Ansible → smoke check `app-staging.devopsdefender.com/health`. +**Staging** (`baremetal-staging-deploy.yml`) — auto-deploys on push to `main`. Deploys the agent as a KVM VM on a dedicated OVH server. Pipeline: build → Packer bake on host → deploy agent VM via KVM → deploy private-llm. -**Production** (`production-deploy.yml`) — manual trigger only (`workflow_dispatch`). Inputs: `num_tiny_agents`, `num_standard_agents`, `num_llm_agents`. Same pipeline as staging. +**Production** (`baremetal-production-deploy.yml`) — manual trigger only (`workflow_dispatch`). Deploys the agent as a KVM VM on dedicated OVH hardware with GPU passthrough. Inputs: `vfio_device`, `agent_memory`, `agent_cpus`. **Health check URLs:** - Staging: `https://app-staging.devopsdefender.com/health` @@ -212,18 +209,18 @@ All infrastructure is managed through GitHub Actions. **Never SSH into hosts, ru **To deploy or fix an environment**, use `gh workflow run` or the GitHub Actions UI: ```bash # Trigger staging deploy -gh workflow run staging-deploy.yml --repo devopsdefender/control-plane +gh workflow run baremetal-staging-deploy.yml # Trigger production deploy -gh workflow run production-deploy.yml --repo devopsdefender/control-plane \ - -f num_tiny_agents=0 -f num_standard_agents=0 -f num_llm_agents=1 +gh workflow run baremetal-production-deploy.yml \ + -f vfio_device=0d:00.0 -f agent_memory=128G -f agent_cpus=32 ``` ## CI/CD Pipelines Each component has its own GitHub Actions workflows: - **agent**: `ci.yml` (check/test/fmt/clippy), `release.yml` (build binary + GitHub release) -- **control-plane**: `ci.yml`, `staging-deploy.yml`, `production-deploy.yml`, `release.yml` +- **control-plane**: `ci.yml`, `baremetal-staging-deploy.yml`, `baremetal-production-deploy.yml`, `release.yml` - **images**: `baremetal-image.yml` (Packer build on self-hosted runner) - **private-llm**: `deploy.yml` (OIDC-authenticated deployment to DD platform) - **website**: `pages.yml` (GitHub Pages deployment) diff --git a/Cargo.lock b/Cargo.lock index a00c175..1b22d23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.12" @@ -14,15 +20,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "aho-corasick" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" -dependencies = [ - "memchr", -] - [[package]] name = "android_system_properties" version = "0.1.5" @@ -50,16 +47,6 @@ dependencies = [ "password-hash", ] -[[package]] -name = "assert-json-diff" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "atomic-waker" version = "1.1.2" @@ -183,50 +170,6 @@ dependencies = [ "cipher", ] -[[package]] -name = "bollard" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97ccca1260af6a459d75994ad5acc1651bcabcbdbc41467cc9786519ab854c30" -dependencies = [ - "base64", - "bollard-stubs", - "bytes", - "futures-core", - "futures-util", - "hex", - "http", - "http-body-util", - "hyper", - "hyper-named-pipe", - "hyper-util", - "hyperlocal", - "log", - "pin-project-lite", - "serde", - "serde_derive", - "serde_json", - "serde_repr", - "serde_urlencoded", - "thiserror", - "tokio", - "tokio-util", - "tower-service", - "url", - "winapi", -] - -[[package]] -name = "bollard-stubs" -version = "1.47.1-rc.27.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f179cfbddb6e77a5472703d4b30436bff32929c0aa8a9008ecf23d1d3cdd0da" -dependencies = [ - "serde", - "serde_repr", - "serde_with", -] - [[package]] name = "bumpalo" version = "3.20.2" @@ -291,15 +234,6 @@ dependencies = [ "inout", ] -[[package]] -name = "colored" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" -dependencies = [ - "windows-sys 0.61.2", -] - [[package]] name = "core-foundation" version = "0.9.4" @@ -335,6 +269,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crypto-common" version = "0.1.7" @@ -350,9 +293,9 @@ name = "dd-agent" version = "0.1.0" dependencies = [ "base64", - "bollard", "chrono", - "futures-util", + "flate2", + "libc", "reqwest", "serde", "serde_json", @@ -370,9 +313,9 @@ dependencies = [ "base64", "bcrypt", "chrono", + "dd-agent", "http", "jsonwebtoken", - "mockito", "rand_core 0.6.4", "reqwest", "rpassword", @@ -393,7 +336,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ "powerfmt", - "serde_core", ] [[package]] @@ -418,12 +360,6 @@ dependencies = [ "syn", ] -[[package]] -name = "dyn-clone" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" - [[package]] name = "encoding_rs" version = "0.8.35" @@ -473,6 +409,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -531,17 +477,6 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" -[[package]] -name = "futures-macro" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "futures-sink" version = "0.3.32" @@ -562,7 +497,6 @@ checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-core", "futures-io", - "futures-macro", "futures-sink", "futures-task", "memchr", @@ -632,19 +566,13 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.13.0", + "indexmap", "slab", "tokio", "tokio-util", "tracing", ] -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.14.5" @@ -684,12 +612,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - [[package]] name = "http" version = "1.4.0" @@ -758,21 +680,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper-named-pipe" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" -dependencies = [ - "hex", - "hyper", - "hyper-util", - "pin-project-lite", - "tokio", - "tower-service", - "winapi", -] - [[package]] name = "hyper-rustls" version = "0.27.7" @@ -831,21 +738,6 @@ dependencies = [ "windows-registry", ] -[[package]] -name = "hyperlocal" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" -dependencies = [ - "hex", - "http-body-util", - "hyper", - "hyper-util", - "pin-project-lite", - "tokio", - "tower-service", -] - [[package]] name = "iana-time-zone" version = "0.1.65" @@ -978,17 +870,6 @@ dependencies = [ "icu_properties", ] -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", - "serde", -] - [[package]] name = "indexmap" version = "2.13.0" @@ -1131,6 +1012,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.1.1" @@ -1142,31 +1033,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "mockito" -version = "1.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90820618712cab19cfc46b274c6c22546a82affcb3c3bdf0f29e3db8e1bb92c0" -dependencies = [ - "assert-json-diff", - "bytes", - "colored", - "futures-core", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-util", - "log", - "pin-project-lite", - "rand", - "regex", - "serde_json", - "serde_urlencoded", - "similar", - "tokio", -] - [[package]] name = "native-tls" version = "0.2.18" @@ -1502,55 +1368,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "ref-cast" -version = "1.0.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" -dependencies = [ - "ref-cast-impl", -] - -[[package]] -name = "ref-cast-impl" -version = "1.0.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "regex" -version = "1.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" - [[package]] name = "reqwest" version = "0.12.28" @@ -1721,30 +1538,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "schemars" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" -dependencies = [ - "dyn-clone", - "ref-cast", - "serde", - "serde_json", -] - -[[package]] -name = "schemars" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" -dependencies = [ - "dyn-clone", - "ref-cast", - "serde", - "serde_json", -] - [[package]] name = "scopeguard" version = "1.2.0" @@ -1834,17 +1627,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "serde_repr" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1857,24 +1639,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_with" -version = "3.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd5414fad8e6907dbdd5bc441a50ae8d6e26151a03b1de04d89a5576de61d01f" -dependencies = [ - "base64", - "chrono", - "hex", - "indexmap 1.9.3", - "indexmap 2.13.0", - "schemars 0.9.0", - "schemars 1.2.1", - "serde_core", - "serde_json", - "time", -] - [[package]] name = "shlex" version = "1.3.0" @@ -1892,10 +1656,10 @@ dependencies = [ ] [[package]] -name = "similar" -version = "2.7.0" +name = "simd-adler32" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" [[package]] name = "simple_asn1" @@ -2392,7 +2156,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap 2.13.0", + "indexmap", "wasm-encoder", "wasmparser", ] @@ -2405,7 +2169,7 @@ checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ "bitflags", "hashbrown 0.15.5", - "indexmap 2.13.0", + "indexmap", "semver", ] @@ -2438,28 +2202,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windows-core" version = "0.62.2" @@ -2723,7 +2465,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck", - "indexmap 2.13.0", + "indexmap", "prettyplease", "syn", "wasm-metadata", @@ -2754,7 +2496,7 @@ checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", "bitflags", - "indexmap 2.13.0", + "indexmap", "log", "serde", "serde_derive", @@ -2773,7 +2515,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap 2.13.0", + "indexmap", "log", "semver", "serde", diff --git a/agent/Cargo.lock b/agent/Cargo.lock index c034fc2..eb5b7af 100644 --- a/agent/Cargo.lock +++ b/agent/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -41,51 +47,6 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" -[[package]] -name = "bollard" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97ccca1260af6a459d75994ad5acc1651bcabcbdbc41467cc9786519ab854c30" -dependencies = [ - "base64", - "bollard-stubs", - "bytes", - "futures-core", - "futures-util", - "hex", - "http", - "http-body-util", - "hyper", - "hyper-named-pipe", - "hyper-util", - "hyperlocal", - "log", - "pin-project-lite", - "serde", - "serde_derive", - "serde_json", - "serde_repr", - "serde_urlencoded", - "thiserror", - "tokio", - "tokio-util", - "tower-service", - "url", - "winapi", -] - -[[package]] -name = "bollard-stubs" -version = "1.47.1-rc.27.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f179cfbddb6e77a5472703d4b30436bff32929c0aa8a9008ecf23d1d3cdd0da" -dependencies = [ - "serde", - "serde_repr", - "serde_with", -] - -[[package]] name = "bumpalo" version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -159,18 +120,27 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "dd-agent" version = "0.1.0" dependencies = [ "base64", - "bollard", "chrono", + "flate2", "futures-util", + "libc", "reqwest", "serde", "serde_json", - "serde_yaml", "thiserror", "tokio", "uuid", @@ -234,6 +204,16 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" diff --git a/agent/Cargo.toml b/agent/Cargo.toml index 9af9289..af8ef9d 100644 --- a/agent/Cargo.toml +++ b/agent/Cargo.toml @@ -13,9 +13,9 @@ path = "src/bin/dd-agent/main.rs" [dependencies] base64 = "0.22" -bollard = "0.18" chrono = { version = "0.4", features = ["serde"] } -futures-util = "0.3" +flate2 = "1" +libc = "0.2" reqwest = { version = "0.12", features = ["blocking", "json", "rustls-tls"] } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/agent/src/api.rs b/agent/src/api.rs index f18ab46..a3db1db 100644 --- a/agent/src/api.rs +++ b/agent/src/api.rs @@ -1,4 +1,4 @@ -use serde::Deserialize; +use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Deserialize, PartialEq)] pub struct AgentChallengeResponse { @@ -12,3 +12,17 @@ pub struct AgentRegisterResponse { pub tunnel_token: String, pub hostname: String, } + +#[derive(Debug, Clone, Deserialize, PartialEq)] +pub struct AgentDeploymentResponse { + pub image: String, + pub env: Vec, + pub cmd: Vec, + pub deployment_id: String, +} + +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct AgentDeploymentStatusRequest { + pub status: String, + pub exit_code: Option, +} diff --git a/agent/src/attestation/tsm.rs b/agent/src/attestation/tsm.rs index b6514a0..bac64b7 100644 --- a/agent/src/attestation/tsm.rs +++ b/agent/src/attestation/tsm.rs @@ -167,7 +167,7 @@ pub fn generate_tdx_quote(report_root: &str, user_data: &[u8]) -> AppResult AppResult { +pub fn generate_tdx_quote_base64(user_data: Option<&[u8]>) -> AppResult { use base64::Engine; // Create a unique report entry under configfs-tsm. @@ -178,7 +178,7 @@ pub fn generate_tdx_quote_base64() -> AppResult { std::fs::create_dir_all(&report_root) .map_err(|e| AppError::External(format!("create tsm report dir: {e}")))?; - let quote_bytes = generate_tdx_quote(&report_root, &[])?; + let quote_bytes = generate_tdx_quote(&report_root, user_data.unwrap_or(&[]))?; // Clean up. let _ = std::fs::remove_dir_all(&report_root); diff --git a/agent/src/bin/dd-agent/config.rs b/agent/src/bin/dd-agent/config.rs index 7f5ae65..19ad43c 100644 --- a/agent/src/bin/dd-agent/config.rs +++ b/agent/src/bin/dd-agent/config.rs @@ -1,33 +1,12 @@ use serde::{Deserialize, Serialize}; -use std::collections::HashMap; use std::path::PathBuf; -/// The three operational modes the agent binary can run in. +/// The two operational modes the agent binary can run in. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub enum AgentMode { Agent, - ControlPlane, - Measure, -} - -impl AgentMode { - fn from_str_loose(s: &str) -> Option { - match s.to_lowercase().replace('_', "-").as_str() { - "agent" => Some(Self::Agent), - "control-plane" | "controlplane" | "cp" => Some(Self::ControlPlane), - "measure" => Some(Self::Measure), - _ => None, - } - } -} - -/// Which provided application the agent should manage. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub enum ProvidedApp { - ControlPlane, - Measure, + BootstrapCp, } /// Runtime configuration for the dd-agent binary. @@ -35,14 +14,18 @@ pub enum ProvidedApp { /// Loaded from a JSON config file with environment variable overrides. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AgentRuntimeConfig { - /// Operating mode (agent / control-plane / measure). - #[serde(default = "default_mode")] + /// Operating mode determined from bootstrap_cp / control_plane_url. + #[serde(skip, default = "default_mode")] pub mode: AgentMode, /// Base URL of the control plane the agent registers with. #[serde(default)] pub control_plane_url: Option, + /// Run as the bootstrap control plane instead of an attached agent. + #[serde(default)] + pub bootstrap_cp: bool, + /// Nominal size of this node (informational label). #[serde(default)] pub node_size: Option, @@ -51,29 +34,9 @@ pub struct AgentRuntimeConfig { #[serde(default)] pub datacenter: Option, - /// Intel Trust Authority API key used for attestation token retrieval. - #[serde(default)] - pub intel_api_key: Option, - - /// OCI image reference for the control-plane workload. - #[serde(default)] - pub control_plane_image: Option, - - /// OCI image reference for the measure workload. - #[serde(default)] - pub measure_app_image: Option, - - /// Which provided application to run, if any. - #[serde(default)] - pub provided_app: Option, - /// Port the workload should listen on. #[serde(default)] pub port: Option, - - /// Catch-all key/value pairs forwarded as environment variables. - #[serde(default)] - pub raw_kv: HashMap, } fn default_mode() -> AgentMode { @@ -85,14 +48,10 @@ impl Default for AgentRuntimeConfig { Self { mode: AgentMode::Agent, control_plane_url: None, + bootstrap_cp: false, node_size: None, datacenter: None, - intel_api_key: None, - control_plane_image: None, - measure_app_image: None, - provided_app: None, port: None, - raw_kv: HashMap::new(), } } } @@ -108,7 +67,8 @@ impl AgentRuntimeConfig { std::env::var("DD_CONFIG").unwrap_or_else(|_| Self::DEFAULT_CONFIG_PATH.to_string()); let mut cfg = Self::load_from_file(&config_path)?; - cfg.apply_env_overrides(); + cfg.apply_env_overrides()?; + cfg.mode = cfg.detect_mode()?; Ok(cfg) } @@ -123,18 +83,14 @@ impl AgentRuntimeConfig { serde_json::from_str(&text).map_err(|e| format!("failed to parse config file {path}: {e}")) } - fn apply_env_overrides(&mut self) { - if let Ok(val) = std::env::var("DD_AGENT_MODE") { - if let Some(mode) = AgentMode::from_str_loose(&val) { - self.mode = mode; - } - } - - // Control plane URL: prefer DD_CP_URL, fall back to AGENT_CP_URL. + fn apply_env_overrides(&mut self) -> Result<(), String> { if let Ok(val) = std::env::var("DD_CP_URL") { self.control_plane_url = Some(val); - } else if let Ok(val) = std::env::var("AGENT_CP_URL") { - self.control_plane_url = Some(val); + } + + if let Ok(val) = std::env::var("DD_BOOTSTRAP_CP") { + self.bootstrap_cp = parse_bool_env(&val) + .map_err(|_| format!("DD_BOOTSTRAP_CP must be a boolean, got {val:?}"))?; } if let Ok(val) = std::env::var("AGENT_NODE_SIZE") { @@ -145,23 +101,34 @@ impl AgentRuntimeConfig { self.datacenter = Some(val); } - if let Ok(val) = std::env::var("DD_INTEL_API_KEY") { - self.intel_api_key = Some(val); - } - - if let Ok(val) = std::env::var("DD_CP_IMAGE") { - self.control_plane_image = Some(val); - } - - if let Ok(val) = std::env::var("DD_MEASURE_IMAGE") { - self.measure_app_image = Some(val); - } - if let Ok(val) = std::env::var("DD_PORT") { if let Ok(p) = val.parse::() { self.port = Some(p); } } + + Ok(()) + } + + fn detect_mode(&self) -> Result { + match (self.bootstrap_cp, self.control_plane_url.as_ref()) { + (true, Some(_)) => Err( + "DD_BOOTSTRAP_CP=true cannot be combined with DD_CP_URL/control_plane_url".into(), + ), + (true, None) => Ok(AgentMode::BootstrapCp), + (false, Some(_)) => Ok(AgentMode::Agent), + (false, None) => { + Err("either DD_CP_URL/control_plane_url or DD_BOOTSTRAP_CP=true must be set".into()) + } + } + } +} + +fn parse_bool_env(val: &str) -> Result { + match val.trim().to_ascii_lowercase().as_str() { + "1" | "true" | "yes" | "on" => Ok(true), + "0" | "false" | "no" | "off" => Ok(false), + _ => Err(format!("invalid boolean value {val:?}")), } } @@ -176,24 +143,38 @@ mod tests { } #[test] - fn mode_from_str_loose() { - assert_eq!(AgentMode::from_str_loose("agent"), Some(AgentMode::Agent)); - assert_eq!( - AgentMode::from_str_loose("control-plane"), - Some(AgentMode::ControlPlane) - ); - assert_eq!( - AgentMode::from_str_loose("control_plane"), - Some(AgentMode::ControlPlane) - ); - assert_eq!( - AgentMode::from_str_loose("cp"), - Some(AgentMode::ControlPlane) - ); - assert_eq!( - AgentMode::from_str_loose("measure"), - Some(AgentMode::Measure) - ); - assert_eq!(AgentMode::from_str_loose("bogus"), None); + fn detect_mode_prefers_bootstrap_cp() { + let cfg = AgentRuntimeConfig { + bootstrap_cp: true, + ..AgentRuntimeConfig::default() + }; + assert_eq!(cfg.detect_mode().unwrap(), AgentMode::BootstrapCp); + } + + #[test] + fn detect_mode_agent_requires_cp_url() { + let cfg = AgentRuntimeConfig { + control_plane_url: Some("https://cp.example".into()), + ..AgentRuntimeConfig::default() + }; + assert_eq!(cfg.detect_mode().unwrap(), AgentMode::Agent); + } + + #[test] + fn detect_mode_rejects_ambiguous_config() { + let cfg = AgentRuntimeConfig { + bootstrap_cp: true, + control_plane_url: Some("https://cp.example".into()), + ..AgentRuntimeConfig::default() + }; + assert!(cfg.detect_mode().is_err()); + } + + #[test] + fn parse_bool_env_accepts_common_values() { + assert!(parse_bool_env("true").unwrap()); + assert!(parse_bool_env("1").unwrap()); + assert!(!parse_bool_env("false").unwrap()); + assert!(parse_bool_env("wat").is_err()); } } diff --git a/agent/src/bin/dd-agent/main.rs b/agent/src/bin/dd-agent/main.rs index 6b00e9b..9e0855e 100644 --- a/agent/src/bin/dd-agent/main.rs +++ b/agent/src/bin/dd-agent/main.rs @@ -1,14 +1,24 @@ mod config; -mod measure; mod oci; use config::{AgentMode, AgentRuntimeConfig}; -use dd_agent::api::{AgentChallengeResponse, AgentRegisterResponse}; - -// ── Entry point ──────────────────────────────────────────────────────────── +use dd_agent::api::{ + AgentChallengeResponse, AgentDeploymentResponse, AgentDeploymentStatusRequest, + AgentRegisterResponse, +}; + +#[derive(Debug)] +struct RunningDeployment { + deployment_id: String, +} #[tokio::main] async fn main() { + if std::env::args().any(|arg| arg == "--measure") { + run_measure_mode(); + return; + } + let cfg = match AgentRuntimeConfig::load() { Ok(c) => c, Err(e) => { @@ -21,23 +31,16 @@ async fn main() { match cfg.mode { AgentMode::Agent => run_agent_mode(cfg).await, - AgentMode::ControlPlane => run_control_plane_mode(cfg), - AgentMode::Measure => measure::run_measure_mode(), + AgentMode::BootstrapCp => run_bootstrap_cp_mode(cfg), } } -// ── Agent mode ───────────────────────────────────────────────────────────── - async fn run_agent_mode(cfg: AgentRuntimeConfig) { - let cp_url = match &cfg.control_plane_url { - Some(url) => url.clone(), - None => { - eprintln!("dd-agent: DD_CP_URL / control_plane_url not set"); - std::process::exit(1); - } - }; + let cp_url = cfg + .control_plane_url + .clone() + .expect("agent mode requires control_plane_url"); - // 1. Build an HTTP client. let http = match reqwest::Client::builder() .danger_accept_invalid_certs(false) .build() @@ -49,65 +52,73 @@ async fn run_agent_mode(cfg: AgentRuntimeConfig) { } }; - // 2. Obtain a challenge nonce from the control plane. - let challenge = match fetch_challenge(&http, &cp_url).await { - Ok(c) => c, + let registration = match register_with_retry(&http, &cp_url, &cfg).await { + Ok(r) => r, Err(e) => { - eprintln!("dd-agent: challenge failed: {e}"); + eprintln!("dd-agent: registration failed: {e}"); std::process::exit(1); } }; - eprintln!( - "dd-agent: received nonce (expires in {}s)", - challenge.expires_in_seconds - ); - - // 3. Generate a TDX quote embedding the nonce as report data. - let quote_b64 = match dd_agent::attestation::tsm::generate_tdx_quote_base64() { - Ok(q) => q, - Err(e) => { - eprintln!("dd-agent: TDX quote generation failed: {e}"); - std::process::exit(1); - } - }; - - // 4. Register with the control plane. - let registration = - match register_agent(&http, &cp_url, &challenge.nonce, "e_b64, &cfg).await { - Ok(r) => r, - Err(e) => { - eprintln!("dd-agent: registration failed: {e}"); - std::process::exit(1); - } - }; - eprintln!( "dd-agent: registered as {} at {}", registration.agent_id, registration.hostname ); - // 5. Start cloudflared tunnel. if let Err(e) = start_cloudflared(®istration.tunnel_token).await { eprintln!("dd-agent: cloudflared start failed: {e}"); - // Non-fatal: continue to workload. } - // 6. Run workload containers. - if let Err(e) = run_workloads(&cfg).await { - eprintln!("dd-agent: workload launch failed: {e}"); + deployment_loop(&http, &cp_url, ®istration.agent_id, cfg.port).await; +} + +async fn register_with_retry( + http: &reqwest::Client, + cp_url: &str, + cfg: &AgentRuntimeConfig, +) -> Result { + let max_retries = 30u32; + + for attempt in 1..=max_retries { + let challenge = match fetch_challenge(http, cp_url).await { + Ok(c) => c, + Err(e) => { + eprintln!("dd-agent: challenge failed (attempt {attempt}/{max_retries}): {e}"); + backoff_sleep(attempt).await; + continue; + } + }; + + let quote_b64 = match dd_agent::attestation::tsm::generate_tdx_quote_base64(Some( + challenge.nonce.as_bytes(), + )) { + Ok(q) => q, + Err(e) => { + eprintln!( + "dd-agent: TDX quote generation failed (attempt {attempt}/{max_retries}): {e}" + ); + backoff_sleep(attempt).await; + continue; + } + }; + + match register_agent(http, cp_url, &challenge.nonce, "e_b64, cfg).await { + Ok(r) => return Ok(r), + Err(e) => { + eprintln!("dd-agent: registration failed (attempt {attempt}/{max_retries}): {e}"); + backoff_sleep(attempt).await; + } + } } - // 7. Heartbeat / reconciliation loop. - let agent_id = registration.agent_id.clone(); - heartbeat_loop(&http, &cp_url, &agent_id).await; + Err(format!("failed after {max_retries} attempts")) } async fn fetch_challenge( http: &reqwest::Client, cp_url: &str, ) -> Result { - let url = format!("{cp_url}/api/agents/challenge"); + let url = format!("{cp_url}/api/v1/agents/challenge"); let resp = http .get(&url) .send() @@ -127,14 +138,14 @@ async fn register_agent( http: &reqwest::Client, cp_url: &str, nonce: &str, - quote_b64: &str, + raw_quote_b64: &str, cfg: &AgentRuntimeConfig, ) -> Result { - let url = format!("{cp_url}/api/agents/register"); + let url = format!("{cp_url}/api/v1/agents/register"); let body = serde_json::json!({ "nonce": nonce, - "quote": quote_b64, + "intel_ta_token": raw_quote_b64, "vm_name": hostname(), "node_size": cfg.node_size, "datacenter": cfg.datacenter, @@ -158,96 +169,224 @@ async fn register_agent( .map_err(|e| format!("parse register response: {e}")) } -async fn start_cloudflared(tunnel_token: &str) -> Result<(), String> { - use tokio::process::Command; +async fn deployment_loop(http: &reqwest::Client, cp_url: &str, agent_id: &str, port: Option) { + let runtime = match oci::NativeOciRuntime::new() { + Ok(r) => r, + Err(e) => { + eprintln!("dd-agent: failed to initialize OCI runtime: {e}"); + std::process::exit(1); + } + }; - eprintln!("dd-agent: starting cloudflared tunnel"); + let heartbeat_interval = std::time::Duration::from_secs(30); + let mut running: Option = None; - let mut child = Command::new("cloudflared") - .args(["tunnel", "--no-autoupdate", "run", "--token", tunnel_token]) - .spawn() - .map_err(|e| format!("spawn cloudflared: {e}"))?; + loop { + if let Err(e) = send_heartbeat(http, cp_url, agent_id).await { + eprintln!("dd-agent: heartbeat failed: {e}"); + } - // Give cloudflared a moment to start, then check it hasn't crashed. - tokio::time::sleep(std::time::Duration::from_secs(2)).await; + check_cloudflared().await; - match child.try_wait() { - Ok(Some(status)) => Err(format!("cloudflared exited immediately: {status}")), - Ok(None) => { - eprintln!("dd-agent: cloudflared running"); - // Detach -- we don't await the child so it keeps running. - Ok(()) + if let Some(current) = &running { + match runtime.try_wait() { + Ok(Some(exit)) => { + eprintln!( + "dd-agent: deployment {} exited success={} code={:?}", + current.deployment_id, exit.success, exit.exit_code + ); + let status = if exit.success { "stopped" } else { "failed" }; + if let Err(e) = report_deployment_status( + http, + cp_url, + agent_id, + ¤t.deployment_id, + status, + exit.exit_code, + ) + .await + { + eprintln!("dd-agent: failed to report deployment exit: {e}"); + } + running = None; + } + Ok(None) => { + tokio::time::sleep(heartbeat_interval).await; + continue; + } + Err(e) => { + eprintln!("dd-agent: failed waiting for workload exit: {e}"); + tokio::time::sleep(heartbeat_interval).await; + continue; + } + } } - Err(e) => Err(format!("cloudflared wait error: {e}")), + + match fetch_deployment(http, cp_url, agent_id).await { + Ok(Some(dep)) => match pull_and_run(&runtime, &dep, port).await { + Ok(pid) => { + eprintln!( + "dd-agent: deployment {} running as pid {}", + dep.deployment_id, pid + ); + if let Err(e) = report_deployment_status( + http, + cp_url, + agent_id, + &dep.deployment_id, + "running", + None, + ) + .await + { + eprintln!("dd-agent: failed to report running deployment: {e}"); + } + running = Some(RunningDeployment { + deployment_id: dep.deployment_id, + }); + } + Err(e) => { + eprintln!("dd-agent: deployment start failed: {e}"); + if let Err(report_err) = report_deployment_status( + http, + cp_url, + agent_id, + &dep.deployment_id, + "failed", + None, + ) + .await + { + eprintln!("dd-agent: failed to report deployment failure: {report_err}"); + } + } + }, + Ok(None) => {} + Err(e) => eprintln!("dd-agent: deployment poll failed: {e}"), + } + + tokio::time::sleep(heartbeat_interval).await; } } -async fn run_workloads(cfg: &AgentRuntimeConfig) -> Result<(), String> { - let runtime = oci::DockerOciRuntime::new()?; - - // Determine which image to launch based on provided_app or mode. - let image = match &cfg.provided_app { - Some(config::ProvidedApp::ControlPlane) => cfg.control_plane_image.as_deref(), - Some(config::ProvidedApp::Measure) => cfg.measure_app_image.as_deref(), - None => None, - }; +async fn send_heartbeat( + http: &reqwest::Client, + cp_url: &str, + agent_id: &str, +) -> Result<(), String> { + let url = format!("{cp_url}/api/v1/agents/{agent_id}/heartbeat"); + let resp = http + .post(&url) + .send() + .await + .map_err(|e| format!("POST {url}: {e}"))?; - let image = match image { - Some(img) => img.to_string(), - None => { - eprintln!("dd-agent: no workload image configured, skipping"); - return Ok(()); - } - }; + if resp.status().is_success() { + Ok(()) + } else { + Err(format!("POST {url}: status {}", resp.status())) + } +} - eprintln!("dd-agent: pulling image {image}"); - runtime.pull_image(&image).await?; +async fn fetch_deployment( + http: &reqwest::Client, + cp_url: &str, + agent_id: &str, +) -> Result, String> { + let url = format!("{cp_url}/api/v1/agents/{agent_id}/deployment"); + let resp = http + .get(&url) + .send() + .await + .map_err(|e| format!("GET {url}: {e}"))?; - let port = cfg.port.unwrap_or(8080); + match resp.status() { + reqwest::StatusCode::OK => resp + .json::() + .await + .map(Some) + .map_err(|e| format!("parse deployment response: {e}")), + reqwest::StatusCode::NO_CONTENT => Ok(None), + status => { + let body = resp.text().await.unwrap_or_default(); + Err(format!("GET {url}: status {status}: {body}")) + } + } +} - let req = oci::LaunchRequest { - image: image.clone(), - name: Some("dd-workload".into()), - env: cfg.raw_kv.iter().map(|(k, v)| format!("{k}={v}")).collect(), - ports: vec![oci::PortMapping { +async fn pull_and_run( + runtime: &oci::NativeOciRuntime, + dep: &AgentDeploymentResponse, + port: Option, +) -> Result { + eprintln!("dd-agent: pulling image {}", dep.image); + runtime.pull_image(&dep.image).await?; + + let mut ports = Vec::new(); + if let Some(port) = port { + ports.push(oci::PortMapping { host_port: port, container_port: port, protocol: "tcp".into(), - }], - cmd: vec![], + }); + } + + let req = oci::LaunchRequest { + image: dep.image.clone(), + name: Some(format!("dd-workload-{}", dep.deployment_id)), + env: dep.env.clone(), + ports, + cmd: dep.cmd.clone(), }; - let container_id = runtime.create_and_start(&req).await?; - eprintln!("dd-agent: workload container started: {container_id}"); - Ok(()) + runtime.create_and_start(&req).await } -async fn heartbeat_loop(http: &reqwest::Client, cp_url: &str, agent_id: &str) { - let url = format!("{cp_url}/api/agents/{agent_id}/heartbeat"); - let mut interval = tokio::time::interval(std::time::Duration::from_secs(30)); +async fn report_deployment_status( + http: &reqwest::Client, + cp_url: &str, + agent_id: &str, + deployment_id: &str, + status: &str, + exit_code: Option, +) -> Result<(), String> { + let url = format!("{cp_url}/api/v1/agents/{agent_id}/deployment/{deployment_id}/status"); + let body = AgentDeploymentStatusRequest { + status: status.to_string(), + exit_code, + }; + let resp = http + .post(&url) + .json(&body) + .send() + .await + .map_err(|e| format!("POST {url}: {e}"))?; - loop { - interval.tick().await; + if resp.status().is_success() { + Ok(()) + } else { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + Err(format!("POST {url}: status {status}: {body}")) + } +} - match http.post(&url).send().await { - Ok(resp) if resp.status().is_success() => { - // Heartbeat acknowledged. - } - Ok(resp) => { - eprintln!( - "dd-agent: heartbeat rejected (status {}), attempting re-registration", - resp.status() - ); - // In a full implementation we would re-register here. - // For now just log and continue. - } - Err(e) => { - eprintln!("dd-agent: heartbeat failed: {e}"); - } - } +async fn start_cloudflared(tunnel_token: &str) -> Result<(), String> { + use tokio::process::Command; - // Check cloudflared is still running. - check_cloudflared().await; + eprintln!("dd-agent: starting cloudflared tunnel"); + + let mut child = Command::new("cloudflared") + .args(["tunnel", "--no-autoupdate", "run", "--token", tunnel_token]) + .spawn() + .map_err(|e| format!("spawn cloudflared: {e}"))?; + + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + + match child.try_wait() { + Ok(Some(status)) => Err(format!("cloudflared exited immediately: {status}")), + Ok(None) => Ok(()), + Err(e) => Err(format!("cloudflared wait error: {e}")), } } @@ -255,55 +394,82 @@ async fn check_cloudflared() { use tokio::process::Command; let output = Command::new("pgrep").arg("cloudflared").output().await; - - match output { - Ok(o) if o.status.success() => { /* still running */ } - _ => { - eprintln!("dd-agent: cloudflared not running (may need restart)"); - } + if !matches!(output, Ok(o) if o.status.success()) { + eprintln!("dd-agent: cloudflared not running (may need restart)"); } } -// ── Control-plane mode ───────────────────────────────────────────────────── +fn run_bootstrap_cp_mode(cfg: AgentRuntimeConfig) { + use std::os::unix::process::CommandExt; -fn run_control_plane_mode(cfg: AgentRuntimeConfig) { - eprintln!("dd-agent: starting control plane (dd-cp)"); + let quote_b64 = match dd_agent::attestation::tsm::generate_tdx_quote_base64(None) { + Ok(quote) => quote, + Err(e) => { + eprintln!("dd-agent: bootstrap CP quote generation failed: {e}"); + std::process::exit(1); + } + }; + let dd_env = match std::env::var("DD_ENV") { + Ok(value) => value, + Err(_) => { + eprintln!("dd-agent: bootstrap CP mode requires DD_ENV to be set"); + std::process::exit(1); + } + }; - let mut cmd = std::process::Command::new("dd-cp"); + let dd_cp = std::env::current_exe() + .ok() + .and_then(|p| p.parent().map(|d| d.join("dd-cp"))) + .filter(|p| p.exists()) + .unwrap_or_else(|| std::path::PathBuf::from("/usr/local/bin/dd-cp")); - // Forward relevant configuration as environment variables. - if let Some(ref dc) = cfg.datacenter { - cmd.env("DD_DATACENTER", dc); - } - if let Some(ref key) = cfg.intel_api_key { - cmd.env("DD_INTEL_API_KEY", key); - } - if let Some(ref port) = cfg.port { + let mut cmd = std::process::Command::new(&dd_cp); + cmd.env("DD_ENV", dd_env); + if let Some(port) = cfg.port { cmd.env("DD_PORT", port.to_string()); } - - // Forward raw key-value pairs. - for (k, v) in &cfg.raw_kv { - cmd.env(k, v); + if let Some(dc) = cfg.datacenter { + cmd.env("DD_DATACENTER", dc); } + cmd.env("DD_SELF_QUOTE_B64", quote_b64); - match cmd.status() { - Ok(status) => { - if !status.success() { - eprintln!("dd-agent: dd-cp exited with {status}"); - std::process::exit(status.code().unwrap_or(1)); + let err = cmd.exec(); + eprintln!("dd-agent: failed to exec dd-cp: {err}"); + std::process::exit(1); +} + +fn run_measure_mode() { + eprintln!("dd-agent: entering measure mode"); + + match dd_agent::attestation::tsm::generate_tdx_quote_base64(None) { + Ok(b64_quote) => match dd_agent::attestation::tsm::parse_tdx_quote_base64(&b64_quote) { + Ok(parsed) => { + println!("mrtd: {}", parsed.mrtd_hex()); + println!("rtmr0: {}", parsed.rtmr_hex(0)); + println!("rtmr1: {}", parsed.rtmr_hex(1)); + println!("rtmr2: {}", parsed.rtmr_hex(2)); + println!("rtmr3: {}", parsed.rtmr_hex(3)); + println!("report_data: {}", parsed.report_data_hex()); + println!("quote_b64: {b64_quote}"); } - } + Err(e) => { + eprintln!("failed to parse generated quote: {e}"); + std::process::exit(1); + } + }, Err(e) => { - eprintln!("dd-agent: failed to start dd-cp: {e}"); + eprintln!("failed to generate TDX quote: {e}"); + eprintln!("(this is expected when not running inside a TDX VM)"); std::process::exit(1); } } } -// ── Helpers ──────────────────────────────────────────────────────────────── +async fn backoff_sleep(attempt: u32) { + let secs = std::cmp::min(5 * 2u64.saturating_pow(attempt.saturating_sub(1)), 60); + tokio::time::sleep(std::time::Duration::from_secs(secs)).await; +} -/// Best-effort hostname for this VM. fn hostname() -> String { std::fs::read_to_string("/etc/hostname") .unwrap_or_else(|_| "unknown".into()) diff --git a/agent/src/bin/dd-agent/measure.rs b/agent/src/bin/dd-agent/measure.rs deleted file mode 100644 index b201767..0000000 --- a/agent/src/bin/dd-agent/measure.rs +++ /dev/null @@ -1,35 +0,0 @@ -use dd_agent::attestation::tsm; - -/// Run the agent in "measure" mode. -/// -/// Generates TDX measurements for the currently running VM and prints -/// them to stdout so they can be captured by tooling. -pub fn run_measure_mode() { - eprintln!("dd-agent: entering measure mode"); - - // In a real TDX VM this would generate a live quote. When running - // outside a TDX environment the tsm calls will fail, which is - // expected during development. - match tsm::generate_tdx_quote_base64() { - Ok(b64_quote) => match tsm::parse_tdx_quote_base64(&b64_quote) { - Ok(parsed) => { - println!("mrtd: {}", parsed.mrtd_hex()); - println!("rtmr0: {}", parsed.rtmr_hex(0)); - println!("rtmr1: {}", parsed.rtmr_hex(1)); - println!("rtmr2: {}", parsed.rtmr_hex(2)); - println!("rtmr3: {}", parsed.rtmr_hex(3)); - println!("report_data: {}", parsed.report_data_hex()); - println!("quote_b64: {b64_quote}"); - } - Err(e) => { - eprintln!("failed to parse generated quote: {e}"); - std::process::exit(1); - } - }, - Err(e) => { - eprintln!("failed to generate TDX quote: {e}"); - eprintln!("(this is expected when not running inside a TDX VM)"); - std::process::exit(1); - } - } -} diff --git a/agent/src/bin/dd-agent/oci/mod.rs b/agent/src/bin/dd-agent/oci/mod.rs index 1465961..cf04cab 100644 --- a/agent/src/bin/dd-agent/oci/mod.rs +++ b/agent/src/bin/dd-agent/oci/mod.rs @@ -1,13 +1,20 @@ -use bollard::container::{ - Config, CreateContainerOptions, ListContainersOptions, LogsOptions, RemoveContainerOptions, - StartContainerOptions, StopContainerOptions, -}; -use bollard::image::CreateImageOptions; -use bollard::models::{HostConfig, PortBinding}; -use bollard::Docker; -use futures_util::StreamExt; +mod registry; +mod unpack; + +use registry::RegistryClient; use serde::{Deserialize, Serialize}; use std::collections::HashMap; +use std::ffi::CString; +use std::fs; +use std::os::unix::process::CommandExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::{Arc, Mutex}; +use unpack::{prepare_rootfs_dir, unpack_layers}; + +const WORKLOAD_DIR: &str = "/var/lib/dd/workload"; +const ROOTFS_DIR: &str = "/var/lib/dd/workload/rootfs"; +const MOUNT_POINTS: [&str; 3] = ["proc", "sys", "dev"]; /// A port mapping from host to container. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -43,170 +50,369 @@ pub struct LaunchRequest { pub cmd: Vec, } -/// OCI runtime backed by the Docker/Podman API via bollard. -#[allow(dead_code)] -pub struct DockerOciRuntime { - client: Docker, +#[derive(Debug, Clone, Default)] +struct PreparedImage { + image: String, + entrypoint: Vec, + cmd: Vec, + env: Vec, + working_dir: Option, +} + +#[derive(Debug, Default)] +struct RuntimeState { + prepared: Option, + child_pid: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ExitStatus { + pub success: bool, + pub exit_code: Option, +} + +/// Native OCI runtime backed by direct registry HTTP access and Linux process primitives. +pub struct NativeOciRuntime { + registry: RegistryClient, + state: Arc>, + rootfs_dir: PathBuf, } -#[allow(dead_code)] -impl DockerOciRuntime { - /// Connect to the local Docker/Podman socket. +impl NativeOciRuntime { pub fn new() -> Result { - let client = Docker::connect_with_local_defaults() - .map_err(|e| format!("failed to connect to container runtime: {e}"))?; - Ok(Self { client }) + fs::create_dir_all(WORKLOAD_DIR) + .map_err(|e| format!("create workload directory {WORKLOAD_DIR}: {e}"))?; + + Ok(Self { + registry: RegistryClient::new()?, + state: Arc::new(Mutex::new(RuntimeState::default())), + rootfs_dir: PathBuf::from(ROOTFS_DIR), + }) } - /// Pull an image from a registry. pub async fn pull_image(&self, image: &str) -> Result<(), String> { - let opts = CreateImageOptions { - from_image: image, - ..Default::default() + self.cleanup_mounts()?; + prepare_rootfs_dir(&self.rootfs_dir)?; + + let pulled = self.registry.pull_image(image).await?; + unpack_layers(&self.rootfs_dir, pulled.layers.iter().map(Vec::as_slice))?; + + let prepared = PreparedImage { + image: image.to_string(), + entrypoint: pulled.entrypoint, + cmd: pulled.cmd, + env: pulled.env, + working_dir: pulled.working_dir, }; - let mut stream = self.client.create_image(Some(opts), None, None); - while let Some(result) = stream.next().await { - match result { - Ok(info) => { - if let Some(status) = &info.status { - eprintln!("pull: {status}"); - } - } - Err(e) => return Err(format!("image pull failed: {e}")), - } - } + let mut state = self + .state + .lock() + .map_err(|_| "runtime state lock poisoned".to_string())?; + state.prepared = Some(prepared); Ok(()) } - /// Create and start a container from a [`LaunchRequest`]. - pub async fn create_and_start(&self, req: &LaunchRequest) -> Result { - // Build port bindings for the host config. - let mut port_bindings: HashMap>> = HashMap::new(); - let mut exposed_port_keys: Vec = Vec::new(); - - for pm in &req.ports { - let container_key = format!("{}/{}", pm.container_port, pm.protocol); - port_bindings.insert( - container_key.clone(), - Some(vec![PortBinding { - host_ip: Some("0.0.0.0".into()), - host_port: Some(pm.host_port.to_string()), - }]), - ); - exposed_port_keys.push(container_key); - } - - let host_config = HostConfig { - port_bindings: Some(port_bindings), - ..Default::default() + pub async fn create_and_start(&self, req: &LaunchRequest) -> Result { + let prepared = { + let state = self + .state + .lock() + .map_err(|_| "runtime state lock poisoned".to_string())?; + state.prepared.clone() }; - let cmd: Vec<&str> = req.cmd.iter().map(|s| s.as_str()).collect(); - - // Build exposed_ports with borrowed keys. - let exposed_ports: HashMap<&str, HashMap<(), ()>> = exposed_port_keys - .iter() - .map(|k| (k.as_str(), HashMap::new())) - .collect(); - - let config = Config { - image: Some(req.image.as_str()), - env: Some(req.env.iter().map(|s| s.as_str()).collect()), - cmd: if cmd.is_empty() { None } else { Some(cmd) }, - exposed_ports: if exposed_ports.is_empty() { - None - } else { - Some(exposed_ports) - }, - host_config: Some(host_config), - ..Default::default() + let prepared = match prepared { + Some(prepared) if prepared.image == req.image => prepared, + _ => { + self.pull_image(&req.image).await?; + let state = self + .state + .lock() + .map_err(|_| "runtime state lock poisoned".to_string())?; + state + .prepared + .clone() + .ok_or_else(|| "image pull completed without prepared state".to_string())? + } }; - let create_opts = req.name.as_ref().map(|n| CreateContainerOptions { - name: n.as_str(), - platform: None, - }); + let argv = build_command_argv(&prepared, req)?; + let program = argv[0].clone(); + let args = &argv[1..]; + let envs = merge_env(&prepared.env, &req.env); - let container = self - .client - .create_container(create_opts, config) - .await - .map_err(|e| format!("create container failed: {e}"))?; + self.mount_rootfs_support()?; - self.client - .start_container(&container.id, None::>) - .await - .map_err(|e| format!("start container failed: {e}"))?; + let mut command = Command::new(&program); + command.args(args); + command.env_clear(); + command.envs(envs); + command.stdin(Stdio::null()); + command.stdout(Stdio::inherit()); + command.stderr(Stdio::inherit()); + let rootfs = self.rootfs_dir.clone(); + let chdir_target = prepared + .working_dir + .clone() + .unwrap_or_else(|| "/".to_string()); + unsafe { + command.pre_exec(move || { + let rootfs_cstr = path_to_cstring(&rootfs)?; + if libc::chroot(rootfs_cstr.as_ptr()) != 0 { + return Err(std::io::Error::last_os_error()); + } + + let chdir_cstr = CString::new(chdir_target.as_str()).map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("working directory contains NUL byte: {chdir_target:?}"), + ) + })?; + if libc::chdir(chdir_cstr.as_ptr()) != 0 { + return Err(std::io::Error::last_os_error()); + } - Ok(container.id) + Ok(()) + }); + } + + let child = command + .spawn() + .map_err(|e| format!("spawn workload {program}: {e}"))?; + + let pid = i32::try_from(child.id()) + .map_err(|_| format!("child PID {} does not fit in i32", child.id()))?; + + let mut state = self + .state + .lock() + .map_err(|_| "runtime state lock poisoned".to_string())?; + state.child_pid = Some(pid); + + Ok(pid) } - /// Stop a running container. - pub async fn stop_container(&self, container_id: &str) -> Result<(), String> { - self.client - .stop_container(container_id, Some(StopContainerOptions { t: 10 })) - .await - .map_err(|e| format!("stop container failed: {e}"))?; - Ok(()) + pub fn try_wait(&self) -> Result, String> { + let pid = { + let state = self + .state + .lock() + .map_err(|_| "runtime state lock poisoned".to_string())?; + state.child_pid + }; + + let Some(pid) = pid else { + return Ok(None); + }; + + let mut raw_status = 0; + let waited = unsafe { libc::waitpid(pid, &mut raw_status, libc::WNOHANG) }; + if waited == 0 { + return Ok(None); + } + if waited < 0 { + return Err(format!( + "waitpid({pid}) failed: {}", + std::io::Error::last_os_error() + )); + } + + let exit = if libc::WIFEXITED(raw_status) { + ExitStatus { + success: libc::WEXITSTATUS(raw_status) == 0, + exit_code: Some(libc::WEXITSTATUS(raw_status)), + } + } else if libc::WIFSIGNALED(raw_status) { + ExitStatus { + success: false, + exit_code: Some(128 + libc::WTERMSIG(raw_status)), + } + } else { + ExitStatus { + success: false, + exit_code: None, + } + }; + + let mut state = self + .state + .lock() + .map_err(|_| "runtime state lock poisoned".to_string())?; + state.child_pid = None; + Ok(Some(exit)) } - /// Remove a container (force). - pub async fn remove_container(&self, container_id: &str) -> Result<(), String> { - self.client - .remove_container( - container_id, - Some(RemoveContainerOptions { - force: true, - ..Default::default() - }), - ) - .await - .map_err(|e| format!("remove container failed: {e}"))?; + fn mount_rootfs_support(&self) -> Result<(), String> { + for mount_name in MOUNT_POINTS { + let target = self.rootfs_dir.join(mount_name); + fs::create_dir_all(&target) + .map_err(|e| format!("create mount point {}: {e}", target.display()))?; + + if is_mountpoint(&target)? { + continue; + } + + let status = Command::new("mount") + .args([ + "--rbind", + &format!("/{mount_name}"), + &target.display().to_string(), + ]) + .status() + .map_err(|e| format!("bind mount {mount_name} into {}: {e}", target.display()))?; + + if !status.success() { + return Err(format!( + "mount --rbind /{mount_name} {} failed with status {status}", + target.display() + )); + } + } + Ok(()) } - /// Retrieve recent logs from a container. - pub async fn logs(&self, container_id: &str, tail: usize) -> Result, String> { - let opts = LogsOptions:: { - stdout: true, - stderr: true, - tail: tail.to_string(), - ..Default::default() - }; + fn cleanup_mounts(&self) -> Result<(), String> { + for mount_name in MOUNT_POINTS.into_iter().rev() { + let target = self.rootfs_dir.join(mount_name); + if !target.exists() || !is_mountpoint(&target)? { + continue; + } - let mut stream = self.client.logs(container_id, Some(opts)); - let mut lines = Vec::new(); + let status = Command::new("umount") + .args(["-l", &target.display().to_string()]) + .status() + .map_err(|e| format!("unmount {}: {e}", target.display()))?; - while let Some(result) = stream.next().await { - match result { - Ok(output) => lines.push(output.to_string()), - Err(e) => return Err(format!("logs failed: {e}")), + if !status.success() { + return Err(format!( + "umount -l {} failed with status {status}", + target.display() + )); } } - Ok(lines) + Ok(()) + } +} + +fn is_mountpoint(path: &Path) -> Result { + let status = Command::new("mountpoint") + .args(["-q", &path.display().to_string()]) + .status() + .map_err(|e| format!("check mountpoint {}: {e}", path.display()))?; + Ok(status.success()) +} + +fn path_to_cstring(path: &Path) -> std::io::Result { + let bytes = path.as_os_str().as_encoded_bytes(); + CString::new(bytes).map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("path contains NUL byte: {}", path.display()), + ) + }) +} + +fn build_command_argv( + prepared: &PreparedImage, + req: &LaunchRequest, +) -> Result, String> { + let mut argv = if prepared.entrypoint.is_empty() { + Vec::new() + } else { + prepared.entrypoint.clone() + }; + + if req.cmd.is_empty() { + argv.extend(prepared.cmd.clone()); + } else { + argv.extend(req.cmd.clone()); + } + + if argv.is_empty() { + return Err("image config did not provide an entrypoint or command".to_string()); } - /// List containers, optionally filtering by label. - pub async fn list_containers(&self, label_filter: Option<&str>) -> Result, String> { - let mut filters = HashMap::new(); - if let Some(label) = label_filter { - filters.insert("label", vec![label]); + argv[0] = normalize_executable(&argv[0], prepared.working_dir.as_deref()); + Ok(argv) +} + +fn normalize_executable(program: &str, working_dir: Option<&str>) -> String { + if program.starts_with('/') { + return program.to_string(); + } + + let base = working_dir.unwrap_or("/"); + let prefix = if base.ends_with('/') { + base.trim_end_matches('/') + } else { + base + }; + + if prefix.is_empty() { + format!("/{program}") + } else { + format!("{prefix}/{program}") + } +} + +fn merge_env(image_env: &[String], request_env: &[String]) -> HashMap { + let mut envs = HashMap::new(); + + for entry in image_env.iter().chain(request_env.iter()) { + if let Some((key, value)) = entry.split_once('=') { + envs.insert(key.to_string(), value.to_string()); } + } - let opts = ListContainersOptions { - all: true, - filters, - ..Default::default() + envs +} + +#[cfg(test)] +mod tests { + use super::{ + build_command_argv, merge_env, normalize_executable, LaunchRequest, PreparedImage, + }; + + #[test] + fn command_override_extends_entrypoint() { + let prepared = PreparedImage { + entrypoint: vec!["/bin/server".into()], + cmd: vec!["--serve".into()], + ..PreparedImage::default() + }; + + let req = LaunchRequest { + image: "ghcr.io/example/app:latest".into(), + name: None, + env: Vec::new(), + ports: Vec::new(), + cmd: vec!["--foreground".into()], }; - let containers = self - .client - .list_containers(Some(opts)) - .await - .map_err(|e| format!("list containers failed: {e}"))?; + let argv = build_command_argv(&prepared, &req).unwrap(); + assert_eq!(argv, vec!["/bin/server", "--foreground"]); + } + + #[test] + fn merge_env_prefers_request_values() { + let envs = merge_env( + &["PATH=/usr/bin".into(), "PORT=8080".into()], + &["PORT=9090".into()], + ); + + assert_eq!(envs.get("PATH").map(String::as_str), Some("/usr/bin")); + assert_eq!(envs.get("PORT").map(String::as_str), Some("9090")); + } - Ok(containers.into_iter().filter_map(|c| c.id).collect()) + #[test] + fn relative_executable_uses_working_directory() { + assert_eq!( + normalize_executable("server", Some("/app/bin")), + "/app/bin/server" + ); + assert_eq!(normalize_executable("server", None), "/server"); } } diff --git a/agent/src/bin/dd-agent/oci/registry.rs b/agent/src/bin/dd-agent/oci/registry.rs new file mode 100644 index 0000000..b3f7fb3 --- /dev/null +++ b/agent/src/bin/dd-agent/oci/registry.rs @@ -0,0 +1,341 @@ +use reqwest::header::{ACCEPT, AUTHORIZATION, WWW_AUTHENTICATE}; +use reqwest::{Client, Response, StatusCode}; +use serde::Deserialize; +use std::collections::HashMap; + +const OCI_MANIFEST_MEDIA_TYPE: &str = "application/vnd.oci.image.manifest.v1+json"; +const DOCKER_MANIFEST_MEDIA_TYPE: &str = "application/vnd.docker.distribution.manifest.v2+json"; + +#[derive(Debug, Clone)] +pub struct RegistryClient { + http: Client, +} + +#[derive(Debug, Clone)] +pub struct PulledImage { + pub layers: Vec>, + pub entrypoint: Vec, + pub cmd: Vec, + pub env: Vec, + pub working_dir: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ImageReference { + scheme: String, + registry: String, + repository: String, + reference: String, +} + +#[derive(Debug, Deserialize)] +struct Manifest { + #[serde(rename = "schemaVersion")] + _schema_version: u32, + config: Descriptor, + layers: Vec, +} + +#[derive(Debug, Deserialize)] +struct Descriptor { + digest: String, +} + +#[derive(Debug, Deserialize)] +struct ImageConfig { + config: Option, + #[serde(default)] + working_dir: Option, +} + +#[derive(Debug, Deserialize)] +struct ConfigSection { + #[serde(default)] + #[serde(rename = "Entrypoint")] + entrypoint: Option>, + #[serde(default)] + #[serde(rename = "Cmd")] + cmd: Option>, + #[serde(default)] + #[serde(rename = "Env")] + env: Option>, + #[serde(default)] + #[serde(rename = "WorkingDir")] + working_dir: Option, +} + +#[derive(Debug, Deserialize)] +struct TokenResponse { + token: Option, + access_token: Option, +} + +#[derive(Debug)] +struct WwwAuthenticate { + realm: String, + params: HashMap, +} + +impl RegistryClient { + pub fn new() -> Result { + let http = Client::builder() + .build() + .map_err(|e| format!("build registry client: {e}"))?; + Ok(Self { http }) + } + + pub async fn pull_image(&self, image: &str) -> Result { + let image = ImageReference::parse(image)?; + let manifest_url = format!( + "{}://{}/v2/{}/manifests/{}", + image.scheme, image.registry, image.repository, image.reference + ); + + let accept = manifest_accept_header(); + let manifest_bytes = self.get_with_auth(&manifest_url, Some(&accept)).await?; + let manifest: Manifest = serde_json::from_slice(&manifest_bytes) + .map_err(|e| format!("parse manifest for {image:?}: {e}"))?; + + let config_url = format!( + "{}://{}/v2/{}/blobs/{}", + image.scheme, image.registry, image.repository, manifest.config.digest + ); + let config_bytes = self.get_with_auth(&config_url, None).await?; + let config: ImageConfig = serde_json::from_slice(&config_bytes) + .map_err(|e| format!("parse image config for {image:?}: {e}"))?; + + let mut layers = Vec::with_capacity(manifest.layers.len()); + for layer in manifest.layers { + let blob_url = format!( + "{}://{}/v2/{}/blobs/{}", + image.scheme, image.registry, image.repository, layer.digest + ); + layers.push(self.get_with_auth(&blob_url, None).await?); + } + + let (entrypoint, cmd, env, working_dir) = match config.config { + Some(config_section) => ( + config_section.entrypoint.unwrap_or_default(), + config_section.cmd.unwrap_or_default(), + config_section.env.unwrap_or_default(), + config_section.working_dir.or(config.working_dir), + ), + None => (Vec::new(), Vec::new(), Vec::new(), config.working_dir), + }; + + Ok(PulledImage { + layers, + entrypoint, + cmd, + env, + working_dir, + }) + } + + async fn get_with_auth(&self, url: &str, accept: Option<&str>) -> Result, String> { + let initial = self.send(url, accept, None).await?; + + let response = if initial.status() == StatusCode::UNAUTHORIZED { + let challenge = initial + .headers() + .get(WWW_AUTHENTICATE) + .ok_or_else(|| format!("401 from {url} without WWW-Authenticate header"))? + .to_str() + .map_err(|e| format!("parse WWW-Authenticate header for {url}: {e}"))? + .to_string(); + + let token = self.fetch_bearer_token(&challenge).await?; + self.send(url, accept, Some(&token)).await? + } else { + initial + }; + + if !response.status().is_success() { + return Err(format!( + "GET {url} failed with status {}", + response.status() + )); + } + + response + .bytes() + .await + .map(|bytes| bytes.to_vec()) + .map_err(|e| format!("read response body for {url}: {e}")) + } + + async fn send( + &self, + url: &str, + accept: Option<&str>, + bearer_token: Option<&str>, + ) -> Result { + let mut request = self.http.get(url); + + if let Some(value) = accept { + request = request.header(ACCEPT, value); + } + if let Some(token) = bearer_token { + request = request.header(AUTHORIZATION, format!("Bearer {token}")); + } + + request.send().await.map_err(|e| format!("GET {url}: {e}")) + } + + async fn fetch_bearer_token(&self, header_value: &str) -> Result { + let challenge = parse_www_authenticate(header_value)?; + let mut request = self.http.get(&challenge.realm); + + for (key, value) in &challenge.params { + request = request.query(&[(key, value)]); + } + + let response = request + .send() + .await + .map_err(|e| format!("GET token endpoint {}: {e}", challenge.realm))?; + + if !response.status().is_success() { + return Err(format!( + "token endpoint {} failed with status {}", + challenge.realm, + response.status() + )); + } + + let token: TokenResponse = response + .json() + .await + .map_err(|e| format!("parse token response from {}: {e}", challenge.realm))?; + + token + .token + .or(token.access_token) + .ok_or_else(|| format!("token endpoint {} returned no token", challenge.realm)) + } +} + +fn manifest_accept_header() -> String { + format!("{OCI_MANIFEST_MEDIA_TYPE}, {DOCKER_MANIFEST_MEDIA_TYPE}, application/json") +} + +fn parse_www_authenticate(header: &str) -> Result { + let (scheme, rest) = header + .split_once(' ') + .ok_or_else(|| format!("unsupported WWW-Authenticate header: {header}"))?; + + if !scheme.eq_ignore_ascii_case("Bearer") { + return Err(format!("unsupported auth scheme {scheme}")); + } + + let mut realm = None; + let mut params = HashMap::new(); + + for field in rest.split(',') { + let (raw_key, raw_value) = field + .trim() + .split_once('=') + .ok_or_else(|| format!("invalid auth challenge component: {field}"))?; + let value = raw_value.trim().trim_matches('"').to_string(); + + if raw_key == "realm" { + realm = Some(value); + } else { + params.insert(raw_key.to_string(), value); + } + } + + let realm = realm.ok_or_else(|| "auth challenge missing realm".to_string())?; + Ok(WwwAuthenticate { realm, params }) +} + +impl ImageReference { + fn parse(input: &str) -> Result { + let (scheme, remainder) = if let Some(value) = input.strip_prefix("https://") { + ("https".to_string(), value) + } else if let Some(value) = input.strip_prefix("http://") { + ("http".to_string(), value) + } else { + ("https".to_string(), input) + }; + + let (registry, repository_and_reference) = match remainder.split_once('/') { + Some((first, rest)) + if first.contains('.') || first.contains(':') || first == "localhost" => + { + (first.to_string(), rest.to_string()) + } + _ => { + let repository = if remainder.contains('/') { + remainder.to_string() + } else { + format!("library/{remainder}") + }; + ("registry-1.docker.io".to_string(), repository) + } + }; + + let (repository, reference) = + if let Some((repo, digest)) = repository_and_reference.rsplit_once('@') { + (repo.to_string(), digest.to_string()) + } else if let Some((repo, tag)) = split_tag(&repository_and_reference) { + (repo.to_string(), tag.to_string()) + } else { + (repository_and_reference, "latest".to_string()) + }; + + Ok(Self { + scheme, + registry, + repository, + reference, + }) + } +} + +fn split_tag(value: &str) -> Option<(&str, &str)> { + let slash = value.rfind('/')?; + let colon = value[slash + 1..].rfind(':')?; + let offset = slash + 1 + colon; + Some((&value[..offset], &value[offset + 1..])) +} + +#[cfg(test)] +mod tests { + use super::{parse_www_authenticate, ImageReference}; + + #[test] + fn parses_ghcr_reference() { + let image = ImageReference::parse("ghcr.io/acme/app:1.2.3").unwrap(); + assert_eq!(image.scheme, "https"); + assert_eq!(image.registry, "ghcr.io"); + assert_eq!(image.repository, "acme/app"); + assert_eq!(image.reference, "1.2.3"); + } + + #[test] + fn defaults_to_docker_hub_and_latest() { + let image = ImageReference::parse("busybox").unwrap(); + assert_eq!(image.registry, "registry-1.docker.io"); + assert_eq!(image.repository, "library/busybox"); + assert_eq!(image.reference, "latest"); + } + + #[test] + fn parses_bearer_challenge() { + let challenge = parse_www_authenticate( + "Bearer realm=\"https://ghcr.io/token\",service=\"ghcr.io\",scope=\"repository:acme/app:pull\"", + ) + .unwrap(); + + assert_eq!(challenge.realm, "https://ghcr.io/token"); + assert_eq!( + challenge.params.get("service").map(String::as_str), + Some("ghcr.io") + ); + assert_eq!( + challenge.params.get("scope").map(String::as_str), + Some("repository:acme/app:pull") + ); + } +} diff --git a/agent/src/bin/dd-agent/oci/unpack.rs b/agent/src/bin/dd-agent/oci/unpack.rs new file mode 100644 index 0000000..a7e7af4 --- /dev/null +++ b/agent/src/bin/dd-agent/oci/unpack.rs @@ -0,0 +1,383 @@ +use flate2::read::GzDecoder; +use std::fs; +use std::io::Read; +use std::os::unix::fs::{symlink, PermissionsExt}; +use std::path::{Component, Path, PathBuf}; + +const TAR_BLOCK_SIZE: usize = 512; + +pub fn prepare_rootfs_dir(rootfs: &Path) -> Result<(), String> { + if rootfs.exists() { + fs::remove_dir_all(rootfs) + .map_err(|e| format!("remove existing rootfs {}: {e}", rootfs.display()))?; + } + + fs::create_dir_all(rootfs).map_err(|e| format!("create rootfs {}: {e}", rootfs.display())) +} + +pub fn unpack_layers<'a, I>(rootfs: &Path, layers: I) -> Result<(), String> +where + I: IntoIterator, +{ + for layer in layers { + unpack_layer(rootfs, layer)?; + } + + Ok(()) +} + +fn unpack_layer(rootfs: &Path, layer_bytes: &[u8]) -> Result<(), String> { + let mut archive = Vec::new(); + GzDecoder::new(layer_bytes) + .read_to_end(&mut archive) + .map_err(|e| format!("decompress layer: {e}"))?; + + let mut offset = 0usize; + while offset + TAR_BLOCK_SIZE <= archive.len() { + let header = &archive[offset..offset + TAR_BLOCK_SIZE]; + if header.iter().all(|byte| *byte == 0) { + break; + } + + let entry = TarEntry::parse(header)?; + let data_start = offset + TAR_BLOCK_SIZE; + let data_end = data_start + .checked_add(entry.size as usize) + .ok_or_else(|| format!("layer entry {} size overflow", entry.path.display()))?; + if data_end > archive.len() { + return Err(format!( + "layer entry {} exceeds archive bounds", + entry.path.display() + )); + } + + if !handle_whiteout(rootfs, &entry.path)? { + unpack_entry(rootfs, &entry, &archive[data_start..data_end])?; + } + + let data_blocks = (entry.size as usize).div_ceil(TAR_BLOCK_SIZE) * TAR_BLOCK_SIZE; + offset = data_start + data_blocks; + } + + Ok(()) +} + +fn unpack_entry(rootfs: &Path, entry: &TarEntry, data: &[u8]) -> Result<(), String> { + let destination = rootfs.join(&entry.path); + let parent = destination + .parent() + .ok_or_else(|| format!("missing parent directory for {}", destination.display()))?; + fs::create_dir_all(parent) + .map_err(|e| format!("create parent directory {}: {e}", parent.display()))?; + + match entry.kind { + TarEntryKind::Regular => { + remove_path_if_exists(&destination)?; + fs::write(&destination, data) + .map_err(|e| format!("write file {}: {e}", destination.display()))?; + fs::set_permissions(&destination, fs::Permissions::from_mode(entry.mode)) + .map_err(|e| format!("set permissions on {}: {e}", destination.display()))?; + } + TarEntryKind::Directory => { + fs::create_dir_all(&destination) + .map_err(|e| format!("create directory {}: {e}", destination.display()))?; + fs::set_permissions(&destination, fs::Permissions::from_mode(entry.mode)) + .map_err(|e| format!("set permissions on {}: {e}", destination.display()))?; + } + TarEntryKind::Symlink => { + let target = entry + .link_name + .as_ref() + .ok_or_else(|| format!("symlink {} missing target", entry.path.display()))?; + remove_path_if_exists(&destination)?; + symlink(target, &destination).map_err(|e| { + format!( + "create symlink {} -> {}: {e}", + destination.display(), + target.display() + ) + })?; + } + TarEntryKind::HardLink => { + let target = entry + .link_name + .as_ref() + .ok_or_else(|| format!("hard link {} missing target", entry.path.display()))?; + let source = rootfs.join(target); + remove_path_if_exists(&destination)?; + fs::hard_link(&source, &destination).map_err(|e| { + format!( + "create hard link {} -> {}: {e}", + destination.display(), + source.display() + ) + })?; + } + TarEntryKind::Unsupported(kind) => { + return Err(format!( + "unsupported tar entry type {kind:?} for {}", + entry.path.display() + )) + } + } + + Ok(()) +} + +fn handle_whiteout(rootfs: &Path, rel_path: &Path) -> Result { + let file_name = rel_path + .file_name() + .and_then(|value| value.to_str()) + .ok_or_else(|| format!("invalid whiteout path {}", rel_path.display()))?; + + if file_name == ".wh..wh..opq" { + let dir = rootfs.join(rel_path.parent().unwrap_or_else(|| Path::new(""))); + clear_directory(&dir)?; + return Ok(true); + } + + let Some(target_name) = file_name.strip_prefix(".wh.") else { + return Ok(false); + }; + + let parent = rel_path.parent().unwrap_or_else(|| Path::new("")); + let target = rootfs.join(parent).join(target_name); + remove_path_if_exists(&target)?; + Ok(true) +} + +fn clear_directory(dir: &Path) -> Result<(), String> { + if !dir.exists() { + return Ok(()); + } + + for child in fs::read_dir(dir).map_err(|e| format!("read directory {}: {e}", dir.display()))? { + let child = child.map_err(|e| format!("read directory entry in {}: {e}", dir.display()))?; + remove_path_if_exists(&child.path())?; + } + + Ok(()) +} + +fn remove_path_if_exists(path: &Path) -> Result<(), String> { + let metadata = match fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(_) => return Ok(()), + }; + + if metadata.is_dir() && !metadata.file_type().is_symlink() { + fs::remove_dir_all(path).map_err(|e| format!("remove directory {}: {e}", path.display())) + } else { + fs::remove_file(path).map_err(|e| format!("remove file {}: {e}", path.display())) + } +} + +#[derive(Debug)] +struct TarEntry { + path: PathBuf, + link_name: Option, + size: u64, + mode: u32, + kind: TarEntryKind, +} + +#[derive(Debug)] +enum TarEntryKind { + Regular, + Directory, + Symlink, + HardLink, + Unsupported(u8), +} + +impl TarEntry { + fn parse(header: &[u8]) -> Result { + let path = sanitize_relative_path(&join_tar_path( + parse_tar_string(&header[0..100])?, + parse_tar_string(&header[345..500])?, + ))?; + let mode = parse_octal(&header[100..108])? as u32; + let size = parse_octal(&header[124..136])?; + let kind = match header[156] { + 0 | b'0' => TarEntryKind::Regular, + b'5' => TarEntryKind::Directory, + b'2' => TarEntryKind::Symlink, + b'1' => TarEntryKind::HardLink, + other => TarEntryKind::Unsupported(other), + }; + let link_name_raw = parse_tar_string(&header[157..257])?; + let link_name = if link_name_raw.is_empty() { + None + } else { + Some(sanitize_relative_path(&PathBuf::from(link_name_raw))?) + }; + + Ok(Self { + path, + link_name, + size, + mode, + kind, + }) + } +} + +fn parse_tar_string(bytes: &[u8]) -> Result { + let end = bytes + .iter() + .position(|byte| *byte == 0) + .unwrap_or(bytes.len()); + let value = + std::str::from_utf8(&bytes[..end]).map_err(|e| format!("invalid tar header: {e}"))?; + Ok(value.trim().to_string()) +} + +fn parse_octal(bytes: &[u8]) -> Result { + let value = parse_tar_string(bytes)?; + let trimmed = value.trim(); + if trimmed.is_empty() { + return Ok(0); + } + + u64::from_str_radix(trimmed, 8).map_err(|e| format!("invalid tar octal value {trimmed:?}: {e}")) +} + +fn join_tar_path(name: String, prefix: String) -> PathBuf { + if prefix.is_empty() { + PathBuf::from(name) + } else { + PathBuf::from(prefix).join(name) + } +} + +fn sanitize_relative_path(path: &Path) -> Result { + let mut sanitized = PathBuf::new(); + for component in path.components() { + match component { + Component::Normal(value) => sanitized.push(value), + Component::CurDir => {} + Component::RootDir | Component::ParentDir | Component::Prefix(_) => { + return Err(format!("refusing to unpack path {}", path.display())) + } + } + } + + Ok(sanitized) +} + +#[cfg(test)] +mod tests { + use super::unpack_layers; + use flate2::write::GzEncoder; + use flate2::Compression; + use std::env; + use std::fs; + use std::io::Write; + use std::path::{Path, PathBuf}; + use std::time::{SystemTime, UNIX_EPOCH}; + + #[test] + fn whiteout_removes_file_from_lower_layer() { + let temp = temp_dir("whiteout_removes_file_from_lower_layer"); + let rootfs = temp.join("rootfs"); + fs::create_dir_all(&rootfs).unwrap(); + + let base = tar_layer(&[TarEntrySpec::file("etc/config", b"hello")]); + let whiteout = tar_layer(&[TarEntrySpec::file("etc/.wh.config", b"")]); + + unpack_layers(&rootfs, [base.as_slice(), whiteout.as_slice()]).unwrap(); + assert!(!rootfs.join("etc/config").exists()); + let _ = fs::remove_dir_all(temp); + } + + #[test] + fn opaque_whiteout_clears_directory_contents() { + let temp = temp_dir("opaque_whiteout_clears_directory_contents"); + let rootfs = temp.join("rootfs"); + fs::create_dir_all(&rootfs).unwrap(); + + let base = tar_layer(&[ + TarEntrySpec::file("app/one.txt", b"one"), + TarEntrySpec::file("app/two.txt", b"two"), + ]); + let whiteout = tar_layer(&[TarEntrySpec::file("app/.wh..wh..opq", b"")]); + + unpack_layers(&rootfs, [base.as_slice(), whiteout.as_slice()]).unwrap(); + assert!(Path::new(&rootfs.join("app")).exists()); + assert_eq!(fs::read_dir(rootfs.join("app")).unwrap().count(), 0); + let _ = fs::remove_dir_all(temp); + } + + struct TarEntrySpec<'a> { + path: &'a str, + data: &'a [u8], + kind: u8, + } + + impl<'a> TarEntrySpec<'a> { + fn file(path: &'a str, data: &'a [u8]) -> Self { + Self { + path, + data, + kind: b'0', + } + } + } + + fn tar_layer(entries: &[TarEntrySpec<'_>]) -> Vec { + let mut archive = Vec::new(); + + for entry in entries { + archive.extend(make_header(entry.path, entry.data.len() as u64, entry.kind)); + archive.extend(entry.data); + + let padding = (512 - (entry.data.len() % 512)) % 512; + archive.extend(vec![0u8; padding]); + } + + archive.extend(vec![0u8; 1024]); + + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(&archive).unwrap(); + encoder.finish().unwrap() + } + + fn make_header(path: &str, size: u64, kind: u8) -> [u8; 512] { + let mut header = [0u8; 512]; + write_bytes(&mut header[0..100], path.as_bytes()); + write_octal(&mut header[100..108], 0o644); + write_octal(&mut header[108..116], 0); + write_octal(&mut header[116..124], 0); + write_octal(&mut header[124..136], size); + write_octal(&mut header[136..148], 0); + header[148..156].fill(b' '); + header[156] = kind; + write_bytes(&mut header[257..263], b"ustar\0"); + write_bytes(&mut header[263..265], b"00"); + + let checksum: u32 = header.iter().map(|byte| *byte as u32).sum(); + write_octal(&mut header[148..156], checksum as u64); + header + } + + fn write_bytes(dst: &mut [u8], src: &[u8]) { + let len = src.len().min(dst.len()); + dst[..len].copy_from_slice(&src[..len]); + } + + fn write_octal(dst: &mut [u8], value: u64) { + let width = dst.len() - 1; + let encoded = format!("{value:0width$o}\0"); + write_bytes(dst, encoded.as_bytes()); + } + + fn temp_dir(test_name: &str) -> PathBuf { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = env::temp_dir().join(format!("dd-agent-{test_name}-{unique}")); + fs::create_dir_all(&path).unwrap(); + path + } +} diff --git a/control-plane/Cargo.toml b/control-plane/Cargo.toml index ec2faac..22549cb 100644 --- a/control-plane/Cargo.toml +++ b/control-plane/Cargo.toml @@ -33,7 +33,7 @@ thiserror = "2" http = "1" tower-http = { version = "0.6", features = ["cors"] } rpassword = "7" +dd-agent = { path = "../agent" } [dev-dependencies] -mockito = "1" tower = { version = "0.5", features = ["util"] } diff --git a/control-plane/migrations/0001_init.sql b/control-plane/migrations/0001_init.sql index 6afd41c..452eb47 100644 --- a/control-plane/migrations/0001_init.sql +++ b/control-plane/migrations/0001_init.sql @@ -12,6 +12,7 @@ CREATE TABLE IF NOT EXISTS agents ( node_size TEXT, datacenter TEXT, github_owner TEXT, + deployment_id TEXT REFERENCES deployments(id), created_at TEXT NOT NULL, last_heartbeat_at TEXT ); @@ -27,10 +28,9 @@ CREATE TABLE IF NOT EXISTS agent_control_credentials ( CREATE TABLE IF NOT EXISTS deployments ( id TEXT PRIMARY KEY, agent_id TEXT NOT NULL REFERENCES agents(id), - app_name TEXT, - app_version TEXT, - compose TEXT NOT NULL, - config TEXT, + image TEXT NOT NULL, + env TEXT NOT NULL, + cmd TEXT NOT NULL DEFAULT '[]', status TEXT NOT NULL DEFAULT 'pending', created_at TEXT NOT NULL, updated_at TEXT NOT NULL diff --git a/control-plane/src/api.rs b/control-plane/src/api.rs index 8559a2a..57046d0 100644 --- a/control-plane/src/api.rs +++ b/control-plane/src/api.rs @@ -46,35 +46,55 @@ pub struct AgentRegisterResponse { } // --------------------------------------------------------------------------- -// Deploy +// Deployments // --------------------------------------------------------------------------- #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct DeployRequest { - pub compose: String, +pub struct AgentDeployRequest { + pub image: String, #[serde(default)] - pub config: Option, + pub env: Vec, #[serde(default)] - pub app_name: Option, - #[serde(default)] - pub app_version: Option, - #[serde(default)] - pub agent_name: Option, - #[serde(default)] - pub node_size: Option, - #[serde(default)] - pub datacenter: Option, - #[serde(default)] - pub dry_run: Option, + pub cmd: Vec, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct DeployResponse { +pub struct AgentDeployResponse { pub deployment_id: Uuid, pub agent_id: Uuid, pub status: DeploymentStatus, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct AgentDeploymentResponse { + pub image: String, + pub env: Vec, + pub cmd: Vec, + pub deployment_id: Uuid, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct AgentDeploymentStatusRequest { + pub status: String, + #[serde(default)] + pub exit_code: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(untagged)] +pub enum CpAttestationResponse { + Attested { + quote_b64: String, + mrtd: String, + tcb_status: String, + attested: bool, + }, + Unattested { + attested: bool, + reason: String, + }, +} + // --------------------------------------------------------------------------- // Agent health check ingestion // --------------------------------------------------------------------------- diff --git a/control-plane/src/attestation/ita.rs b/control-plane/src/attestation/ita.rs index d6089b8..33668be 100644 --- a/control-plane/src/attestation/ita.rs +++ b/control-plane/src/attestation/ita.rs @@ -21,14 +21,39 @@ pub struct AttestationClaims { pub exp: u64, #[serde(default)] pub nbf: u64, - #[serde(default, rename = "tdx.mrtd")] - pub tdx_mrtd: Option, + /// TDX measurements — ITA nests these under a "tdx" object. + #[serde(default)] + pub tdx: Option, #[serde(default, rename = "attester_tcb_status")] pub attester_tcb_status: Option, #[serde(flatten)] pub extra: HashMap, } +/// Nested TDX-specific claims from the ITA attestation token. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct TdxClaims { + #[serde(default)] + pub mrtd: Option, + #[serde(default)] + pub rtmr0: Option, + #[serde(default)] + pub rtmr1: Option, + #[serde(default)] + pub rtmr2: Option, + #[serde(default)] + pub rtmr3: Option, + #[serde(flatten)] + pub extra: HashMap, +} + +impl AttestationClaims { + /// Helper to get MRTD regardless of nesting. + pub fn tdx_mrtd(&self) -> Option<&str> { + self.tdx.as_ref().and_then(|t| t.mrtd.as_deref()) + } +} + // --------------------------------------------------------------------------- // JWKS types // --------------------------------------------------------------------------- @@ -226,18 +251,44 @@ mod tests { use super::*; use jsonwebtoken::{encode, EncodingKey, Header}; - fn make_hs256_jwks_and_key() -> (String, Vec) { + fn make_hs256_jwks_and_key() -> (JwksDocument, Vec) { let secret = b"test-secret-key-for-ita-verification-32b"; let k = URL_SAFE_NO_PAD.encode(secret); - let jwks_json = serde_json::json!({ - "keys": [{ - "kid": "test-kid-1", - "kty": "oct", - "alg": "HS256", - "k": k, - }] - }); - (serde_json::to_string(&jwks_json).unwrap(), secret.to_vec()) + ( + JwksDocument { + keys: vec![Jwk { + kid: Some("test-kid-1".into()), + kty: "oct".into(), + alg: Some("HS256".into()), + n: None, + e: None, + k: Some(k), + }], + }, + secret.to_vec(), + ) + } + + async fn verifier_with_cached_jwks( + expected_issuer: Option, + expected_audience: Option, + ) -> (ItaVerifier, Vec) { + let (jwks, secret) = make_hs256_jwks_and_key(); + let verifier = ItaVerifier::new( + "https://unused.test/jwks".into(), + expected_issuer, + expected_audience, + ); + + { + let mut cache = verifier.cache.write().await; + *cache = Some(CachedJwks { + document: jwks, + fetched_at: std::time::Instant::now(), + }); + } + + (verifier, secret) } fn make_token(secret: &[u8], claims: &AttestationClaims) -> String { @@ -252,7 +303,10 @@ mod tests { aud: serde_json::Value::String("devopsdefender".into()), exp: (chrono::Utc::now().timestamp() + 3600) as u64, nbf: 0, - tdx_mrtd: Some("abc123".into()), + tdx: Some(TdxClaims { + mrtd: Some("abc123".into()), + ..Default::default() + }), attester_tcb_status: Some("UpToDate".into()), extra: HashMap::new(), } @@ -260,21 +314,11 @@ mod tests { #[tokio::test] async fn verify_valid_token_with_mock_jwks() { - let (jwks_body, secret) = make_hs256_jwks_and_key(); - let mut server = mockito::Server::new_async().await; - let mock = server - .mock("GET", "/jwks") - .with_status(200) - .with_header("content-type", "application/json") - .with_body(&jwks_body) - .create_async() - .await; - - let verifier = ItaVerifier::new( - format!("{}/jwks", server.url()), + let (verifier, secret) = verifier_with_cached_jwks( Some("https://portal.trustauthority.intel.com".into()), Some("devopsdefender".into()), - ); + ) + .await; let claims = valid_claims(); let token = make_token(&secret, &claims); @@ -282,23 +326,12 @@ mod tests { let result = verifier.verify_attestation_token(&token).await; assert!(result.is_ok()); let verified = result.unwrap(); - assert_eq!(verified.tdx_mrtd, Some("abc123".into())); - mock.assert_async().await; + assert_eq!(verified.tdx_mrtd(), Some("abc123")); } #[tokio::test] async fn reject_expired_token() { - let (jwks_body, secret) = make_hs256_jwks_and_key(); - let mut server = mockito::Server::new_async().await; - let _mock = server - .mock("GET", "/jwks") - .with_status(200) - .with_header("content-type", "application/json") - .with_body(&jwks_body) - .create_async() - .await; - - let verifier = ItaVerifier::new(format!("{}/jwks", server.url()), None, None); + let (verifier, secret) = verifier_with_cached_jwks(None, None).await; let mut claims = valid_claims(); claims.exp = 1000; // way in the past @@ -310,21 +343,8 @@ mod tests { #[tokio::test] async fn reject_wrong_audience() { - let (jwks_body, secret) = make_hs256_jwks_and_key(); - let mut server = mockito::Server::new_async().await; - let _mock = server - .mock("GET", "/jwks") - .with_status(200) - .with_header("content-type", "application/json") - .with_body(&jwks_body) - .create_async() - .await; - - let verifier = ItaVerifier::new( - format!("{}/jwks", server.url()), - None, - Some("wrong-audience".into()), - ); + let (verifier, secret) = + verifier_with_cached_jwks(None, Some("wrong-audience".into())).await; let claims = valid_claims(); let token = make_token(&secret, &claims); @@ -335,18 +355,7 @@ mod tests { #[tokio::test] async fn jwks_cache_reuse() { - let (jwks_body, secret) = make_hs256_jwks_and_key(); - let mut server = mockito::Server::new_async().await; - let mock = server - .mock("GET", "/jwks") - .with_status(200) - .with_header("content-type", "application/json") - .with_body(&jwks_body) - .expect(1) // should only be called once - .create_async() - .await; - - let verifier = ItaVerifier::new(format!("{}/jwks", server.url()), None, None); + let (verifier, secret) = verifier_with_cached_jwks(None, None).await; let claims = valid_claims(); let token1 = make_token(&secret, &claims); @@ -356,8 +365,6 @@ mod tests { assert!(r1.is_ok()); let r2 = verifier.verify_attestation_token(&token2).await; assert!(r2.is_ok()); - - mock.assert_async().await; } #[tokio::test] @@ -371,18 +378,8 @@ mod tests { #[tokio::test] async fn allow_missing_audience_when_not_configured() { - let (jwks_body, secret) = make_hs256_jwks_and_key(); - let mut server = mockito::Server::new_async().await; - let _mock = server - .mock("GET", "/jwks") - .with_status(200) - .with_header("content-type", "application/json") - .with_body(&jwks_body) - .create_async() - .await; - // No audience configured -- should accept any audience in token - let verifier = ItaVerifier::new(format!("{}/jwks", server.url()), None, None); + let (verifier, secret) = verifier_with_cached_jwks(None, None).await; let claims = valid_claims(); let token = make_token(&secret, &claims); diff --git a/control-plane/src/bin/dd-cp/main.rs b/control-plane/src/bin/dd-cp/main.rs index 611c3c2..19e0abc 100644 --- a/control-plane/src/bin/dd-cp/main.rs +++ b/control-plane/src/bin/dd-cp/main.rs @@ -1,13 +1,51 @@ use dd_control_plane::config::CpConfig; use dd_control_plane::db; use dd_control_plane::routes; -use dd_control_plane::services::attestation::{AttestationService, RuntimeEnv}; +use dd_control_plane::services::attestation::AttestationService; use dd_control_plane::services::github_oidc::GithubOidcService; use dd_control_plane::services::tunnel::TunnelService; use dd_control_plane::state::AppState; +use serde::Deserialize; + +#[derive(Debug, Deserialize)] +struct FileConfig { + #[serde(default)] + raw_kv: std::collections::HashMap, +} + +fn load_raw_env_from_config_file() { + let config_path = match std::env::var("DD_CONFIG") { + Ok(path) => path, + Err(_) => return, + }; + + let text = match std::fs::read_to_string(&config_path) { + Ok(text) => text, + Err(err) => { + eprintln!(" warning: failed to read DD_CONFIG file {config_path}: {err}"); + return; + } + }; + + let file_cfg: FileConfig = match serde_json::from_str(&text) { + Ok(cfg) => cfg, + Err(err) => { + eprintln!(" warning: failed to parse DD_CONFIG file {config_path}: {err}"); + return; + } + }; + + for (key, value) in file_cfg.raw_kv { + if std::env::var_os(&key).is_none() { + unsafe { std::env::set_var(key, value) }; + } + } +} #[tokio::main] async fn main() { + load_raw_env_from_config_file(); + let config = CpConfig::from_env(); eprintln!("DevOps Defender Control Plane starting..."); @@ -23,13 +61,12 @@ async fn main() { let mut state = AppState::from_env(db); // Override with real services if configured - let env = RuntimeEnv::detect(); if let Ok(ita_url) = std::env::var("DD_CP_ITA_JWKS_URL") { let ita_issuer = std::env::var("DD_CP_ITA_ISSUER").ok(); let ita_audience = std::env::var("DD_CP_ITA_AUDIENCE").ok(); let verifier = dd_control_plane::attestation::ita::ItaVerifier::new(ita_url, ita_issuer, ita_audience); - state.attestation = AttestationService::new(verifier, env); + state.attestation = AttestationService::new(verifier); } if std::env::var("DD_CP_CF_API_TOKEN").is_ok() { diff --git a/control-plane/src/routes/agents.rs b/control-plane/src/routes/agents.rs index ca73df0..8085833 100644 --- a/control-plane/src/routes/agents.rs +++ b/control-plane/src/routes/agents.rs @@ -7,6 +7,7 @@ use crate::api::{ AgentRegisterRequest, AgentRegisterResponse, }; use crate::common::error::AppError; +use crate::services::nonce::ConsumeResult; use crate::state::AppState; use crate::stores::{agent as agent_store, health as health_store}; @@ -24,13 +25,53 @@ pub async fn agent_register( State(state): State, Json(req): Json, ) -> Result<(StatusCode, Json), AppError> { - // Verify attestation token + match state.nonce.consume(&req.nonce).await { + ConsumeResult::Ok => {} + ConsumeResult::Missing => { + return Err(AppError::InvalidInput( + "registration nonce is missing or already used".into(), + )); + } + ConsumeResult::Expired => { + return Err(AppError::InvalidInput( + "registration nonce has expired".into(), + )); + } + } + + // Verify attestation quote. let attestation = state .attestation - .verify_registration_token(&req.intel_ta_token) + .verify_registration_token(&req.intel_ta_token, &req.nonce) .await?; - // Create tunnel + // Re-register: if an agent with the same vm_name already exists, reuse its + // tunnel and update its record instead of creating a brand-new tunnel every + // time (avoids Cloudflare rate limits on tunnel creation). + if let Some(existing) = agent_store::find_agent_by_vm_name(&state.db, &req.vm_name)? { + let agent_id: uuid::Uuid = existing.id.parse().map_err(|_| AppError::Internal)?; + let hostname = existing.hostname.unwrap_or_default(); + let tunnel_token = state + .tunnel + .get_tunnel_token_for_agent(agent_id, &req.vm_name) + .await + .unwrap_or_else(|_| format!("reuse-tunnel-token-{agent_id}")); + + // Reset registration state + heartbeat + agent_store::update_registration_state(&state.db, &existing.id, "ready")?; + agent_store::update_heartbeat(&state.db, &existing.id)?; + + return Ok(( + StatusCode::CREATED, + Json(AgentRegisterResponse { + agent_id, + tunnel_token, + hostname, + }), + )); + } + + // New agent: create tunnel let agent_id = uuid::Uuid::new_v4(); let tunnel_info = state .tunnel @@ -50,6 +91,7 @@ pub async fn agent_register( node_size: req.node_size, datacenter: req.datacenter, github_owner: req.github_owner, + deployment_id: None, created_at: chrono::Utc::now().to_rfc3339(), last_heartbeat_at: None, }; @@ -208,15 +250,16 @@ mod tests { } #[tokio::test] - async fn register_and_list_agents() { + async fn register_rejects_without_attestation_config() { let state = test_state(); + let nonce = state.nonce.issue().await; let app = build_router(state); - // Register an agent + // Without DD_INTEL_API_KEY, registration must fail — no insecure mode. let register_req = AgentRegisterRequest { intel_ta_token: "fake-token".into(), vm_name: "test-vm".into(), - nonce: "test-nonce".into(), + nonce, node_size: None, datacenter: None, github_owner: None, @@ -230,7 +273,7 @@ mod tests { .unwrap(); let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::CREATED); + assert_eq!(resp.status(), StatusCode::INTERNAL_SERVER_ERROR); } #[tokio::test] diff --git a/control-plane/src/routes/attestation.rs b/control-plane/src/routes/attestation.rs new file mode 100644 index 0000000..c4ce675 --- /dev/null +++ b/control-plane/src/routes/attestation.rs @@ -0,0 +1,69 @@ +use axum::extract::State; +use axum::Json; + +use crate::api::CpAttestationResponse; +use crate::state::AppState; + +/// GET /api/v1/attestation +pub async fn get_attestation(State(_state): State) -> Json { + let Some(quote_b64) = std::env::var("DD_SELF_QUOTE_B64").ok() else { + return Json(CpAttestationResponse::Unattested { + attested: false, + reason: "CP is not running in a TDX environment".into(), + }); + }; + + match dd_agent::attestation::tsm::parse_tdx_quote_base64("e_b64) { + Ok(parsed) => Json(CpAttestationResponse::Attested { + quote_b64, + mrtd: parsed.mrtd_hex(), + tcb_status: "self-reported".into(), + attested: true, + }), + Err(e) => Json(CpAttestationResponse::Unattested { + attested: false, + reason: format!("DD_SELF_QUOTE_B64 is present but invalid: {e}"), + }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db; + use crate::routes::build_router; + use axum::body::Body; + use axum::http::{Request, StatusCode}; + use tower::ServiceExt; + + fn test_state() -> AppState { + let db = db::connect_and_migrate("sqlite://:memory:").unwrap(); + AppState::for_testing(db) + } + + #[tokio::test] + async fn reports_unattested_without_self_quote() { + std::env::remove_var("DD_SELF_QUOTE_B64"); + + let app = build_router(test_state()); + let req = Request::builder() + .uri("/api/v1/attestation") + .body(Body::empty()) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let body = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + let parsed: CpAttestationResponse = serde_json::from_slice(&body).unwrap(); + assert_eq!( + parsed, + CpAttestationResponse::Unattested { + attested: false, + reason: "CP is not running in a TDX environment".into(), + } + ); + } +} diff --git a/control-plane/src/routes/deploy.rs b/control-plane/src/routes/deploy.rs deleted file mode 100644 index 7e4530a..0000000 --- a/control-plane/src/routes/deploy.rs +++ /dev/null @@ -1,130 +0,0 @@ -use axum::extract::{Path, Query, State}; -use axum::http::StatusCode; -use axum::Json; - -use crate::api::{DeployRequest, DeployResponse}; -use crate::common::error::AppError; -use crate::state::AppState; -use crate::stores::{agent as agent_store, deployment as deployment_store}; -use crate::types::DeploymentStatus; - -/// POST /api/v1/deploy -pub async fn deploy( - State(state): State, - Json(req): Json, -) -> Result<(StatusCode, Json), AppError> { - // Find an available agent - let agent = agent_store::find_available_agent( - &state.db, - req.node_size.as_deref(), - req.datacenter.as_deref(), - )? - .ok_or(AppError::NotFound)?; - - let agent_id: uuid::Uuid = agent.id.parse().map_err(|_| AppError::Internal)?; - - let dry_run = req.dry_run.unwrap_or(false); - if dry_run { - return Ok(( - StatusCode::OK, - Json(DeployResponse { - deployment_id: uuid::Uuid::new_v4(), - agent_id, - status: DeploymentStatus::Pending, - }), - )); - } - - // Create deployment record - let deployment_id = uuid::Uuid::new_v4(); - let now = chrono::Utc::now().to_rfc3339(); - let dep = deployment_store::DeploymentRow { - id: deployment_id.to_string(), - agent_id: agent.id.clone(), - app_name: req.app_name, - app_version: req.app_version, - compose: req.compose, - config: req.config, - status: "deploying".into(), - created_at: now.clone(), - updated_at: now, - }; - deployment_store::insert_deployment(&state.db, &dep)?; - - // Update agent status - agent_store::update_agent_status(&state.db, &agent.id, "deploying")?; - - Ok(( - StatusCode::CREATED, - Json(DeployResponse { - deployment_id, - agent_id, - status: DeploymentStatus::Deploying, - }), - )) -} - -#[derive(Debug, serde::Deserialize)] -pub struct DeploymentListQuery { - pub agent_id: Option, -} - -/// GET /api/v1/deployments -pub async fn list_deployments( - State(state): State, - Query(query): Query, -) -> Result>, AppError> { - let deps = deployment_store::list_deployments(&state.db, query.agent_id.as_deref())?; - Ok(Json(deps)) -} - -/// GET /api/v1/deployments/{id} -pub async fn get_deployment( - State(state): State, - Path(id): Path, -) -> Result, AppError> { - let dep = deployment_store::get_deployment(&state.db, &id)?.ok_or(AppError::NotFound)?; - Ok(Json(dep)) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::db; - use crate::routes::build_router; - use axum::body::Body; - use axum::http::Request; - use tower::ServiceExt; - - fn test_state() -> AppState { - let db = db::connect_and_migrate("sqlite://:memory:").unwrap(); - AppState::for_testing(db) - } - - #[tokio::test] - async fn deploy_no_agents_returns_not_found() { - let state = test_state(); - let app = build_router(state); - - let deploy_req = DeployRequest { - compose: "version: '3'".into(), - config: None, - app_name: None, - app_version: None, - agent_name: None, - node_size: None, - datacenter: None, - dry_run: None, - }; - - let req = Request::builder() - .uri("/api/v1/deploy") - .method("POST") - .header("content-type", "application/json") - .body(Body::from(serde_json::to_string(&deploy_req).unwrap())) - .unwrap(); - - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - } -} diff --git a/control-plane/src/routes/deployments.rs b/control-plane/src/routes/deployments.rs new file mode 100644 index 0000000..a1f852a --- /dev/null +++ b/control-plane/src/routes/deployments.rs @@ -0,0 +1,300 @@ +use axum::extract::{Path, Query, State}; +use axum::http::{header, HeaderMap, StatusCode}; +use axum::Json; + +use crate::api::{ + AgentDeployRequest, AgentDeployResponse, AgentDeploymentResponse, AgentDeploymentStatusRequest, +}; +use crate::auth::admin_session; +use crate::common::error::AppError; +use crate::state::AppState; +use crate::stores::{ + agent as agent_store, deployment as deployment_store, session as session_store, +}; +use crate::types::{AgentStatus, DeploymentStatus}; + +/// POST /api/v1/agents/{id}/deploy +pub async fn deploy_to_agent( + State(state): State, + Path(agent_id): Path, + headers: HeaderMap, + Json(req): Json, +) -> Result<(StatusCode, Json), AppError> { + require_admin_session(&state, &headers)?; + + let agent = agent_store::get_agent(&state.db, &agent_id)?.ok_or(AppError::NotFound)?; + if agent.deployment_id.is_some() || agent.status != AgentStatus::Undeployed.to_string() { + return Err(AppError::Conflict( + "agent already has an active deployment".into(), + )); + } + if req.image.trim().is_empty() { + return Err(AppError::InvalidInput( + "deployment image is required".into(), + )); + } + + let deployment_id = uuid::Uuid::new_v4(); + let now = chrono::Utc::now().to_rfc3339(); + let dep = deployment_store::DeploymentRow { + id: deployment_id.to_string(), + agent_id: agent_id.clone(), + image: req.image, + env: req.env, + cmd: req.cmd, + status: DeploymentStatus::Deploying.to_string(), + created_at: now.clone(), + updated_at: now, + }; + deployment_store::insert_deployment(&state.db, &dep)?; + agent_store::update_agent_deployment(&state.db, &agent_id, Some(&dep.id))?; + agent_store::update_agent_status(&state.db, &agent_id, &AgentStatus::Deploying.to_string())?; + + Ok(( + StatusCode::CREATED, + Json(AgentDeployResponse { + deployment_id, + agent_id: agent_id.parse().map_err(|_| AppError::Internal)?, + status: DeploymentStatus::Deploying, + }), + )) +} + +/// GET /api/v1/agents/{id}/deployment +pub async fn get_agent_deployment( + State(state): State, + Path(agent_id): Path, +) -> Result, AppError> { + let agent = agent_store::get_agent(&state.db, &agent_id)?.ok_or(AppError::NotFound)?; + let Some(deployment_id) = agent.deployment_id else { + return Ok(StatusCodeOrJson::Status(StatusCode::NO_CONTENT)); + }; + + let dep = + deployment_store::get_deployment(&state.db, &deployment_id)?.ok_or(AppError::NotFound)?; + Ok(StatusCodeOrJson::Json(Json(AgentDeploymentResponse { + image: dep.image, + env: dep.env, + cmd: dep.cmd, + deployment_id: dep.id.parse().map_err(|_| AppError::Internal)?, + }))) +} + +/// POST /api/v1/agents/{id}/deployment/{deployment_id}/status +pub async fn update_agent_deployment_status( + State(state): State, + Path((agent_id, deployment_id)): Path<(String, String)>, + Json(req): Json, +) -> Result { + let agent = agent_store::get_agent(&state.db, &agent_id)?.ok_or(AppError::NotFound)?; + if agent.deployment_id.as_deref() != Some(deployment_id.as_str()) { + return Err(AppError::NotFound); + } + + let deployment = + deployment_store::get_deployment(&state.db, &deployment_id)?.ok_or(AppError::NotFound)?; + if deployment.agent_id != agent_id { + return Err(AppError::NotFound); + } + + match req.status.as_str() { + "running" => { + deployment_store::update_deployment_status( + &state.db, + &deployment_id, + &DeploymentStatus::Running.to_string(), + )?; + agent_store::update_agent_status( + &state.db, + &agent_id, + &AgentStatus::Deployed.to_string(), + )?; + } + "stopped" | "failed" => { + let deployment_status = if req.status == "stopped" { + DeploymentStatus::Stopped + } else { + DeploymentStatus::Failed + }; + deployment_store::update_deployment_status( + &state.db, + &deployment_id, + &deployment_status.to_string(), + )?; + agent_store::update_agent_deployment(&state.db, &agent_id, None)?; + agent_store::update_agent_status( + &state.db, + &agent_id, + &AgentStatus::Undeployed.to_string(), + )?; + } + other => { + return Err(AppError::InvalidInput(format!( + "unsupported deployment status {other:?}" + ))); + } + } + + Ok(StatusCode::OK) +} + +#[derive(Debug, serde::Deserialize)] +pub struct DeploymentListQuery { + pub agent_id: Option, +} + +/// GET /api/v1/deployments +pub async fn list_deployments( + State(state): State, + Query(query): Query, +) -> Result>, AppError> { + let deps = deployment_store::list_deployments(&state.db, query.agent_id.as_deref())?; + Ok(Json(deps)) +} + +/// GET /api/v1/deployments/{id} +pub async fn get_deployment( + State(state): State, + Path(id): Path, +) -> Result, AppError> { + let dep = deployment_store::get_deployment(&state.db, &id)?.ok_or(AppError::NotFound)?; + Ok(Json(dep)) +} + +fn require_admin_session(state: &AppState, headers: &HeaderMap) -> Result<(), AppError> { + let auth_value = headers + .get(header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .ok_or(AppError::Unauthorized)?; + + let token = auth_value + .strip_prefix("Bearer ") + .ok_or(AppError::Unauthorized)?; + let prefix = admin_session::token_prefix_from_raw(token); + let session = + session_store::find_by_prefix(&state.db, &prefix)?.ok_or(AppError::Unauthorized)?; + + if !admin_session::verify_session_token(token, &session.token_hash) { + return Err(AppError::Unauthorized); + } + + let expires_at = chrono::DateTime::parse_from_rfc3339(&session.expires_at) + .map_err(|_| AppError::Internal)?; + if expires_at < chrono::Utc::now().fixed_offset() { + return Err(AppError::Unauthorized); + } + + Ok(()) +} + +pub enum StatusCodeOrJson { + Status(StatusCode), + Json(Json), +} + +impl axum::response::IntoResponse for StatusCodeOrJson +where + Json: axum::response::IntoResponse, +{ + fn into_response(self) -> axum::response::Response { + match self { + StatusCodeOrJson::Status(status) => status.into_response(), + StatusCodeOrJson::Json(json) => json.into_response(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::auth::admin_session; + use crate::db; + use crate::routes::build_router; + use crate::stores::{agent as agent_store, session as session_store}; + use axum::body::Body; + use axum::http::Request; + use tower::ServiceExt; + + fn test_state() -> AppState { + let db = db::connect_and_migrate("sqlite://:memory:").unwrap(); + AppState::for_testing(db) + } + + fn insert_agent(state: &AppState, id: &str) { + let agent = agent_store::AgentRow { + id: id.into(), + vm_name: format!("vm-{id}"), + status: AgentStatus::Undeployed.to_string(), + registration_state: "ready".into(), + hostname: None, + tunnel_id: None, + mrtd: None, + tcb_status: None, + node_size: None, + datacenter: None, + github_owner: None, + deployment_id: None, + created_at: chrono::Utc::now().to_rfc3339(), + last_heartbeat_at: None, + }; + agent_store::insert_agent(&state.db, &agent).unwrap(); + } + + fn admin_header(state: &AppState) -> String { + let raw = admin_session::issue_session_token(); + let session = session_store::SessionRow { + id: uuid::Uuid::new_v4().to_string(), + token_hash: admin_session::hash_session_token(&raw), + token_prefix: admin_session::token_prefix_from_raw(&raw), + created_at: chrono::Utc::now().to_rfc3339(), + expires_at: (chrono::Utc::now() + chrono::Duration::hours(1)).to_rfc3339(), + }; + session_store::insert_session(&state.db, &session).unwrap(); + format!("Bearer {raw}") + } + + #[tokio::test] + async fn deploy_and_fetch_agent_deployment() { + let state = test_state(); + insert_agent(&state, "11111111-1111-1111-1111-111111111111"); + let auth_header = admin_header(&state); + let app = build_router(state.clone()); + + let deploy_req = AgentDeployRequest { + image: "ghcr.io/devopsdefender/workload:latest".into(), + env: vec!["KEY=VALUE".into()], + cmd: vec![], + }; + + let req = Request::builder() + .uri("/api/v1/agents/11111111-1111-1111-1111-111111111111/deploy") + .method("POST") + .header("content-type", "application/json") + .header("authorization", auth_header) + .body(Body::from(serde_json::to_string(&deploy_req).unwrap())) + .unwrap(); + let resp = app.clone().oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::CREATED); + + let req = Request::builder() + .uri("/api/v1/agents/11111111-1111-1111-1111-111111111111/deployment") + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + } + + #[tokio::test] + async fn deployment_poll_returns_no_content_when_unassigned() { + let state = test_state(); + insert_agent(&state, "11111111-1111-1111-1111-111111111111"); + let app = build_router(state); + + let req = Request::builder() + .uri("/api/v1/agents/11111111-1111-1111-1111-111111111111/deployment") + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + } +} diff --git a/control-plane/src/routes/mod.rs b/control-plane/src/routes/mod.rs index aa12152..6df9365 100644 --- a/control-plane/src/routes/mod.rs +++ b/control-plane/src/routes/mod.rs @@ -1,8 +1,9 @@ pub mod accounts; pub mod admin; pub mod agents; +pub mod attestation; pub mod auth; -pub mod deploy; +pub mod deployments; pub mod health; pub mod stats; pub mod ui; @@ -32,12 +33,25 @@ pub fn build_router(state: AppState) -> Router { "/api/v1/agents/{id}/heartbeat", post(agents::agent_heartbeat), ) + .route( + "/api/v1/agents/{id}/deployment", + get(deployments::get_agent_deployment), + ) + .route( + "/api/v1/agents/{id}/deploy", + post(deployments::deploy_to_agent), + ) + .route( + "/api/v1/agents/{id}/deployment/{deployment_id}/status", + post(deployments::update_agent_deployment_status), + ) .route("/api/v1/agents/{id}/checks", post(agents::ingest_check)) .route("/api/v1/agents/{id}/checks", get(agents::list_checks)) - // Deploy - .route("/api/v1/deploy", post(deploy::deploy)) - .route("/api/v1/deployments", get(deploy::list_deployments)) - .route("/api/v1/deployments/{id}", get(deploy::get_deployment)) + // Deployments + .route("/api/v1/deployments", get(deployments::list_deployments)) + .route("/api/v1/deployments/{id}", get(deployments::get_deployment)) + // Control plane self-attestation + .route("/api/v1/attestation", get(attestation::get_attestation)) // Stats .route("/api/v1/stats/apps", get(stats::app_stats)) .route("/api/v1/stats/agents", get(stats::agent_stats)) diff --git a/control-plane/src/routes/ui_root.html b/control-plane/src/routes/ui_root.html index 19a613b..dff6e59 100644 --- a/control-plane/src/routes/ui_root.html +++ b/control-plane/src/routes/ui_root.html @@ -66,7 +66,8 @@

Quick Links

  • GET /health - Health check
  • GET /api/v1/agents - List agents
    • -
  • POST /api/v1/deploy - Deploy workload
    • +
  • POST /api/v1/agents/{id}/deploy - Assign a deployment
    • +
  • GET /api/v1/attestation - CP self-attestation
  • GET /api/v1/apps - List applications
diff --git a/control-plane/src/services/attestation.rs b/control-plane/src/services/attestation.rs index 3f55662..728ad83 100644 --- a/control-plane/src/services/attestation.rs +++ b/control-plane/src/services/attestation.rs @@ -1,24 +1,26 @@ use crate::attestation::ita::{AttestationClaims, ItaVerifier}; use crate::common::error::{AppError, AppResult}; +use crate::services::ita_client::ItaClient; +use dd_agent::attestation::tsm; /// Runtime environment classification. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RuntimeEnv { - Local, Staging, Production, } impl RuntimeEnv { /// Detect runtime environment from DD_ENV or DD_CP_ENV env vars. + /// Panics if neither is set — every deployment must declare its environment. pub fn detect() -> Self { let val = std::env::var("DD_ENV") .or_else(|_| std::env::var("DD_CP_ENV")) - .unwrap_or_default(); + .expect("DD_ENV or DD_CP_ENV must be set (staging or production)"); match val.to_lowercase().as_str() { "production" | "prod" => RuntimeEnv::Production, "staging" | "stage" => RuntimeEnv::Staging, - _ => RuntimeEnv::Local, + other => panic!("unknown DD_ENV value '{other}' — must be 'staging' or 'production'"), } } } @@ -31,64 +33,99 @@ pub struct VerifiedAttestation { pub rtmrs: Vec, } -/// High-level attestation service that wraps ItaVerifier with environment-specific policy. +/// High-level attestation service that wraps ItaVerifier. +/// There is no insecure mode — every environment requires real attestation. #[derive(Clone)] pub struct AttestationService { verifier: Option, - env: RuntimeEnv, + ita_client: Option, } impl AttestationService { - pub fn new(verifier: ItaVerifier, env: RuntimeEnv) -> Self { + pub fn new(verifier: ItaVerifier) -> Self { Self { verifier: Some(verifier), - env, + ita_client: ItaClient::from_env().ok(), } } - /// Create a service that skips real verification (for tests / local dev). - pub fn insecure_for_tests() -> Self { + /// Build an AttestationService with no verifier — only for tests. + /// All registration attempts will be rejected. + #[cfg(test)] + pub fn reject_all() -> Self { Self { verifier: None, - env: RuntimeEnv::Local, + ita_client: None, } } - /// Validate that runtime requirements are met for the current environment. - pub fn validate_runtime_requirements(&self) -> AppResult<()> { - match self.env { - RuntimeEnv::Production => { - if self.verifier.is_none() { - return Err(AppError::Config( - "production requires a configured ITA verifier".into(), - )); - } - Ok(()) + /// Build from environment. Uses ITA verifier when DD_INTEL_API_KEY is set. + /// Without the key, attestation verification will reject all tokens — + /// this is intentional: there is no "insecure" mode. + /// + /// Panics if DD_ENV is not set (ensures every deployment declares its environment). + pub fn from_env() -> Self { + // Force environment declaration — no silent fallback to "local". + let _env = RuntimeEnv::detect(); + + if let Ok(api_key) = std::env::var("DD_INTEL_API_KEY") { + let jwks_url = std::env::var("DD_ITA_JWKS_URL") + .unwrap_or_else(|_| "https://portal.trustauthority.intel.com/certs".into()); + let issuer = std::env::var("DD_ITA_ISSUER").ok(); + let audience = std::env::var("DD_ITA_AUDIENCE").ok().or(Some(api_key)); + Self { + verifier: Some(ItaVerifier::new(jwks_url, issuer, audience)), + ita_client: ItaClient::from_env().ok(), } - RuntimeEnv::Staging => { - // Staging allows missing verifier but logs a warning - Ok(()) + } else { + eprintln!("dd-cp: DD_INTEL_API_KEY not set — attestation verification will reject all registration attempts"); + Self { + verifier: None, + ita_client: None, } - RuntimeEnv::Local => Ok(()), } } - /// Verify an agent registration token, returning attestation data. - pub async fn verify_registration_token(&self, token: &str) -> AppResult { - match &self.verifier { - Some(v) => { - let claims = v.verify_attestation_token(token).await?; - Ok(extract_attestation(&claims)) - } - None => { - // Insecure mode: accept anything - Ok(VerifiedAttestation { - mrtd: Some("insecure-local-mrtd".into()), - tcb_status: Some("UpToDate".into()), - rtmrs: vec![], - }) - } + /// Validate that runtime requirements are met for the current environment. + /// All environments require a configured ITA verifier — there is no insecure mode. + pub fn validate_runtime_requirements(&self) -> AppResult<()> { + if self.verifier.is_none() { + return Err(AppError::Config( + "DD_INTEL_API_KEY is required — attestation verifier must be configured".into(), + )); } + if self.ita_client.is_none() { + return Err(AppError::Config( + "DD_INTEL_API_KEY is required — ITA client must be configured".into(), + )); + } + Ok(()) + } + + /// Verify an agent registration quote, returning attestation data. + pub async fn verify_registration_token( + &self, + raw_quote_b64: &str, + expected_nonce: &str, + ) -> AppResult { + if raw_quote_b64.is_empty() { + return Err(AppError::Config("attestation quote is required".into())); + } + + let verifier = self.verifier.as_ref().ok_or_else(|| { + AppError::Config( + "attestation verifier not configured (DD_INTEL_API_KEY required)".into(), + ) + })?; + let ita_client = self.ita_client.as_ref().ok_or_else(|| { + AppError::Config("ITA client not configured (DD_INTEL_API_KEY required)".into()) + })?; + + verify_quote_freshness(raw_quote_b64, expected_nonce)?; + + let token = ita_client.attest(raw_quote_b64).await?; + let claims = verifier.verify_attestation_token(&token).await?; + Ok(extract_attestation(&claims)) } } @@ -104,40 +141,66 @@ fn extract_attestation(claims: &AttestationClaims) -> VerifiedAttestation { } VerifiedAttestation { - mrtd: claims.tdx_mrtd.clone(), + mrtd: claims.tdx_mrtd().map(String::from), tcb_status: claims.attester_tcb_status.clone(), rtmrs, } } +fn verify_quote_freshness(raw_quote_b64: &str, expected_nonce: &str) -> AppResult<()> { + let quote = tsm::parse_tdx_quote_base64(raw_quote_b64) + .map_err(|e| AppError::InvalidInput(format!("invalid TDX quote: {e}")))?; + let report_data_prefix = + "e.report_data[..expected_nonce.len().min(quote.report_data.len())]; + + if report_data_prefix != expected_nonce.as_bytes() { + return Err(AppError::InvalidInput( + "TDX quote report data does not match the control-plane challenge nonce".into(), + )); + } + + Ok(()) +} + #[cfg(test)] mod tests { use super::*; - - #[tokio::test] - async fn insecure_mode_accepts_anything() { - let svc = AttestationService::insecure_for_tests(); - let result = svc.verify_registration_token("fake-token").await; - assert!(result.is_ok()); - let att = result.unwrap(); - assert_eq!(att.mrtd, Some("insecure-local-mrtd".into())); - } + use base64::Engine; #[test] - fn staging_allows_no_verifier() { + fn no_verifier_is_rejected() { let svc = AttestationService { verifier: None, - env: RuntimeEnv::Staging, + ita_client: None, }; - assert!(svc.validate_runtime_requirements().is_ok()); + assert!(svc.validate_runtime_requirements().is_err()); } #[test] - fn production_requires_verifier() { + fn empty_token_is_rejected() { let svc = AttestationService { verifier: None, - env: RuntimeEnv::Production, + ita_client: None, }; - assert!(svc.validate_runtime_requirements().is_err()); + let result = tokio::runtime::Runtime::new() + .unwrap() + .block_on(svc.verify_registration_token("", "")); + assert!(result.is_err()); + } + + #[test] + fn quote_freshness_rejects_mismatched_nonce() { + let report_data = b"expected-nonce"; + let mut quote = vec![0u8; tsm::MIN_QUOTE_SIZE]; + quote[0] = 4; + quote[1] = 0; + let body = tsm::QUOTE_HEADER_SIZE; + let report_data_offset = body + 520; + quote[report_data_offset..report_data_offset + report_data.len()] + .copy_from_slice(report_data); + let quote_b64 = base64::engine::general_purpose::STANDARD.encode(quote); + + let result = verify_quote_freshness("e_b64, "other-nonce"); + assert!(result.is_err()); } } diff --git a/control-plane/src/services/github_oidc.rs b/control-plane/src/services/github_oidc.rs index 0ea1b78..bccfff8 100644 --- a/control-plane/src/services/github_oidc.rs +++ b/control-plane/src/services/github_oidc.rs @@ -22,11 +22,6 @@ impl GithubOidcService { } } - /// Create a disabled service for tests. - pub fn disabled_for_tests() -> Self { - Self { verifier: None } - } - /// Verify a GitHub OIDC token and return the claims. pub async fn verify_token(&self, token: &str) -> AppResult { match &self.verifier { diff --git a/control-plane/src/services/ita_client.rs b/control-plane/src/services/ita_client.rs new file mode 100644 index 0000000..f1f8488 --- /dev/null +++ b/control-plane/src/services/ita_client.rs @@ -0,0 +1,110 @@ +use crate::common::error::{AppError, AppResult}; +use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, CONTENT_TYPE}; +use serde::{Deserialize, Serialize}; + +const DEFAULT_ITA_API_URL: &str = "https://api.trustauthority.intel.com"; + +#[derive(Debug, Serialize)] +struct ItaAttestRequest<'a> { + quote: &'a str, +} + +#[derive(Debug, Deserialize)] +struct ItaTokenResponse { + token: String, +} + +#[derive(Clone)] +pub struct ItaClient { + api_url: String, + http: reqwest::Client, +} + +impl ItaClient { + pub fn from_env() -> AppResult { + let api_key = std::env::var("DD_INTEL_API_KEY") + .map_err(|_| AppError::Config("DD_INTEL_API_KEY is required".into()))?; + let api_url = + std::env::var("DD_ITA_API_URL").unwrap_or_else(|_| DEFAULT_ITA_API_URL.to_string()); + Self::new(api_url, api_key) + } + + pub fn new(api_url: impl Into, api_key: impl AsRef) -> AppResult { + let mut headers = HeaderMap::new(); + let api_key = HeaderValue::from_str(api_key.as_ref()) + .map_err(|e| AppError::Config(format!("invalid Intel API key header value: {e}")))?; + headers.insert("x-api-key", api_key); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + headers.insert(ACCEPT, HeaderValue::from_static("application/json")); + + let http = reqwest::Client::builder() + .default_headers(headers) + .build() + .map_err(|e| AppError::External(format!("build ITA HTTP client: {e}")))?; + + Ok(Self { + api_url: api_url.into().trim_end_matches('/').to_string(), + http, + }) + } + + pub async fn attest(&self, quote_b64: &str) -> AppResult { + let url = format!("{}/appraisal/v1/attest", self.api_url); + let body = ItaAttestRequest { quote: quote_b64 }; + + let resp = self + .http + .post(&url) + .json(&body) + .send() + .await + .map_err(|e| AppError::External(format!("POST {url}: {e}")))?; + + if !resp.status().is_success() { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + return Err(AppError::External(format!( + "POST {url}: status {status}: {body}" + ))); + } + + let body = resp + .text() + .await + .map_err(|e| AppError::External(format!("read ITA attestation response: {e}")))?; + + parse_token_response(&body) + } +} + +fn parse_token_response(body: &str) -> AppResult { + let trimmed = body.trim(); + if trimmed.is_empty() { + return Err(AppError::External("empty ITA attestation response".into())); + } + + if let Ok(token) = serde_json::from_str::(trimmed) { + return Ok(token); + } + + if let Ok(resp) = serde_json::from_str::(trimmed) { + return Ok(resp.token); + } + + Ok(trimmed.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_token_response_accepts_raw_or_json() { + assert_eq!(parse_token_response("token-abc").unwrap(), "token-abc"); + assert_eq!(parse_token_response("\"token-abc\"").unwrap(), "token-abc"); + assert_eq!( + parse_token_response("{\"token\":\"token-abc\"}").unwrap(), + "token-abc" + ); + } +} diff --git a/control-plane/src/services/mod.rs b/control-plane/src/services/mod.rs index 61e76a6..7afbed1 100644 --- a/control-plane/src/services/mod.rs +++ b/control-plane/src/services/mod.rs @@ -1,4 +1,5 @@ pub mod attestation; pub mod github_oidc; +pub mod ita_client; pub mod nonce; pub mod tunnel; diff --git a/control-plane/src/services/tunnel.rs b/control-plane/src/services/tunnel.rs index b827c2c..add64b4 100644 --- a/control-plane/src/services/tunnel.rs +++ b/control-plane/src/services/tunnel.rs @@ -8,7 +8,6 @@ pub struct TunnelService { account_id: Option, zone_id: Option, domain: String, - enabled: bool, } /// Result of creating a tunnel. @@ -28,18 +27,6 @@ impl TunnelService { zone_id: std::env::var("DD_CP_CF_ZONE_ID").ok(), domain: std::env::var("DD_CP_CF_DOMAIN") .unwrap_or_else(|_| "devopsdefender.com".to_string()), - enabled: true, - } - } - - /// Create a no-op tunnel service for tests / local dev. - pub fn disabled_for_tests() -> Self { - Self { - api_token: None, - account_id: None, - zone_id: None, - domain: "test.devopsdefender.com".into(), - enabled: false, } } @@ -49,15 +36,6 @@ impl TunnelService { agent_id: Uuid, vm_name: &str, ) -> AppResult { - if !self.enabled { - // Return fake tunnel info for tests - return Ok(TunnelInfo { - tunnel_id: Uuid::new_v4().to_string(), - tunnel_token: format!("test-tunnel-token-{agent_id}"), - hostname: format!("{vm_name}.{}", self.domain), - }); - } - let api_token = self .api_token .as_ref() @@ -122,12 +100,57 @@ impl TunnelService { }) } - /// Delete a Cloudflare tunnel. - pub async fn delete_tunnel(&self, tunnel_id: &str) -> AppResult<()> { - if !self.enabled { - return Ok(()); + /// Retrieve a tunnel token for an existing agent (for re-registration). + /// When tunnels are disabled, returns a synthetic token. + pub async fn get_tunnel_token_for_agent( + &self, + agent_id: Uuid, + vm_name: &str, + ) -> AppResult { + let api_token = self + .api_token + .as_ref() + .ok_or_else(|| AppError::Config("DD_CP_CF_API_TOKEN not set".into()))?; + let account_id = self + .account_id + .as_ref() + .ok_or_else(|| AppError::Config("DD_CP_CF_ACCOUNT_ID not set".into()))?; + + let tunnel_name = format!("dd-agent-{agent_id}"); + let client = reqwest::Client::new(); + + // List tunnels to find existing one + let list_resp = client + .get(format!( + "https://api.cloudflare.com/client/v4/accounts/{account_id}/tunnels?name={tunnel_name}&is_deleted=false" + )) + .header("Authorization", format!("Bearer {api_token}")) + .send() + .await + .map_err(|e| AppError::External(format!("CF tunnel list failed: {e}")))?; + + let body: serde_json::Value = list_resp + .json() + .await + .map_err(|e| AppError::External(format!("CF tunnel list parse failed: {e}")))?; + + if let Some(tunnels) = body["result"].as_array() { + if let Some(tunnel) = tunnels.first() { + if let Some(token) = tunnel["token"].as_str() { + return Ok(token.to_string()); + } + } } + // Fallback: can't retrieve token, return a placeholder + // The agent will need a fresh registration with a new tunnel + Err(AppError::External(format!( + "No existing tunnel found for agent {agent_id} ({vm_name})" + ))) + } + + /// Delete a Cloudflare tunnel, first cleaning up any active connections. + pub async fn delete_tunnel(&self, tunnel_id: &str) -> AppResult<()> { let api_token = self .api_token .as_ref() @@ -138,6 +161,17 @@ impl TunnelService { .ok_or_else(|| AppError::Config("DD_CP_CF_ACCOUNT_ID not set".into()))?; let client = reqwest::Client::new(); + + // First: clean up active connections so the delete doesn't fail. + let _ = client + .delete(format!( + "https://api.cloudflare.com/client/v4/accounts/{account_id}/cfd_tunnel/{tunnel_id}/connections" + )) + .header("Authorization", format!("Bearer {api_token}")) + .send() + .await; + + // Then: delete the tunnel itself. let resp = client .delete(format!( "https://api.cloudflare.com/client/v4/accounts/{account_id}/cfd_tunnel/{tunnel_id}" @@ -165,14 +199,6 @@ impl TunnelService { hostname: &str, local_port: u16, ) -> AppResult { - if !self.enabled { - return Ok(TunnelInfo { - tunnel_id: "disabled".into(), - tunnel_token: "disabled".into(), - hostname: hostname.to_string(), - }); - } - let api_token = self .api_token .as_ref() @@ -187,7 +213,87 @@ impl TunnelService { let client = reqwest::Client::new(); - // 1. Create the tunnel. + // 1. Try to find an existing tunnel with this name first. + let existing = self + .find_tunnel_by_name(&client, api_token, account_id, &tunnel_name) + .await?; + + let (tunnel_id, tunnel_token) = if let Some(existing_id) = existing { + // Tunnel already exists — delete and recreate so we get a fresh token. + // (CF doesn't expose the token after initial creation.) + let _ = self.delete_tunnel(&existing_id).await; + + let (id, token) = self + .create_tunnel_raw(&client, api_token, account_id, &tunnel_name, &tunnel_secret) + .await?; + (id, token) + } else { + self.create_tunnel_raw(&client, api_token, account_id, &tunnel_name, &tunnel_secret) + .await? + }; + + // 2. Configure tunnel ingress to route to localhost. + self.configure_tunnel_ingress( + &tunnel_id, + hostname, + &format!("http://localhost:{local_port}"), + ) + .await?; + + // 3. Create DNS CNAME record. + self.create_dns_record(&tunnel_id, hostname).await?; + + // 4. Spawn cloudflared. + Self::spawn_cloudflared(&tunnel_token)?; + + Ok(TunnelInfo { + tunnel_id, + tunnel_token, + hostname: hostname.to_string(), + }) + } + + /// Look up an existing tunnel by name, returning its ID if found. + async fn find_tunnel_by_name( + &self, + client: &reqwest::Client, + api_token: &str, + account_id: &str, + tunnel_name: &str, + ) -> AppResult> { + let resp = client + .get(format!( + "https://api.cloudflare.com/client/v4/accounts/{account_id}/cfd_tunnel?name={tunnel_name}&is_deleted=false" + )) + .header("Authorization", format!("Bearer {api_token}")) + .send() + .await; + + match resp { + Ok(r) if r.status().is_success() => { + let body: serde_json::Value = r + .json() + .await + .map_err(|e| AppError::External(format!("CF tunnel list parse: {e}")))?; + Ok(body["result"] + .as_array() + .and_then(|arr| arr.first()) + .and_then(|t| t["id"].as_str()) + .map(|s| s.to_string())) + } + _ => Ok(None), // Non-fatal: if lookup fails, fall through to create. + } + } + + /// Low-level tunnel creation — returns (tunnel_id, tunnel_token). + async fn create_tunnel_raw( + &self, + client: &reqwest::Client, + api_token: &str, + account_id: &str, + tunnel_name: &str, + tunnel_secret: &str, + ) -> AppResult<(String, String)> { let create_resp = client .post(format!( "https://api.cloudflare.com/client/v4/accounts/{account_id}/cfd_tunnel" @@ -222,38 +328,16 @@ impl TunnelService { .to_string(); let tunnel_token = resp_body["result"]["token"] .as_str() - .unwrap_or(&tunnel_secret) + .unwrap_or(tunnel_secret) .to_string(); - // 2. Configure tunnel ingress to route to localhost. - self.configure_tunnel_ingress( - &tunnel_id, - hostname, - &format!("http://localhost:{local_port}"), - ) - .await?; - - // 3. Create DNS CNAME record. - self.create_dns_record(&tunnel_id, hostname).await?; - - // 4. Spawn cloudflared. - Self::spawn_cloudflared(&tunnel_token)?; - - Ok(TunnelInfo { - tunnel_id, - tunnel_token, - hostname: hostname.to_string(), - }) + Ok((tunnel_id, tunnel_token)) } /// Create or update a CNAME DNS record pointing to a tunnel. /// /// If a CNAME record for `hostname` already exists, it is updated in place. pub async fn create_dns_record(&self, tunnel_id: &str, hostname: &str) -> AppResult<()> { - if !self.enabled { - return Ok(()); - } - let api_token = self .api_token .as_ref() @@ -425,20 +509,27 @@ mod tests { use super::*; #[tokio::test] - async fn disabled_service_returns_fake_tunnel() { - let svc = TunnelService::disabled_for_tests(); + async fn unconfigured_service_errors_on_create() { + let svc = TunnelService { + api_token: None, + account_id: None, + zone_id: None, + domain: "devopsdefender.com".into(), + }; let agent_id = Uuid::new_v4(); let result = svc.create_tunnel_for_agent(agent_id, "test-vm").await; - assert!(result.is_ok()); - let info = result.unwrap(); - assert!(info.hostname.contains("test-vm")); - assert!(info.hostname.contains("devopsdefender.com")); + assert!(result.is_err()); } #[tokio::test] - async fn disabled_service_delete_is_noop() { - let svc = TunnelService::disabled_for_tests(); + async fn unconfigured_service_errors_on_delete() { + let svc = TunnelService { + api_token: None, + account_id: None, + zone_id: None, + domain: "devopsdefender.com".into(), + }; let result = svc.delete_tunnel("fake-tunnel-id").await; - assert!(result.is_ok()); + assert!(result.is_err()); } } diff --git a/control-plane/src/state.rs b/control-plane/src/state.rs index 0598b5f..a73d022 100644 --- a/control-plane/src/state.rs +++ b/control-plane/src/state.rs @@ -55,9 +55,9 @@ impl AppState { db: db.clone(), settings: SettingsStore::new(db), nonce: NonceService::new(env_u64("DD_CP_NONCE_TTL_SECONDS", 300)), - attestation: AttestationService::insecure_for_tests(), - github_oidc: GithubOidcService::disabled_for_tests(), - tunnel: TunnelService::disabled_for_tests(), + attestation: AttestationService::from_env(), + github_oidc: GithubOidcService::from_env(), + tunnel: TunnelService::from_env(), check_ingest_token: std::env::var("DD_CP_CHECK_INGEST_TOKEN").ok(), heartbeat_interval_seconds: env_u64("DD_CP_HEARTBEAT_INTERVAL_SECONDS", 30), check_timeout_seconds: env_u64("DD_CP_CHECK_TIMEOUT_SECONDS", 10), @@ -70,6 +70,8 @@ impl AppState { } /// Build AppState suitable for testing with an in-memory DB. + /// Uses reject_all() for attestation — no env vars needed. + #[cfg(test)] pub fn for_testing(db: Db) -> Self { Self { boot_id: "test-boot-id".into(), @@ -79,9 +81,9 @@ impl AppState { db: db.clone(), settings: SettingsStore::new(db), nonce: NonceService::new(300), - attestation: AttestationService::insecure_for_tests(), - github_oidc: GithubOidcService::disabled_for_tests(), - tunnel: TunnelService::disabled_for_tests(), + attestation: AttestationService::reject_all(), + github_oidc: GithubOidcService::from_env(), + tunnel: TunnelService::from_env(), check_ingest_token: Some("test-ingest-token".into()), heartbeat_interval_seconds: 30, check_timeout_seconds: 10, diff --git a/control-plane/src/stores/agent.rs b/control-plane/src/stores/agent.rs index d000141..e83dae7 100644 --- a/control-plane/src/stores/agent.rs +++ b/control-plane/src/stores/agent.rs @@ -19,6 +19,7 @@ pub struct AgentRow { pub node_size: Option, pub datacenter: Option, pub github_owner: Option, + pub deployment_id: Option, pub created_at: String, pub last_heartbeat_at: Option, } @@ -36,6 +37,7 @@ fn row_to_agent(row: &rusqlite::Row<'_>) -> rusqlite::Result { node_size: row.get("node_size")?, datacenter: row.get("datacenter")?, github_owner: row.get("github_owner")?, + deployment_id: row.get("deployment_id")?, created_at: row.get("created_at")?, last_heartbeat_at: row.get("last_heartbeat_at")?, }) @@ -46,8 +48,8 @@ pub fn insert_agent(db: &Db, agent: &AgentRow) -> AppResult<()> { let conn = db.lock().unwrap(); conn.execute( "INSERT INTO agents (id, vm_name, status, registration_state, hostname, tunnel_id, \ - mrtd, tcb_status, node_size, datacenter, github_owner, created_at, last_heartbeat_at) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)", + mrtd, tcb_status, node_size, datacenter, github_owner, deployment_id, created_at, \ + last_heartbeat_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)", params![ agent.id, agent.vm_name, @@ -60,6 +62,7 @@ pub fn insert_agent(db: &Db, agent: &AgentRow) -> AppResult<()> { agent.node_size, agent.datacenter, agent.github_owner, + agent.deployment_id, agent.created_at, agent.last_heartbeat_at, ], @@ -75,8 +78,8 @@ pub fn get_agent(db: &Db, id: &str) -> AppResult> { let mut stmt = conn .prepare( "SELECT id, vm_name, status, registration_state, hostname, tunnel_id, \ - mrtd, tcb_status, node_size, datacenter, github_owner, created_at, last_heartbeat_at \ - FROM agents WHERE id = ?1", + mrtd, tcb_status, node_size, datacenter, github_owner, deployment_id, created_at, \ + last_heartbeat_at FROM agents WHERE id = ?1", ) .map_err(|_| AppError::Internal)?; @@ -88,14 +91,33 @@ pub fn get_agent(db: &Db, id: &str) -> AppResult> { Ok(result) } +/// Find an agent by VM name (most recent first). +pub fn find_agent_by_vm_name(db: &Db, vm_name: &str) -> AppResult> { + let conn = db.lock().unwrap(); + let mut stmt = conn + .prepare( + "SELECT id, vm_name, status, registration_state, hostname, tunnel_id, \ + mrtd, tcb_status, node_size, datacenter, github_owner, deployment_id, created_at, \ + last_heartbeat_at FROM agents WHERE vm_name = ?1 ORDER BY created_at DESC LIMIT 1", + ) + .map_err(|_| AppError::Internal)?; + + let result = stmt + .query_row(params![vm_name], row_to_agent) + .optional() + .map_err(|_| AppError::Internal)?; + + Ok(result) +} + /// List all agents. pub fn list_agents(db: &Db) -> AppResult> { let conn = db.lock().unwrap(); let mut stmt = conn .prepare( "SELECT id, vm_name, status, registration_state, hostname, tunnel_id, \ - mrtd, tcb_status, node_size, datacenter, github_owner, created_at, last_heartbeat_at \ - FROM agents ORDER BY created_at DESC", + mrtd, tcb_status, node_size, datacenter, github_owner, deployment_id, created_at, \ + last_heartbeat_at FROM agents ORDER BY created_at DESC", ) .map_err(|_| AppError::Internal)?; @@ -131,6 +153,18 @@ pub fn update_agent_status(db: &Db, id: &str, status: &str) -> AppResult { Ok(count > 0) } +/// Assign or clear the deployment currently associated with an agent. +pub fn update_agent_deployment(db: &Db, id: &str, deployment_id: Option<&str>) -> AppResult { + let conn = db.lock().unwrap(); + let count = conn + .execute( + "UPDATE agents SET deployment_id = ?1 WHERE id = ?2", + params![deployment_id, id], + ) + .map_err(|_| AppError::Internal)?; + Ok(count > 0) +} + /// Update agent registration state. pub fn update_registration_state(db: &Db, id: &str, state: &str) -> AppResult { let conn = db.lock().unwrap(); @@ -165,8 +199,9 @@ pub fn find_available_agent( let conn = db.lock().unwrap(); let mut query = String::from( "SELECT id, vm_name, status, registration_state, hostname, tunnel_id, \ - mrtd, tcb_status, node_size, datacenter, github_owner, created_at, last_heartbeat_at \ - FROM agents WHERE status = 'undeployed' AND registration_state = 'ready'", + mrtd, tcb_status, node_size, datacenter, github_owner, deployment_id, created_at, \ + last_heartbeat_at FROM agents WHERE status = 'undeployed' AND registration_state = 'ready' \ + AND deployment_id IS NULL", ); let mut bind_values: Vec = Vec::new(); @@ -232,6 +267,7 @@ mod tests { node_size: None, datacenter: None, github_owner: None, + deployment_id: None, created_at: Utc::now().to_rfc3339(), last_heartbeat_at: None, } diff --git a/control-plane/src/stores/deployment.rs b/control-plane/src/stores/deployment.rs index 4704de9..ef8e62d 100644 --- a/control-plane/src/stores/deployment.rs +++ b/control-plane/src/stores/deployment.rs @@ -8,43 +8,40 @@ use crate::db::Db; pub struct DeploymentRow { pub id: String, pub agent_id: String, - pub app_name: Option, - pub app_version: Option, - pub compose: String, - pub config: Option, + pub image: String, + pub env: Vec, + pub cmd: Vec, pub status: String, pub created_at: String, pub updated_at: String, } fn row_to_deployment(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let env_json: String = row.get("env")?; + let cmd_json: String = row.get("cmd")?; Ok(DeploymentRow { id: row.get("id")?, agent_id: row.get("agent_id")?, - app_name: row.get("app_name")?, - app_version: row.get("app_version")?, - compose: row.get("compose")?, - config: row.get("config")?, + image: row.get("image")?, + env: serde_json::from_str(&env_json).map_err(serde_to_sql_error)?, + cmd: serde_json::from_str(&cmd_json).map_err(serde_to_sql_error)?, status: row.get("status")?, created_at: row.get("created_at")?, updated_at: row.get("updated_at")?, }) } -/// Insert a new deployment. pub fn insert_deployment(db: &Db, dep: &DeploymentRow) -> AppResult<()> { let conn = db.lock().unwrap(); conn.execute( - "INSERT INTO deployments (id, agent_id, app_name, app_version, compose, config, \ - status, created_at, updated_at) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + "INSERT INTO deployments (id, agent_id, image, env, cmd, status, created_at, updated_at) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", params![ dep.id, dep.agent_id, - dep.app_name, - dep.app_version, - dep.compose, - dep.config, + dep.image, + serde_json::to_string(&dep.env).map_err(|_| AppError::Internal)?, + serde_json::to_string(&dep.cmd).map_err(|_| AppError::Internal)?, dep.status, dep.created_at, dep.updated_at, @@ -55,13 +52,11 @@ pub fn insert_deployment(db: &Db, dep: &DeploymentRow) -> AppResult<()> { Ok(()) } -/// Get a deployment by ID. pub fn get_deployment(db: &Db, id: &str) -> AppResult> { let conn = db.lock().unwrap(); let mut stmt = conn .prepare( - "SELECT id, agent_id, app_name, app_version, compose, config, \ - status, created_at, updated_at \ + "SELECT id, agent_id, image, env, cmd, status, created_at, updated_at \ FROM deployments WHERE id = ?1", ) .map_err(|_| AppError::Internal)?; @@ -74,28 +69,24 @@ pub fn get_deployment(db: &Db, id: &str) -> AppResult> { Ok(result) } -/// List deployments, optionally filtering by agent_id. pub fn list_deployments(db: &Db, agent_id: Option<&str>) -> AppResult> { let conn = db.lock().unwrap(); let (query, bind_val) = if let Some(aid) = agent_id { ( - "SELECT id, agent_id, app_name, app_version, compose, config, \ - status, created_at, updated_at \ + "SELECT id, agent_id, image, env, cmd, status, created_at, updated_at \ FROM deployments WHERE agent_id = ?1 ORDER BY created_at DESC", Some(aid.to_string()), ) } else { ( - "SELECT id, agent_id, app_name, app_version, compose, config, \ - status, created_at, updated_at \ + "SELECT id, agent_id, image, env, cmd, status, created_at, updated_at \ FROM deployments ORDER BY created_at DESC", None, ) }; let mut stmt = conn.prepare(query).map_err(|_| AppError::Internal)?; - let rows = if let Some(ref v) = bind_val { stmt.query_map(params![v], row_to_deployment) .map_err(|_| AppError::Internal)? @@ -111,7 +102,6 @@ pub fn list_deployments(db: &Db, agent_id: Option<&str>) -> AppResult AppResult { let now = chrono::Utc::now().to_rfc3339(); let conn = db.lock().unwrap(); @@ -138,6 +128,10 @@ impl OptionalExt for Result { } } +fn serde_to_sql_error(err: serde_json::Error) -> rusqlite::Error { + rusqlite::Error::FromSqlConversionFailure(0, rusqlite::types::Type::Text, Box::new(err)) +} + #[cfg(test)] mod tests { use super::*; @@ -161,10 +155,10 @@ mod tests { node_size: None, datacenter: None, github_owner: None, + deployment_id: None, created_at: chrono::Utc::now().to_rfc3339(), last_heartbeat_at: None, }; - // Ignore error if agent already exists let _ = agent_store::insert_agent(db, &agent); } @@ -173,10 +167,9 @@ mod tests { DeploymentRow { id: id.into(), agent_id: agent_id.into(), - app_name: Some("test-app".into()), - app_version: Some("1.0.0".into()), - compose: "version: '3'\nservices: {}".into(), - config: None, + image: "ghcr.io/devopsdefender/workload:latest".into(), + env: vec!["KEY=VALUE".into()], + cmd: vec!["/bin/server".into()], status: "pending".into(), created_at: now.clone(), updated_at: now, @@ -190,9 +183,9 @@ mod tests { let dep = make_deployment("d1", "a1"); insert_deployment(&db, &dep).unwrap(); - let fetched = get_deployment(&db, "d1").unwrap(); - assert!(fetched.is_some()); - assert_eq!(fetched.unwrap().app_name, Some("test-app".into())); + let fetched = get_deployment(&db, "d1").unwrap().unwrap(); + assert_eq!(fetched.image, "ghcr.io/devopsdefender/workload:latest"); + assert_eq!(fetched.env, vec!["KEY=VALUE"]); } #[test] diff --git a/images/packer/baremetal-agent-image.pkr.hcl b/images/packer/baremetal-agent-image.pkr.hcl index 29376e6..36104b6 100644 --- a/images/packer/baremetal-agent-image.pkr.hcl +++ b/images/packer/baremetal-agent-image.pkr.hcl @@ -107,15 +107,20 @@ build { name = "dd-baremetal-agent" sources = ["source.qemu.baremetal"] + provisioner "shell-local" { + inline = [ + "if [ -n \"${var.cp_binary_path}\" ] && [ -f \"${var.cp_binary_path}\" ]; then cp \"${var.cp_binary_path}\" /tmp/dd-cp-upload; else touch /tmp/dd-cp-upload; fi" + ] + } + provisioner "file" { source = var.agent_binary_path destination = "/tmp/dd-agent" } provisioner "file" { - source = var.cp_binary_path + source = "/tmp/dd-cp-upload" destination = "/tmp/dd-cp" - only = [for s in ["source.qemu.baremetal"] : s if var.cp_binary_path != ""] } provisioner "shell" { diff --git a/images/packer/provision-agent-image.sh b/images/packer/provision-agent-image.sh index 02b8d5c..6a0563d 100755 --- a/images/packer/provision-agent-image.sh +++ b/images/packer/provision-agent-image.sh @@ -97,7 +97,7 @@ Type=simple User=root Environment=DD_AGENT_MODE=control-plane Environment=DD_CONFIG=/etc/devopsdefender/control-plane.json -ExecStart=/usr/local/bin/dd-agent +ExecStart=/usr/local/bin/dd-cp Restart=on-failure RestartSec=5 StandardOutput=journal+console diff --git a/images/packer/provision-baremetal-image.sh b/images/packer/provision-baremetal-image.sh index a6ad1b5..4a26b84 100755 --- a/images/packer/provision-baremetal-image.sh +++ b/images/packer/provision-baremetal-image.sh @@ -73,6 +73,22 @@ apt-get update apt-get install -y --no-install-recommends nvidia-container-toolkit nvidia-ctk runtime configure --runtime=docker || true +# ── TDX guest support (attestation via configfs-tsm) ───────────────────── +# The generic kernel has CONFIG_TDX_GUEST_DRIVER=m but the module lives in +# linux-modules-extra, which cloud images sometimes omit. Install it to +# ensure tdx_guest.ko is present for quote generation. +GUEST_KERNEL="$(ls /boot/vmlinuz-* 2>/dev/null | head -1 | sed 's|/boot/vmlinuz-||')" +if [ -n "$GUEST_KERNEL" ]; then + apt-get update + apt-get install -y --no-install-recommends "linux-modules-extra-${GUEST_KERNEL}" || true +fi + +cat > /etc/modules-load.d/tdx-guest.conf <<'MODULES' +# Load TDX guest modules for attestation quote generation. +# These are no-ops on non-TDX VMs. +tdx_guest +MODULES + # ── Create config directory ─────────────────────────────────────────────── install -d -m 0755 /etc/devopsdefender @@ -109,7 +125,8 @@ Type=simple User=root Environment=DD_AGENT_MODE=control-plane Environment=DD_CONFIG=/etc/devopsdefender/control-plane.json -ExecStart=/usr/local/bin/dd-agent +Environment=PATH=/usr/local/bin:/usr/bin:/bin +ExecStart=/usr/local/bin/dd-cp Restart=on-failure RestartSec=5 StandardOutput=journal+console diff --git a/infra/README.md b/infra/README.md new file mode 100644 index 0000000..c14badb --- /dev/null +++ b/infra/README.md @@ -0,0 +1,45 @@ +# Infrastructure + +Staging is intentionally split: + +1. The staging control plane runs on GCP. +2. The first/bootstrap validation agent also runs on GCP. +3. That first GCP agent is disposable after staging validation. +4. The example app runs on the OVH host inside a libvirt-managed VM. + +Cloudflare remains in use for the public staging and production hostnames. Only stale DNS and tunnel resources were cleaned up manually while unwinding the previous staging experiment. + +## GCP staging + +`.github/workflows/staging-deploy.yml` is the canonical staging control-plane workflow. It: + +1. Builds `dd-agent` and `dd-cp`. +2. Bakes the GCP image with Packer. +3. Launches the staging control plane on GCP. +4. Launches exactly one tiny bootstrap/test agent on GCP. +5. Smoke-checks `https://app-staging.devopsdefender.com/health`. + +The supporting playbooks live under `infra/ansible/playbooks/gcp-*.yml`. + +## OVH app VM + +`infra/ansible/playbooks/baremetal-agent-deploy.yml` maintains the OVH VM that the example app targets. It: + +1. Installs `libvirt`, `virt-install`, QEMU, and cloud-init tooling. +2. Bakes a qcow2 image on the OVH host. +3. Replaces the old VM with a fresh libvirt-managed VM. +4. Points that VM at the external control plane via `cp_url`. + +The VM lifecycle scripts are intentionally `virsh`-centric: + +- `infra/scripts/vm-launch.sh`: defines and starts the domain in libvirt. +- `infra/scripts/vm-stop.sh`: shuts down and undefines the domain. +- `infra/scripts/vm-status.sh`: shows the repo-managed VMs and `virsh list --all`. + +The expected operator view is `virsh list --all`, not a detached raw `qemu-system-*` process. + +## Inventories + +`infra/ansible/inventory/staging.yml` is the OVH staging app host and points at the external GCP staging control plane. + +`infra/ansible/inventory/production.yml` is the OVH production app host and points at the external production control plane. diff --git a/infra/STATUS.md b/infra/STATUS.md new file mode 100644 index 0000000..be3741c --- /dev/null +++ b/infra/STATUS.md @@ -0,0 +1,48 @@ +# Infra Status + +Current target state: + +- Staging control plane: GCP +- Staging bootstrap/test agent: GCP +- Staging example-app VM: OVH host, managed by libvirt and visible in `virsh list` +- Production app VM: OVH host, managed by libvirt and pointed at the external production control plane +- Cloudflare: still active; only stale records/resources were deleted manually + +## Structure + +``` +infra/ +├── ansible/ +│ ├── ansible.cfg # Default inventory: staging.yml +│ ├── inventory/ +│ │ ├── staging.yml # OVH staging app VM host + external GCP cp_url +│ │ └── production.yml # OVH production app VM host + external cp_url +│ └── playbooks/ +│ ├── baremetal-agent-deploy.yml # Bake image + deploy libvirt-managed OVH VM +│ ├── gcp-control-plane-new.yml # Launch staging control plane on GCP +│ ├── gcp-deploy.yml # Launch staging control plane + first GCP agent +│ ├── gcp-image-bake.yml # Bake GCP image +│ ├── gcp-vm-fleet-new.yml # Launch GCP agents +│ └── templates/ +│ ├── agent-startup.sh.j2 # GCP agent startup metadata +│ └── control-plane-startup.sh.j2 # GCP control-plane startup metadata +└── scripts/ + ├── vm-launch.sh # Define/start libvirt VM from qcow2 image + ├── vm-stop.sh # Shutdown + undefine libvirt VM + └── vm-status.sh # Repo VM summary + virsh list --all +``` + +## Workflows + +| Workflow | Trigger | Playbook | +|---|---|---| +| `staging-deploy.yml` | Push to `main` / manual | `gcp-deploy.yml` | +| `baremetal-staging-deploy.yml` | PR / manual | `baremetal-agent-deploy.yml` | +| `baremetal-production-deploy.yml` | Push to `main` / manual | `baremetal-agent-deploy.yml` | + +## Deploy Flow + +1. `staging-deploy.yml` keeps the control plane and one disposable bootstrap agent on GCP. +2. `baremetal-staging-deploy.yml` keeps the OVH example-app VM aligned with the current branch. +3. `baremetal-production-deploy.yml` does the same for production. +4. OVH VMs are expected to be visible and operable through `virsh list --all`. diff --git a/infra/ansible/inventory/production.yml b/infra/ansible/inventory/production.yml index 57c8f36..e19c51e 100644 --- a/infra/ansible/inventory/production.yml +++ b/infra/ansible/inventory/production.yml @@ -4,10 +4,10 @@ all: ansible_host: 162.222.34.121 ansible_user: ubuntu ansible_ssh_private_key_file: "{{ lookup('env', 'SSH_KEY_PATH') | default('/tmp/deploy-key', true) }}" + cp_url: "https://app.devopsdefender.com" vars: dd_env: production - cp_memory: 8G - cp_cpus: 8 agent_memory: 64G agent_cpus: 16 agent_node_size: llm + agent_tdx: true # Production runs TDX confidential VMs diff --git a/infra/ansible/inventory/staging.yml b/infra/ansible/inventory/staging.yml index fc20baf..e22da50 100644 --- a/infra/ansible/inventory/staging.yml +++ b/infra/ansible/inventory/staging.yml @@ -1,13 +1,14 @@ all: hosts: staging: - ansible_host: 57.130.10.246 + ansible_host: 57.130.10.246 # OVH dedicated host for the staging example-app VM ansible_user: ubuntu ansible_ssh_private_key_file: "{{ lookup('env', 'SSH_KEY_PATH') | default('/tmp/deploy-key', true) }}" + cp_url: "https://app-staging.devopsdefender.com" vars: dd_env: staging - cp_memory: 4G - cp_cpus: 4 + datacenter: ovh-eu agent_memory: 8G agent_cpus: 4 - agent_node_size: standard + agent_node_size: standard # The example app lands on this OVH VM + agent_tdx: true # Keep staging attestation aligned with production diff --git a/infra/ansible/playbooks/baremetal-agent-deploy.yml b/infra/ansible/playbooks/baremetal-agent-deploy.yml index 2647a1a..8247602 100644 --- a/infra/ansible/playbooks/baremetal-agent-deploy.yml +++ b/infra/ansible/playbooks/baremetal-agent-deploy.yml @@ -3,22 +3,20 @@ hosts: all gather_facts: false vars: - dd_env: staging - cp_url: "" - dd_skip_attestation: true - agent_node_size: standard - agent_memory: 8G - agent_cpus: 4 + dd_env: "{{ hostvars[inventory_hostname].dd_env | default('staging') }}" + cp_url: "{{ hostvars[inventory_hostname].cp_url | default('') }}" + agent_node_size: "{{ hostvars[inventory_hostname].agent_node_size | default('standard') }}" + agent_memory: "{{ hostvars[inventory_hostname].agent_memory | default('8G') }}" + agent_cpus: "{{ hostvars[inventory_hostname].agent_cpus | default('4') }}" + agent_tdx: "{{ hostvars[inventory_hostname].agent_tdx | default(true) }}" vfio_device: "" vm_scripts_dir: "{{ playbook_dir }}/../../scripts" devopsdefender_base_dir: /var/lib/devopsdefender - # Image build vars (set by workflow) agent_binary_local: "" cp_binary_local: "" packer_template_dir: "" image_name: dd-baremetal-agent tasks: - # ── Prerequisites ─────────────────────────────────────────────────── - name: Install prerequisites ansible.builtin.apt: name: @@ -28,10 +26,55 @@ - genisoimage - jq - curl + - libvirt-clients + - libvirt-daemon-system + - virtinst state: present update_cache: true become: true + - name: Start libvirt + ansible.builtin.systemd: + name: libvirtd + enabled: true + state: started + become: true + + - name: Probe default libvirt network + ansible.builtin.command: + argv: + - virsh + - net-info + - default + register: libvirt_default_net + failed_when: false + changed_when: false + become: true + + - name: Start default libvirt network + ansible.builtin.command: + argv: + - virsh + - net-start + - default + register: libvirt_net_start + failed_when: false + changed_when: libvirt_net_start.rc == 0 + become: true + when: libvirt_default_net.rc == 0 + + - name: Autostart default libvirt network + ansible.builtin.command: + argv: + - virsh + - net-autostart + - default + register: libvirt_net_autostart + failed_when: false + changed_when: libvirt_net_autostart.rc == 0 + become: true + when: libvirt_default_net.rc == 0 + - name: Check /dev/kvm exists ansible.builtin.stat: path: /dev/kvm @@ -42,6 +85,11 @@ msg: "/dev/kvm not found - KVM support is required" when: not kvm_check.stat.exists + - name: Fail if cp_url is missing + ansible.builtin.fail: + msg: "cp_url must point at the external control plane" + when: cp_url | trim | length == 0 + - name: Create system directories ansible.builtin.file: path: "{{ item }}" @@ -67,7 +115,6 @@ - /tmp/dd-deploy/scripts - /tmp/dd-deploy/packer-build - # ── Build image on target (if binaries provided) ──────────────────── - name: Install Packer when: agent_binary_local | length > 0 block: @@ -176,7 +223,6 @@ become: true changed_when: true - # ── Deploy agent VM ───────────────────────────────────────────────── - name: Copy VM scripts ansible.builtin.copy: src: "{{ vm_scripts_dir }}/{{ item }}" @@ -221,6 +267,11 @@ - --cpus - "{{ agent_cpus }}" + - name: Add TDX flag to agent VM args + ansible.builtin.set_fact: + agent_launch_args: "{{ agent_launch_args + ['--tdx'] }}" + when: agent_tdx | bool + - name: Add VFIO device to agent VM args ansible.builtin.set_fact: agent_launch_args: "{{ agent_launch_args + ['--vfio-device', vfio_device] }}" @@ -231,6 +282,42 @@ argv: "{{ agent_launch_args }}" become: true + - name: Wait for agent to register with control plane + ansible.builtin.uri: + url: "{{ cp_url }}/api/v1/agents" + method: GET + return_content: true + status_code: 200 + register: agents_poll + retries: 60 + delay: 5 + until: >- + ( + ( + ((agents_poll.json | default({})).agents | default([])) + if ((agents_poll.json | default({})) is mapping) + else (agents_poll.json | default([])) + ) + | selectattr('registration_state', 'equalto', 'ready') + | selectattr('vm_name', 'equalto', 'dd-agent-' + dd_env) + | list + | length + ) > 0 + + - name: Print registered agents + ansible.builtin.debug: + msg: >- + {{ + ( + ((agents_poll.json | default({})).agents | default([])) + if ((agents_poll.json | default({})) is mapping) + else (agents_poll.json | default([])) + ) + | selectattr('registration_state', 'equalto', 'ready') + | selectattr('vm_name', 'equalto', 'dd-agent-' + dd_env) + | list + }} + - name: Show VM status ansible.builtin.command: argv: diff --git a/infra/ansible/playbooks/baremetal-deploy.yml b/infra/ansible/playbooks/baremetal-deploy.yml deleted file mode 100644 index b3bb6cd..0000000 --- a/infra/ansible/playbooks/baremetal-deploy.yml +++ /dev/null @@ -1,217 +0,0 @@ ---- -- name: Deploy control plane and agent on baremetal - hosts: all - gather_facts: false - vars: - image_path_local: "" - dd_env: staging - dd_cp_admin_password: "" - dd_cp_cf_api_token: "" - dd_cp_cf_account_id: "" - dd_cp_cf_zone_id: "" - dd_cp_cf_domain: devopsdefender.com - dd_cp_git_sha: "" - dd_cp_nonce_enforcement: optional - dd_cp_tcb_enforcement: warn - dd_cp_rtmr_enforcement: warn - dd_skip_attestation: true - agent_node_size: standard - cp_memory: 4G - cp_cpus: 4 - agent_memory: 8G - agent_cpus: 4 - vfio_device: "" - vm_scripts_dir: "{{ playbook_dir }}/../../scripts" - devopsdefender_base_dir: /var/lib/devopsdefender - cp_port_forward: "8080:8080" - tasks: - - name: Install prerequisites - ansible.builtin.apt: - name: - - qemu-system-x86 - - qemu-utils - - cloud-image-utils - - genisoimage - - jq - - curl - state: present - update_cache: true - become: true - - - name: Check /dev/kvm exists - ansible.builtin.stat: - path: /dev/kvm - register: kvm_check - - - name: Fail if /dev/kvm is missing - ansible.builtin.fail: - msg: "/dev/kvm not found - KVM support is required" - when: not kvm_check.stat.exists - - - name: Create required directories - ansible.builtin.file: - path: "{{ item }}" - state: directory - mode: "0755" - loop: - - "{{ devopsdefender_base_dir }}/images" - - "{{ devopsdefender_base_dir }}/vms" - - /tmp/dd-deploy/scripts - become: true - - - name: Copy VM scripts to remote host - ansible.builtin.copy: - src: "{{ vm_scripts_dir }}/{{ item }}" - dest: "/tmp/dd-deploy/scripts/{{ item }}" - mode: "0755" - loop: - - vm-launch.sh - - vm-stop.sh - - vm-status.sh - - - name: Copy baked qcow2 image to remote host - ansible.builtin.copy: - src: "{{ image_path_local }}" - dest: "{{ devopsdefender_base_dir }}/images/dd-baremetal.qcow2" - mode: "0644" - become: true - when: image_path_local | length > 0 - - - name: Template control-plane config - ansible.builtin.template: - src: "{{ playbook_dir }}/templates/control-plane.json.j2" - dest: /tmp/dd-deploy/control-plane.json - mode: "0600" - - - name: Template agent config - ansible.builtin.template: - src: "{{ playbook_dir }}/templates/agent.json.j2" - dest: /tmp/dd-deploy/agent.json - mode: "0600" - - - name: Stop old CP VM - ansible.builtin.command: - argv: - - /tmp/dd-deploy/scripts/vm-stop.sh - - "dd-cp-{{ dd_env }}" - - --clean - failed_when: false - changed_when: stop_cp.rc == 0 - register: stop_cp - become: true - - - name: Stop old agent VM - ansible.builtin.command: - argv: - - /tmp/dd-deploy/scripts/vm-stop.sh - - "dd-agent-{{ dd_env }}" - - --clean - failed_when: false - changed_when: stop_agent.rc == 0 - register: stop_agent - become: true - - - name: Launch CP VM - ansible.builtin.command: - argv: - - /tmp/dd-deploy/scripts/vm-launch.sh - - --image - - "{{ devopsdefender_base_dir }}/images/dd-baremetal.qcow2" - - --name - - "dd-cp-{{ dd_env }}" - - --config - - /tmp/dd-deploy/control-plane.json - - --config-mode - - control-plane - - --memory - - "{{ cp_memory }}" - - --cpus - - "{{ cp_cpus }}" - - --port-forward - - "{{ cp_port_forward }}" - become: true - - - name: Wait for CP health - ansible.builtin.uri: - url: "http://localhost:8080/health" - method: GET - status_code: 200 - return_content: true - register: cp_health - retries: 60 - delay: 5 - until: cp_health.status == 200 - - - name: Build agent VM launch args - ansible.builtin.set_fact: - agent_launch_args: - - /tmp/dd-deploy/scripts/vm-launch.sh - - --image - - "{{ devopsdefender_base_dir }}/images/dd-baremetal.qcow2" - - --name - - "dd-agent-{{ dd_env }}" - - --config - - /tmp/dd-deploy/agent.json - - --config-mode - - agent - - --memory - - "{{ agent_memory }}" - - --cpus - - "{{ agent_cpus }}" - - - name: Add VFIO device to agent VM args - ansible.builtin.set_fact: - agent_launch_args: "{{ agent_launch_args + ['--vfio-device', vfio_device] }}" - when: vfio_device | length > 0 - - - name: Launch agent VM - ansible.builtin.command: - argv: "{{ agent_launch_args }}" - become: true - - - name: Wait for agent registration - ansible.builtin.uri: - url: "http://localhost:8080/api/v1/agents" - method: GET - return_content: true - status_code: 200 - register: agents_poll - retries: 60 - delay: 5 - until: (agents_poll.json | default([]) | length) > 0 - - - name: Smoke test - health - ansible.builtin.uri: - url: "http://localhost:8080/health" - method: GET - status_code: 200 - return_content: true - register: smoke_health - - - name: Smoke test - agent list - ansible.builtin.uri: - url: "http://localhost:8080/api/v1/agents" - method: GET - status_code: 200 - return_content: true - register: smoke_agents - - - name: Print smoke test results - ansible.builtin.debug: - msg: - - "Health: {{ smoke_health.json }}" - - "Agents: {{ smoke_agents.json | length }} registered" - - - name: Show VM status - ansible.builtin.command: - argv: - - /tmp/dd-deploy/scripts/vm-status.sh - register: vm_status - changed_when: false - failed_when: false - become: true - - - name: Print VM status - ansible.builtin.debug: - var: vm_status.stdout_lines - when: vm_status.stdout_lines is defined diff --git a/infra/ansible/playbooks/gcp-control-plane-new.yml b/infra/ansible/playbooks/gcp-control-plane-new.yml index 8a6a211..a950fe6 100644 --- a/infra/ansible/playbooks/gcp-control-plane-new.yml +++ b/infra/ansible/playbooks/gcp-control-plane-new.yml @@ -112,7 +112,15 @@ - name: Resolve control-plane config env inputs ansible.builtin.set_fact: - admin_password_env: "{{ lookup('env', 'CP_ADMIN_PASSWORD') | default(lookup('env', 'ADMIN_PASSWORD'), true) }}" + admin_password_env: >- + {{ + lookup('env', 'CP_ADMIN_PASSWORD') + | default( + lookup('env', 'DD_CP_ADMIN_PASSWORD') + | default(lookup('env', 'ADMIN_PASSWORD'), true), + true + ) + }} dd_agent_ita_api_key_env: "{{ lookup('env', 'DD_AGENT_ITA_API_KEY') | default(lookup('env', 'ITA_API_KEY') | default(lookup('env', 'INTEL_API_KEY'), true), true) }}" - name: Build control-plane raw_kv payload (env vars forwarded to dd-cp) diff --git a/infra/ansible/playbooks/gcp-deploy.yml b/infra/ansible/playbooks/gcp-deploy.yml index 278af26..bd02fc7 100644 --- a/infra/ansible/playbooks/gcp-deploy.yml +++ b/infra/ansible/playbooks/gcp-deploy.yml @@ -29,7 +29,15 @@ else (repo_root ~ '/infra/ansible/ansible.cfg') }} ita_api_key: "{{ lookup('env', 'ITA_API_KEY') | default(lookup('env', 'INTEL_API_KEY'), true) }}" - admin_password: "{{ lookup('env', 'ADMIN_PASSWORD') | default(lookup('env', 'CP_ADMIN_PASSWORD'), true) }}" + admin_password: >- + {{ + lookup('env', 'CP_ADMIN_PASSWORD') + | default( + lookup('env', 'DD_CP_ADMIN_PASSWORD') + | default(lookup('env', 'ADMIN_PASSWORD'), true), + true + ) + }} tasks: - name: Validate required deploy inputs ansible.builtin.assert: @@ -284,7 +292,7 @@ ansible.builtin.assert: that: - admin_password | length > 0 - fail_msg: "ADMIN_PASSWORD (or CP_ADMIN_PASSWORD) is required for trusted measurement approval" + fail_msg: "ADMIN_PASSWORD, CP_ADMIN_PASSWORD, or DD_CP_ADMIN_PASSWORD is required for trusted measurement approval" when: admin_api_available | bool - name: Admin login diff --git a/infra/ansible/playbooks/templates/agent.json.j2 b/infra/ansible/playbooks/templates/agent.json.j2 index d495cdd..37830f8 100644 --- a/infra/ansible/playbooks/templates/agent.json.j2 +++ b/infra/ansible/playbooks/templates/agent.json.j2 @@ -1,7 +1,5 @@ { - "mode": "agent", - "control_plane_url": "{{ cp_url | default('http://localhost:8080') }}", - "skip_attestation": {{ dd_skip_attestation | default(true) | lower }}, + "control_plane_url": "{{ cp_url }}", "node_size": "{{ agent_node_size | default('standard') }}", "datacenter": "{{ dd_env }}" } diff --git a/infra/ansible/playbooks/templates/control-plane.json.j2 b/infra/ansible/playbooks/templates/control-plane.json.j2 index f0f1eff..7fbc0fb 100644 --- a/infra/ansible/playbooks/templates/control-plane.json.j2 +++ b/infra/ansible/playbooks/templates/control-plane.json.j2 @@ -1,17 +1,19 @@ { - "mode": "control-plane", + "bootstrap_cp": true, "raw_kv": { "DD_CP_BIND_ADDR": "0.0.0.0:8080", "DD_CP_DATABASE_URL": "sqlite://devopsdefender.db?mode=rwc", - "DD_CP_ADMIN_PASSWORD": "{{ dd_cp_admin_password }}", - "DD_CP_CF_API_TOKEN": "{{ dd_cp_cf_api_token }}", - "DD_CP_CF_ACCOUNT_ID": "{{ dd_cp_cf_account_id }}", - "DD_CP_CF_ZONE_ID": "{{ dd_cp_cf_zone_id }}", - "DD_CP_CF_DOMAIN": "{{ dd_cp_cf_domain | default('devopsdefender.com') }}", - "DD_ENV": "{{ dd_env }}", - "DD_CP_GIT_SHA": "{{ dd_cp_git_sha | default('') }}", - "DD_CP_NONCE_ENFORCEMENT_MODE": "{{ dd_cp_nonce_enforcement | default('optional') }}", - "DD_CP_TCB_ENFORCEMENT_MODE": "{{ dd_cp_tcb_enforcement | default('warn') }}", - "DD_CP_RTMR_ENFORCEMENT_MODE": "{{ dd_cp_rtmr_enforcement | default('warn') }}" + "DD_CP_ADMIN_PASSWORD": {{ dd_cp_admin_password | to_json }}, + "DD_CP_CF_API_TOKEN": {{ dd_cp_cf_api_token | to_json }}, + "DD_CP_CF_ACCOUNT_ID": {{ dd_cp_cf_account_id | to_json }}, + "DD_CP_CF_ZONE_ID": {{ dd_cp_cf_zone_id | to_json }}, + "DD_CP_CF_DOMAIN": {{ dd_cp_cf_domain | to_json }}, + "DD_CP_PUBLIC_HOSTNAME": {{ cp_public_hostname | to_json }}, + "DD_ENV": {{ dd_env | to_json }}, + "DD_INTEL_API_KEY": {{ dd_intel_api_key | to_json }}, + "DD_CP_GIT_SHA": {{ (dd_cp_git_sha | default('')) | to_json }}, + "DD_CP_NONCE_ENFORCEMENT_MODE": {{ (dd_cp_nonce_enforcement | default('optional')) | to_json }}, + "DD_CP_TCB_ENFORCEMENT_MODE": {{ (dd_cp_tcb_enforcement | default('warn')) | to_json }}, + "DD_CP_RTMR_ENFORCEMENT_MODE": {{ (dd_cp_rtmr_enforcement | default('warn')) | to_json }} } } diff --git a/infra/scripts/vm-launch.sh b/infra/scripts/vm-launch.sh index 3bd459b..5ded744 100755 --- a/infra/scripts/vm-launch.sh +++ b/infra/scripts/vm-launch.sh @@ -1,10 +1,5 @@ #!/usr/bin/env bash -# Launch a QEMU/KVM VM from a baked qcow2 image. -# -# Usage: -# ./vm-launch.sh --image /path/to/base.qcow2 --name dd-cp-staging \ -# --config /path/to/config.json --memory 8G --cpus 4 \ -# --port-forward 8080:8080 +# Launch a libvirt-managed VM from a baked qcow2 image. set -euo pipefail IMAGE="" @@ -14,8 +9,10 @@ MEMORY="4G" CPUS="2" PORT_FORWARDS=() VFIO_DEVICE="" +TDX="false" VM_DIR="/var/lib/devopsdefender/vms" -CONFIG_MODE="agent" # agent or control-plane +CONFIG_MODE="agent" +LIBVIRT_NETWORK="${LIBVIRT_NETWORK:-default}" usage() { cat </dev/null 2>&1 || { + echo "Error: required command not found: $1" >&2 + exit 1 + } +} + +detect_libvirt_qemu_owner() { + if [ -n "${LIBVIRT_QEMU_USER:-}" ]; then + LIBVIRT_QEMU_GROUP="${LIBVIRT_QEMU_GROUP:-$(id -gn "$LIBVIRT_QEMU_USER" 2>/dev/null || true)}" + return 0 + fi + + for candidate in libvirt-qemu qemu; do + if id -u "$candidate" >/dev/null 2>&1; then + LIBVIRT_QEMU_USER="$candidate" + LIBVIRT_QEMU_GROUP="$(id -gn "$candidate")" + return 0 + fi + done + + LIBVIRT_QEMU_USER="" + LIBVIRT_QEMU_GROUP="" +} + +prepare_runtime_permissions() { + if [ "$(id -u)" -ne 0 ]; then + return 0 + fi + + detect_libvirt_qemu_owner + if [ -z "$LIBVIRT_QEMU_USER" ] || [ -z "$LIBVIRT_QEMU_GROUP" ]; then + return 0 + fi + + chown "$LIBVIRT_QEMU_USER:$LIBVIRT_QEMU_GROUP" "$VM_WORK_DIR" + chmod 0770 "$VM_WORK_DIR" + + chown "$LIBVIRT_QEMU_USER:$LIBVIRT_QEMU_GROUP" "$OVERLAY" "$CIDATA_ISO" + chmod 0660 "$OVERLAY" "$CIDATA_ISO" + + touch "$SERIAL_LOG" + chown "$LIBVIRT_QEMU_USER:$LIBVIRT_QEMU_GROUP" "$SERIAL_LOG" + chmod 0660 "$SERIAL_LOG" +} + +to_mib() { + local value number unit + value="${1^^}" + if [[ "$value" =~ ^([0-9]+)([GM])I?B?$ ]]; then + number="${BASH_REMATCH[1]}" + unit="${BASH_REMATCH[2]}" + elif [[ "$value" =~ ^([0-9]+)$ ]]; then + echo "$value" + return 0 + else + echo "Error: unsupported memory value '$1' (use 4096, 4G, 8192M)" >&2 + exit 1 + fi + + if [ "$unit" = "G" ]; then + echo $((number * 1024)) + else + echo "$number" + fi +} + +escape_xml() { + sed \ + -e 's/&/\&/g' \ + -e 's//\>/g' \ + -e "s/'/\'/g" \ + -e 's/"/\"/g' +} + while [[ $# -gt 0 ]]; do case "$1" in --image) IMAGE="$2"; shift 2 ;; @@ -42,6 +116,7 @@ while [[ $# -gt 0 ]]; do --cpus) CPUS="$2"; shift 2 ;; --port-forward) PORT_FORWARDS+=("$2"); shift 2 ;; --vfio-device) VFIO_DEVICE="$2"; shift 2 ;; + --tdx) TDX="true"; shift ;; --help|-h) usage ;; *) echo "Unknown option: $1" >&2; usage ;; esac @@ -52,28 +127,21 @@ if [ -z "$IMAGE" ] || [ -z "$VM_NAME" ] || [ -z "$CONFIG_FILE" ]; then usage fi -if [ ! -f "$IMAGE" ]; then - echo "Error: base image not found: $IMAGE" >&2 - exit 1 -fi +require_cmd virsh +require_cmd qemu-img -if [ ! -f "$CONFIG_FILE" ]; then - echo "Error: config file not found: $CONFIG_FILE" >&2 - exit 1 -fi +[ -f "$IMAGE" ] || { echo "Error: base image not found: $IMAGE" >&2; exit 1; } +[ -f "$CONFIG_FILE" ] || { echo "Error: config file not found: $CONFIG_FILE" >&2; exit 1; } -# Create VM working directory. VM_WORK_DIR="${VM_DIR}/${VM_NAME}" mkdir -p "$VM_WORK_DIR" -# Create copy-on-write overlay from base image. OVERLAY="${VM_WORK_DIR}/${VM_NAME}.qcow2" if [ ! -f "$OVERLAY" ]; then echo "==> Creating overlay image from base" qemu-img create -b "$(realpath "$IMAGE")" -F qcow2 -f qcow2 "$OVERLAY" fi -# Determine config target path inside cloud-init. if [ "$CONFIG_MODE" = "control-plane" ]; then CONFIG_DEST="/etc/devopsdefender/control-plane.json" SYSTEMD_ENABLE="devopsdefender-control-plane.service" @@ -84,11 +152,8 @@ else SYSTEMD_DISABLE="devopsdefender-control-plane.service" fi -# Generate cloud-init ISO for config injection. CIDATA_DIR="${VM_WORK_DIR}/cidata" mkdir -p "$CIDATA_DIR" - -# Escape JSON for embedding in cloud-init write_files. CONFIG_CONTENT="$(cat "$CONFIG_FILE")" cat > "${CIDATA_DIR}/user-data" </dev/null; then +if command -v cloud-localds >/dev/null 2>&1; then cloud-localds "$CIDATA_ISO" "${CIDATA_DIR}/user-data" "${CIDATA_DIR}/meta-data" -elif command -v genisoimage &>/dev/null; then +elif command -v genisoimage >/dev/null 2>&1; then genisoimage -output "$CIDATA_ISO" -volid cidata -joliet -rock \ "${CIDATA_DIR}/user-data" "${CIDATA_DIR}/meta-data" else @@ -121,77 +186,143 @@ else exit 1 fi -# Build QEMU command. -QEMU_ARGS=( - qemu-system-x86_64 - -enable-kvm - -machine q35 - -cpu host - -m "$MEMORY" - -smp "$CPUS" - -drive "file=${OVERLAY},format=qcow2,if=virtio" - -drive "file=${CIDATA_ISO},format=raw,if=virtio,readonly=on" - -display none - -serial file:${VM_WORK_DIR}/${VM_NAME}.log - -daemonize - -pidfile "${VM_WORK_DIR}/${VM_NAME}.pid" -) - -# Pass through VFIO device (GPU) if requested. -if [ -n "$VFIO_DEVICE" ]; then - QEMU_ARGS+=(-device "vfio-pci,host=${VFIO_DEVICE}") +MEMORY_MIB="$(to_mib "$MEMORY")" +DOMAIN_XML="${VM_WORK_DIR}/${VM_NAME}.xml" +SERIAL_LOG="${VM_WORK_DIR}/${VM_NAME}.log" + +prepare_runtime_permissions + +QEMU_NS="" +LAUNCH_SECURITY="" +QEMU_COMMANDLINE="" +FEATURES_EXTRA="" +CLOCK_XML=" \n" +PM_XML="" +MEMORY_BACKING_XML="" +OS_OPEN_TAG=" " +LOADER_XML="" +if [ "$TDX" = "true" ]; then + MEMORY_BACKING_XML+=" \n" + MEMORY_BACKING_XML+=" \n" + MEMORY_BACKING_XML+=" \n" + MEMORY_BACKING_XML+=" \n" + OS_OPEN_TAG=" " + LOADER_XML+=" /usr/share/qemu/OVMF.fd\n" + LAUNCH_SECURITY+=" \n" + LAUNCH_SECURITY+=" 0x10000000\n" + LAUNCH_SECURITY+=" \n" + LAUNCH_SECURITY+=" \n" + LAUNCH_SECURITY+=" \n" + LAUNCH_SECURITY+=" \n" + FEATURES_EXTRA+=" \n" + CLOCK_XML=" \n" + CLOCK_XML+=" \n" + CLOCK_XML+=" \n" + PM_XML+=" \n" + PM_XML+=" \n" + PM_XML+=" \n" + PM_XML+=" \n" fi -# Build port forwarding netdev. -HOSTFWD_ARGS="" -for pf in "${PORT_FORWARDS[@]+"${PORT_FORWARDS[@]}"}"; do - host_port="${pf%%:*}" - guest_port="${pf##*:}" - HOSTFWD_ARGS="${HOSTFWD_ARGS},hostfwd=tcp::${host_port}-:${guest_port}" -done +HOSTDEV_XML="" +if [ -n "$VFIO_DEVICE" ]; then + domain_hex="${VFIO_DEVICE%%:*}" + remainder="${VFIO_DEVICE#*:}" + bus_hex="${remainder%%.*}" + function_hex="${remainder##*.}" + HOSTDEV_XML+=" \n" + HOSTDEV_XML+=" \n" + HOSTDEV_XML+="
\n" + HOSTDEV_XML+=" \n" + HOSTDEV_XML+=" \n" +fi -# Always forward SSH on a high port for debugging. -SSH_PORT=$((10000 + RANDOM % 50000)) -HOSTFWD_ARGS="${HOSTFWD_ARGS},hostfwd=tcp::${SSH_PORT}-:22" +cat > "$DOMAIN_XML" < + ${VM_NAME} + ${MEMORY_MIB} + ${MEMORY_MIB} +$(printf "%b" "$MEMORY_BACKING_XML") ${CPUS} +$(printf "%b" "$OS_OPEN_TAG") + hvm +$(printf "%b" "$LOADER_XML") + +$(printf "%b" "$LAUNCH_SECURITY") + + +$(printf "%b" "$FEATURES_EXTRA") + +$(printf "%b" "$CLOCK_XML") destroy + restart + destroy +$(printf "%b" "$PM_XML") + /usr/bin/qemu-system-x86_64 + + + + + + + + + + + + + + + + + + + + + + + + /dev/urandom + +$(printf "%b" "$HOSTDEV_XML") +$(printf "%b" "$QEMU_COMMANDLINE") +EOF -QEMU_ARGS+=(-netdev "user,id=net0${HOSTFWD_ARGS}" -device "virtio-net-pci,netdev=net0") +if virsh dominfo "$VM_NAME" >/dev/null 2>&1; then + echo "Error: domain '$VM_NAME' already exists; stop it first" >&2 + exit 1 +fi -echo "==> Launching VM: ${VM_NAME}" +echo "==> Defining libvirt domain: ${VM_NAME}" echo " Memory: ${MEMORY}, CPUs: ${CPUS}" echo " Overlay: ${OVERLAY}" echo " Config mode: ${CONFIG_MODE}" -echo " SSH port: ${SSH_PORT}" +echo " Libvirt network: ${LIBVIRT_NETWORK}" +echo " TDX: ${TDX}" if [ -n "$VFIO_DEVICE" ]; then echo " VFIO device: ${VFIO_DEVICE}" fi for pf in "${PORT_FORWARDS[@]+"${PORT_FORWARDS[@]}"}"; do - echo " Port forward: ${pf}" + echo " Recorded port forward hint: ${pf}" done -"${QEMU_ARGS[@]}" /dev/null +virsh start "$VM_NAME" >/dev/null -PID_FILE="${VM_WORK_DIR}/${VM_NAME}.pid" -if [ -f "$PID_FILE" ]; then - PID="$(cat "$PID_FILE")" - echo "==> VM started with PID ${PID}" - echo " PID file: ${PID_FILE}" - echo " SSH: ssh -p ${SSH_PORT} ubuntu@localhost" -else - echo "Warning: PID file not created, VM may not have started" >&2 -fi +STATE="$(virsh domstate "$VM_NAME" | tr -d '\r' | xargs)" +echo "==> VM started with libvirt state: ${STATE}" +echo " Inspect with: virsh list --all" -# Write metadata for vm-status.sh / vm-stop.sh. cat > "${VM_WORK_DIR}/vm-info.json" </dev/null 2>&1 || { + echo "Error: virsh is required" >&2 + exit 1 +} + if [ ! -d "$VM_DIR" ]; then echo "No VMs found (${VM_DIR} does not exist)" exit 0 fi -printf "%-25s %-8s %-15s %-6s %-6s %-8s %s\n" \ - "NAME" "PID" "STATUS" "MEM" "CPUS" "SSH" "STARTED" -printf "%s\n" "$(printf '%.0s-' {1..100})" +printf "%-25s %-12s %-6s %-6s %-12s %s\n" \ + "NAME" "STATE" "MEM" "CPUS" "NETWORK" "STARTED" +printf "%s\n" "$(printf '%.0s-' {1..90})" found=0 for vm_dir in "${VM_DIR}"/*/; do @@ -23,27 +27,25 @@ for vm_dir in "${VM_DIR}"/*/; do found=1 name="$(jq -r '.name // "unknown"' "$info_file")" - pid_file="$(jq -r '.pid_file // ""' "$info_file")" memory="$(jq -r '.memory // "?"' "$info_file")" cpus="$(jq -r '.cpus // "?"' "$info_file")" - ssh_port="$(jq -r '.ssh_port // "?"' "$info_file")" + network="$(jq -r '.libvirt_network // "?"' "$info_file")" started="$(jq -r '.started_at // "?"' "$info_file")" - status="stopped" - pid="-" - if [ -n "$pid_file" ] && [ -f "$pid_file" ]; then - pid="$(cat "$pid_file")" - if kill -0 "$pid" 2>/dev/null; then - status="running" - else - status="dead" - fi + state="undefined" + if virsh dominfo "$name" >/dev/null 2>&1; then + state="$(virsh domstate "$name" | tr -d '\r' | xargs)" fi - printf "%-25s %-8s %-15s %-6s %-6s %-8s %s\n" \ - "$name" "$pid" "$status" "$memory" "$cpus" "$ssh_port" "$started" + printf "%-25s %-12s %-6s %-6s %-12s %s\n" \ + "$name" "$state" "$memory" "$cpus" "$network" "$started" done if [ "$found" -eq 0 ]; then echo "No VMs found" + exit 0 fi + +echo +echo "virsh list --all" +virsh list --all diff --git a/infra/scripts/vm-stop.sh b/infra/scripts/vm-stop.sh index 5c67ff4..7b8e2e4 100755 --- a/infra/scripts/vm-stop.sh +++ b/infra/scripts/vm-stop.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash -# Stop a VM by name. -# Usage: ./vm-stop.sh [--clean] +# Stop a libvirt-managed VM by name. set -euo pipefail VM_DIR="/var/lib/devopsdefender/vms" @@ -22,39 +21,39 @@ if [ -z "$VM_NAME" ]; then fi VM_WORK_DIR="${VM_DIR}/${VM_NAME}" -PID_FILE="${VM_WORK_DIR}/${VM_NAME}.pid" -if [ ! -f "$PID_FILE" ]; then - echo "No PID file found for VM '${VM_NAME}' at ${PID_FILE}" >&2 +command -v virsh >/dev/null 2>&1 || { + echo "Error: virsh is required" >&2 exit 1 -fi +} -PID="$(cat "$PID_FILE")" +if ! virsh dominfo "$VM_NAME" >/dev/null 2>&1; then + echo "No libvirt domain found for VM '${VM_NAME}'" >&2 + exit 1 +fi -if kill -0 "$PID" 2>/dev/null; then - echo "==> Stopping VM '${VM_NAME}' (PID ${PID})" - kill "$PID" +STATE="$(virsh domstate "$VM_NAME" | tr -d '\r' | xargs)" +echo "==> Stopping VM '${VM_NAME}' (state: ${STATE})" - # Wait for process to exit (up to 30s). +if [ "$STATE" = "running" ] || [ "$STATE" = "paused" ] || [ "$STATE" = "in shutdown" ]; then + virsh shutdown "$VM_NAME" >/dev/null || true for _ in $(seq 1 30); do - if ! kill -0 "$PID" 2>/dev/null; then + STATE="$(virsh domstate "$VM_NAME" | tr -d '\r' | xargs)" + if [ "$STATE" = "shut off" ]; then break fi sleep 1 done +fi - # Force kill if still running. - if kill -0 "$PID" 2>/dev/null; then - echo " Force killing PID ${PID}" - kill -9 "$PID" 2>/dev/null || true - fi - - echo "==> VM '${VM_NAME}' stopped" -else - echo "VM '${VM_NAME}' is not running (PID ${PID})" +STATE="$(virsh domstate "$VM_NAME" | tr -d '\r' | xargs)" +if [ "$STATE" != "shut off" ]; then + echo " Force destroying domain ${VM_NAME}" + virsh destroy "$VM_NAME" >/dev/null || true fi -rm -f "$PID_FILE" +virsh undefine "$VM_NAME" --nvram >/dev/null 2>&1 || virsh undefine "$VM_NAME" >/dev/null +echo "==> VM '${VM_NAME}' undefined" if [ "$CLEAN" = true ]; then echo "==> Cleaning up VM directory: ${VM_WORK_DIR}"