From 6f6351cec65d19cfb0a8a0a7d506e942af785159 Mon Sep 17 00:00:00 2001 From: Konstantins Feofantovs Date: Mon, 10 Aug 2026 20:38:21 +0300 Subject: [PATCH] =?UTF-8?q?test:=20full-path=20go-inventory=20e2e=20in=20k?= =?UTF-8?q?ind=20=E2=80=94=20DaemonSet=20=E2=86=92=20controller=20?= =?UTF-8?q?=E2=86=92=20spool=20(ADR=200010)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Asserts the whole node→controller inventory path against a real cluster: a known Go workload (a Deployment) runs, the node DaemonSet scans it and ships the fact to the controller over HTTP with a projected controller-audience token, the controller validates the token locally against the cluster JWKS, joins the fact against its workload inventory, and writes a go_inventory payload to its spool. The test asserts on the spool contents — the payload the backend would receive, not logs (docs/development.md): the sample's module path, a go1.x version, a sha256 image digest, pgo=false, container and workload resolved. - deploy/controller.yaml: the controller's raw manifest for this path — read RBAC (security.md §4) plus the read-only JWKS-discovery nonResourceURL grant (no TokenReview, no write verb), the node-intake Service, ConfigMap, and an emptyDir spool. Scoped to the inventory path; backend enrollment/identity is a later slice. - test/e2e/inventory_test.go: deploys the sample, the controller (with a shell sidecar sharing the spool volume, since the agent image is distroless), and the node DaemonSet pointed at the controller Service; reads the payload via exec in the sidecar. Cluster-scoped RBAC objects are per-run named and cleaned up. - deployNodeDaemonSet gains a controller-endpoint parameter; clusterClient is refactored to expose the REST config for exec. New `make inventory-e2e` target (loads a busybox sidecar image). go.mod gains the remotecommand transitive deps, used only under the e2e build tag. Ran locally against kind: inventory-e2e, node-e2e, and the non-image e2e suite all pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- Makefile | 17 +- deploy/controller.yaml | 147 ++++++++++++++ go.mod | 3 + go.sum | 8 + test/e2e/inventory_test.go | 388 +++++++++++++++++++++++++++++++++++++ test/e2e/nodescan_test.go | 19 +- test/e2e/pods_test.go | 13 +- 7 files changed, 586 insertions(+), 9 deletions(-) create mode 100644 deploy/controller.yaml create mode 100644 test/e2e/inventory_test.go diff --git a/Makefile b/Makefile index d4ad384..75fb5ab 100644 --- a/Makefile +++ b/Makefile @@ -9,8 +9,11 @@ SMOKE_SECONDS ?= 5 IMAGE ?= runtime-agent IMAGE_TAG ?= $(VERSION) SAMPLE_IMAGE ?= rebuildstack-e2e-goworkload:latest +# A shell-bearing image loaded into kind as a sidecar so the inventory e2e can +# read the controller's spool (the agent image is distroless — no shell). +SPOOL_READER_IMAGE ?= busybox:1.37 -.PHONY: build test lint tidy clean cluster-up cluster-down e2e smoke image sample-image kind-load node-e2e +.PHONY: build test lint tidy clean cluster-up cluster-down e2e smoke image sample-image kind-load node-e2e inventory-e2e build: ## Build the agent binary into bin/ go build -ldflags '$(LDFLAGS)' -o bin/agent ./cmd/agent @@ -63,5 +66,17 @@ node-e2e: kind-load ## Deploy the node DaemonSet in kind and assert Go-binary de go test -tags e2e -count=1 -timeout 15m -v ./test/e2e/ -run TestNodeScanner 2>&1 \ | tee test/e2e/logs/node-e2e-$$(date +%Y%m%d-%H%M%S).log +inventory-e2e: kind-load ## Deploy controller + node DaemonSet + sample in kind and assert the go_inventory payload in the controller spool; log goes to test/e2e/logs/ + @mkdir -p test/e2e/logs + docker pull $(SPOOL_READER_IMAGE) + go tool kind load docker-image $(SPOOL_READER_IMAGE) --name $(E2E_CLUSTER) + set -o pipefail; \ + E2E_KUBE_CONTEXT=kind-$(E2E_CLUSTER) \ + E2E_AGENT_IMAGE=$(IMAGE):$(IMAGE_TAG) \ + E2E_SAMPLE_IMAGE=$(SAMPLE_IMAGE) \ + E2E_SPOOL_READER_IMAGE=$(SPOOL_READER_IMAGE) \ + go test -tags e2e -count=1 -timeout 15m -v ./test/e2e/ -run TestGoInventoryEndToEnd 2>&1 \ + | tee test/e2e/logs/inventory-e2e-$$(date +%Y%m%d-%H%M%S).log + clean: rm -rf bin diff --git a/deploy/controller.yaml b/deploy/controller.yaml new file mode 100644 index 0000000..a8e3dda --- /dev/null +++ b/deploy/controller.yaml @@ -0,0 +1,147 @@ +# Controller — raw manifest for the node-intake receiver (ADR 0010). No Helm +# chart lives in this repo yet; this file is what the e2e deploys and a reference +# for the eventual chart. It is deliberately scoped to what the node→controller +# inventory path needs: the read RBAC the collectors use (security.md §4), the +# read-only JWKS discovery grant for local token validation (no TokenReview, no +# write verb), the receiver Service, and a spool volume. It does NOT include the +# backend enrollment/identity machinery (ADR 0005/0008) — that is a later slice. +--- +apiVersion: v1 +kind: Namespace +metadata: + name: runtime-agent +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: runtime-agent-controller + namespace: runtime-agent +--- +# The controller's access is read-only (security.md §4). The two nonResourceURL +# GETs are OIDC discovery + JWKS, used to validate node tokens locally — a read, +# never a write, and specifically NOT `create tokenreviews`. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: runtime-agent-controller +rules: + - apiGroups: [""] + resources: [pods, namespaces, nodes] + verbs: [get, list, watch] + - apiGroups: [""] + resources: [nodes/proxy] + verbs: [get] + - apiGroups: [apps] + resources: [replicasets, deployments, statefulsets, daemonsets] + verbs: [get, list, watch] + - apiGroups: [batch] + resources: [jobs, cronjobs] + verbs: [get, list, watch] + - nonResourceURLs: ["/openid/v1/jwks", "/.well-known/openid-configuration"] + verbs: [get] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: runtime-agent-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: runtime-agent-controller +subjects: + - kind: ServiceAccount + name: runtime-agent-controller + namespace: runtime-agent +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: runtime-agent-controller + namespace: runtime-agent +data: + config.yaml: | + spool: + dir: /var/spool/runtime-agent + nodeIntake: + enabled: true + listenAddress: ":8080" + # Must match the audience the node DaemonSet's projected token requests. + audience: rebuildstack-controller + # Pin the accepted caller to the node role's ServiceAccount. + expectedSubject: "system:serviceaccount:runtime-agent:runtime-agent-node" +--- +apiVersion: v1 +kind: Service +metadata: + name: runtime-agent-controller + namespace: runtime-agent + labels: + app.kubernetes.io/name: runtime-agent + app.kubernetes.io/component: controller +spec: + selector: + app.kubernetes.io/name: runtime-agent + app.kubernetes.io/component: controller + ports: + - name: node-intake + port: 8080 + targetPort: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: runtime-agent-controller + namespace: runtime-agent + labels: + app.kubernetes.io/name: runtime-agent + app.kubernetes.io/component: controller +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: runtime-agent + app.kubernetes.io/component: controller + template: + metadata: + labels: + app.kubernetes.io/name: runtime-agent + app.kubernetes.io/component: controller + spec: + serviceAccountName: runtime-agent-controller + containers: + - name: controller + image: runtime-agent:latest + args: + - -config + - /etc/runtime-agent/config.yaml + ports: + - name: node-intake + containerPort: 8080 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + volumeMounts: + - name: config + mountPath: /etc/runtime-agent + readOnly: true + - name: spool + mountPath: /var/spool/runtime-agent + volumes: + - name: config + configMap: + name: runtime-agent-controller + # emptyDir by default (ADR 0007): unshipped payloads are loss-harmless. + - name: spool + emptyDir: {} diff --git a/go.mod b/go.mod index c91c8b1..8a6bcf1 100644 --- a/go.mod +++ b/go.mod @@ -27,11 +27,13 @@ require ( github.com/go-openapi/swag v0.23.0 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/moby/spdystream v0.5.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -55,6 +57,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect + k8s.io/streaming v0.36.3 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/kind v0.32.0 // indirect sigs.k8s.io/randfill v1.0.0 // indirect diff --git a/go.sum b/go.sum index 1d40226..a9fe6fb 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ al.essio.dev/pkg/shellescape v1.5.1 h1:86HrALUujYS/h+GtqoB26SBEdkWfmMI6FubjXlsXy al.essio.dev/pkg/shellescape v1.5.1/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890= github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -35,6 +37,8 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaU github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -53,6 +57,8 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= +github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -136,6 +142,8 @@ k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hk k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= k8s.io/kubelet v0.36.3 h1:dRzEnhHk35Opy6wjWR4YBcN5RI9lB2npUY37TghFuPU= k8s.io/kubelet v0.36.3/go.mod h1:4USFGr21Ioka+b964Beq0NvV5b5aca3RWJ1/kfq+RLw= +k8s.io/streaming v0.36.3 h1:9rAaqBk0C0Pc7+/fqGekj07NV+/Xrew58p647A0JT8w= +k8s.io/streaming v0.36.3/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= diff --git a/test/e2e/inventory_test.go b/test/e2e/inventory_test.go new file mode 100644 index 0000000..27d14a6 --- /dev/null +++ b/test/e2e/inventory_test.go @@ -0,0 +1,388 @@ +//go:build e2e + +package e2e + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilyaml "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + k8syaml "sigs.k8s.io/yaml" +) + +const controllerManifestPath = "../../deploy/controller.yaml" + +// spoolReaderImage is a shell-bearing image loaded into kind as a sidecar in +// the controller pod: the agent image is distroless (no shell), so the sidecar +// is how the test reads the spool file the controller writes. `make +// inventory-e2e` loads it. +func spoolReaderImage() string { + if img := os.Getenv("E2E_SPOOL_READER_IMAGE"); img != "" { + return img + } + return "busybox:1.37" +} + +const spoolPath = "/var/spool/runtime-agent/go-inventory.json" + +// TestGoInventoryEndToEnd exercises the whole node→controller inventory path in +// a real cluster (ADR 0010): a known Go workload runs, the node DaemonSet scans +// it and ships the fact to the controller over HTTP with a projected token, the +// controller validates the token against the cluster JWKS, joins the fact +// against its workload inventory, and writes a go_inventory payload to its +// spool. The test asserts on the spool contents — the payload the backend would +// receive — not on logs (docs/development.md). +// +// Gated on E2E_AGENT_IMAGE / E2E_SAMPLE_IMAGE (kind-loaded); use +// `make inventory-e2e`. +func TestGoInventoryEndToEnd(t *testing.T) { + agentImage := os.Getenv("E2E_AGENT_IMAGE") + sampleImage := os.Getenv("E2E_SAMPLE_IMAGE") + if agentImage == "" || sampleImage == "" { + t.Skip("E2E_AGENT_IMAGE / E2E_SAMPLE_IMAGE not set; run `make inventory-e2e`") + } + config := clusterConfig(t) + clientset := clusterClient(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + ns := fmt.Sprintf("runtime-agent-inv-e2e-%d", os.Getpid()) + if _, err := clientset.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + }, metav1.CreateOptions{}); err != nil { + t.Fatalf("creating namespace: %v", err) + } + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cleanupCancel() + _ = clientset.CoreV1().Namespaces().Delete(cleanupCtx, ns, metav1.DeleteOptions{}) + }) + + // The known Go workload as a Deployment, so the controller resolves it to a + // workload (Pod → ReplicaSet → Deployment) — exercising the owner chain. + deploySampleWorkload(ctx, t, clientset, ns, sampleImage) + + // The controller: read RBAC, JWKS discovery grant, node-intake receiver, and + // a spool with a shell sidecar to read it. + deployController(ctx, t, clientset, ns, agentImage) + controllerPod := waitDeploymentPod(ctx, t, clientset, ns, "controller") + t.Logf("controller pod: %s", controllerPod) + + // The node DaemonSet, pointed at the controller Service in this namespace. + endpoint := fmt.Sprintf("http://runtime-agent-controller.%s.svc.cluster.local:8080/v1/node-inventory", ns) + deployNodeDaemonSet(ctx, t, clientset, ns, agentImage, endpoint) + nodePod := waitDaemonSetPod(ctx, t, clientset, ns) + t.Logf("node pod: %s", nodePod) + + // Poll the controller's spool until the go_inventory payload carries the + // sample workload. Delivery is periodic (the node's scan interval and the + // controller's flush cadence), so allow generous time. + deadline := time.Now().Add(6 * time.Minute) + for { + if rec, ok := findSampleRecord(ctx, t, config, clientset, ns, controllerPod); ok { + if !strings.HasPrefix(rec.GoVersion, "go1.") { + t.Errorf("go_version = %q, want a go1.x version", rec.GoVersion) + } + if rec.ImageDigest == "" { + t.Errorf("image_digest empty; want the sample image content digest") + } + if !strings.HasPrefix(rec.ImageDigest, "sha256:") { + t.Errorf("image_digest = %q, want a sha256: digest", rec.ImageDigest) + } + if rec.PGO { + t.Errorf("pgo = true; the sample is built without PGO") + } + if rec.Container != "goworkload" { + t.Errorf("container = %q, want goworkload", rec.Container) + } + if rec.WorkloadKind != "Deployment" || rec.WorkloadName != "goworkload" { + t.Errorf("workload = %s/%s, want Deployment/goworkload", rec.WorkloadKind, rec.WorkloadName) + } + return + } + if time.Now().After(deadline) { + t.Fatalf("go_inventory payload never carried the sample workload (%s) before timeout", sampleModulePath) + } + time.Sleep(5 * time.Second) + } +} + +// goInventoryRecord mirrors one record of the go_inventory payload. +type goInventoryRecord struct { + Namespace string `json:"namespace"` + WorkloadKind string `json:"workload_kind"` + WorkloadName string `json:"workload_name"` + Container string `json:"container"` + GoVersion string `json:"go_version"` + ModulePath string `json:"module_path"` + ImageDigest string `json:"image_digest"` + PGO bool `json:"pgo"` +} + +// findSampleRecord reads the controller's spool file via the sidecar and returns +// the record for the sample module, if the payload exists yet. +func findSampleRecord(ctx context.Context, t *testing.T, config *rest.Config, cs kubernetes.Interface, ns, pod string) (goInventoryRecord, bool) { + t.Helper() + raw, ok := readSpoolFile(ctx, t, config, cs, ns, pod) + if !ok { + return goInventoryRecord{}, false + } + var payload struct { + Kind string `json:"kind"` + Records []goInventoryRecord `json:"records"` + } + if err := json.Unmarshal([]byte(raw), &payload); err != nil { + t.Logf("spool file not valid JSON yet (will retry): %v", err) + return goInventoryRecord{}, false + } + if payload.Kind != "go_inventory" { + t.Errorf("payload kind = %q, want go_inventory", payload.Kind) + } + for _, r := range payload.Records { + if r.ModulePath == sampleModulePath { + return r, true + } + } + return goInventoryRecord{}, false +} + +// readSpoolFile cats the go_inventory file from the controller pod's spool via +// the shell sidecar. A non-zero exit (file not written yet) returns ok=false so +// the caller retries. +func readSpoolFile(ctx context.Context, t *testing.T, config *rest.Config, cs kubernetes.Interface, ns, pod string) (string, bool) { + t.Helper() + req := cs.CoreV1().RESTClient().Post(). + Resource("pods").Name(pod).Namespace(ns).SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: "spool-reader", + Command: []string{"cat", spoolPath}, + Stdout: true, + Stderr: true, + }, scheme.ParameterCodec) + exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + if err != nil { + t.Fatalf("building exec: %v", err) + } + var stdout, stderr bytes.Buffer + if err := exec.StreamWithContext(ctx, remotecommand.StreamOptions{Stdout: &stdout, Stderr: &stderr}); err != nil { + // cat exits non-zero until the controller writes the file — expected + // during the poll, not a test failure. + return "", false + } + return stdout.String(), true +} + +// deploySampleWorkload creates the known Go workload as a Deployment and waits +// for its pod to run. +func deploySampleWorkload(ctx context.Context, t *testing.T, cs kubernetes.Interface, ns, image string) { + t.Helper() + replicas := int32(1) + labels := map[string]string{"app": "goworkload"} + dep := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "goworkload"}, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "goworkload", + Image: image, + ImagePullPolicy: corev1.PullNever, + }}, + }, + }, + }, + } + if _, err := cs.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}); err != nil { + t.Fatalf("creating sample deployment: %v", err) + } + waitDeploymentPod(ctx, t, cs, ns, "goworkload") +} + +// deployController decodes deploy/controller.yaml and applies it into ns. It +// retargets the namespace, uniquely renames the cluster-scoped RBAC objects (so +// concurrent runs and cleanup are safe), points them at this run's +// ServiceAccount, overrides the image, renders the ConfigMap for this namespace, +// and adds the shell sidecar that shares the spool volume so the test can read +// the payload the controller writes. +func deployController(ctx context.Context, t *testing.T, cs kubernetes.Interface, ns, image string) { + t.Helper() + data, err := os.ReadFile(controllerManifestPath) + if err != nil { + t.Fatalf("reading manifest %s: %v", controllerManifestPath, err) + } + clusterRoleName := "runtime-agent-controller-" + ns + reader := utilyaml.NewYAMLReader(bufio.NewReader(bytes.NewReader(data))) + for { + doc, err := reader.Read() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("reading manifest doc: %v", err) + } + if strings.TrimSpace(string(doc)) == "" { + continue + } + var tm metav1.TypeMeta + if err := k8syaml.Unmarshal(doc, &tm); err != nil { + t.Fatalf("decoding doc TypeMeta: %v", err) + } + switch tm.Kind { + case "", "Namespace": + // The test manages its own namespace. + case "ServiceAccount": + var sa corev1.ServiceAccount + mustUnmarshal(t, doc, &sa) + sa.Namespace = ns + create(ctx, t, "ServiceAccount", func() error { + _, err := cs.CoreV1().ServiceAccounts(ns).Create(ctx, &sa, metav1.CreateOptions{}) + return err + }) + case "ClusterRole": + var cr rbacv1.ClusterRole + mustUnmarshal(t, doc, &cr) + cr.Name = clusterRoleName + create(ctx, t, "ClusterRole", func() error { + _, err := cs.RbacV1().ClusterRoles().Create(ctx, &cr, metav1.CreateOptions{}) + return err + }) + t.Cleanup(func() { + delCtx, c := context.WithTimeout(context.Background(), time.Minute) + defer c() + _ = cs.RbacV1().ClusterRoles().Delete(delCtx, clusterRoleName, metav1.DeleteOptions{}) + }) + case "ClusterRoleBinding": + var crb rbacv1.ClusterRoleBinding + mustUnmarshal(t, doc, &crb) + crb.Name = clusterRoleName + crb.RoleRef.Name = clusterRoleName + for i := range crb.Subjects { + crb.Subjects[i].Namespace = ns + } + create(ctx, t, "ClusterRoleBinding", func() error { + _, err := cs.RbacV1().ClusterRoleBindings().Create(ctx, &crb, metav1.CreateOptions{}) + return err + }) + t.Cleanup(func() { + delCtx, c := context.WithTimeout(context.Background(), time.Minute) + defer c() + _ = cs.RbacV1().ClusterRoleBindings().Delete(delCtx, clusterRoleName, metav1.DeleteOptions{}) + }) + case "ConfigMap": + var cm corev1.ConfigMap + mustUnmarshal(t, doc, &cm) + cm.Namespace = ns + // Render the config for this namespace: the expected subject is the + // node role's ServiceAccount in ns. + cm.Data["config.yaml"] = renderControllerConfig(ns) + create(ctx, t, "ConfigMap", func() error { + _, err := cs.CoreV1().ConfigMaps(ns).Create(ctx, &cm, metav1.CreateOptions{}) + return err + }) + case "Service": + var svc corev1.Service + mustUnmarshal(t, doc, &svc) + svc.Namespace = ns + create(ctx, t, "Service", func() error { + _, err := cs.CoreV1().Services(ns).Create(ctx, &svc, metav1.CreateOptions{}) + return err + }) + case "Deployment": + var dep appsv1.Deployment + mustUnmarshal(t, doc, &dep) + dep.Namespace = ns + c := &dep.Spec.Template.Spec.Containers[0] + c.Image = image + c.ImagePullPolicy = corev1.PullNever + // Add the shell sidecar sharing the spool volume (read-only), so the + // test can cat the payload the distroless controller cannot serve. + dep.Spec.Template.Spec.Containers = append(dep.Spec.Template.Spec.Containers, corev1.Container{ + Name: "spool-reader", + Image: spoolReaderImage(), + ImagePullPolicy: corev1.PullNever, + Command: []string{"sleep", "86400"}, + VolumeMounts: []corev1.VolumeMount{{ + Name: "spool", MountPath: "/var/spool/runtime-agent", ReadOnly: true, + }}, + }) + create(ctx, t, "Deployment", func() error { + _, err := cs.AppsV1().Deployments(ns).Create(ctx, &dep, metav1.CreateOptions{}) + return err + }) + default: + t.Fatalf("unexpected manifest kind %q", tm.Kind) + } + } +} + +func renderControllerConfig(ns string) string { + return fmt.Sprintf(`spool: + dir: /var/spool/runtime-agent +nodeIntake: + enabled: true + listenAddress: ":8080" + audience: rebuildstack-controller + expectedSubject: "system:serviceaccount:%s:runtime-agent-node" +`, ns) +} + +func mustUnmarshal(t *testing.T, doc []byte, into any) { + t.Helper() + if err := k8syaml.Unmarshal(doc, into); err != nil { + t.Fatalf("decoding manifest doc: %v", err) + } +} + +func create(ctx context.Context, t *testing.T, kind string, fn func() error) { + t.Helper() + if err := fn(); err != nil { + t.Fatalf("creating %s: %v", kind, err) + } +} + +// waitDeploymentPod waits for a Running pod of the named component (matched by +// the app label the Deployment sets) and returns its name. +func waitDeploymentPod(ctx context.Context, t *testing.T, cs kubernetes.Interface, ns, name string) string { + t.Helper() + selectors := []string{ + "app.kubernetes.io/component=" + name, + "app=" + name, + } + deadline := time.Now().Add(4 * time.Minute) + for time.Now().Before(deadline) { + for _, sel := range selectors { + pods, err := cs.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: sel}) + if err != nil { + continue + } + for _, p := range pods.Items { + if p.Status.Phase == corev1.PodRunning { + return p.Name + } + } + } + time.Sleep(2 * time.Second) + } + t.Fatalf("no Running pod for %q in %s before timeout", name, ns) + return "" +} diff --git a/test/e2e/nodescan_test.go b/test/e2e/nodescan_test.go index 073cf45..eeed96c 100644 --- a/test/e2e/nodescan_test.go +++ b/test/e2e/nodescan_test.go @@ -80,8 +80,9 @@ func TestNodeScannerAgainstRealCluster(t *testing.T) { waitPodRunning(ctx, t, clientset, ns, "goworkload") // Deploy the node role from the shipped manifest, retargeted at this - // namespace and the kind-loaded image. - deployNodeDaemonSet(ctx, t, clientset, ns, agentImage) + // namespace and the kind-loaded image. No controller endpoint here — this + // test asserts on the node's own log, log-only mode. + deployNodeDaemonSet(ctx, t, clientset, ns, agentImage, "") nodePod := waitDaemonSetPod(ctx, t, clientset, ns) t.Logf("node pod: %s", nodePod) @@ -158,10 +159,13 @@ func scanLog(ctx context.Context, t *testing.T, cs kubernetes.Interface, ns, pod // deployNodeDaemonSet decodes the shipped manifest and applies its // ServiceAccount and DaemonSet into ns, overriding only the namespace, the -// image, the pull policy, and the scan interval. The zero-RBAC ServiceAccount -// (no Role/RoleBinding is created anywhere) and the securityContext are applied -// exactly as written. -func deployNodeDaemonSet(ctx context.Context, t *testing.T, cs kubernetes.Interface, ns, image string) { +// image, the pull policy, and the scan interval. When controllerEndpoint is +// non-empty, the -controller-endpoint arg is added so the node ships its +// findings there (ADR 0010); empty runs the node log-only. The zero-RBAC +// ServiceAccount (no Role/RoleBinding is created anywhere), the projected +// controller-audience token volume, and the securityContext are applied exactly +// as written. +func deployNodeDaemonSet(ctx context.Context, t *testing.T, cs kubernetes.Interface, ns, image, controllerEndpoint string) { t.Helper() data, err := os.ReadFile(nodeManifestPath) if err != nil { @@ -208,6 +212,9 @@ func deployNodeDaemonSet(ctx context.Context, t *testing.T, cs kubernetes.Interf // Shorten the interval so the test observes repeated passes quickly; // the first pass runs at startup regardless. c.Args = []string{"node", "-proc", "/host/proc", "-interval", "15s"} + if controllerEndpoint != "" { + c.Args = append(c.Args, "-controller-endpoint", controllerEndpoint) + } if _, err := cs.AppsV1().DaemonSets(ns).Create(ctx, &ds, metav1.CreateOptions{}); err != nil { t.Fatalf("creating DaemonSet: %v", err) } diff --git a/test/e2e/pods_test.go b/test/e2e/pods_test.go index 974b261..6453a32 100644 --- a/test/e2e/pods_test.go +++ b/test/e2e/pods_test.go @@ -23,6 +23,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/utils/ptr" @@ -31,7 +32,10 @@ import ( const pauseImage = "registry.k8s.io/pause:3.10" -func clusterClient(t *testing.T) kubernetes.Interface { +// clusterConfig builds the REST config for the kind context named in +// E2E_KUBE_CONTEXT, refusing any non-kind context. It is the single place the +// e2e resolves a cluster to talk to. +func clusterConfig(t *testing.T) *rest.Config { t.Helper() contextName := os.Getenv("E2E_KUBE_CONTEXT") if contextName == "" { @@ -46,7 +50,12 @@ func clusterClient(t *testing.T) kubernetes.Interface { if err != nil { t.Fatalf("loading kubeconfig context %q: %v", contextName, err) } - clientset, err := kubernetes.NewForConfig(config) + return config +} + +func clusterClient(t *testing.T) kubernetes.Interface { + t.Helper() + clientset, err := kubernetes.NewForConfig(clusterConfig(t)) if err != nil { t.Fatalf("building clientset: %v", err) }