Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 0 additions & 12 deletions charts/runtime-agent/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -61,21 +61,9 @@ be half-enabled.
{{/*
Validation the JSON schema cannot express, called from every template that
renders a workload so it fires whatever is being rendered.

The allow-list rule is not defensiveness. The sample shipped in this repository
before the chart existed enabled the profiler with `allowedModulePrefixes: []`,
which admits nothing: profiling ran, filtered everything out, and produced no
profile ever — the same shape of trap ADR 0025 found in `eligibleNamespaces`,
where a knob read as deny-by-default and was inert. An install that cannot
produce its own output should not start.
*/}}
{{- define "runtime-agent.validate" -}}
{{- if not (has .Values.profile (list "metrics-only" "inventory" "ebpf")) -}}
{{- fail (printf "unknown profile %q: expected metrics-only, inventory or ebpf" .Values.profile) -}}
{{- end -}}
{{- if eq .Values.profile "ebpf" -}}
{{- if not .Values.profiling.allowedModulePrefixes -}}
{{- fail "profile \"ebpf\" requires profiling.allowedModulePrefixes to name at least one Go module prefix. The allow-list admits your own domain-bearing modules; without it a profile keeps your main package and the standard library and drops your entire service layer, which is a worse profile with no warning attached (ADR 0011 §4, ADR 0041)." -}}
{{- end -}}
{{- end -}}
{{- end -}}
22 changes: 8 additions & 14 deletions charts/runtime-agent/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -98,23 +98,17 @@ profiling:
# that rests on it.
pull: true

# The Go module-path prefixes of your own code whose stack frames may leave.
# Extra Go module-path prefixes of your own code whose stack frames may leave.
# Everything else is redacted before anything is sent.
#
# This adds to what the agent can already tell is yours: a pulled profile is
# also reduced against the modules its build states it was compiled from — its
# main module and anything a `replace` redirects — which the binary carries and
# no list can be wrong about (ADR 0058 §5). Set this for shared internal
# modules a build merely requires.
# You usually need none. Every profile is reduced against the modules its own
# build states it was compiled from — the main module, and anything a `replace`
# redirects — read from the binary, where no list can be wrong about it. Set
# this for a shared internal module a build merely requires (ADR 0059).
#
# This is the load-bearing control of the whole profiling path (ADR 0011 §4).
# It is enforced on the node, from the node's own file, and no reply from the
# controller can widen it — which is why it lives here rather than arriving
# over the wire.
#
# Required by profile `ebpf`, and required to be non-empty: an empty list
# admits nothing, so profiling would run and produce nothing, forever and
# silently. The chart refuses to render that rather than install it.
# It is enforced from the agent's own file, on both paths, and no reply from
# the controller can widen it — which is why it lives here rather than
# arriving over the wire (ADR 0011 §4).
allowedModulePrefixes: []
# "drop" (default) or "keep": whether third-party dependency frames survive
# the filter. Empty selects the agent's default.
Expand Down
22 changes: 21 additions & 1 deletion cmd/agent/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (

"github.com/RebuildStackCo/runtime-agent/internal/config"
"github.com/RebuildStackCo/runtime-agent/internal/ebpfgate"
"github.com/RebuildStackCo/runtime-agent/internal/nodeprofile"
"github.com/RebuildStackCo/runtime-agent/internal/nodescan"
)

Expand Down Expand Up @@ -86,6 +87,10 @@ func runNode(ctx context.Context, logger *slog.Logger, args []string) error {
// degrades to scanner-only, never an escalation to privileged. ebpfMetrics
// is the in-process counter, surfaced in the node's own log.
ebpfMetrics := newEBPFGateMetrics()
// What the scanner learns about whose code each container runs, for the
// profiler beside it. Published on every pass and read per window; the two
// are different goroutines in this one process (ADR 0059 §1).
moduleIndex := &nodeprofile.ModuleIndex{}
if *enableEBPF {
logger.Info("ebpf profiling config",
"allowed_module_prefixes", len(profiling.AllowedModulePrefixes),
Expand All @@ -103,7 +108,8 @@ func runNode(ctx context.Context, logger *slog.Logger, args []string) error {
// executable the scanner is not allowed to open was an asymmetry,
// and stack traces are the more sensitive of the two (ADR 0025).
go runProfilingPipeline(ctx, logger, profiling, *procRoot, node,
targetsCl, newScopeClient(*scopeEndpoint, *tokenPath), profileSh, ebpfMetrics)
targetsCl, newScopeClient(*scopeEndpoint, *tokenPath), profileSh,
ebpfMetrics, moduleIndex)
}
}

Expand Down Expand Up @@ -150,6 +156,19 @@ func runNode(ctx context.Context, logger *slog.Logger, args []string) error {
"container_id", b.ContainerID,
)
}
// The pass's own answer to whose code each container runs, replacing the
// last one wholesale so a container no live process belongs to leaves
// with it (ADR 0059 §1).
byContainer := map[string][]string{}
for _, b := range res.Binaries {
if b.ContainerID == "" {
continue
}
byContainer[b.ContainerID] = append(byContainer[b.ContainerID],
nodescan.OwnModules(b.MainModule, b.Dependencies)...)
}
moduleIndex.Publish(byContainer)

// Aggregate-only line for everything not kept: no identities of
// filtered-out or unreadable binaries ever appear (CLAUDE.md
// invariant 6, docs/security.md §8).
Expand All @@ -160,6 +179,7 @@ func runNode(ctx context.Context, logger *slog.Logger, args []string) error {
"filtered_scope", res.Counters.FilteredScope,
"filtered_infra", res.Counters.FilteredInfra,
"unreadable", res.Counters.Unreadable,
"containers_with_known_modules", moduleIndex.Size(),
)

// Ship to the controller when configured. Best-effort: a delivery
Expand Down
3 changes: 2 additions & 1 deletion cmd/agent/node_ebpf_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (

"github.com/RebuildStackCo/runtime-agent/internal/config"
"github.com/RebuildStackCo/runtime-agent/internal/ebpfgate"
"github.com/RebuildStackCo/runtime-agent/internal/nodeprofile"
)

// writeGateFixture builds a proc/sys pair the gate can probe: an osrelease file
Expand Down Expand Up @@ -106,7 +107,7 @@ func TestRunProfilingPipelineGraceful(t *testing.T) {
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
m := newEBPFGateMetrics()
runProfilingPipeline(context.Background(), logger, config.NodeProfiling{}.Normalized(),
t.TempDir(), "node", nil, nil, nil, m)
t.TempDir(), "node", nil, nil, nil, m, &nodeprofile.ModuleIndex{})
if m.refusals[ebpfgate.ReasonProgramLoadFailed] != 1 {
t.Errorf("program_load_failed = %d, want 1", m.refusals[ebpfgate.ReasonProgramLoadFailed])
}
Expand Down
25 changes: 20 additions & 5 deletions cmd/agent/pipeline.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ func samplesPerSecond(overheadCeilingPercent int) int {
// cluster (ADR 0015). Both must admit a container, and scope fails closed.
func runProfilingPipeline(ctx context.Context, logger *slog.Logger, p config.NodeProfiling,
procRoot, node string, targets *targetsClient, scoper *scopeClient,
shipper *profileShipper, m *ebpfGateMetrics) {
shipper *profileShipper, m *ebpfGateMetrics, modules *nodeprofile.ModuleIndex) {

sps := samplesPerSecond(p.OverheadCeilingPercent)
session, err := nodeprofile.Start(ctx, logger, nodeprofile.Config{SamplesPerSecond: sps})
Expand All @@ -57,7 +57,21 @@ func runProfilingPipeline(ctx context.Context, logger *slog.Logger, p config.Nod
if p.ThirdPartySymbols == config.ThirdPartySymbolsKeep {
thirdParty = nodeprofile.ThirdPartyKeep
}
filter := nodeprofile.NewSymbolFilter(p.AllowedModulePrefixes, thirdParty)
// One filter per container, not one per node. The configured prefixes are
// what the operator added; the modules a build states it was compiled from
// are what the scanner read from the binary beside this profiler, and
// without them a service under its own domain-bearing module is redacted as
// third-party — the whole service layer, on a default install (ADR 0059).
filterFor := func(containerID string) *nodeprofile.SymbolFilter {
own := modules.Modules(containerID)
if len(own) == 0 {
return nodeprofile.NewSymbolFilter(p.AllowedModulePrefixes, thirdParty)
}
allowed := make([]string, 0, len(p.AllowedModulePrefixes)+len(own))
allowed = append(allowed, p.AllowedModulePrefixes...)
allowed = append(allowed, own...)
return nodeprofile.NewSymbolFilter(allowed, thirdParty)
}

window := time.Duration(p.CaptureDurationSeconds) * time.Second // ship cadence / window length
refresh := time.Duration(p.IntervalSeconds) * time.Second // targets-refresh cadence
Expand Down Expand Up @@ -96,7 +110,7 @@ func runProfilingPipeline(ctx context.Context, logger *slog.Logger, p config.Nod
} else {
scope = nodescan.NewScope(uids)
}
processWindow(ctx, logger, filter, sps, procRoot, node,
processWindow(ctx, logger, filterFor, sps, procRoot, node,
start, end, session.Drain(), targetSet, scope, shipper)
start = time.Now()
}
Expand All @@ -110,7 +124,8 @@ func runProfilingPipeline(ctx context.Context, logger *slog.Logger, p config.Nod
// which pods the filters admit at all. A container in the first and not the
// second is one whose executable the scanner may not even open (ADR 0015). How
// many a window may ship is the controller's TopN alone (ADR 0025).
func processWindow(ctx context.Context, logger *slog.Logger, filter *nodeprofile.SymbolFilter,
func processWindow(ctx context.Context, logger *slog.Logger,
filterFor func(containerID string) *nodeprofile.SymbolFilter,
sps int, procRoot, node string, start, end time.Time, samples []nodeprofile.Sample,
targetSet map[string]struct{}, scope nodescan.Scope, shipper *profileShipper) {

Expand Down Expand Up @@ -170,7 +185,7 @@ func processWindow(ctx context.Context, logger *slog.Logger, filter *nodeprofile
for _, cid := range cids {
g := groups[cid]

filtered, _ := filter.Filter(g.samples)
filtered, _ := filterFor(cid).Filter(g.samples)
pprof, err := nodeprofile.Serialize(filtered, sps)
if err != nil {
logger.Error("serializing profile failed", "error", err)
Expand Down
101 changes: 97 additions & 4 deletions cmd/agent/pipeline_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ import (
"testing"
"time"

"github.com/google/pprof/profile"

"github.com/RebuildStackCo/runtime-agent/internal/nodeintake"
"github.com/RebuildStackCo/runtime-agent/internal/nodeprofile"
"github.com/RebuildStackCo/runtime-agent/internal/nodescan"
Expand Down Expand Up @@ -74,7 +76,7 @@ func TestProcessWindowShipsOnlyTargetedContainers(t *testing.T) {
targetSet := map[string]struct{}{cid1: {}}

logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))
processWindow(context.Background(), logger, filter, 20, procRoot, "node",
processWindow(context.Background(), logger, staticFilter(filter), 20, procRoot, "node",
time.Unix(100, 0), time.Unix(160, 0), samples, targetSet,
nodescan.NewScope([]string{podUID}), shipper)

Expand All @@ -91,13 +93,95 @@ func TestProcessWindowShipsOnlyTargetedContainers(t *testing.T) {
}
}

// The defect ADR 0059 closes: with the configured allow-list empty — the chart's
// default — a service under its own domain-bearing module was redacted as
// third-party, and everything below `main` disappeared. The binary states its
// module, so the filter admits it without anything being configured.
func TestOwnCodeSurvivesWithNothingConfigured(t *testing.T) {
const (
podUID = "1234abcd-12ab-34cd-56ef-1234567890ab"
ownFunc = "github.com/acme/web/svc.Handle"
depFunc = "github.com/vendor/lib.Do"
)
cid := strings.Repeat("a", 64)
procRoot := t.TempDir()
writeCgroup(t, procRoot, 100, podUID, cid)

var mu sync.Mutex
var shipped []nodeintake.ProfileReport
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var rep nodeintake.ProfileReport
_ = json.NewDecoder(r.Body).Decode(&rep)
mu.Lock()
shipped = append(shipped, rep)
mu.Unlock()
w.WriteHeader(http.StatusAccepted)
}))
t.Cleanup(srv.Close)

samples := []nodeprofile.Sample{{PID: 100, Value: 5, Frames: []nodeprofile.Frame{
{Function: ownFunc, Kind: "go"},
{Function: depFunc, Kind: "go"},
}}}

index := &nodeprofile.ModuleIndex{}
index.Publish(map[string][]string{cid: {"github.com/acme/web"}})
// No configured prefixes at all: whatever survives, survives because the
// scanner read the module off the binary.
filterFor := func(containerID string) *nodeprofile.SymbolFilter {
return nodeprofile.NewSymbolFilter(index.Modules(containerID), nodeprofile.ThirdPartyDrop)
}

logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))
processWindow(context.Background(), logger, filterFor, 20, procRoot, "node",
time.Unix(100, 0), time.Unix(160, 0), samples, map[string]struct{}{cid: {}},
nodescan.NewScope([]string{podUID}), newProfileShipper(srv.URL, writeToken(t, "tok")))

mu.Lock()
defer mu.Unlock()
if len(shipped) != 1 {
t.Fatalf("shipped %d profiles, want 1 — the workload's own frame was redacted", len(shipped))
}
p, err := profile.ParseData(shipped[0].Pprof)
if err != nil {
t.Fatalf("shipped profile does not parse: %v", err)
}
names := map[string]bool{}
for _, fn := range p.Function {
names[fn.Name] = true
}
if !names[ownFunc] {
t.Error("the workload's own frame did not survive; the build states its module")
}
if names[depFunc] {
t.Error("a dependency frame survived under the drop policy")
}
}

// And without the index the same window ships nothing, which is what the trap
// looked like: a profile of `main` and the runtime, rejected as unshippable.
func TestWithoutTheIndexTheSameWindowShipsNothing(t *testing.T) {
const podUID = "1234abcd-12ab-34cd-56ef-1234567890ab"
cid := strings.Repeat("a", 64)
procRoot := t.TempDir()
writeCgroup(t, procRoot, 100, podUID, cid)

samples := []nodeprofile.Sample{{PID: 100, Value: 5, Frames: []nodeprofile.Frame{
{Function: "github.com/acme/web/svc.Handle", Kind: "go"},
}}}
logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))
processWindow(context.Background(), logger, staticFilter(nil), 20, procRoot, "node",
time.Unix(100, 0), time.Unix(160, 0), samples, map[string]struct{}{cid: {}},
nodescan.NewScope([]string{podUID}), nil)
}

func TestProcessWindowNoTargetsShipsNothing(t *testing.T) {
procRoot := t.TempDir()
writeCgroup(t, procRoot, 100, "1234abcd-12ab-34cd-56ef-1234567890ab", strings.Repeat("a", 64))
samples := []nodeprofile.Sample{{PID: 100, Value: 1, Frames: []nodeprofile.Frame{{Function: "main.hot", Kind: "go"}}}}
logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))
// empty target set -> nothing shipped, no shipper touched
processWindow(context.Background(), logger, nodeprofile.NewSymbolFilter(nil, nodeprofile.ThirdPartyDrop),
processWindow(context.Background(), logger, staticFilter(nil),
20, procRoot, "node", time.Unix(1, 0), time.Unix(2, 0), samples, map[string]struct{}{},
nodescan.NewScope([]string{"1234abcd-12ab-34cd-56ef-1234567890ab"}), nil)
}
Expand Down Expand Up @@ -128,7 +212,7 @@ func TestProcessWindowShipsNothingOutsideTheScanScope(t *testing.T) {
logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))

// Targeted, but the scope admits a different pod.
processWindow(context.Background(), logger, nodeprofile.NewSymbolFilter(nil, nodeprofile.ThirdPartyDrop),
processWindow(context.Background(), logger, staticFilter(nil),
20, procRoot, "node", time.Unix(100, 0), time.Unix(160, 0), samples,
map[string]struct{}{cid: {}},
nodescan.NewScope([]string{"1234abcd-12ab-34cd-56ef-1234567890ab"}),
Expand Down Expand Up @@ -160,7 +244,7 @@ func TestProcessWindowFailsClosedWithoutAScope(t *testing.T) {
}
logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))

processWindow(context.Background(), logger, nodeprofile.NewSymbolFilter(nil, nodeprofile.ThirdPartyDrop),
processWindow(context.Background(), logger, staticFilter(nil),
20, procRoot, "node", time.Unix(100, 0), time.Unix(160, 0), samples,
map[string]struct{}{cid: {}}, nodescan.DenyAll(),
newProfileShipper(srv.URL, writeToken(t, "tok")))
Expand All @@ -169,3 +253,12 @@ func TestProcessWindowFailsClosedWithoutAScope(t *testing.T) {
t.Error("shipped a profile with no scan scope; profiling must fail closed")
}
}

// staticFilter is the pre-ADR-0059 shape: one filter for every container. Tests
// that are not about which code is whose use it.
func staticFilter(f *nodeprofile.SymbolFilter) func(string) *nodeprofile.SymbolFilter {
if f == nil {
f = nodeprofile.NewSymbolFilter(nil, nodeprofile.ThirdPartyDrop)
}
return func(string) *nodeprofile.SymbolFilter { return f }
}
2 changes: 1 addition & 1 deletion docs/adr/0011-node-ebpf-cpu-profiling.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ Date: 2026-08-11

Status: Accepted
Amends: 0009, 0010
Amended by: 0022, 0023, 0025, 0039, 0041, 0058
Amended by: 0022, 0023, 0025, 0039, 0041, 0058, 0059

Extends the node role of 0009 and the node→controller channel of 0010; scopes a
narrow, config-bounded exception to 0010 §1's "one level down" reply discipline
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ Date: 2026-08-26

Status: Accepted

Amended by: 0059

Amends: 0011 §4, 0039 §5

Cuts the compiler-recorded source path down to a base name before it enters the
Expand Down
Loading