Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 26 additions & 12 deletions cmd/context-guru-proxy/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -1170,25 +1170,39 @@ func buildHostedTenantConfig(doc []byte, e components.Emitter) (proxy.BuiltConfi
// The mapping lives here rather than in `proxy` for the reason ConfigBuilder exists: the
// proxy package does not depend on the configuration loader.
func cachePolicy(c config.CacheConfig) proxy.CachePolicy {
r := c.Resolved()
// The per-wait ping plan (#858) rides on EVERY policy, a parked one too, because a manager
// strategy can switch the keep-alive on for an account whose own block has it off, and the plan
// then still has to apply. These fields alone never turn the keep-alive on (Policy.On reads
// KeepAlive).
plan := proxy.CachePolicy{
WaitPlan: c.WaitPlanOn(),
ToolWaitMaxPings: r.KeepAliveToolWaitMaxPings,
BackgroundExtraPings: r.KeepAliveBackgroundExtraPings,
HumanExtraPings: r.KeepAliveHumanExtraPings,
}
// Resolve the defaults only when something is actually switched on. A parked block
// (`keepalive: false` with a tuned interval) must not come out looking enabled, and a
// zero HeadTTLMinBytes must stay zero so a host that never configured this asks for
// nothing rather than asking on every request.
if !c.KeepAlive && !c.HeadTTL1h {
return proxy.CachePolicy{}
return plan
}
r := c.Resolved()
return proxy.CachePolicy{
KeepAlive: c.KeepAlive,
Idle: time.Duration(r.KeepAliveIdleSeconds) * time.Second,
OpenAIIdle: time.Duration(r.KeepAliveOpenAIIdleSeconds) * time.Second,
MaxPings: r.KeepAliveMaxPings,
Idle1h: time.Duration(r.KeepAliveIdleSeconds1h) * time.Second,
MaxPings1h: r.KeepAliveMaxPings1h,
MaxUSDPerPing: r.KeepAliveMaxUSDPerPing,
MinPrefixTokens: r.KeepAliveMinPrefixTokens,
HeadTTL1h: c.HeadTTL1h,
HeadTTLMinBytes: headTTLMin(c.HeadTTL1h, r.HeadTTLMinBytes),
WaitPlan: plan.WaitPlan,
ToolWaitMaxPings: plan.ToolWaitMaxPings,
BackgroundExtraPings: plan.BackgroundExtraPings,
HumanExtraPings: plan.HumanExtraPings,
KeepAlive: c.KeepAlive,
Idle: time.Duration(r.KeepAliveIdleSeconds) * time.Second,
OpenAIIdle: time.Duration(r.KeepAliveOpenAIIdleSeconds) * time.Second,
MaxPings: r.KeepAliveMaxPings,
Idle1h: time.Duration(r.KeepAliveIdleSeconds1h) * time.Second,
MaxPings1h: r.KeepAliveMaxPings1h,
MaxUSDPerPing: r.KeepAliveMaxUSDPerPing,
MinPrefixTokens: r.KeepAliveMinPrefixTokens,
HeadTTL1h: c.HeadTTL1h,
HeadTTLMinBytes: headTTLMin(c.HeadTTL1h, r.HeadTTLMinBytes),
}
}

Expand Down
15 changes: 15 additions & 0 deletions cmd/context-guru-proxy/terminal_tools_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"testing"

"github.com/rossoctl/context-guru/components"
"github.com/rossoctl/context-guru/config"
)

const docWithKey = "preset: off\ncache:\n keepalive_terminal_tools: []\n"
Expand Down Expand Up @@ -34,3 +35,17 @@ func TestHostConfigWithTerminalToolsStillBuilds(t *testing.T) {
t.Fatalf("local rebuild of a host config with the key: %v", err)
}
}

// #858: the wait-plan fields reach the policy, also for a parked account block (a manager strategy can
// switch the keep-alive on for it), and the off switch reaches it.
func TestCachePolicyCarriesTheWaitPlan(t *testing.T) {
p := cachePolicy(config.CacheConfig{})
if !p.WaitPlan || p.On() || p.ToolWaitMaxPings != 8 || p.BackgroundExtraPings != 3 || p.HumanExtraPings != 1 {
t.Errorf("parked block: %+v", p)
}
off := false
p = cachePolicy(config.CacheConfig{KeepAlive: true, KeepAliveWaitPlan: &off})
if p.WaitPlan || !p.On() || p.MinPrefixTokens != 20000 {
t.Errorf("switch off: %+v", p)
}
}
54 changes: 51 additions & 3 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,26 @@ type CacheConfig struct {
// (StructuredOutput). An empty list (`keepalive_terminal_tools: []`) turns the rule off, which
// is the behavior before #857: such a thread gets its pings.
KeepAliveTerminalTools []string `yaml:"keepalive_terminal_tools"`
// KeepAliveWaitPlan turns on the per-wait ping plan (#858). Absent: on. `false` is the one switch
// that brings back the behavior before #858: MaxPings for every wait, time windows for every wait,
// sweep questions during tool waits, and pings for a thread with no tool call yet. It does not
// bring back the 80k minimum prefix, which is only a default (keepalive_min_prefix_tokens).
//
// With the plan on, the ping count follows what the thread waits for. "Base" is
// keepalive_max_pings:
// - its own tool (main or subagent): keep pinging until the result comes, at most
// keepalive_tool_wait_max_pings (default 8);
// - main, end_turn, background work: base + keepalive_background_extra_pings (default 3);
// - main, end_turn, a person: base + keepalive_human_extra_pings (default 1);
// - main waiting for a teammate, or a subagent waiting for its main agent: base;
// - a finished thread, or one with no tool call yet: 0.
// Manager-strategy time windows then apply only to a wait for a person, and the sweep question is
// not asked while a thread waits for its own tool. It applies to the 5-minute tier. 0 on the three
// counts means the default. The three counts are at most 11, the highest strategy ping count.
KeepAliveWaitPlan *bool `yaml:"keepalive_wait_plan"`
KeepAliveToolWaitMaxPings int `yaml:"keepalive_tool_wait_max_pings"`
KeepAliveBackgroundExtraPings int `yaml:"keepalive_background_extra_pings"`
KeepAliveHumanExtraPings int `yaml:"keepalive_human_extra_pings"`

// HeadTTL1h asks for the ONE-HOUR tier on the request's HEAD breakpoints (`tools` and
// `system`) while leaving the trailing message breakpoint at 5 minutes — the provider's
Expand Down Expand Up @@ -236,8 +256,17 @@ const (
DefaultKeepAliveIdle1h = int(modelinfo.AnthropicDefaultKeepAliveIdle1h / time.Second)
DefaultKeepAliveMaxPings1h = modelinfo.AnthropicDefaultKeepAliveMaxPings1h
DefaultKeepAliveMaxUSDPerPing = 0.50
DefaultKeepAliveMinPrefix = 80000
DefaultHeadTTLMinBytes = 200000
// DefaultKeepAliveMinPrefix is 20000 since #858. No long subagent idle span had a prefix under
// 20k, so any floor at or under 20k gives the same result there. It was 80000 (#435).
DefaultKeepAliveMinPrefix = 20000
// The per-wait ping plan defaults (#858). They mirror keepalive.DefaultToolWaitMaxPings,
// DefaultBackgroundExtraPings and DefaultHumanExtraPings, pinned equal by a test.
DefaultKeepAliveToolWaitMaxPings = 8
DefaultKeepAliveBackgroundExtraPings = 3
DefaultKeepAliveHumanExtraPings = 1
// MaxKeepAliveWaitPings bounds each of the three counts: the highest strategy ping count.
MaxKeepAliveWaitPings = 11
DefaultHeadTTLMinBytes = 200000
// DefaultHeadTTLMinTokens is DEPRECATED, kept for old callers: the old default of 50,000.
DefaultHeadTTLMinTokens = DefaultHeadTTLMinBytes / HeadTTLBytesPerToken
// HeadTTLBytesPerToken converts the deprecated head_ttl_min_tokens into bytes. It is the
Expand Down Expand Up @@ -279,6 +308,15 @@ func (c CacheConfig) Resolved() CacheConfig {
if c.KeepAliveMinPrefixTokens <= 0 {
c.KeepAliveMinPrefixTokens = DefaultKeepAliveMinPrefix
}
if c.KeepAliveToolWaitMaxPings <= 0 {
c.KeepAliveToolWaitMaxPings = DefaultKeepAliveToolWaitMaxPings
}
if c.KeepAliveBackgroundExtraPings <= 0 {
c.KeepAliveBackgroundExtraPings = DefaultKeepAliveBackgroundExtraPings
}
if c.KeepAliveHumanExtraPings <= 0 {
c.KeepAliveHumanExtraPings = DefaultKeepAliveHumanExtraPings
}
if c.HeadTTLMinBytes <= 0 {
if c.HeadTTLMinTokens > 0 {
c.HeadTTLMinBytes = c.HeadTTLMinTokens * HeadTTLBytesPerToken
Expand All @@ -298,9 +336,16 @@ func (c CacheConfig) validate() error {
if c.KeepAliveIdleSeconds < 0 || c.KeepAliveOpenAIIdleSeconds < 0 ||
c.KeepAliveMaxPings < 0 || c.HeadTTLMinTokens < 0 || c.HeadTTLMinBytes < 0 ||
c.KeepAliveMaxUSDPerPing < 0 || c.KeepAliveMinPrefixTokens < 0 ||
c.KeepAliveIdleSeconds1h < 0 || c.KeepAliveMaxPings1h < 0 {
c.KeepAliveIdleSeconds1h < 0 || c.KeepAliveMaxPings1h < 0 ||
c.KeepAliveToolWaitMaxPings < 0 || c.KeepAliveBackgroundExtraPings < 0 || c.KeepAliveHumanExtraPings < 0 {
return fmt.Errorf("config: cache: negative values are not meaningful")
}
if c.KeepAliveToolWaitMaxPings > MaxKeepAliveWaitPings || c.KeepAliveBackgroundExtraPings > MaxKeepAliveWaitPings ||
c.KeepAliveHumanExtraPings > MaxKeepAliveWaitPings {
return fmt.Errorf("config: cache: keepalive_tool_wait_max_pings, keepalive_background_extra_pings and "+
"keepalive_human_extra_pings must be at most %d (the credential hold is (K+1) x the idle time)",
MaxKeepAliveWaitPings)
}
if c.HeadTTLMinBytes > MaxHeadTTLMinBytes || c.HeadTTLMinTokens > MaxHeadTTLMinBytes/HeadTTLBytesPerToken {
return fmt.Errorf("config: cache: head_ttl_min_bytes must be at most %d (head_ttl_min_tokens at most %d)",
MaxHeadTTLMinBytes, MaxHeadTTLMinBytes/HeadTTLBytesPerToken)
Expand Down Expand Up @@ -1011,3 +1056,6 @@ func (c *Config) NewStore() store.Store {
}
return store.NewMemory(c.Store)
}

// WaitPlanOn reports whether the per-wait ping plan is on (#858). Absent is on.
func (c CacheConfig) WaitPlanOn() bool { return c.KeepAliveWaitPlan == nil || *c.KeepAliveWaitPlan }
6 changes: 3 additions & 3 deletions config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -266,12 +266,12 @@ func TestPresetLadderSummarizerModes(t *testing.T) {
}
}

// The recommended keep-alive policy (#435, B-policy-r2) is the default; pin the four knobs.
// The recommended keep-alive policy (#435, B-policy-r2; floor 20000 since #858) is the default; pin the four knobs.
func TestKeepAliveRecommendedDefaults(t *testing.T) {
r := CacheConfig{}.Resolved()
if r.KeepAliveIdleSeconds != 240 || r.KeepAliveMaxPings != 2 ||
r.KeepAliveMinPrefixTokens != 80000 || r.KeepAliveMaxUSDPerPing != 0.50 {
t.Errorf("Resolved() = idle %d, K %d, floor %d, ceiling %v; want 240, 2, 80000, 0.50",
r.KeepAliveMinPrefixTokens != 20000 || r.KeepAliveMaxUSDPerPing != 0.50 {
t.Errorf("Resolved() = idle %d, K %d, floor %d, ceiling %v; want 240, 2, 20000, 0.50",
r.KeepAliveIdleSeconds, r.KeepAliveMaxPings, r.KeepAliveMinPrefixTokens, r.KeepAliveMaxUSDPerPing)
}
}
Expand Down
42 changes: 42 additions & 0 deletions config/terminal_tools_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,45 @@ func TestKeepAliveTerminalToolsYAML(t *testing.T) {
})
}
}

// #858: the wait-plan keys. Absent: the plan is on with the shipped counts. Present: kept.
func TestKeepAliveWaitPlanYAML(t *testing.T) {
cfg, err := LoadBytes([]byte("preset: off\ncache:\n keepalive: true\n"))
if err != nil {
t.Fatal(err)
}
if !cfg.Cache.WaitPlanOn() || cfg.Cache.KeepAliveWaitPlan != nil {
t.Errorf("absent: plan on = %v, raw %v; want on and nil", cfg.Cache.WaitPlanOn(), cfg.Cache.KeepAliveWaitPlan)
}
r := cfg.Cache.Resolved()
if r.KeepAliveToolWaitMaxPings != 8 || r.KeepAliveBackgroundExtraPings != 3 || r.KeepAliveHumanExtraPings != 1 {
t.Errorf("absent: resolved = %d/%d/%d, want 8/3/1", r.KeepAliveToolWaitMaxPings,
r.KeepAliveBackgroundExtraPings, r.KeepAliveHumanExtraPings)
}
cfg, err = LoadBytes([]byte("preset: off\ncache:\n keepalive: true\n keepalive_wait_plan: false\n" +
" keepalive_tool_wait_max_pings: 5\n keepalive_background_extra_pings: 4\n keepalive_human_extra_pings: 2\n"))
if err != nil {
t.Fatal(err)
}
r = cfg.Cache.Resolved()
if cfg.Cache.WaitPlanOn() || r.KeepAliveToolWaitMaxPings != 5 || r.KeepAliveBackgroundExtraPings != 4 ||
r.KeepAliveHumanExtraPings != 2 {
t.Errorf("present: on=%v %d/%d/%d, want off and 5/4/2", cfg.Cache.WaitPlanOn(),
r.KeepAliveToolWaitMaxPings, r.KeepAliveBackgroundExtraPings, r.KeepAliveHumanExtraPings)
}
for _, bad := range []string{"keepalive_tool_wait_max_pings: 12", "keepalive_human_extra_pings: -1"} {
if _, err := LoadBytes([]byte("preset: off\ncache:\n " + bad + "\n")); err == nil {
t.Errorf("%q was accepted", bad)
}
}
}

// #858: the default minimum prefix is 20000; an explicit value stays.
func TestKeepAliveMinPrefixDefaultIs20k(t *testing.T) {
if got := (CacheConfig{}).Resolved().KeepAliveMinPrefixTokens; got != 20000 {
t.Errorf("default = %d, want 20000", got)
}
if got := (CacheConfig{KeepAliveMinPrefixTokens: 80000}).Resolved().KeepAliveMinPrefixTokens; got != 80000 {
t.Errorf("explicit 80000 became %d", got)
}
}
30 changes: 20 additions & 10 deletions dash/keepalive.go
Original file line number Diff line number Diff line change
Expand Up @@ -550,7 +550,7 @@ type KeepAliveBehaviour struct {
Gaps []AccountGaps `json:"account_gaps"`
// PrefixP10/P50/P90 in tokens, over the lapsed entries. p50 is 292,527 on the corpus and
// 347 of 356 are past 20k — which is the measured reason a `prefix >= 20k` gate
// costs almost nothing at 20k (the pre-#435 gate; the shipped floor is now 80k), and the panel says so.
// costs almost nothing at 20k (the pre-#435 gate; the shipped floor is 20k since #858), and the panel says so.
PrefixP10 int64 `json:"prefix_p10"`
PrefixP50 int64 `json:"prefix_p50"`
PrefixP90 int64 `json:"prefix_p90"`
Expand All @@ -570,7 +570,7 @@ type KeepAliveBehaviour struct {
// gapEdges are the idle-gap bands' fixed edges in seconds. The last is unbounded.
var gapEdges = []float64{0, 300, 580, 600, 1800, 3600, 14400}

// prefixEdges are the billed-prefix bands' fixed edges in tokens. 20,000 was the pre-#435 gate (the shipped floor is now 80,000).
// prefixEdges are the billed-prefix bands' fixed edges in tokens. 20,000 was the pre-#435 gate and is the shipped floor again since #858 (80,000 in between).
var prefixEdges = []float64{0, 20000, 50000, 100000, 200000, 400000, 800000}

// bandLabels renders the edge list as human labels, with a unit formatter.
Expand Down Expand Up @@ -1262,9 +1262,9 @@ func PingsPerSpan(gap, idleSeconds float64, maxPings int) int {
}

// kaGateMinPrefix is the billed-prefix floor a replay gates on: the shipped default, mirroring
// config.DefaultKeepAliveMinPrefix, which this package may not import. Pinned by
// config.DefaultKeepAliveMinPrefix (20000 since #858), which this package may not import. Pinned by
// TestTheReplayGateMatchesTheShippedPolicy.
const kaGateMinPrefix = 80000
const kaGateMinPrefix = 20000

// pingSpan is one idle span a live keep-alive would send pings in.
type pingSpan struct {
Expand Down Expand Up @@ -1306,9 +1306,15 @@ func (s pingSpan) pings(idleSeconds float64, maxPings int) int {
// names, so it MOVES when the gate or K moves instead of silently ignoring both.
//
// The gate is the request path's, in the same order: proxy/keepalive.go's kaEntry.pingable(). It
// gates at 20k (the pre-#435 default; the shipped floor is now 80k) and not at the account's own configured floor, because this
// gates at 20k (the shipped default since #858) and not at the account's own configured floor, because this
// package does not read tenant configuration — an account that tuned its floor sees a replay of
// the default policy, and the panel's copy names 80k so the page stays self-consistent.
// the default policy, and the panel's copy names 20k so the page stays self-consistent.
//
// THE COUNT IS A FIXED K (#858). The live policy now picks the count per wait (keepalive.Policy.WaitPings:
// a tool wait up to 8, background K+3, a person K+1, no tool call yet 0). This replay does not model
// that: it charges one K per span. So the ladder and the recommendation describe the PLAN-OFF policy
// (keepalive_wait_plan: false), and no rung is marked "current". Modelling the plan needs the wait cause
// and tool history of each span and of each expiry's previous request; that is a follow-up.
func (d *DB) pingSpans(f Filter, minPrefix int64) ([]pingSpan, error) {
cond, args := f.where()
// LEAD, not LAG: a span belongs to the request that OPENS it, which is the request the gate
Expand Down Expand Up @@ -1377,6 +1383,8 @@ type KeepAliveCalc struct {
Addressable int64 `json:"addressable_misses"`
AddressableUSD float64 `json:"addressable_usd"`
Rows []CalcRow `json:"rows"`
// Plan says which policy the ladder models: "fixed_k", the policy with keepalive_wait_plan off (#858).
Plan string `json:"plan"`
// PrefixSource says where the prefill came from, so nobody reads a number as measured when
// it was typed: "session", "account_median" or "given".
PrefixSource string `json:"prefix_source,omitempty"`
Expand All @@ -1393,11 +1401,11 @@ type KeepAliveCalc struct {
// The rates themselves are the operator's price list; a model missing from it leaves every
// dollar field omitted (Priced above).
func (d *DB) KeepAliveCalc(f Filter, idleSeconds float64, prefix int64, model string,
price func(string) (modelinfo.Price, bool), currentPings int) (*KeepAliveCalc, error) {
price func(string) (modelinfo.Price, bool), _ int) (*KeepAliveCalc, error) {
if idleSeconds <= 0 {
idleSeconds = recIdleSeconds
}
out := &KeepAliveCalc{IdleSeconds: idleSeconds, Prefix: prefix, Model: model, Rows: []CalcRow{}}
out := &KeepAliveCalc{IdleSeconds: idleSeconds, Prefix: prefix, Model: model, Rows: []CalcRow{}, Plan: "fixed_k"}
var p modelinfo.Price
if price != nil && model != "" {
if pr, ok := price(model); ok && !pr.Zero() {
Expand Down Expand Up @@ -1460,7 +1468,8 @@ func (d *DB) KeepAliveCalc(f Filter, idleSeconds float64, prefix int64, model st
// Coverage is the account-wide policy figure (the 5-minute default, no row). The test
// below uses each expiry's OWN lifetime, so a GPT-5.6 gap is judged against its 30 min.
row := CalcRow{MaxPings: k, Coverage: CoverageSeconds(idleSeconds, k),
Current: k == currentPings}
// Never marked: the live policy picks its count per wait (#858), so no fixed-K rung is it.
Current: false}
for i, g := range gaps {
if g > idleSeconds && g <= CoverageSecondsFor(idleSeconds, k, lifetimes[i]) {
row.Convertible++
Expand Down Expand Up @@ -1604,7 +1613,8 @@ type KeepAliveRecommendation struct {
KeepAliveCoverage
}

// The shipped policy, which is also what is recommended when the rule admits an account.
// The fixed-count policy the recommendation models (240 s, K=2): the policy with keepalive_wait_plan off.
// The live default picks its count per wait (#858) and is not modelled here.
const (
recIdleSeconds = 240
recMaxPings = 2
Expand Down
20 changes: 13 additions & 7 deletions dash/keepalivetab_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -507,9 +507,15 @@ func TestCalculatorGoldenAgainstTheProductionBands(t *testing.T) {
if g3 > g1 {
t.Errorf("the ladder does not flatten: K1->K2 gains %d, K3->K4 gains %d", g1, g3)
}
// And the current row is marked, which is what the panel emphasises.
if !out.Rows[1].Current {
t.Error("K=2 was not marked as the current policy")
// No rung is marked current (#858): the live policy picks its count per wait, and the ladder
// models the fixed-K (plan off) policy, which it says on the wire.
for _, r := range out.Rows {
if r.Current {
t.Errorf("K=%d is marked current; no fixed-K rung is the live policy", r.MaxPings)
}
}
if out.Plan != "fixed_k" {
t.Errorf("plan = %q, want fixed_k", out.Plan)
}
}

Expand Down Expand Up @@ -822,12 +828,12 @@ func TestTheCalculatorChargesThePingsAPolicyWouldActuallySend(t *testing.T) {
const t0 = int64(1_700_000_000_000)
sec := func(n int64) int64 { return n * 1000 }
evs := []*Event{
// A gated session: prefix 100k throughout (above the 80k floor). turn 0, then a 300 s gap, then a 1000 s gap,
// A gated session: prefix 100k throughout (above the 20k floor). turn 0, then a 300 s gap, then a 1000 s gap,
// then nothing — the last request opens a span a live policy pings in and never closes.
bigPrefix(kaAgent(t0, "big", 0.10)),
bigPrefix(kaAgent(t0+sec(300), "big", 0.10)),
bigPrefix(kaAgent(t0+sec(1300), "big", 0.10)),
// Below the 80k prefix floor: the shipped gate never pings this session at all.
// Below the 20k prefix floor: the shipped gate never pings this session at all.
smallPrefix(kaAgent(t0, "small", 0.10)),
smallPrefix(kaAgent(t0+sec(1000), "small", 0.10)),
}
Expand Down Expand Up @@ -879,8 +885,8 @@ func smallPrefix(e *Event) *Event {
// The replay gate is the request path's gate. A drift here is a calculator modelling a policy
// nobody runs, which is the whole of F5.
func TestTheReplayGateMatchesTheShippedPolicy(t *testing.T) {
if kaGateMinPrefix != 80000 {
t.Errorf("kaGateMinPrefix = %d; config.DefaultKeepAliveMinPrefix is 80000 and the replay "+
if kaGateMinPrefix != 20000 {
t.Errorf("kaGateMinPrefix = %d; config.DefaultKeepAliveMinPrefix is 20000 and the replay "+
"has to gate on what the request path gates on", kaGateMinPrefix)
}
}
Expand Down
Loading
Loading