diff --git a/cmd/router/main.go b/cmd/router/main.go index d68412933..c36069f6c 100644 --- a/cmd/router/main.go +++ b/cmd/router/main.go @@ -586,13 +586,20 @@ func main() { // registered, so the Service's nil-check disables Tier-3 correctly (a // typed-nil concrete pointer would defeat it). var compactionSz proxy.CompactionSummarizer - if client, ok := providerMap[handoverProviderName]; ok { - ps := proxy.NewProviderSummarizer(client, handoverModel, handoverTimeout) + client, handoverProviderRegistered := providerMap[handoverProviderName] + // The summarizer emits Anthropic Messages bodies directly, with no + // translator in front of it. + handoverFamilyOK := providers.FamilyFor(handoverProviderName) == providers.FamilyAnthropic + switch { + case !handoverProviderRegistered: + logger.Info("Handover summarizer disabled (provider not registered); switch turns will preserve full history instead", "requested_provider", handoverProviderName) + case !handoverFamilyOK: + logger.Warn("Handover summarizer disabled (provider does not speak the Anthropic Messages format); switch turns will preserve full history instead", "requested_provider", handoverProviderName) + default: + ps := proxy.NewProviderSummarizer(client, handoverProviderName, handoverModel, handoverTimeout) summarizer = ps compactionSz = ps logger.Info("Handover summarizer wired", "provider", handoverProviderName, "model", handoverModel, "timeout_ms", handoverTimeout.Milliseconds()) - } else { - logger.Info("Handover summarizer disabled (provider not registered); switch turns will preserve full history instead", "requested_provider", handoverProviderName) } compactionPct := parseEnvFloat("ROUTER_COMPACTION_PCT", proxy.DefaultCompactionTriggerPct) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 414234693..3ff1cf6b4 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -117,6 +117,9 @@ Set `DATABASE_URL` directly, or compose it from the individual vars: | `ROUTER_SUBAGENT_MODEL` | *(none)* | Route Claude Code Task-tool sub-agent turns to a distinct model, independent of `ROUTER_HARD_PIN_MODEL` — e.g. a local/self-hosted OpenAI-compatible model (point `OPENROUTER_BASE_URL` at your local server) while the main loop keeps using Anthropic/whatever the scorer picks. Requires `ROUTER_SUBAGENT_PROVIDER`; either alone is ignored. Takes effect regardless of `ROUTER_HARD_PIN_EXPLORE`, but the HMM strategy keeps its own sub-agent handling and isn't affected. | | `ROUTER_SUBAGENT_PROVIDER` | *(none)* | Pair with `ROUTER_SUBAGENT_MODEL`. | | `ROUTER_TRANSLATION_COMPATIBILITY_MODE` | `shadow` | Translation representability rollout: `off` disables broad filtering, `shadow` records candidate exclusions without changing routes, and `enforce` makes declared semantic requirements hard routing constraints. Native-only safety paths (such as unsupported Responses tool unions and native Gemini ingress) remain protected unless mode is `off`. | +| `ROUTER_HANDOVER_PROVIDER` | `anthropic` | Provider that summarizes prior history when the planner switches models, and for the compaction cascade's structured summary. Must be registered and speak the Anthropic Messages format, or summarization stays off and switch turns forward full history. | +| `ROUTER_HANDOVER_MODEL` | `claude-haiku-4-5` | Model used for that summary. | +| `ROUTER_HANDOVER_TIMEOUT_MS` | `8000` | Deadline for the summary call. On timeout the full prior history is forwarded unchanged. | | `ROUTER_COMPACTION_PCT` | `0.85` | Fraction of the largest eligible model's context window at which the proactive compaction cascade engages (clear old tool results → structured summary → trim). Range `(0,1]`; `0` disables compaction (over-window requests then 413). Mirrors Claude Code's ~0.85 auto-compact trigger. | | `ROUTER_ONNX_ASSETS_DIR` | `/opt/router/assets` | Directory containing `model.onnx` + `tokenizer.json`. | | `ROUTER_ONNX_LIBRARY_DIR` | *(system default)* | Path to `libonnxruntime` (e.g. `/opt/homebrew/lib` on Apple Silicon). | @@ -152,6 +155,13 @@ permitted binding left is forced normally and served through that binding. A live session whose forced pin is later excluded fails the same way rather than quietly reverting to automatic routing — clear it with `/unforce-model`. +The router's own synthetic calls obey the same lists. The handover and +compaction summarizers ship the entire prior conversation upstream, so an +excluded `ROUTER_HANDOVER_PROVIDER` (or `ROUTER_HANDOVER_MODEL`) means no +summary is requested at all: a switch turn forwards full history and +compaction falls through to trimming, exactly as on a summarizer error. +Compaction's window-aware model choice skips excluded models too. + Excluding every provider that serves the models you route to leaves requests with nowhere to go (HTTP 503 from the scorer), so exclude deliberately. diff --git a/internal/proxy/AGENTS.md b/internal/proxy/AGENTS.md index df3f682fb..554915f98 100644 --- a/internal/proxy/AGENTS.md +++ b/internal/proxy/AGENTS.md @@ -37,6 +37,8 @@ The per-action flow is more than "scorer → dispatch". Pinned session, planner The provider-backed `Summarizer` implementation for handover lives in [`handover.go`](handover.go); the inner-ring `handover` package only defines the contract. On summarizer timeout or error, proxy keeps the full prior history unchanged (it does **not** trim) — a pricier switch action beats silently dropping the conversation the switched-to model needs. +A summary carries the whole prior conversation, so it goes through [`gateSummarizerCall`](summarizer_policy.go) first: excluded providers/models and the tenant credential boundary refuse it outright, and a refusal takes the same full-history / trim fallback as a summarizer error. `Summarizer` therefore exposes `Provider()`/`Model()` — the identity credential resolution and exclusions key off. + ## Proactive context-window compaction `ProxyMessages` / `ProxyOpenAIChatCompletion` call [`maybeCompact`](compaction.go) **before** routing so an over-long session is compacted rather than dead-ending in the scorer with no eligible provider. It engages when the estimate reaches `ROUTER_COMPACTION_PCT` (default 0.85) of the largest eligible model's window and runs Claude Code's tiered cascade: (1) `ClearOldToolResults` — local, clears stale tool results; (2) structured 9-section summary via a **window-aware** Anthropic-family summarizer (`SummarizeForCompaction`; haiku when the history fits, `claude-fable-5` for larger) rewritten with `RewriteForCompaction(summary, recentTurns)`; (3) progressive `TrimLastNMessages` rescue. If even the trimmed floor overflows, it returns `ErrContextWindowExceeded` → HTTP 413 (distinct from the "no provider keys" `ErrNoEligibleProvider`). The summary call is billed as a `_precompaction_summary` ledger row. Trigger below the window (not at overflow) is load-bearing: a summarizer can only ingest a history that still fits *some* model. diff --git a/internal/proxy/CLAUDE.md b/internal/proxy/CLAUDE.md index cbf04fe88..c464b33a6 100644 --- a/internal/proxy/CLAUDE.md +++ b/internal/proxy/CLAUDE.md @@ -37,6 +37,8 @@ The per-action flow is more than "scorer → dispatch". Pinned session, planner The provider-backed `Summarizer` implementation for handover lives in [`handover.go`](handover.go); the inner-ring `handover` package only defines the contract. On summarizer timeout or error, proxy keeps the full prior history unchanged (it does **not** trim) — a pricier switch action beats silently dropping the conversation the switched-to model needs. +A summary carries the whole prior conversation, so it goes through [`gateSummarizerCall`](summarizer_policy.go) first: excluded providers/models and the tenant credential boundary refuse it outright, and a refusal takes the same full-history / trim fallback as a summarizer error. `Summarizer` therefore exposes `Provider()`/`Model()` — the identity credential resolution and exclusions key off. + ## Proactive context-window compaction `ProxyMessages` / `ProxyOpenAIChatCompletion` call [`maybeCompact`](compaction.go) **before** routing so an over-long session is compacted rather than dead-ending in the scorer with no eligible provider. It engages when the estimate reaches `ROUTER_COMPACTION_PCT` (default 0.85) of the largest eligible model's window and runs Claude Code's tiered cascade: (1) `ClearOldToolResults` — local, clears stale tool results; (2) structured 9-section summary via a **window-aware** Anthropic-family summarizer (`SummarizeForCompaction`; haiku when the history fits, `claude-fable-5` for larger) rewritten with `RewriteForCompaction(summary, recentTurns)`; (3) progressive `TrimLastNMessages` rescue. If even the trimmed floor overflows, it returns `ErrContextWindowExceeded` → HTTP 413 (distinct from the "no provider keys" `ErrNoEligibleProvider`). The summary call is billed as a `_precompaction_summary` ledger row. Trigger below the window (not at overflow) is load-bearing: a summarizer can only ingest a history that still fits *some* model. diff --git a/internal/proxy/authoritative_turnloop_internal_test.go b/internal/proxy/authoritative_turnloop_internal_test.go index 6acf11560..13ba03863 100644 --- a/internal/proxy/authoritative_turnloop_internal_test.go +++ b/internal/proxy/authoritative_turnloop_internal_test.go @@ -45,6 +45,10 @@ func (s *authoritativeHandoverSummarizer) Provider() string { return providers.ProviderAnthropic } +func (s *authoritativeHandoverSummarizer) Model() string { + return DefaultHandoverModel +} + func TestAuthoritativePolicySelectsEveryEligibleTurn(t *testing.T) { strategy := router.Strategy("authoritative-test") tests := []struct { diff --git a/internal/proxy/compaction.go b/internal/proxy/compaction.go index 43a137f33..5bd6a08b4 100644 --- a/internal/proxy/compaction.go +++ b/internal/proxy/compaction.go @@ -92,12 +92,17 @@ func (s *Service) maxEligibleContextWindow(policyExcluded map[string]struct{}, s return maxWindow } -// selectCompactionSummarizer returns the cheapest configured summarizer model +// selectCompactionSummarizer returns the cheapest permitted summarizer model // whose context window can ingest historyTokens plus summary headroom, or "" -// when none can (caller falls back to trimming). -func (s *Service) selectCompactionSummarizer(historyTokens int) string { +// when none can (caller falls back to trimming). Excluded models are skipped: +// the summary call would ship the whole history to a banned model. +func (s *Service) selectCompactionSummarizer(ctx context.Context, historyTokens int) string { need := historyTokens + compactionSummaryOutputReserve + excluded := s.excludedModelsForRequest(ctx) for _, m := range compactionSummarizerModels { + if _, banned := excluded[m]; banned { + continue + } if catalog.ContextWindowFor(m) >= need { return m } @@ -204,33 +209,27 @@ func (s *Service) billCompactionSummary(ctx context.Context, requestID, external } // runCompactionSummary picks a window-aware summarizer model and dispatches the -// structured summary call, honoring the tenant-boundary credential rules used -// by the switch-handover path. Returns ok=false (and logs) when no summarizer -// fits the history, the tenant boundary forbids the call, or the call fails — -// in every such case the caller falls through to trimming. +// structured summary call, honoring the exclusion and tenant-boundary rules +// used by the switch-handover path. Returns ok=false (and logs) when no +// permitted summarizer fits the history, policy forbids the call, or the call +// fails — in every such case the caller falls through to trimming. func (s *Service) runCompactionSummary(ctx context.Context, env *translate.RequestEnvelope, reqHeaders http.Header) (string, handover.Usage, string, bool) { log := observability.FromContext(ctx) - model := s.selectCompactionSummarizer(env.ContextOverflowTokenEstimate()) + model := s.selectCompactionSummarizer(ctx, env.ContextOverflowTokenEstimate()) if model == "" { - log.Info("Compaction Tier-3 skipped: history exceeds every summarizer window", "history", env.ContextOverflowTokenEstimate()) + log.Info("Compaction Tier-3 skipped: no permitted summarizer window fits the history", "history", env.ContextOverflowTokenEstimate()) return "", handover.Usage{}, "", false } sumProvider := s.compactionSummarizer.Provider() - sumCreds := resolveSummarizerCreds(ctx, sumProvider, reqHeaders) - if sumCreds == nil && s.requestUsesNonDeploymentCreds(ctx, reqHeaders) { - log.Info("Compaction Tier-3 skipped: would cross tenant boundary", "sum_provider", sumProvider) + gate := s.gateSummarizerCall(ctx, sumProvider, model, reqHeaders) + if !gate.Allowed { + log.Info("Compaction Tier-3 skipped by policy", "skip_reason", gate.SkipReason, "sum_provider", sumProvider, "summary_model", model) return "", handover.Usage{}, "", false } - summCtx := ctx - if sumCreds != nil { - summCtx = context.WithValue(ctx, CredentialsContextKey{}, sumCreds) - } else { - summCtx = clearCredentials(ctx) - } - summary, usage, err := s.compactionSummarizer.SummarizeForCompaction(summCtx, env, model, DefaultCompactionMaxTokens) + summary, usage, err := s.compactionSummarizer.SummarizeForCompaction(gate.summarizerContext(ctx), env, model, DefaultCompactionMaxTokens) if err != nil { log.Warn("Compaction summarizer failed; falling back to trim", "err", err, "model", model) return "", handover.Usage{}, "", false diff --git a/internal/proxy/compaction_test.go b/internal/proxy/compaction_test.go index 8fc35413e..b30424fea 100644 --- a/internal/proxy/compaction_test.go +++ b/internal/proxy/compaction_test.go @@ -190,9 +190,10 @@ func TestWithCompaction_ZeroPctDisables(t *testing.T) { func TestSelectCompactionSummarizer_WindowAware(t *testing.T) { s := &Service{} - assert.Equal(t, DefaultHandoverModel, s.selectCompactionSummarizer(1_000), "small history → cheap model") - assert.Equal(t, largeWindowSummarizerModel, s.selectCompactionSummarizer(300_000), "history over the cheap model's window → large-window model") - assert.Equal(t, "", s.selectCompactionSummarizer(5_000_000), "history over every window → none") + ctx := context.Background() + assert.Equal(t, DefaultHandoverModel, s.selectCompactionSummarizer(ctx, 1_000), "small history → cheap model") + assert.Equal(t, largeWindowSummarizerModel, s.selectCompactionSummarizer(ctx, 300_000), "history over the cheap model's window → large-window model") + assert.Equal(t, "", s.selectCompactionSummarizer(ctx, 5_000_000), "history over every window → none") } func TestMaxEligibleContextWindow(t *testing.T) { diff --git a/internal/proxy/handover.go b/internal/proxy/handover.go index 83e9021d4..2d91b93ee 100644 --- a/internal/proxy/handover.go +++ b/internal/proxy/handover.go @@ -65,16 +65,23 @@ const compactionInstruction = "The conversation is being compacted to fit the mo // ProviderSummarizer adapts a providers.Client to handover.Summarizer by // building a small Anthropic Messages request from the prior conversation. +// The client must speak the Anthropic Messages format natively; provider names +// the upstream it dispatches to so credential resolution, exclusion policy and +// billing attribution all key off the same identity. type ProviderSummarizer struct { client providers.Client + provider string model string timeout time.Duration maxTokens int } -// NewProviderSummarizer constructs a summarizer adapter. Empty/zero args -// fall back to defaults. -func NewProviderSummarizer(client providers.Client, model string, timeout time.Duration) *ProviderSummarizer { +// NewProviderSummarizer constructs a summarizer adapter dispatching to the +// named provider. Empty/zero args fall back to defaults. +func NewProviderSummarizer(client providers.Client, provider, model string, timeout time.Duration) *ProviderSummarizer { + if provider == "" { + provider = providers.ProviderAnthropic + } if model == "" { model = DefaultHandoverModel } @@ -83,6 +90,7 @@ func NewProviderSummarizer(client providers.Client, model string, timeout time.D } return &ProviderSummarizer{ client: client, + provider: provider, model: model, timeout: timeout, maxTokens: DefaultHandoverMaxTokens, @@ -100,7 +108,12 @@ func (s *ProviderSummarizer) WithMaxTokens(n int) *ProviderSummarizer { // Provider returns the upstream provider this summarizer dispatches to. func (s *ProviderSummarizer) Provider() string { - return providers.ProviderAnthropic + return s.provider +} + +// Model returns the model the switch-handover summary is dispatched to. +func (s *ProviderSummarizer) Model() string { + return s.model } // ErrEmptySummary is returned when the upstream call succeeded but no @@ -162,7 +175,7 @@ func (s *ProviderSummarizer) summarize(ctx context.Context, env *translate.Reque prep.Headers.Set("anthropic-version", "2023-06-01") decision := router.Decision{ - Provider: providers.ProviderAnthropic, + Provider: s.provider, Model: model, Reason: kind + "_summary", } @@ -193,7 +206,7 @@ func (s *ProviderSummarizer) summarize(ctx context.Context, env *translate.Reque } usage := extractAnthropicUsage(respBody) usage.Model = model - usage.Provider = providers.ProviderAnthropic + usage.Provider = s.provider return text, usage, nil } @@ -309,34 +322,23 @@ func (s *Service) runCompactionHandover(ctx context.Context, env *translate.Requ out.Invoked = true var ( - sumProvider string - sumCreds *Credentials - canCallSummarizer bool + sumProvider string + gate summarizerGate ) if s.summarizer != nil { sumProvider = s.summarizer.Provider() - sumCreds = resolveSummarizerCreds(ctx, sumProvider, reqHeaders) - nonDepCreds := s.requestUsesNonDeploymentCreds(ctx, reqHeaders) - canCallSummarizer = sumCreds != nil || !nonDepCreds + gate = s.gateSummarizerCall(ctx, sumProvider, s.summarizer.Model(), reqHeaders) } switch { case s.summarizer == nil: out.FallbackToFullHistory = true log.Info("Compaction handover: summarizer not wired; preserved compacted body instead", "decision_model", decisionModel) - case !canCallSummarizer: + case !gate.Allowed: out.FallbackToFullHistory = true - log.Info("Compaction handover: summarizer skipped (tenant boundary); preserved compacted body instead", "decision_model", decisionModel) + log.Info("Compaction handover: summarizer skipped by policy; preserved compacted body instead", "skip_reason", gate.SkipReason, "sum_provider", sumProvider, "decision_model", decisionModel) default: - summCtx := ctx - if sumCreds != nil { - summCtx = context.WithValue(ctx, CredentialsContextKey{}, sumCreds) - } else { - // Strip any request credential (e.g. subscription OAuth token) so - // this synthetic call runs on the deployment key instead of - // inheriting one that could 401 or cross a tenant boundary. - summCtx = clearCredentials(ctx) - } + summCtx := gate.summarizerContext(ctx) start := time.Now() summary, summaryUsage, sumErr := s.summarizer.Summarize(summCtx, env) out.LatencyMS = time.Since(start).Milliseconds() diff --git a/internal/proxy/handover_internal_test.go b/internal/proxy/handover_internal_test.go index 251a83a3d..6570c4eaa 100644 --- a/internal/proxy/handover_internal_test.go +++ b/internal/proxy/handover_internal_test.go @@ -89,7 +89,7 @@ func TestProviderSummarizer_SuccessReturnsAssistantText(t *testing.T) { respBody: canonicalAnthropicResponse, respStatus: http.StatusOK, } - s := NewProviderSummarizer(fake, "", 200*time.Millisecond) + s := NewProviderSummarizer(fake, providers.ProviderAnthropic, "", 200*time.Millisecond) got, _, err := s.Summarize(context.Background(), env) require.NoError(t, err) @@ -107,7 +107,7 @@ func TestProviderSummarizer_TimeoutReturnsError(t *testing.T) { // Sleep longer than the summarizer's timeout. sleep: 200 * time.Millisecond, } - s := NewProviderSummarizer(fake, "", 25*time.Millisecond) + s := NewProviderSummarizer(fake, providers.ProviderAnthropic, "", 25*time.Millisecond) got, _, err := s.Summarize(context.Background(), env) require.Error(t, err) @@ -127,7 +127,7 @@ func TestProviderSummarizer_Non2xxReturnsError(t *testing.T) { respBody: `{"error":"oops"}`, respStatus: http.StatusInternalServerError, } - s := NewProviderSummarizer(fake, "", 200*time.Millisecond) + s := NewProviderSummarizer(fake, providers.ProviderAnthropic, "", 200*time.Millisecond) got, _, err := s.Summarize(context.Background(), env) require.Error(t, err) @@ -147,7 +147,7 @@ func TestProviderSummarizer_EmptyContentReturnsErrEmptySummary(t *testing.T) { respBody: `{"id":"msg_empty","content":[]}`, respStatus: http.StatusOK, } - s := NewProviderSummarizer(fake, "", 200*time.Millisecond) + s := NewProviderSummarizer(fake, providers.ProviderAnthropic, "", 200*time.Millisecond) got, _, err := s.Summarize(context.Background(), env) require.Error(t, err) @@ -159,7 +159,7 @@ func TestProviderSummarizer_NilEnvelopeReturnsError(t *testing.T) { t.Parallel() fake := &fakeHandoverProvider{} - s := NewProviderSummarizer(fake, "", 200*time.Millisecond) + s := NewProviderSummarizer(fake, providers.ProviderAnthropic, "", 200*time.Millisecond) _, _, err := s.Summarize(context.Background(), nil) require.Error(t, err) diff --git a/internal/proxy/summarizer_policy.go b/internal/proxy/summarizer_policy.go new file mode 100644 index 000000000..1b115630b --- /dev/null +++ b/internal/proxy/summarizer_policy.go @@ -0,0 +1,58 @@ +package proxy + +import ( + "context" + "net/http" +) + +// Skip reasons reported by gateSummarizerCall, used as log fields so an +// operator can tell a policy refusal apart from a tenant-boundary one. +const ( + summarizerSkipProviderExcluded = "provider_excluded" + summarizerSkipModelExcluded = "model_excluded" + summarizerSkipTenantBoundary = "tenant_boundary" +) + +// summarizerGate is the resolved verdict for one synthetic summarizer call: +// whether it may run at all, and under whose credentials. +type summarizerGate struct { + // Creds are the caller's own forwarded credentials for the summarizer's + // provider, or nil to run on the deployment key. + Creds *Credentials + // Allowed reports whether the call may be dispatched. + Allowed bool + // SkipReason is one of the summarizerSkip* constants when Allowed is false. + SkipReason string +} + +// gateSummarizerCall decides whether a summary call may be dispatched to +// provider/model. Summaries ship the full prior conversation, so policy +// exclusions apply — policyExcludedProviders, not session strike-outs, which +// are transient evidence. On BYOK/client-keyed requests without matching +// forwarded creds, skip rather than spend the deployment key across the tenant +// boundary. +func (s *Service) gateSummarizerCall(ctx context.Context, provider, model string, headers http.Header) summarizerGate { + if _, excluded := s.policyExcludedProviders(ctx)[provider]; excluded { + return summarizerGate{SkipReason: summarizerSkipProviderExcluded} + } + if model != "" { + if _, excluded := s.excludedModelsForRequest(ctx)[model]; excluded { + return summarizerGate{SkipReason: summarizerSkipModelExcluded} + } + } + creds := resolveSummarizerCreds(ctx, provider, headers) + if creds == nil && s.requestUsesNonDeploymentCreds(ctx, headers) { + return summarizerGate{SkipReason: summarizerSkipTenantBoundary} + } + return summarizerGate{Creds: creds, Allowed: true} +} + +// summarizerContext returns the context for the summary call: caller creds when +// available, otherwise stripped so a subscription OAuth token can't 401 or +// cross a tenant boundary. +func (g summarizerGate) summarizerContext(ctx context.Context) context.Context { + if g.Creds != nil { + return context.WithValue(ctx, CredentialsContextKey{}, g.Creds) + } + return clearCredentials(ctx) +} diff --git a/internal/proxy/summarizer_policy_internal_test.go b/internal/proxy/summarizer_policy_internal_test.go new file mode 100644 index 000000000..e89532d3a --- /dev/null +++ b/internal/proxy/summarizer_policy_internal_test.go @@ -0,0 +1,127 @@ +package proxy + +import ( + "context" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "workweave/router/internal/providers" +) + +func excludedModelsCtx(names ...string) context.Context { + return context.WithValue(context.Background(), InstallationExcludedModelsContextKey{}, names) +} + +// A summary ships the whole prior conversation upstream, so an excluded +// provider must be refused even though the summarizer is wired deployment-wide. +func TestGateSummarizerCall_RefusesExcludedProvider(t *testing.T) { + s := &Service{} + + gate := s.gateSummarizerCall(excludedProvidersCtx(providers.ProviderAnthropic), + providers.ProviderAnthropic, DefaultHandoverModel, http.Header{}) + + assert.False(t, gate.Allowed) + assert.Equal(t, summarizerSkipProviderExcluded, gate.SkipReason) +} + +// Deployment-wide exclusions bind the summarizer too. +func TestGateSummarizerCall_RefusesDeploymentExcludedProvider(t *testing.T) { + s := (&Service{}).WithExcludedProvidersOverride([]string{providers.ProviderAnthropic}) + + gate := s.gateSummarizerCall(context.Background(), + providers.ProviderAnthropic, DefaultHandoverModel, http.Header{}) + + assert.False(t, gate.Allowed) + assert.Equal(t, summarizerSkipProviderExcluded, gate.SkipReason) +} + +func TestGateSummarizerCall_RefusesExcludedModel(t *testing.T) { + s := &Service{} + + gate := s.gateSummarizerCall(excludedModelsCtx(DefaultHandoverModel), + providers.ProviderAnthropic, DefaultHandoverModel, http.Header{}) + + assert.False(t, gate.Allowed) + assert.Equal(t, summarizerSkipModelExcluded, gate.SkipReason) +} + +// An unrelated exclusion must not disable summarization. +func TestGateSummarizerCall_AllowsPermittedBinding(t *testing.T) { + s := &Service{} + + gate := s.gateSummarizerCall(excludedProvidersCtx(providers.ProviderOpenAI), + providers.ProviderAnthropic, DefaultHandoverModel, http.Header{}) + + require.True(t, gate.Allowed) + assert.Empty(t, gate.SkipReason) + assert.Nil(t, gate.Creds, "no caller creds forwarded → run on the deployment key") +} + +// Transient 529 strike-outs are evidence, not policy: they must not silently +// disable summarization for the rest of the session. +func TestGateSummarizerCall_IgnoresSessionStrikeOuts(t *testing.T) { + s := &Service{} + ctx := context.WithValue(context.Background(), + SessionDisabledProvidersContextKey{}, []string{providers.ProviderAnthropic}) + + gate := s.gateSummarizerCall(ctx, providers.ProviderAnthropic, DefaultHandoverModel, http.Header{}) + + assert.True(t, gate.Allowed) +} + +// The tenant boundary still applies: a client-keyed request with no matching +// forwarded credential must not spend the deployment key. +func TestGateSummarizerCall_RefusesTenantBoundaryCrossing(t *testing.T) { + s := &Service{} + headers := http.Header{} + headers.Set("Authorization", "Bearer sk-customer-openai-key") + + gate := s.gateSummarizerCall(context.Background(), providers.ProviderAnthropic, DefaultHandoverModel, headers) + + assert.False(t, gate.Allowed) + assert.Equal(t, summarizerSkipTenantBoundary, gate.SkipReason) +} + +// The exclusion check runs before credential resolution: forwarding the +// caller's own key does not buy egress to a provider the operator excluded. +func TestGateSummarizerCall_ExclusionOutranksCallerCreds(t *testing.T) { + s := &Service{} + headers := http.Header{} + headers.Set("x-api-key", "sk-ant-customer-byok-key") + + gate := s.gateSummarizerCall(excludedProvidersCtx(providers.ProviderAnthropic), + providers.ProviderAnthropic, DefaultHandoverModel, headers) + + assert.False(t, gate.Allowed) + assert.Equal(t, summarizerSkipProviderExcluded, gate.SkipReason) + assert.Nil(t, gate.Creds) +} + +// Compaction picks its own model, so the window-aware selector must skip an +// excluded one rather than hand it the history. +func TestSelectCompactionSummarizer_SkipsExcludedModel(t *testing.T) { + s := &Service{} + + got := s.selectCompactionSummarizer(excludedModelsCtx(DefaultHandoverModel), 1_000) + + assert.Equal(t, largeWindowSummarizerModel, got, "cheap model excluded → next permitted window") + + got = s.selectCompactionSummarizer( + excludedModelsCtx(DefaultHandoverModel, largeWindowSummarizerModel), 1_000) + + assert.Empty(t, got, "every summarizer model excluded → no summarization") +} + +// The summarizer must report the provider it was actually built for, or +// credential resolution keys off the wrong one. +func TestProviderSummarizer_ReportsConfiguredProvider(t *testing.T) { + s := NewProviderSummarizer(nil, providers.ProviderAnthropicGateway, "gateway-model", 0) + + assert.Equal(t, providers.ProviderAnthropicGateway, s.Provider()) + assert.Equal(t, "gateway-model", s.Model()) + + assert.Equal(t, providers.ProviderAnthropic, NewProviderSummarizer(nil, "", "", 0).Provider()) +} diff --git a/internal/proxy/turnloop.go b/internal/proxy/turnloop.go index e6de8f7ab..a3abd0951 100644 --- a/internal/proxy/turnloop.go +++ b/internal/proxy/turnloop.go @@ -940,11 +940,10 @@ func (s *Service) runTurnLoop( // summarizer failure keeps the full prior history rather than trimming — // an expensive switch turn beats silently dropping context. // - // Privacy guard: the summarizer runs on deployment-level creds by default, - // which would cross the tenant boundary for a BYOK/client request. Prefer - // the caller's own forwarded creds for the summarizer's provider when - // available; skip summarization (pass full history through) only when the - // request is BYOK/client-keyed with no matching creds forwarded. + // Privacy guard: a summary ships the whole prior conversation to the + // summarizer's own binding, so gateSummarizerCall withholds it from an + // excluded provider/model and keeps a BYOK/client request off the + // deployment key. A refusal passes the full history through. if pinFound && prefixBroken { // Client already trimmed its own history — summarizing again is pure // cost, so forward unchanged. @@ -955,34 +954,24 @@ func (s *Service) runTurnLoop( } if pinFound && !prefixBroken { var ( - sumProvider string - sumCreds *Credentials - canCallSummarizer bool + sumProvider string + gate summarizerGate ) if s.summarizer != nil { sumProvider = s.summarizer.Provider() - sumCreds = resolveSummarizerCreds(ctx, sumProvider, reqHeaders) - nonDepCreds := s.requestUsesNonDeploymentCreds(ctx, reqHeaders) - canCallSummarizer = sumCreds != nil || !nonDepCreds + gate = s.gateSummarizerCall(ctx, sumProvider, s.summarizer.Model(), reqHeaders) } switch { case s.summarizer == nil: res.Handover.Invoked = true res.Handover.FallbackToFullHistory = true log.Info("Handover summarizer not wired; preserved full history instead", "pin_model", pin.Model, "fresh_model", fresh.Model) - case !canCallSummarizer: + case !gate.Allowed: res.Handover.Invoked = true res.Handover.FallbackToFullHistory = true - log.Info("Handover summarizer skipped to preserve tenant boundary; preserved full history instead", "pin_model", pin.Model, "fresh_model", fresh.Model, "sum_provider", sumProvider) + log.Info("Handover summarizer skipped by policy; preserved full history instead", "skip_reason", gate.SkipReason, "pin_model", pin.Model, "fresh_model", fresh.Model, "sum_provider", sumProvider) default: - summCtx := ctx - if sumCreds != nil { - summCtx = context.WithValue(ctx, CredentialsContextKey{}, sumCreds) - } else { - // Strip any request credential (e.g. subscription OAuth token) - // so this synthetic call doesn't inherit it and 401/cross tenants. - summCtx = clearCredentials(ctx) - } + summCtx := gate.summarizerContext(ctx) start := time.Now() summary, summaryUsage, sumErr := s.summarizer.Summarize(summCtx, env) res.Handover.Invoked = true diff --git a/internal/proxy/turnloop_test.go b/internal/proxy/turnloop_test.go index 0bc49be24..b0067ba7f 100644 --- a/internal/proxy/turnloop_test.go +++ b/internal/proxy/turnloop_test.go @@ -28,6 +28,7 @@ import ( // or errOnCall if set. calls counts invocations. type fakeSummarizer struct { summary string + provider string errOnCall error calls atomic.Int32 } @@ -40,7 +41,14 @@ func (f *fakeSummarizer) Summarize(ctx context.Context, env *translate.RequestEn return f.summary, handover.Usage{}, nil } -func (f *fakeSummarizer) Provider() string { return providers.ProviderAnthropic } +func (f *fakeSummarizer) Provider() string { + if f.provider != "" { + return f.provider + } + return providers.ProviderAnthropic +} + +func (f *fakeSummarizer) Model() string { return proxy.DefaultHandoverModel } // usageProvider writes an Anthropic response with the configured token // usage so the OTel UsageExtractor surfaces it to the cache-stats writeback. @@ -748,6 +756,35 @@ func TestTurnLoop_HandoverSkippedWhenClientCredsCrossProvider(t *testing.T) { assert.Equal(t, "claude-haiku-4-5", rec.Header().Get(proxy.HeaderRouterModel), "switch must still happen with full history passed through") } +// An excluded summarizer provider must not receive the conversation even when +// the routed model is permitted and the deployment holds a key for it. +func TestTurnLoop_HandoverSkippedWhenSummarizerProviderExcluded(t *testing.T) { + store := newFakePinStore() + store.hasPin = true + store.pin = sessionpin.Pin{ + Provider: providers.ProviderAnthropic, + Model: "claude-opus-4-7", + Reason: "cluster:v0.2", + PinnedUntil: time.Now().Add(time.Hour), + LastInputTokens: 5000, + LastTurnEndedAt: time.Now().Add(-30 * time.Second), + } + fr := &fakeRouter{decision: router.Decision{Provider: providers.ProviderAnthropic, Model: "claude-haiku-4-5", Reason: "cluster:v0.2"}} + sz := &fakeSummarizer{summary: "Should not be invoked.", provider: providers.ProviderOpenAI} + svc, provider := newPinSvcCapturing(fr, store) + svc.WithSummarizer(sz) + + ctx := context.WithValue(authedCtx(uuid.New().String()), + proxy.InstallationExcludedProvidersContextKey{}, []string{providers.ProviderOpenAI}) + rec := httptest.NewRecorder() + httpReq := httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader("")) + require.NoError(t, svc.ProxyMessages(ctx, largeMultiTurnBody(t), rec, httpReq)) + + assert.Equal(t, int32(0), sz.calls.Load(), "summarizer must NOT receive the conversation when its provider is excluded") + assert.Equal(t, "claude-haiku-4-5", rec.Header().Get(proxy.HeaderRouterModel), "switch must still happen") + assert.Equal(t, 6, forwardedMessageCount(t, provider), "refused summary must preserve full history, not trim it") +} + // ROUTER_PLANNER_ENABLED kill switch: an existing pin wins outright without // consulting the scorer, mirroring legacy stickiness. func TestTurnLoop_PlannerDisabledPreservesFirstDecisionWins(t *testing.T) { diff --git a/internal/router/handover/AGENTS.md b/internal/router/handover/AGENTS.md index b2b7f2b19..90765bce4 100644 --- a/internal/router/handover/AGENTS.md +++ b/internal/router/handover/AGENTS.md @@ -17,3 +17,4 @@ When the planner decides SWITCH, proxy asks a small model to summarize prior con - **`Summarizer` implementations MUST respect the context deadline.** On summarizer timeout or error, proxy keeps the full prior history unchanged (no trim) — a pricier switch action beats silently dropping context. Do not "fix" by waiting longer, and do not reintroduce a silent trim fallback. - **No I/O in this package.** All I/O lives in the proxy-side implementation. +- **`Provider()`/`Model()` must report what the implementation will actually call.** Proxy keys credential resolution and exclusion checks off them; a stale or hardcoded answer sends a tenant's key, or their conversation, to the wrong upstream. diff --git a/internal/router/handover/CLAUDE.md b/internal/router/handover/CLAUDE.md index 2245fd4a6..08745a3da 100644 --- a/internal/router/handover/CLAUDE.md +++ b/internal/router/handover/CLAUDE.md @@ -17,3 +17,4 @@ When the planner decides SWITCH, proxy asks a small model to summarize prior con - **`Summarizer` implementations MUST respect the context deadline.** On summarizer timeout or error, proxy keeps the full prior history unchanged (no trim) — a pricier switch action beats silently dropping context. Do not "fix" by waiting longer, and do not reintroduce a silent trim fallback. - **No I/O in this package.** All I/O lives in the proxy-side implementation. +- **`Provider()`/`Model()` must report what the implementation will actually call.** Proxy keys credential resolution and exclusion checks off them; a stale or hardcoded answer sends a tenant's key, or their conversation, to the wrong upstream. diff --git a/internal/router/handover/summarizer.go b/internal/router/handover/summarizer.go index 593e2662c..7d5a894d1 100644 --- a/internal/router/handover/summarizer.go +++ b/internal/router/handover/summarizer.go @@ -33,12 +33,14 @@ type Usage struct { // Implementations SHOULD respect the context deadline; on timeout or error, // callers keep the full prior history unchanged instead of dropping it. // -// Provider identifies the upstream this summarizer dispatches to (e.g. -// "anthropic"), so the orchestrator can plumb matching BYOK creds through -// and keep tenant data from crossing the deployment key boundary. +// Provider and Model identify the upstream this summarizer dispatches to (e.g. +// "anthropic"), so the orchestrator can plumb matching BYOK creds through, keep +// tenant data from crossing the deployment key boundary, and withhold the +// conversation from a binding the installation excluded. type Summarizer interface { Summarize(ctx context.Context, env *translate.RequestEnvelope) (summary string, usage Usage, err error) Provider() string + Model() string } // RewriteEnvelope mutates env in-place: keeps system blocks, replaces