feat: jev-scored context compaction — shadow-first, opt-in, fail-open (reference-faithful port)

README.md

@@ -203,8 +203,15 @@ export OPENAI_MODEL="model-name"
 * **True cl100k BPE Offline Tokenizer:** Embedded tokenizer calculates real BPE token counts offline with zero heuristic guesswork.
 * **Native Context-Aware Search:** High-performance codebase search with globster filtering that respects `.gitignore` and `.llmignore`.
 * **Agent Skills & MCP Support:** Natively consume external Model Context Protocol (MCP) servers and third-party Agent Skills with zero configuration overhead.
+* **Context Compaction with an Expand Tool (staged: shadow → enabled):** Oversized tool outputs are segmented and scored; low-scoring segments are elided into `[[elided id=r:…]]` content-addressed pointers backed by a persistent record store (`~/.local/share/late/compaction-store.jsonl`), with the originals retrievable via the `expand` tool. Gate safety knobs (`compaction-max-elide-percent` tripwire, `compaction-protected-floor` for stacktraces/diffs) and a three-stage `-check-compaction` preflight ship with it. The elision score cutoff resolves as explicit `-compaction-threshold` flag > `compaction-threshold` in `config.json` > the 0.35 default (see the Quickstart).
+* **413 / Context-Too-Large Recovery:** When a provider rejects the request body outright (`413 Request Entity Too Large`), the error carries actionable recovery guidance, and — with `compaction-mode: enabled` — the TUI automatically runs one recovery compaction pass (once per conversation, never a loop). The Quickstart ships a short playbook: replay the shadow log at candidate thresholds (`-replay-shadow=0.35,0.5,0.65,0.8`), set `compaction-threshold` accordingly, restart.
+* **Full-History Compaction (`/jev-compact-context` + autocompact):** One command — or an automatic trigger at a configurable context-usage percent (`jev-autocompact`, `jev-autocompact-percent`) — scores the whole conversation after an append-only frozen prefix (persisted high-water mark; never re-scores what an earlier run froze) and elides low-scoring segments into pointers, originals retrievable via the `expand` tool (see the Quickstart).
+* **Retrieved Context (`compaction-retrieval`, default off):** The read side of the compaction store — before each request its summaries are scored against the current task and the top matches are appended to the request's work area (never the frozen prefix, never persisted), with `kind=retrieve` decisions logged for replay (see the Quickstart).
+* **Offline Demo & Shadow Replay Tooling:** `-replay-shadow=0.10,0.35` re-decides the shadow log at your own thresholds (kept/relocated/tokens-saved table + false-negative rate, no scorer round trips), and `compaction-backend: "offline"` runs the whole compaction flow — including `-check-compaction` — against a deterministic local scripted scorer with no API key and no network. Demo/tests only, never a production default (see the Quickstart).
 * **Git Worktree Support:** Run independent, parallel agent instances across multiple branches simultaneously with zero context bleeding.
 
+The context-compaction feature set is a Go port of [jev-compaction](https://github.com/Waxmell114514/jev-compaction) (MIT); the port follows that repo's staged `shadow` → `enabled` rollout, so scoring starts out as pure observation and only ever elides once you explicitly opt into `enabled` mode.
+
 ---
 
 ## License

cmd/late/check_compaction_test.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"io"
+	"strings"
+	"testing"
+
+	"late/internal/compaction"
+)
+
+// TestCheckCompactionFlagParses pins the -check-compaction flag's CLI shape:
+// it is a boolean (no value name in help), it defaults to false, it pairs
+// with -compaction-mode, and it renders in the grouped help under Context
+// compaction.
+func TestCheckCompactionFlagParses(t *testing.T) {
+	newFlagSet := func() (*flag.FlagSet, *bool) {
+		fs := flag.NewFlagSet("check-test", flag.ContinueOnError)
+		fs.SetOutput(io.Discard)
+		check := fs.Bool("check-compaction", false, "usage of check-compaction")
+		fs.String("compaction-mode", "", "usage of compaction-mode")
+		return fs, check
+	}
+
+	// Default: off.
+	fs, check := newFlagSet()
+	if err := fs.Parse(nil); err != nil {
+		t.Fatalf("Parse(nil): %v", err)
+	}
+	if *check {
+		t.Error("-check-compaction defaults to true, want false")
+	}
+
+	// Set, alone.
+	fs, check = newFlagSet()
+	if err := fs.Parse([]string{"-check-compaction"}); err != nil {
+		t.Fatalf("Parse(-check-compaction): %v", err)
+	}
+	if !*check {
+		t.Error("-check-compaction did not parse as true")
+	}
+
+	// Set, paired with -compaction-mode (the documented pairing: the check
+	// resolves the backend the same way a run with that mode would).
+	fs, check = newFlagSet()
+	if err := fs.Parse([]string{"-check-compaction", "-compaction-mode=enabled"}); err != nil {
+		t.Fatalf("Parse(-check-compaction -compaction-mode=enabled): %v", err)
+	}
+	if !*check {
+		t.Error("-check-compaction did not parse as true when paired with -compaction-mode")
+	}
+	if got := fs.Lookup("compaction-mode").Value.String(); got != "enabled" {
+		t.Errorf("-compaction-mode = %q, want enabled", got)
+	}
+
+	// Help rendering: the flag shows up exactly once in the grouped output.
+	var buf bytes.Buffer
+	writeHelp(&buf, newHelpTestFlagSet(t))
+	if n := countRenderedFlagLines(buf.String(), "check-compaction"); n != 1 {
+		t.Errorf("-check-compaction rendered %d times in help, want exactly 1:\n%s", n, buf.String())
+	}
+}
+
+// TestNoBackendCheckReport pins the stage-0 failure report: the guidance
+// sentence, the FAIL verdict naming the backend stage, and no question/gate/
+// expand rows — the real stages cannot run without a backend.
+func TestNoBackendCheckReport(t *testing.T) {
+	results := noBackendCheckResults(errNoBackendForTest())
+	if len(results) != 1 {
+		t.Fatalf("got %d results, want exactly the stage-0 backend failure", len(results))
+	}
+	if results[0].Stage != compaction.CheckStageBackend || results[0].OK {
+		t.Fatalf("results[0] = %+v, want a failing %q stage", results[0], compaction.CheckStageBackend)
+	}
+
+	out := compaction.FormatCheckReport(results, false)
+	for _, want := range []string{
+		"late compaction preflight",
+		"[FAIL] " + compaction.CheckStageBackend,
+		"no compaction backend configured (set the provider key or run with -compaction-mode pointing at a gateway)",
+		"no System One backend available", // the resolver's typed reason
+		`result: FAIL (stage "backend" failed)`,
+		"cost: n/a (the decisions client does not track token usage)",
+	} {
+		if !strings.Contains(out, want) {
+			t.Errorf("report missing %q:\n%s", want, out)
+		}
+	}
+	for _, banned := range []string{"[ok  ] " + compaction.CheckStageQuestions, "[FAIL] " + compaction.CheckStageGate, "[FAIL] " + compaction.CheckStageExpand} {
+		if strings.Contains(out, banned) {
+			t.Errorf("report must not run the %q stage without a backend:\n%s", banned, out)
+		}
+	}
+}
+
+// errNoBackendForTest builds the resolver's no-backend error without touching
+// the process environment (a test machine may carry real provider keys, and
+// runCompactionCheck must never be exercised against them).
+func errNoBackendForTest() error {
+	return &compaction.NoBackendError{
+		Detail: "typesafe: no API key for backend \"typesafe\": set TYPESAFE_API_KEY=<key> in the environment, or write the key to /keys/compaction-typesafe.key",
+	}
+}

cmd/late/compaction_offline_test.go

@@ -0,0 +1,149 @@
+package main
+
+import (
+	"context"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"late/internal/compaction"
+	appconfig "late/internal/config"
+	"late/internal/session"
+)
+
+// The offline compaction tests never construct a server: compaction-backend
+// "offline" is the no-key, no-network demo path, so every assertion below
+// runs against the ScriptedScorer with nothing to connect to — an attempted
+// request would fail and fail the test.
+
+// TestOfflineBackendNeedsNoEnvKeys is the wiring smoke test (Step 18): in an
+// environment where NO System One backend resolves (empty env lookup, empty
+// key dir), the config offline selection still builds a working pipeline and
+// scores — no key, no env, no network.
+func TestOfflineBackendNeedsNoEnvKeys(t *testing.T) {
+	// Negative control: the env-based resolution must fail in this machine
+	// state, so the offline path below is demonstrably not leaning on it.
+	if _, err := compaction.ResolveBackendIn(t.TempDir(), "", func(string) string { return "" }); err == nil {
+		t.Fatal("no backend should resolve with an empty environment and no key files")
+	}
+
+	// The config selects offline; the resolver honors it with no warning.
+	backend, warning := appconfig.ResolveCompactionBackend(&appconfig.Config{CompactionBackend: appconfig.CompactionBackendOffline})
+	if backend != appconfig.CompactionBackendOffline {
+		t.Fatalf("ResolveCompactionBackend() = %q (warning %q), want %q", backend, warning, appconfig.CompactionBackendOffline)
+	}
+	if warning != "" {
+		t.Fatalf("ResolveCompactionBackend() warning = %q, want none", warning)
+	}
+
+	// The wiring that follows never resolves a backend: it builds the
+	// offline pipeline directly (mirroring main()'s offline branch) and
+	// scores through it.
+	pipeline := compaction.NewOfflinePipeline(compaction.PipelineOptions{})
+	output := strings.Repeat("a", 200) + "\n\n" + strings.Repeat("b", 200) + "\n\n" + strings.Repeat("c", 200)
+	got, err := pipeline.ScoreToolOutput(context.Background(), "Bash", output)
+	if err != nil {
+		t.Fatalf("ScoreToolOutput() error = %v, want nil (offline scoring never touches the network)", err)
+	}
+	if len(got.Scores) != len(got.Segments) || len(got.Segments) != 3 {
+		t.Fatalf("got %d segments / %d scores, want 3/3", len(got.Segments), len(got.Scores))
+	}
+	if len(got.Errors) != 0 {
+		t.Errorf("Errors = %v, want none", got.Errors)
+	}
+}
+
+// TestCheckCompactionOfflinePassPath pins the -check-compaction offline
+// branch at unit level: the three stages (questions, gate, expand) run
+// against the offline scripted scorer and pass, and the flag's exit-code
+// wrapper returns 0.
+func TestCheckCompactionOfflinePassPath(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	results, ok := compaction.RunPreflightOffline(ctx)
+	if !ok {
+		t.Fatalf("offline preflight failed:\n%s", compaction.FormatCheckReport(results, ok))
+	}
+	for _, r := range results {
+		if !r.OK {
+			t.Errorf("stage %q failed: %s", r.Stage, r.Detail)
+		}
+	}
+	stages := map[string]bool{}
+	for _, r := range results {
+		stages[r.Stage] = true
+	}
+	for _, want := range []string{compaction.CheckStageQuestions, compaction.CheckStageGate, compaction.CheckStageExpand} {
+		if !stages[want] {
+			t.Errorf("the three stages must all run offline; missing %q", want)
+		}
+	}
+
+	// The flag wrapper: exit code 0 (it prints the report to stdout).
+	if code := runCompactionCheck(true); code != 0 {
+		t.Errorf("runCompactionCheck(offline) = %d, want 0", code)
+	}
+}
+
+// TestOfflineSessionCompactContextEndToEnd is the history-side half of the
+// offline end-to-end pin: the offline pipeline's scorer drives
+// session.CompactContext (the /jev-compact-context flow) enabled-mode, the
+// rewritten history carries [[elided id=r:…]] content-id pointers, and
+// Reconstruct over the store restores the original message byte for byte.
+// No network anywhere.
+func TestOfflineSessionCompactContextEndToEnd(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "history.json")
+	sess := compactionTestSession(t, path)
+	original := sess.History[1].Content.Text
+
+	pipeline := compaction.NewOfflinePipeline(compaction.PipelineOptions{})
+	store := compaction.NewStore()
+
+	report, err := sess.CompactContext(context.Background(), pipeline.HistoryScorer(), store, session.CompactionOptions{
+		Threshold: 1.0, // below 1.0 always elides: scripted scores are < 1
+	})
+	if err != nil {
+		t.Fatalf("CompactContext() error = %v, want a clean offline walk", err)
+	}
+	if report.ShadowOnly {
+		t.Error("a mutating offline run must not report ShadowOnly")
+	}
+	if report.MessagesCompacted != 1 || report.SegmentsElided < 1 {
+		t.Fatalf("report = %+v, want the assistant message compacted with ≥1 segment elided", report)
+	}
+
+	// The user message is never compacted; the assistant message carries a
+	// content-id pointer.
+	if got := sess.History[0].Content.Text; got != "Please analyze this build log." {
+		t.Errorf("user message mutated:\n%s", got)
+	}
+	compacted := sess.History[1].Content.Text
+	if !strings.Contains(compacted, "[[elided id=r:") {
+		t.Fatalf("compacted history carries no pointer:\n%s", compacted)
+	}
+
+	// Byte-for-byte inverse through the same store.
+	if expanded := compaction.Reconstruct(compacted, store); expanded != original {
+		t.Errorf("Reconstruct did not restore the original byte for byte (%d vs %d bytes)", len(expanded), len(original))
+	}
+
+	// The expand tool's read side (what a real enabled-mode session wires)
+	// resolves the pointer from the same store.
+	if text, ok := store.Get(pointerID(t, compacted)); !ok || text != original {
+		t.Errorf("store.Get(pointer) = (%q, %v), want the original message", text, ok)
+	}
+}
+
+// pointerID extracts the first [[elided id=…]] pointer id from text via the
+// compaction package's own parser.
+func pointerID(t *testing.T, text string) string {
+	t.Helper()
+	for _, line := range strings.Split(text, "\n") {
+		if p, ok := compaction.ParsePointer(line); ok {
+			return p.ID
+		}
+	}
+	t.Fatalf("no pointer in:\n%s", text)
+	return ""
+}