From 39a3093e7750e052036b1babf4e4e57c41fc34e1 Mon Sep 17 00:00:00 2001 From: Emasoft <713559+Emasoft@users.noreply.github.com> Date: Fri, 18 Sep 2026 17:58:53 +0200 Subject: [PATCH 1/4] feat: resilient LLM streaming & recovery, --continue-project, grouped help Focused rework of the streaming/retry PR after owner review (5716775820 on PR #127): the OTP permission mode and install-dev.sh were split out into their own independent PR, so this PR now contains only the streaming/retry work and related improvements. Contents (net diff vs main): - Two-tier stream retry with independent budgets (infrastructure: -max-stream-retries / LATE_MAX_STREAM_RETRIES, default 10, 0 disables; bad-body HTTP 400: 3), 500ms doubling backoff, 30s cap, full jitter; server Retry-After honored as a floor (capped 5 min, cancelable); permanent network causes (x509 trust/hostname/chain, TLS record header, unsupported scheme, HTTP-on-HTTPS) fail fast. - Typed errors: client.StatusError (provider type/code, bounded sanitized diagnostics) and client.StreamInterruptedError for mid-body transport failures (HTTP/2 RST_STREAM, GOAWAY, resets, truncation); the session relay drains the terminal error so a select race can never commit a truncated attempt as a clean turn. - Terminal-400 rollback persisted (pop-to-empty removes the stale history file, meta sidecar kept); per-request history sanitizer (dangling tool_calls closed, empty-ID calls stripped). - Dedicated RecoveryEvent + failure-class-matched recovery toasts; retry status "retry N/M after Xs backoff" (no live-countdown implication); stale retry state cleared on error/stop/close. - --continue restored to global-latest; new --continue-project resolves the repo root (works from subdirs) and matches project dirs by os.SameFile identity; session lookup loads the exact enumerated sidecar, skips vanished metadata, guards nil meta. - Grouped -h with true defaults even when flags precede -h; status-bar agent type; docs (en + zh-CN) updated; plugin hook tests de-flaked (deadline-based pid-file poll, 60s watchdog). Gates: go build ./..., go vet ./..., go test ./... -race -count=1 all green; flagship mid-stream retry tests stress-verified 150/150, 50/50 and 30x under -race. --- cmd/late/continue_project_test.go | 170 +++ cmd/late/help.go | 129 ++ cmd/late/help_test.go | 210 ++++ cmd/late/main.go | 191 ++- cmd/late/main_test.go | 100 +- docs/architecture.md | 21 +- docs/architecture.zh-CN.md | 21 +- docs/quickstart.md | 60 +- docs/quickstart.zh-CN.md | 60 +- internal/client/client.go | 188 ++- internal/client/client_test.go | 240 ++++ internal/client/status_error_test.go | 429 +++++++ internal/common/interfaces.go | 32 +- internal/config/config.go | 51 + internal/config/config_test.go | 137 +++ internal/executor/executor.go | 131 +- internal/executor/stream_retry.go | 239 ++++ .../executor/stream_retry_integration_test.go | 1071 +++++++++++++++++ internal/executor/stream_retry_test.go | 551 +++++++++ internal/git/repo_root_test.go | 95 ++ internal/git/worktree.go | 21 + internal/orchestrator/base.go | 112 ++ internal/orchestrator/base_errors_test.go | 56 + internal/orchestrator/base_retry_test.go | 337 ++++++ internal/plugin/commands_tools_test.go | 7 +- internal/plugin/hooks_unix_test.go | 47 +- internal/plugin/project_test.go | 39 +- internal/plugin/regression_test.go | 6 +- internal/plugin/sandbox_test.go | 37 + internal/plugin/security_test.go | 19 +- internal/session/history_sanitize.go | 103 ++ internal/session/history_sanitize_test.go | 196 +++ internal/session/models.go | 123 +- internal/session/models_test.go | 422 +++++++ internal/session/session.go | 73 +- internal/session/session_pop_test.go | 191 +++ internal/session/session_startstream_test.go | 122 ++ internal/session/ttystyle.go | 9 +- internal/session/ttystyle_test.go | 43 + internal/tool/ast/policy.go | 1 + internal/tool/ast/policy_test.go | 6 +- internal/tool/ast/snapshot_test.go | 2 +- internal/tool/implementations.go | 2 +- internal/tool/implementations_test.go | 1 - internal/tool/search_test.go | 2 - internal/tui/agent_type_test.go | 73 ++ internal/tui/retry_test.go | 538 +++++++++ internal/tui/state.go | 16 + internal/tui/update.go | 89 ++ internal/tui/view.go | 57 +- 50 files changed, 6728 insertions(+), 148 deletions(-) create mode 100644 cmd/late/continue_project_test.go create mode 100644 cmd/late/help.go create mode 100644 cmd/late/help_test.go create mode 100644 internal/client/status_error_test.go create mode 100644 internal/executor/stream_retry.go create mode 100644 internal/executor/stream_retry_integration_test.go create mode 100644 internal/executor/stream_retry_test.go create mode 100644 internal/git/repo_root_test.go create mode 100644 internal/orchestrator/base_errors_test.go create mode 100644 internal/orchestrator/base_retry_test.go create mode 100644 internal/plugin/sandbox_test.go create mode 100644 internal/session/history_sanitize.go create mode 100644 internal/session/history_sanitize_test.go create mode 100644 internal/session/session_pop_test.go create mode 100644 internal/session/session_startstream_test.go create mode 100644 internal/session/ttystyle_test.go create mode 100644 internal/tui/agent_type_test.go create mode 100644 internal/tui/retry_test.go diff --git a/cmd/late/continue_project_test.go b/cmd/late/continue_project_test.go new file mode 100644 index 00000000..6e68f2bc --- /dev/null +++ b/cmd/late/continue_project_test.go @@ -0,0 +1,170 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// initGitRepo creates a git repository at dir, mirroring the plugin package's +// git-fixture tests (which run `git init` directly and fail loudly). +func initGitRepo(t *testing.T, dir string) { + t.Helper() + if out, err := exec.Command("git", "init", dir).CombinedOutput(); err != nil { + t.Fatalf("git init %s: %v: %s", dir, err, out) + } +} + +// sameDir reports whether two paths refer to the same directory on disk, +// tolerating symlink-resolved aliases (e.g. macOS /tmp -> /private/tmp, which +// makes `git rev-parse --show-toplevel` report a different spelling than +// t.TempDir()). +func sameDir(t *testing.T, a, b string) bool { + t.Helper() + ai, err := os.Stat(a) + if err != nil { + t.Fatalf("Stat(%s): %v", a, err) + } + bi, err := os.Stat(b) + if err != nil { + t.Fatalf("Stat(%s): %v", b, err) + } + return os.SameFile(ai, bi) +} + +// TestResolveContinueProjectDir_UsesRepoRoot guards the project resolution +// rule: --continue-project scopes to the git repository root of the current +// working directory, so it also works from inside a subdirectory. +func TestResolveContinueProjectDir_UsesRepoRoot(t *testing.T) { + repo := t.TempDir() + initGitRepo(t, repo) + + sub := filepath.Join(repo, "internal", "deep") + if err := os.MkdirAll(sub, 0700); err != nil { + t.Fatalf("creating subdirectory: %v", err) + } + t.Chdir(sub) + + dir, err := resolveContinueProjectDir() + if err != nil { + t.Fatalf("resolveContinueProjectDir(): %v", err) + } + if !sameDir(t, dir, repo) { + t.Errorf("resolveContinueProjectDir() = %q, want the repo root %q", dir, repo) + } +} + +// TestResolveContinueProjectDir_FallsBackToCwdOutsideRepo guards the +// fallback: outside a git repository, the project is the working directory +// itself. +func TestResolveContinueProjectDir_FallsBackToCwdOutsideRepo(t *testing.T) { + plain := t.TempDir() + if root, ok := gitRepoRootForTest(t, plain); ok && !sameDir(t, root, plain) { + t.Skipf("temp dir %s resolves inside git repo %s; cannot test the no-repo fallback here", plain, root) + } + t.Chdir(plain) + + dir, err := resolveContinueProjectDir() + if err != nil { + t.Fatalf("resolveContinueProjectDir(): %v", err) + } + if !sameDir(t, dir, plain) { + t.Errorf("resolveContinueProjectDir() = %q, want the working directory %q", dir, plain) + } +} + +// gitRepoRootForTest exposes the raw repo-root probe so the fallback test can +// detect a temp directory that unexpectedly lives inside a repository. +func gitRepoRootForTest(t *testing.T, dir string) (string, bool) { + t.Helper() + cmd := exec.Command("git", "-C", dir, "rev-parse", "--show-toplevel") + out, err := cmd.Output() + if err != nil { + return "", false + } + return strings.TrimSpace(string(out)), true +} + +// TestResolveContinueProjectSession_FindsSessionFromSubdirectory covers the +// end-to-end --continue-project lookup: the session was started at the repo +// root, the user runs from a subdirectory, and the repo root (possibly a +// symlink-resolved spelling of the recorded path) matches via directory +// identity. +func TestResolveContinueProjectSession_FindsSessionFromSubdirectory(t *testing.T) { + repo := t.TempDir() + initGitRepo(t, repo) + + sub := filepath.Join(repo, "cmd", "late") + if err := os.MkdirAll(sub, 0700); err != nil { + t.Fatalf("creating subdirectory: %v", err) + } + + sessionsDir := injectSessionDir(t) + writeTestSession(t, sessionsDir, "session-20250101-100000", repo) + + t.Chdir(sub) + + meta, err := resolveContinueProjectSession() + if err != nil { + t.Fatalf("resolveContinueProjectSession(): %v", err) + } + if meta == nil { + t.Fatal("resolveContinueProjectSession() returned nil, want the repo-root session found from a subdirectory") + } + if meta.ID != "session-20250101-100000" { + t.Errorf("resolveContinueProjectSession() = %q, want session-20250101-100000", meta.ID) + } +} + +// TestResolveContinueProjectSession_IgnoresOtherProjects guards the scoping: +// sessions belonging to other projects are never returned, and a project with +// no recorded session resolves to (nil, nil) without an error. +func TestResolveContinueProjectSession_IgnoresOtherProjects(t *testing.T) { + repo := t.TempDir() + initGitRepo(t, repo) + + sub := filepath.Join(repo, "pkg") + if err := os.MkdirAll(sub, 0700); err != nil { + t.Fatalf("creating subdirectory: %v", err) + } + + sessionsDir := injectSessionDir(t) + otherProject := filepath.Join(sessionsDir, "other-project") + if err := os.MkdirAll(otherProject, 0700); err != nil { + t.Fatalf("creating other-project: %v", err) + } + writeTestSession(t, sessionsDir, "session-20250101-100000", otherProject) + + t.Chdir(sub) + + meta, err := resolveContinueProjectSession() + if err != nil { + t.Fatalf("resolveContinueProjectSession(): %v", err) + } + if meta != nil { + t.Fatalf("resolveContinueProjectSession() = %+v, want nil when only other projects have sessions", meta) + } +} + +// TestValidateContinueFlags_MutuallyExclusive guards the --continue / +// --continue-project exclusivity rule. +func TestValidateContinueFlags_MutuallyExclusive(t *testing.T) { + err := validateContinueFlags(true, true) + if err == nil { + t.Fatal("validateContinueFlags(true, true) = nil, want an error") + } + if !strings.Contains(err.Error(), "mutually exclusive") { + t.Errorf("error = %q, want it to mention mutual exclusivity", err) + } + for _, tc := range []struct{ cont, contProject bool }{ + {true, false}, + {false, true}, + {false, false}, + } { + if err := validateContinueFlags(tc.cont, tc.contProject); err != nil { + t.Errorf("validateContinueFlags(%v, %v) = %v, want nil", tc.cont, tc.contProject, err) + } + } +} diff --git a/cmd/late/help.go b/cmd/late/help.go new file mode 100644 index 00000000..7b6a2580 --- /dev/null +++ b/cmd/late/help.go @@ -0,0 +1,129 @@ +package main + +import ( + "flag" + "fmt" + "io" +) + +// flagGroups defines the display order and scope grouping of the root flags +// in -h output, plus an optional note rendered right after a group's flag +// block (note lines are pre-indented to align with the flag names). Keep it +// in sync with the registrations in main(): any flag registered but missing +// here is still rendered, under "Other:", by writeGroupedFlags, so new flags +// can never silently vanish from the help. +var flagGroups = []struct { + heading string + flags []string + note string +}{ + {"General", []string{"help", "version"}, ""}, + {"Session & startup", []string{"continue", "continue-project", "prompt", "theme", "show-cwd"}, ""}, + {"System prompt", []string{"system-prompt", "system-prompt-file", "append-system-prompt", "inject-cwd", "gemma-thinking"}, ""}, + {"Model & streaming", []string{"logit-bias", "suppress-thinking-words", "max-stream-retries"}, ""}, + {"Subagents", []string{"enable-subagents", "subagent-max-turns", "subagent-logit-bias", "save-subagent-histories"}, ""}, + {"Tools", []string{"use-tools", "enable-bash", "enable-images", "enable-sqz"}, ""}, + {"Supervision & safety", []string{"ask-for-user-approval", "i-promise-i-have-backups-and-will-not-file-issues"}, + "These two flags are mutually exclusive: pass at most one. The default\n (ask-for-user-approval) can be changed by adding a \"permission-mode\"\n entry to late's config.json with one of the values above."}, +} + +// writeHelp renders the full `late -h` output. src is the FlagSet whose +// flags are rendered (flag.CommandLine in production; isolated FlagSets in +// tests). Every usage string rendered here must contain no back-quoted +// word: PrintDefaults would render it as the flag's value name. +func writeHelp(w io.Writer, src *flag.FlagSet) { + fmt.Fprintln(w, "Late — the AI agent that always stays sharp.") + fmt.Fprintln(w, "Isolates execution steps to keep the model's context clean during long workflows.") + fmt.Fprintln(w) + fmt.Fprintln(w, "Usage:") + fmt.Fprintln(w, " late [flags]") + fmt.Fprintln(w, " late session [args]") + fmt.Fprintln(w, " late plugin [args]") + fmt.Fprintln(w, " late worktree [args]") + fmt.Fprintln(w) + fmt.Fprintln(w, "Commands:") + fmt.Fprintln(w, " session list [-v] List saved sessions (-v for details)") + fmt.Fprintln(w, " session load Resume a session in the TUI (exact ID or unique prefix)") + fmt.Fprintln(w, " session delete Delete a session by ID") + fmt.Fprintln(w, " plugin list | ls List installed plugins") + fmt.Fprintln(w, " plugin install | i [--project] Install from npm, git, a local path, or a marketplace name") + fmt.Fprintln(w, " plugin remove | rm | uninstall [--project] Remove a plugin") + fmt.Fprintln(w, " plugin link [--project] Symlink a local plugin directory for development") + fmt.Fprintln(w, " plugin update [] Update one plugin, or all when the name is omitted") + fmt.Fprintln(w, " plugin enable Enable a plugin") + fmt.Fprintln(w, " plugin disable Disable a plugin") + fmt.Fprintln(w, " worktree list List git worktrees") + fmt.Fprintln(w, " worktree create [branch] Create a worktree (branch defaults to the current one)") + fmt.Fprintln(w, " worktree remove Remove a worktree") + fmt.Fprintln(w, " worktree active Show the active worktree") + fmt.Fprintln(w) + fmt.Fprintln(w, "Flags:") + writeGroupedFlags(w, src) + fmt.Fprintln(w, "🌟 Enjoying Late? Consider leaving a star on GitHub: https://github.com/mlhher/late-cli") +} + +// registerDisplayFlag registers src's flag f on the display-only FlagSet dfs +// used for help rendering. It keeps the display flag sharing f's real +// registered Value — PrintDefaults derives value names ("string", "int", …) +// and zero-value checks from the Value's concrete type — but restores +// DefValue from src: Var snapshots Value.String() at registration time, which +// would advertise a value the caller mutated before -h (e.g. +// `late -show-cwd=false -h`) instead of the flag's true default. DefValue is +// captured once at registration and is never changed by parsing, so src is +// the single source of truth for what -h must advertise. +func registerDisplayFlag(dfs *flag.FlagSet, f *flag.Flag) { + dfs.Var(f.Value, f.Name, f.Usage) + dfs.Lookup(f.Name).DefValue = f.DefValue +} + +// writeGroupedFlags renders src's flags grouped by scope, in flagGroups +// order, reusing the flag package's PrintDefaults formatting per group, and +// appends each group's note (when non-empty) right after its flag block. +// Each group is rendered through a display-only FlagSet sharing the real +// registered flag Values, so value names stay correct and single-sourced with +// the registrations in main(); defaults are restored from src's DefValue by +// registerDisplayFlag, so they stay true even when flags were parsed before +// -h. +func writeGroupedFlags(w io.Writer, src *flag.FlagSet) { + listed := make(map[string]bool) + for _, g := range flagGroups { + gfs := flag.NewFlagSet(g.heading, flag.ContinueOnError) + gfs.SetOutput(w) + count := 0 + for _, name := range g.flags { + f := src.Lookup(name) + if f == nil { + continue + } + registerDisplayFlag(gfs, f) + listed[name] = true + count++ + } + if count == 0 { + continue + } + fmt.Fprintln(w, g.heading+":") + gfs.PrintDefaults() + if g.note != "" { + fmt.Fprintln(w, " "+g.note) + } + fmt.Fprintln(w) + } + var others []string + src.VisitAll(func(f *flag.Flag) { + if !listed[f.Name] { + others = append(others, f.Name) + } + }) + if len(others) > 0 { + ofs := flag.NewFlagSet("Other", flag.ContinueOnError) + ofs.SetOutput(w) + for _, name := range others { + f := src.Lookup(name) + registerDisplayFlag(ofs, f) + } + fmt.Fprintln(w, "Other:") + ofs.PrintDefaults() + fmt.Fprintln(w) + } +} diff --git a/cmd/late/help_test.go b/cmd/late/help_test.go new file mode 100644 index 00000000..ada41ebd --- /dev/null +++ b/cmd/late/help_test.go @@ -0,0 +1,210 @@ +package main + +import ( + "bytes" + "flag" + "strings" + "testing" +) + +// newHelpTestFlagSet mirrors main()'s root flag registrations (names and +// kinds only) on an isolated FlagSet so rendering can be tested without +// running main(). +func newHelpTestFlagSet(t *testing.T) *flag.FlagSet { + t.Helper() + fs := flag.NewFlagSet("help-test", flag.ContinueOnError) + bools := []string{ + "help", "version", "continue", "continue-project", "show-cwd", "inject-cwd", "gemma-thinking", + "suppress-thinking-words", "save-subagent-histories", "enable-sqz", + "ask-for-user-approval", "i-promise-i-have-backups-and-will-not-file-issues", + "enable-images", + "use-tools", "enable-bash", "enable-subagents", + } + for _, name := range bools { + def := name == "use-tools" || name == "enable-bash" || name == "enable-subagents" + fs.Bool(name, def, "usage of "+name) + } + strs := []string{"system-prompt", "system-prompt-file", "append-system-prompt", "theme", "prompt", "logit-bias", "subagent-logit-bias"} + for _, name := range strs { + fs.String(name, "", "usage of "+name) + } + fs.Int("subagent-max-turns", 500, "usage of subagent-max-turns") + fs.Int("max-stream-retries", 100, "usage of max-stream-retries") + return fs +} + +// countRenderedFlagLines counts output lines that render the flag `name`: +// a line starting with " -" whose flag-name token (up to the first space +// or tab) equals name exactly. Token-boundary matching keeps "-system-prompt" +// from being miscounted against the "-system-prompt-file" line, which a +// plain substring count (" -"+name) would wrongly attribute to both. +func countRenderedFlagLines(out, name string) int { + n := 0 + for _, line := range strings.Split(out, "\n") { + rest, ok := strings.CutPrefix(line, " -") + if !ok { + continue + } + if i := strings.IndexAny(rest, " \t"); i >= 0 { + rest = rest[:i] + } + if rest == name { + n++ + } + } + return n +} + +func TestWriteGroupedFlagsCoversAllGroupedFlagsOnce(t *testing.T) { + var buf bytes.Buffer + writeGroupedFlags(&buf, newHelpTestFlagSet(t)) + out := buf.String() + for _, g := range flagGroups { + for _, name := range g.flags { + if n := countRenderedFlagLines(out, name); n != 1 { + t.Errorf("flag -%s rendered %d times, want exactly 1", name, n) + } + } + } + // Headings appear, in the declared order. + last := -1 + for _, g := range flagGroups { + idx := strings.Index(out, g.heading+":") + if idx < 0 { + t.Fatalf("missing heading %q in output:\n%s", g.heading, out) + } + if idx < last { + t.Errorf("heading %q appears out of order", g.heading) + } + last = idx + } + if strings.Contains(out, "Other:") { + t.Errorf("all grouped flags were listed, unexpected Other section:\n%s", out) + } + // Boolean flags must not render a value name; string flags render "string". + if !strings.Contains(out, " -enable-bash\n") { + t.Errorf("bool flag should render with no value name:\n%s", out) + } + if !strings.Contains(out, " -system-prompt string") { + t.Errorf("string flag should render with 'string' value name:\n%s", out) + } + if !strings.Contains(out, "(default 500)") { + t.Errorf("expected '(default 500)' for subagent-max-turns:\n%s", out) + } +} + +// TestWriteGroupedFlagsShowsTrueDefaultsWhenMutated guards the grouped-help +// default rendering: flag values mutated before -h (e.g. `late +// -show-cwd=false -h`) must not change what the help advertises. Defaults are +// rendered from the DefValue captured at registration in the source FlagSet — +// which parsing never touches — so a render after mutation must be +// byte-identical to an unmutated render of the same flags. +func TestWriteGroupedFlagsShowsTrueDefaultsWhenMutated(t *testing.T) { + newTestFlagSet := func() *flag.FlagSet { + fs := newHelpTestFlagSet(t) + // Extra ungrouped flag so the Other: section is exercised too. + fs.String("zzz-future-flag", "", "usage of zzz-future-flag") + return fs + } + + var baseline bytes.Buffer + writeGroupedFlags(&baseline, newTestFlagSet()) + + // Mirror production order: values are parsed (mutated) before -h renders. + fs := newTestFlagSet() + for _, m := range []struct{ name, value string }{ + {"use-tools", "false"}, // test default true + {"show-cwd", "true"}, // test default false + {"subagent-max-turns", "1"}, // test default 500 + {"theme", "gruvbox"}, // test default "" + {"zzz-future-flag", "later"}, // renders under Other: + } { + if err := fs.Set(m.name, m.value); err != nil { + t.Fatalf("Set(%s, %s): %v", m.name, m.value, err) + } + } + + var mutated bytes.Buffer + writeGroupedFlags(&mutated, fs) + + if got, want := mutated.String(), baseline.String(); got != want { + t.Fatalf("help rendered after mutating flag values must equal the unmutated render\n--- mutated ---\n%s\n--- unmutated ---\n%s", got, want) + } + + out := mutated.String() + // True non-zero defaults must survive the mutation; usage strings are + // "usage of ", so each pattern matches exactly one flag's line. + for _, want := range []string{ + "\tusage of use-tools (default true)\n", + "\tusage of subagent-max-turns (default 500)\n", + } { + if !strings.Contains(out, want) { + t.Errorf("mutated render missing %q:\n%s", want, out) + } + } + // Flags whose true default is the zero value must still render none. + for _, want := range []string{ + "\tusage of show-cwd\n", + "\tusage of theme\n", + "\tusage of zzz-future-flag\n", + } { + if !strings.Contains(out, want) { + t.Errorf("mutated render missing %q:\n%s", want, out) + } + } + // Parsed values must not leak in as advertised defaults ("(default 1)" + // cannot false-match "(default 100)" because of the closing paren). + for _, banned := range []string{"(default \"gruvbox\")", "(default \"later\")", "(default 1)"} { + if strings.Contains(out, banned) { + t.Errorf("mutated render advertises a parsed value as a default (%s):\n%s", banned, out) + } + } +} + +func TestWriteGroupedFlagsUncategorizedFallToOther(t *testing.T) { + fs := newHelpTestFlagSet(t) + fs.String("zzz-future-flag", "", "a flag added later and not yet grouped") + var buf bytes.Buffer + writeGroupedFlags(&buf, fs) + out := buf.String() + if !strings.Contains(out, "Other:") || !strings.Contains(out, " -zzz-future-flag") { + t.Fatalf("ungrouped flag must still be rendered under Other:\n%s", out) + } +} + +func TestWriteHelpSections(t *testing.T) { + var buf bytes.Buffer + writeHelp(&buf, newHelpTestFlagSet(t)) + out := buf.String() + for _, want := range []string{ + "Usage:", "Commands:", "Flags:", + "session list [-v]", "session load ", "session delete ", + "plugin list | ls", "plugin install | i", "plugin remove | rm | uninstall", + "plugin update []", "worktree create [branch]", "worktree active", + "ask-for-user-approval", + } { + if !strings.Contains(out, want) { + t.Errorf("writeHelp output missing %q", want) + } + } +} + +// TestWriteHelp_ShowsPermissionModeNote guards the Supervision & safety note: +// the three mutually exclusive permission flags are grouped together with a +// note explaining that the default (ask-for-user-approval) can be overridden +// via the permission-mode entry in late's config.json. +func TestWriteHelp_ShowsPermissionModeNote(t *testing.T) { + var buf bytes.Buffer + writeHelp(&buf, newHelpTestFlagSet(t)) + out := buf.String() + for _, want := range []string{ + "-ask-for-user-approval", + "mutually exclusive", + "permission-mode", + "config.json", + } { + if !strings.Contains(out, want) { + t.Errorf("writeHelp output missing %q", want) + } + } +} diff --git a/cmd/late/main.go b/cmd/late/main.go index d312724a..bb16d27e 100644 --- a/cmd/late/main.go +++ b/cmd/late/main.go @@ -13,6 +13,7 @@ import ( "os/exec" "path/filepath" "runtime" + "strconv" "strings" "sync" "time" @@ -34,6 +35,13 @@ import ( "golang.org/x/term" ) +// askForUserApprovalUsage is the -h description of -ask-for-user-approval. +// +// This string must contain no back-quoted word: flag.PrintDefaults renders +// the first back-quoted word as the flag's value name, which would advertise +// this boolean flag as taking an argument. +const askForUserApprovalUsage = "Require explicit user approval before running potentially dangerous commands (default; overrides config.json permission-mode)." + // pluginInlineTool adapts a plugin.InlineTool (defined in internal/plugin/tools.go) // into a common.Tool so the CLI's session registry can dispatch invocations to // plugin-declared runners. It exists because upstream repurposed @@ -76,52 +84,44 @@ func (p pluginInlineTool) CallString(args json.RawMessage) string { func main() { // Parse flags - helpReq := flag.Bool("help", false, "Show help") - systemPromptReq := flag.String("system-prompt", "", "Set the system prompt (literal string)") - systemPromptFileReq := flag.String("system-prompt-file", "", "Set the system prompt from a file") - useToolsReq := flag.Bool("use-tools", true, "Enable tool usage (allows LLM to call tools)") - enableBashReq := flag.Bool("enable-bash", true, "Enable bash tool execution") - injectCWDReq := flag.Bool("inject-cwd", true, "Replace ${{CWD}} in system prompt with current working directory") - enableSubagentsReq := flag.Bool("enable-subagents", true, "Enable subagent usage") - gemmaThinkingReq := flag.Bool("gemma-thinking", false, "Prepend <|think|> token to system prompt for Gemma 4 models") - subagentMaxTurns := flag.Int("subagent-max-turns", 500, "Maximum number of turns for subagents (default: 500)") - saveSubagentHistoriesReq := flag.Bool("save-subagent-histories", false, "Persist subagent conversation histories to disk (default: off)") - enableSqzReq := flag.Bool("enable-sqz", false, "Enable sqz context compression (if available)") - appendSystemPromptReq := flag.String("append-system-prompt", "", "Append text to the system prompt after processing") - versionReq := flag.Bool("version", false, "Show version") - unsupervisedReq := flag.Bool("i-promise-i-have-backups-and-will-not-file-issues", false, "Unsupported: Execute all tools without supervision. Do not use this, bad things will happen. You have been warned.") - enableImagesReq := flag.Bool("enable-images", false, "Force enable support for image attachments for unsupported servers.") - continueReq := flag.Bool("continue", false, "Load and start the latest session") - showCWDReq := flag.Bool("show-cwd", true, "Show current working directory in status bar") - themeReq := flag.String("theme", "", "Plugin theme id (':'); falls back to $LATE_THEME") - promptReq := flag.String("prompt", "", "Start the agent immediately with the given prompt") - logitBiasReq := flag.String("logit-bias", "", "Token-bias mappings as raw JSON or key-value pairs (e.g., TOKEN_ID:BIAS,TOKEN_ID:BIAS)") - suppressThinkingWordsReq := flag.Bool("suppress-thinking-words", false, "Apply standard anti-overthinking bias map (dynamically resolved via /tokenize)") - subagentLogitBiasReq := flag.String("subagent-logit-bias", "", "Token-bias mappings for subagents as raw JSON or key-value pairs (e.g., TOKEN_ID:BIAS,TOKEN_ID:BIAS)") + helpReq := flag.Bool("help", false, "Show this help and exit.") + systemPromptReq := flag.String("system-prompt", "", "Replace the built-in system prompt with this text.") + systemPromptFileReq := flag.String("system-prompt-file", "", "Replace the built-in system prompt with a file's contents (highest priority).") + useToolsReq := flag.Bool("use-tools", true, "Offer tools to the main agent at all.") + enableBashReq := flag.Bool("enable-bash", true, "Enable the bash tool.") + injectCWDReq := flag.Bool("inject-cwd", true, "Replace ${{CWD}} in the system prompt with the working directory.") + enableSubagentsReq := flag.Bool("enable-subagents", true, "Allow the agent to spawn subagents.") + gemmaThinkingReq := flag.Bool("gemma-thinking", false, "Prepend the Gemma <|think|> token to the system prompt.") + subagentMaxTurns := flag.Int("subagent-max-turns", 500, "Maximum turns per subagent.") + // LATE_MAX_STREAM_RETRIES optionally overrides the default retry budget + // for LLM stream errors; an explicit -max-stream-retries flag wins over it. + maxStreamRetriesDefault := executor.DefaultMaxStreamRetries + if v := os.Getenv("LATE_MAX_STREAM_RETRIES"); v != "" { + if parsed, err := strconv.Atoi(v); err == nil { + maxStreamRetriesDefault = parsed + } else { + fmt.Fprintf(os.Stderr, "Warning: ignoring invalid LATE_MAX_STREAM_RETRIES %q: %v\n", v, err) + } + } + maxStreamRetries := flag.Int("max-stream-retries", maxStreamRetriesDefault, "Retries for LLM stream errors with backoff; 0 disables. Env: LATE_MAX_STREAM_RETRIES") + saveSubagentHistoriesReq := flag.Bool("save-subagent-histories", false, "Persist subagent histories to disk (overrides session and config).") + enableSqzReq := flag.Bool("enable-sqz", false, "Compress bash tool output with the external 'sqz' binary if available.") + appendSystemPromptReq := flag.String("append-system-prompt", "", "Append this text to the final system prompt.") + versionReq := flag.Bool("version", false, "Print the version and exit.") + unsupervisedReq := flag.Bool("i-promise-i-have-backups-and-will-not-file-issues", false, "UNSUPPORTED: run every tool without user confirmation.") + askForUserApprovalReq := flag.Bool("ask-for-user-approval", false, askForUserApprovalUsage) + enableImagesReq := flag.Bool("enable-images", false, "Force-enable image attachments even if the backend does not advertise vision support.") + continueReq := flag.Bool("continue", false, "Resume the most recently updated session, regardless of which project directory it was started in.") + continueProjectReq := flag.Bool("continue-project", false, "Resume the most recently updated session for the current project (git repo root of the working directory, or the working directory outside a repo); mutually exclusive with -continue.") + showCWDReq := flag.Bool("show-cwd", true, "Show the git branch / working directory in the status bar.") + themeReq := flag.String("theme", "", "Plugin theme id ('plugin:name' or bare name); env: LATE_THEME.") + promptReq := flag.String("prompt", "", "Start the agent immediately with this prompt.") + logitBiasReq := flag.String("logit-bias", "", "Main-agent token bias: JSON object or comma-separated TOKEN_ID:BIAS pairs.") + suppressThinkingWordsReq := flag.Bool("suppress-thinking-words", false, "Bias anti-overthinking tokens (requires the same model for main agent and subagents).") + subagentLogitBiasReq := flag.String("subagent-logit-bias", "", "Subagent token bias: JSON object or comma-separated TOKEN_ID:BIAS pairs.") flag.Usage = func() { - fmt.Fprintf(os.Stderr, "Usage of late:\n") - fmt.Fprintf(os.Stderr, " late [flags]\n") - fmt.Fprintf(os.Stderr, " late session [args]\n") - fmt.Fprintf(os.Stderr, " late plugin [args]\n") - fmt.Fprintf(os.Stderr, " late worktree [args]\n\n") - fmt.Fprintf(os.Stderr, "Commands:\n") - fmt.Fprintf(os.Stderr, " session list [-v] List all saved sessions (use -v for verbose/detailed view)\n") - fmt.Fprintf(os.Stderr, " session load Load a session by ID\n") - fmt.Fprintf(os.Stderr, " session delete Delete a session by ID\n") - fmt.Fprintf(os.Stderr, " plugin list, ls List installed plugins\n") - fmt.Fprintf(os.Stderr, " plugin install [--project] Install a plugin from npm/git/local\n") - fmt.Fprintf(os.Stderr, " plugin remove [--project] Remove a plugin\n") - fmt.Fprintf(os.Stderr, " plugin link [--project] Link a local plugin directory\n") - fmt.Fprintf(os.Stderr, " plugin update [] Update all or a specific plugin\n") - fmt.Fprintf(os.Stderr, " plugin enable Enable a plugin\n") - fmt.Fprintf(os.Stderr, " plugin disable Disable a plugin\n") - fmt.Fprintf(os.Stderr, " worktree list List all worktrees\n") - fmt.Fprintf(os.Stderr, " worktree create [branch] Create a new worktree\n") - fmt.Fprintf(os.Stderr, " worktree remove Remove a worktree\n") - fmt.Fprintf(os.Stderr, " worktree active Show current worktree\n\n") - flag.PrintDefaults() - fmt.Fprintf(os.Stderr, "\n🌟 Enjoying Late? Consider leaving a star on GitHub: https://github.com/mlhher/late-cli\n") + writeHelp(os.Stderr, flag.CommandLine) } flag.Parse() @@ -137,24 +137,58 @@ func main() { return } + // --continue and --continue-project are mutually exclusive: both select + // the session to resume, so asking for two is ambiguous (same rule and + // messaging style as the permission flags). + if err := validateContinueFlags(*continueReq, *continueProjectReq); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + var loadedHistoryPath string var resumedSessionTitle string var loadedSessionMeta *session.SessionMeta - if *continueReq { - meta, err := session.GetLatestSession() + switch { + case *continueReq: + // --continue: resume the most recently updated session overall, + // regardless of the project directory it was started in. + meta, err := resolveContinueSession() if err != nil { fmt.Fprintf(os.Stderr, "Error getting latest session: %v\n", err) os.Exit(1) } if meta == nil { fmt.Fprintln(os.Stderr, "No sessions found to continue.") + fmt.Fprintln(os.Stderr, "Use `late session list` to see saved sessions, or `late session load ` to resume one directly.") os.Exit(1) } loadedHistoryPath = meta.HistoryPath resumedSessionTitle = fmt.Sprintf("Resumed session: %s (%s)", meta.ID, meta.Title) loadedSessionMeta = meta - } else if flag.NArg() > 0 && flag.Arg(0) == "session" { + case *continueProjectReq: + // --continue-project: resume the most recently updated session of + // the current project (git repo root of the working directory, or + // the working directory outside a repo). It works from inside a + // subdirectory because the repo root is matched, not the CWD. + meta, err := resolveContinueProjectSession() + if err != nil { + fmt.Fprintf(os.Stderr, "Error getting latest session: %v\n", err) + os.Exit(1) + } + if meta == nil { + if projectDir, dirErr := resolveContinueProjectDir(); dirErr == nil { + fmt.Fprintf(os.Stderr, "No sessions found to continue in project %s.\n", projectDir) + } else { + fmt.Fprintln(os.Stderr, "No sessions found to continue in the current project.") + } + fmt.Fprintln(os.Stderr, "Use `late session list` to see sessions started in other projects, or `late session load ` to resume one directly.") + os.Exit(1) + } + loadedHistoryPath = meta.HistoryPath + resumedSessionTitle = fmt.Sprintf("Resumed session: %s (%s)", meta.ID, meta.Title) + loadedSessionMeta = meta + case flag.NArg() > 0 && flag.Arg(0) == "session": sessCmdResult := handleSessionCommand(flag.Args()[1:]) if sessCmdResult.ShouldExit { return @@ -375,6 +409,17 @@ func main() { } saveSubagentHistories := appconfig.ResolveSaveSubagentHistories(appConfig, saveSubagentHistoriesCLI, *saveSubagentHistoriesReq, storedSubagentHistoryPreference) + // Resolve the effective permission mode + // (explicit CLI flag > config.json permission-mode > ask-for-user-approval). + permissionMode, permissionModeWarning, err := appconfig.ResolvePermissionMode(appConfig, *askForUserApprovalReq, *unsupervisedReq) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + if permissionModeWarning != "" { + fmt.Fprintf(os.Stderr, "Warning: %s\n", permissionModeWarning) + } + // Initialize Core Components resolvedOpenAIConfig := appconfig.ResolveOpenAISettings(appConfig) resolvedClientConfig := client.Config{ @@ -438,6 +483,9 @@ func main() { sess := session.New(c, historyPath, history, systemPrompt, *useToolsReq) if loadedSessionMeta != nil { sess.SetSubagentMetadata(loadedSessionMeta.SubagentSeq, loadedSessionMeta.SaveSubagentHistories) + if loadedSessionMeta.WorkingDir != "" { + sess.SetWorkingDir(loadedSessionMeta.WorkingDir) + } } else { sess.SetSubagentMetadata(0, &saveSubagentHistories) } @@ -653,9 +701,11 @@ func main() { // Create context with InputProvider ctx := context.WithValue(context.Background(), common.InputProviderKey, tui.NewTUIInputProvider(p)) - if *unsupervisedReq { + switch permissionMode { + case appconfig.PermissionModeUnsupervised: ctx = context.WithValue(ctx, common.SkipConfirmationKey, true) } + ctx = context.WithValue(ctx, common.MaxStreamRetriesKey, *maxStreamRetries) rootAgent.SetContext(ctx) // Set middlewares (see buildMiddlewares for ordering rationale). @@ -907,6 +957,51 @@ type sessionCommandResult struct { ShouldExit bool } +// validateContinueFlags enforces that at most one of --continue and +// --continue-project is passed: both select the session to resume, so +// requesting both is ambiguous. The messaging mirrors the permission-flag +// exclusivity error. +func validateContinueFlags(continueFlag, continueProjectFlag bool) error { + if continueFlag && continueProjectFlag { + return fmt.Errorf("continue flags are mutually exclusive; pass at most one of -continue, -continue-project") + } + return nil +} + +// resolveContinueSession returns the session to resume for --continue: the +// most recently updated session overall, regardless of which project +// directory it was started in. It returns (nil, nil) when no sessions exist. +func resolveContinueSession() (*session.SessionMeta, error) { + return session.GetLatestSession() +} + +// resolveContinueProjectDir returns the project directory that scopes +// --continue-project: the git repository root containing the current working +// directory (so the flag also works from inside a subdirectory), or the +// working directory itself when it is not inside a git repository. +func resolveContinueProjectDir() (string, error) { + cwd, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("determining current directory: %w", err) + } + if root, ok := git.RepoRoot(cwd); ok { + return root, nil + } + return cwd, nil +} + +// resolveContinueProjectSession returns the session to resume for +// --continue-project: the most recently updated session whose recorded +// project directory is the current project. It returns (nil, nil) when no +// matching session exists. +func resolveContinueProjectSession() (*session.SessionMeta, error) { + projectDir, err := resolveContinueProjectDir() + if err != nil { + return nil, err + } + return session.GetLatestSessionForDir(projectDir) +} + // handleSessionCommand processes session subcommands. func handleSessionCommand(args []string) sessionCommandResult { if len(args) == 0 { diff --git a/cmd/late/main_test.go b/cmd/late/main_test.go index 3b89c29c..815d00e6 100644 --- a/cmd/late/main_test.go +++ b/cmd/late/main_test.go @@ -2,10 +2,12 @@ package main import ( "encoding/json" + "flag" "net/http" "net/http/httptest" "os" "path/filepath" + "strings" "testing" "time" @@ -70,7 +72,8 @@ func TestToolEnabled_BareNameFallback(t *testing.T) { // writeTestSession creates a flat session in the injected sessions directory: // /.json (history) and /.meta.json. It returns both paths. -func writeTestSession(t *testing.T, sessionsDir, id string) (metaPath, historyPath string) { +// An optional workingDir argument records the session's working directory. +func writeTestSession(t *testing.T, sessionsDir, id string, workingDir ...string) (metaPath, historyPath string) { t.Helper() historyPath = filepath.Join(sessionsDir, id+".json") @@ -80,14 +83,18 @@ func writeTestSession(t *testing.T, sessionsDir, id string) (metaPath, historyPa t.Fatalf("SaveHistory(%s): %v", id, err) } - if err := session.SaveSessionMeta(session.SessionMeta{ + meta := session.SessionMeta{ ID: id, Title: "Test session " + id, CreatedAt: time.Now(), LastUpdated: time.Now(), HistoryPath: historyPath, MessageCount: 1, - }); err != nil { + } + if len(workingDir) > 0 { + meta.WorkingDir = workingDir[0] + } + if err := session.SaveSessionMeta(meta); err != nil { t.Fatalf("SaveSessionMeta(%s): %v", id, err) } @@ -159,6 +166,72 @@ func TestHandleSessionDelete_LegacyFlatSession(t *testing.T) { assertFileGone(t, historyC) } +// TestResolveContinueSession_ReturnsGlobalLatest guards the --continue +// resolution rule: pick the most recently updated session overall, regardless +// of which project directory it was started in — even when the working +// directory belongs to a different project. Project-scoped resume is +// --continue-project's job. +func TestResolveContinueSession_ReturnsGlobalLatest(t *testing.T) { + tmp := injectSessionDir(t) + + // os.Getwd needs real directories, so the "projects" live inside the temp area. + projA := filepath.Join(tmp, "proj-a") + projB := filepath.Join(tmp, "proj-b") + if err := os.MkdirAll(projA, 0700); err != nil { + t.Fatalf("creating proj-a: %v", err) + } + if err := os.MkdirAll(projB, 0700); err != nil { + t.Fatalf("creating proj-b: %v", err) + } + + _, metaA1 := writeTestSession(t, tmp, "session-20250101-100000", projA) + _, metaA2 := writeTestSession(t, tmp, "session-20250102-100000", projA) + _, metaB1 := writeTestSession(t, tmp, "session-20250103-100000", projB) + + // The helper writes all three back-to-back; pin the meta mtimes so the + // ordering is deterministic. The /proj-b session is the global newest. + base := time.Now().Add(-time.Hour) + for i, metaPath := range []string{metaA1, metaA2, metaB1} { + at := base.Add(time.Duration(i) * time.Hour) + if err := os.Chtimes(metaPath, at, at); err != nil { + t.Fatalf("Chtimes(%s): %v", metaPath, err) + } + } + + // Run from proj-a even though the newest session belongs to proj-b: + // --continue must ignore the current directory entirely. + t.Chdir(projA) + + meta, err := resolveContinueSession() + if err != nil { + t.Fatalf("resolveContinueSession(): %v", err) + } + if meta == nil { + t.Fatal("resolveContinueSession() returned nil, want the globally newest session") + } + if meta.ID != "session-20250103-100000" { + t.Errorf("resolveContinueSession() = %q, want session-20250103-100000 (globally newest session, regardless of directory)", meta.ID) + } +} + +// TestResolveContinueSession_NoMatchReturnsNil guards the empty case: with no +// saved sessions at all, --continue resolves to (nil, nil) rather than an +// error. +func TestResolveContinueSession_NoMatchReturnsNil(t *testing.T) { + injectSessionDir(t) + + empty := t.TempDir() + t.Chdir(empty) + + meta, err := resolveContinueSession() + if err != nil { + t.Fatalf("resolveContinueSession(): %v", err) + } + if meta != nil { + t.Fatalf("resolveContinueSession() = %+v, want nil when no sessions exist", meta) + } +} + func TestDeriveEffectiveSessionID(t *testing.T) { tests := []struct { name string @@ -392,3 +465,24 @@ func TestRunBootstrap_DynamicLogitBias(t *testing.T) { t.Errorf("user bias 999 bled into subagentClient: %v", subBiases) } } + +// TestPermissionFlagUsageRendersWithoutValueName guards the -h output of +// -ask-for-user-approval: its usage string must contain no back-quoted word, +// because flag.UnquoteUsage turns the first back-quoted word into the flag's +// value name and PrintDefaults would then render the boolean flag as taking +// an argument (e.g. "-ask-for-user-approval something"), wrongly implying the +// value is passed on the CLI. +func TestPermissionFlagUsageRendersWithoutValueName(t *testing.T) { + // -ask-for-user-approval's usage string is the package-level + // askForUserApprovalUsage const, single-sourced with the flag + // registration in main(). + fs := flag.NewFlagSet("usage-test", flag.ContinueOnError) + fs.Bool("ask-for-user-approval", false, askForUserApprovalUsage) + if strings.ContainsRune(askForUserApprovalUsage, '`') { + t.Fatalf("ask-for-user-approval usage must not contain backquotes (flag.UnquoteUsage would render the quoted word as the flag's value name): %q", askForUserApprovalUsage) + } + name, _ := flag.UnquoteUsage(fs.Lookup("ask-for-user-approval")) + if name != "" { + t.Errorf("expected no rendered value name for this boolean flag, got %q (help would show -ask-for-user-approval %s)", name, name) + } +} diff --git a/docs/architecture.md b/docs/architecture.md index 55ae79df..a3e8c83c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -119,9 +119,28 @@ Unlike systems where subagent delegation is merely prompt-recommended or optiona --- +## Stream Retry Policy + +Late wraps each LLM stream call in two independent retry tiers, each with its own budget and counter: + +- **Infrastructure tier:** Covers transport errors (connection refused/reset, timeouts, mid-body disconnects) and HTTP 408/429/5xx. Budgeted via `-max-stream-retries` / `LATE_MAX_STREAM_RETRIES` (default: 10; `0` or a negative value disables stream retrying). Precedence is CLI flag > environment variable > built-in default. +- **Mid-stream interruptions:** Mid-body transport failures arrive after the server has already accepted the request (HTTP 200) — HTTP/2 RST_STREAM / INTERNAL_ERROR (classic error text: `stream error: stream ID N; INTERNAL_ERROR; received from peer`), GOAWAY, connection resets, and truncated bodies — so the client wraps them in a typed `StreamInterruptedError`, which is retried from the infrastructure budget. The single exception is an SSE line exceeding the 1 MB scanner cap (`bufio.ErrTooLong`), which fails fast because retrying cannot shrink the line. +- **Bad-body tier:** Covers HTTP 400 only, with a dedicated small budget (3 attempts, `DefaultMaxBadBodyRetries`; not yet flag-configurable). Strict OpenAI-compatible gateways (e.g. z.ai/GLM) frequently fail transiently while reading the request body ("read body failed"), which a few quick retries resolve; genuinely malformed requests still terminate after this small bounded budget. +- **Backoff:** Exponential — 500 ms base doubling per attempt, capped at 30 s, with full jitter (uniform over `[0, cap]`). A server `Retry-After` header is honored as a backoff floor — the combined wait is never shorter than the server requested — capped at 5 minutes so a hostile or buggy server cannot hang an interactive session; the wait remains cancelable throughout. +- **Fail-fast errors:** Errors that retrying cannot help are never retried: TLS certificate/trust failures (untrusted authority, hostname mismatch, invalid or expired chains), non-TLS bytes on a TLS connection, unsupported URL schemes, HTTP-on-HTTPS, context cancellation, and permanent client errors (401/403/404). Unknown errors also fail fast, exactly like the pre-retry behavior. +- **Independent counters:** 400 retries never consume the infrastructure budget, and vice versa. +- **Retry status and recovery:** While an attempt is being retried, the status line reads `retry N/M after Xs backoff` — a statement of the backoff applied before the next attempt, rendered once and never updated, so it does not imply a live countdown. A dedicated `RecoveryEvent` announces recovery the moment a retried attempt succeeds — exactly once per retried turn — instead of waiting for the next turn's "thinking" status. + +When retries are exhausted, two guarantees keep the session usable: + +- **History sanitization:** Per request, histories interrupted mid-tool-run are repaired — dangling assistant `tool_calls` receive synthesized tool results. Only the outgoing request copy is repaired; the saved history is untouched. +- **Terminal 400 rollback:** If the API still rejects the request body after bad-body retries, the last user message is rolled back, with the rollback persisted in all cases — including when the rollback empties the history (the stale history file is removed from disk; the `.meta.json` sidecar is kept so `--continue` scoping still finds the session) — returning the session to its pre-submit state instead of leaving it blocked. + +--- + ## Persistence -- **Root Session History:** Orchestrator conversation history is persisted to disk under `/.json` alongside a `.meta.json` sidecar for state resumption. +- **Root Session History:** Orchestrator conversation history is persisted to disk under `/.json` alongside a `.meta.json` sidecar for state resumption. The sidecar records the project directory where the session was started (`working_dir`), which `--continue-project` uses to scope resume to the current project — the git repository root of the working directory, falling back to the working directory itself outside a repository — with a same-file identity check so symlinked paths still match. - **Optional Subagent Histories:** Active subagent contexts are ephemeral in memory during execution. When enabled, subagent transcripts are persisted to `//subagents/.json` for auditing and debugging. - **Ephemeral Context vs. Disk Audit:** Workers do not leak their raw context into the orchestrator; debugging and post-mortem analysis rely on on-disk transcripts rather than an overloaded central KV cache. diff --git a/docs/architecture.zh-CN.md b/docs/architecture.zh-CN.md index d7738c42..dec47701 100644 --- a/docs/architecture.zh-CN.md +++ b/docs/architecture.zh-CN.md @@ -119,9 +119,28 @@ Late 将规划与执行分离：**主编排器（Lead Orchestrator）**仅保留 --- +## 流式重试策略 + +Late 将每次 LLM 流式调用包裹在两个相互独立的重试层级中，各自拥有独立的预算和计数器： + +- **基础设施层（Infrastructure tier）：** 覆盖传输类错误（连接被拒绝/重置、超时、响应体中途断开）以及 HTTP 408/429/5xx。预算通过 `-max-stream-retries` / `LATE_MAX_STREAM_RETRIES` 控制（默认 10；`0` 或负值会完全禁用流式重试）。优先级为 CLI 标志 > 环境变量 > 内置默认值。 +- **流式中断：** 响应体中途的传输故障发生在服务器已经接受请求（HTTP 200）之后——包括 HTTP/2 RST_STREAM / INTERNAL_ERROR（典型报错文本：`stream error: stream ID N; INTERNAL_ERROR; received from peer`）、GOAWAY、连接重置以及响应体被截断——因此客户端会将其包装为类型化的 `StreamInterruptedError`，并从基础设施层预算中进行重试。唯一的例外是超出 1 MB 扫描器上限（`bufio.ErrTooLong`）的 SSE 行——它会快速失败，因为重试无法缩短该行。 +- **无效请求体层（Bad-body tier）：** 仅覆盖 HTTP 400，使用一个专用的小预算（3 次，常量 `DefaultMaxBadBodyRetries`；暂不支持通过命令行标志配置）。严格的 OpenAI 兼容网关（如 z.ai/GLM）在读取请求体时经常发生瞬时故障（"read body failed"），少量快速重试即可解决；而真正格式错误的请求仍会在这一小额有界预算耗尽后终止。 +- **退避（Backoff）：** 指数退避——以 500 ms 为基数逐次翻倍，上限 30 s，并叠加完全抖动（full jitter，在 `[0, cap]` 区间内均匀取值）。服务器返回的 `Retry-After` 会被作为退避下限遵守——合并后的等待时间绝不短于服务器要求的时长——并设有 5 分钟上限，以防止恶意或有缺陷的服务器挂起交互会话；等待过程始终可以取消。 +- **快速失败（Fail-fast）：** 重试无法解决的错误绝不会重试：TLS 证书/信任故障（不受信任的颁发机构、主机名不匹配、无效或过期的证书链）、TLS 连接上的非 TLS 字节、不支持的 URL scheme、HTTPS 端点上的纯 HTTP、上下文取消，以及永久性的客户端错误（401/403/404）。未知错误同样会快速失败，与引入重试之前的行为完全一致。 +- **计数器相互独立：** 400 的重试不会消耗基础设施层的预算，反之亦然。 +- **重试状态与恢复：** 当某次尝试正在被重试时，状态行会显示为 `retry N/M after Xs backoff`——即本次失败后、下次尝试前应用的退避时长，只渲染一次且不会更新，因此并不表示实时倒计时。一个专用的 `RecoveryEvent` 会在重试后的尝试成功的瞬间宣布恢复——每个发生重试的轮次恰好触发一次——而不是等待下一个轮次的 "thinking" 状态。 + +当重试预算耗尽时，以下两项保证让会话仍然可用： + +- **历史净化（History sanitization）：** 每次请求前会修复在工具执行中途被打断的历史——为悬空的助手 `tool_calls` 合成对应的工具结果。只有发往 API 的请求副本会被修复，已保存的历史保持不变。 +- **终态 400 回滚：** 如果 API 在无效请求体重试之后仍然拒绝请求体，最后一条用户消息会被回滚，且回滚结果在所有情况下都会持久化——包括回滚导致历史为空的情况（过期的历史文件会从磁盘删除，同时保留 `.meta.json` 文件以便 `--continue` 仍能定位该会话）——使会话回到提交前的状态，而不是陷入阻塞。 + +--- + ## 持久化 -- **根会话历史：** 编排器的对话历史持久化到磁盘上的 `/.json` 中，并配有用于状态恢复的 `.meta.json` 文件。 +- **根会话历史：** 编排器的对话历史持久化到磁盘上的 `/.json` 中，并配有用于状态恢复的 `.meta.json` 文件。该文件记录会话启动时所在的项目目录（`working_dir`），`--continue-project` 用它将恢复范围限定为当前项目——当前工作目录所在的 git 仓库根目录，不在仓库中时回退为当前工作目录本身——并通过同文件（same-file）身份检查使符号链接路径同样可以匹配。 - **可选的子智能体历史：** 执行期间，活动的子智能体上下文临时驻留在内存中。启用后，子智能体的完整会话记录会被持久化到 `//subagents/.json` 以供审计和调试。 - **临时上下文 vs 磁盘审计：** 工作智能体不会将其原始上下文传回编排器；调试和事后分析依赖于磁盘上的会话记录，而不是过载的核心 KV 缓存。 diff --git a/docs/quickstart.md b/docs/quickstart.md index 1259a580..9493131a 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -77,6 +77,27 @@ Configuration precedence is: For the standard local `llama-server` setup on `localhost:8080`, you do not need to create a configuration file. +### Tool Approval Mode (`permission-mode`) + +You can choose how much supervision Late applies to dangerous commands by adding a `permission-mode` entry to the `config.json` file for your platform (see the locations above): + +```json +{ + "permission-mode": "ask-for-user-approval" +} +``` + +The two allowed values are: + +* `ask-for-user-approval` — the default. Potentially dangerous commands require your approval. +* `i-promise-i-have-backups-and-will-not-file-issues` — run every tool without user confirmation. + +Notes: + +* The two CLI flags of the same names (`--ask-for-user-approval`, `--i-promise-i-have-backups-and-will-not-file-issues`) are mutually exclusive and override the `config.json` value. +* Omitting the entry (and any flag) defaults to `ask-for-user-approval`. +* An invalid value is ignored with a warning and the safe default applies. + ### Advanced Model Configuration (`models` and `agent_models`) By default, Late uses the same model for the orchestrator and its subagents. However, you can map different models to specific agent roles (e.g., using a massive frontier model for planning, and a faster local model for execution). @@ -211,16 +232,23 @@ late-podman -- --prompt "Refactor this package and verify all tests." Late automatically saves sessions. -Resume the previous session: +Resume the most recently updated session, no matter which project it belongs to: ```bash late --continue ``` -Or inspect saved sessions: +Resume the most recently updated session for the **current project**. The project is resolved as the git repository root containing your working directory (falling back to the working directory itself outside a repository), so this also works from inside a subdirectory: + +```bash +late --continue-project +``` + +The two flags are mutually exclusive: pass at most one. Sessions from other projects — or ones created before the project directory was recorded — can be found with `late session list` (use `-v` to see each session's project folder) and resumed with `late session load `: ```bash -late session list +late session list -v +late session load ``` --- @@ -290,13 +318,32 @@ You can also create an `.llmignore` file alongside your `.gitignore` to specific --- +## Stream Retries + +Transient LLM API failures are retried automatically, so a flaky gateway rarely interrupts a run: + +* Transport errors — connection refused/reset, timeouts, and mid-stream disconnects (the server accepted the request, then the body died) — are retried. +* HTTP 408, 429, and 5xx responses are retried; HTTP 400 gets a few quick retries. +* Each retry waits a jittered exponential backoff (500 ms base, capped at 30 s). A server `Retry-After` header is honored as a floor, capped at 5 minutes. +* Failures that retrying cannot fix fail fast: TLS/certificate errors, unsupported URL schemes, and plain HTTP on an HTTPS endpoint. + +The retry budget is resolved as CLI flag > environment variable > built-in default: + +* `--max-stream-retries ` — maximum retries per stream call (default: 10) +* `LATE_MAX_STREAM_RETRIES` — environment variable, used when the flag is not passed + +Setting `0` (or a negative value) disables stream retrying entirely. Run `late -h` to see all flags. + +--- + ## Common Flags | Flag | Description | | --- | --- | | `--help` | Show all flags and commands | | `--version` | Show version information | -| `--continue` | Resume the previous session | +| `--continue` | Resume the latest session, regardless of project directory | +| `--continue-project` | Resume the latest session in the current project (git repo root of the working directory; falls back to the working directory outside a repo) | | `--prompt "..."` | Start the agent immediately with the given prompt | | `--suppress-thinking-words` | Apply a default Logit bias map of overthinking words (`llama.cpp` only) | | `--logit-bias` and `--subagent-logit-bias` | Manually set the logit biases for specific models (`llama.cpp` only) | @@ -305,4 +352,9 @@ You can also create an `.llmignore` file alongside your `.gitignore` to specific | `--append-system-prompt "..."` | Append text to the system prompt (e.g. further instructions) | | `--enable-images` | Treat models as supporting images (for non llama.cpp servers) | | `--save-subagent-histories` | Persist subagent conversation histories to disk | +| `--max-stream-retries ` | Max retries per LLM stream call with jittered exponential backoff (default: 10); `0` disables stream retrying. Env: `LATE_MAX_STREAM_RETRIES` | +| `--ask-for-user-approval` | Require user approval for dangerous commands (default; overrides `config.json` `permission-mode`) | +| `--i-promise-i-have-backups-and-will-not-file-issues` | Run every tool without user confirmation (overrides `config.json` `permission-mode`) | + +The two permission flags above are mutually exclusive: pass at most one of `--ask-for-user-approval` or `--i-promise-i-have-backups-and-will-not-file-issues`. diff --git a/docs/quickstart.zh-CN.md b/docs/quickstart.zh-CN.md index 7e7b8399..5e93884f 100644 --- a/docs/quickstart.zh-CN.md +++ b/docs/quickstart.zh-CN.md @@ -77,6 +77,27 @@ late 对于在 `localhost:8080` 上运行的标准本地 `llama-server`，你不需要创建配置文件。 +### 工具授权模式（`permission-mode`） + +你可以通过在所用平台对应的 `config.json`（位置见上文）中添加 `permission-mode` 条目，来选择 Late 对危险命令的监督程度： + +```json +{ + "permission-mode": "ask-for-user-approval" +} +``` + +有两个可选值： + +* `ask-for-user-approval` — 默认值。可能具有破坏性的命令需要你的批准。 +* `i-promise-i-have-backups-and-will-not-file-issues` — 运行所有工具而不需要用户确认。 + +注意事项： + +* 两个同名的 CLI 标志（`--ask-for-user-approval`、`--i-promise-i-have-backups-and-will-not-file-issues`）互斥，且会覆盖 `config.json` 中的值。 +* 省略该条目（以及任何标志）时，默认为 `ask-for-user-approval`。 +* 无效的值会被忽略并给出警告，同时应用安全的默认值。 + ### 高级模型配置（`models` 和 `agent_models`） 默认情况下，Late 为主编排器和子智能体使用同一个模型。不过，你可以将不同的模型映射到特定的智能体角色（例如，使用庞大的前沿模型进行规划，使用快速的本地模型进行执行）。 @@ -211,16 +232,23 @@ late-podman -- --prompt "Refactor this package and verify all tests." Late 会自动保存会话。 -恢复上一个会话： +恢复最近更新的会话，无论它属于哪个项目： ```bash late --continue ``` -或者查看已保存的会话： +恢复**当前项目**中最近更新的会话。项目会解析为包含当前工作目录的 git 仓库根目录（不在仓库中时回退为当前工作目录本身），因此在子目录中同样有效： + +```bash +late --continue-project +``` + +这两个标志互斥：最多只能传入一个。其他项目中的会话——或在此功能引入之前创建的会话——可以使用 `late session list` 查找（使用 `-v` 查看每个会话的项目目录），并使用 `late session load ` 恢复： ```bash -late session list +late session list -v +late session load ``` --- @@ -289,13 +317,32 @@ Late 的原生搜索工具会自动遵守你项目的 `.gitignore`，通过排 --- +## 流式重试 + +瞬时的 LLM API 故障会被自动重试，因此不稳定的网关很少会中断一次运行： + +* 传输类错误——连接被拒绝/重置、超时，以及流式中断（服务器已接受请求，但响应体随后断开）——会被重试。 +* HTTP 408、429 和 5xx 响应会被重试；HTTP 400 会获得少量快速重试。 +* 每次重试都会等待叠加抖动的指数退避（以 500 ms 为基数，上限 30 s）。服务器返回的 `Retry-After` 会被作为等待下限遵守，并设有 5 分钟上限。 +* 重试无法解决的故障会立即失败：TLS/证书错误、不支持的 URL scheme，以及在 HTTPS 端点上使用纯 HTTP。 + +重试预算按 CLI 标志 > 环境变量 > 内置默认值 的优先级解析： + +* `--max-stream-retries ` — 每次流式调用的最大重试次数（默认：10） +* `LATE_MAX_STREAM_RETRIES` — 环境变量，在未传入标志时生效 + +设置为 `0`（或负值）会完全禁用流式重试。运行 `late -h` 可查看所有标志。 + +--- + ## 常用标志 (Common Flags) | 标志 | 描述 | | --- | --- | | `--help` | 显示所有标志和命令 | | `--version` | 显示版本信息 | -| `--continue` | 恢复上一个会话 | +| `--continue` | 恢复最近更新的会话，不限项目目录 | +| `--continue-project` | 恢复当前项目中最近更新的会话（解析为当前工作目录所在的 git 仓库根目录；不在仓库中时回退为当前工作目录） | | `--prompt "..."` | 使用给定的 prompt 立即启动智能体 | | `--suppress-thinking-words` | 应用关于过度思考词汇的默认 Logit 偏置映射（仅限 `llama.cpp`） | | `--logit-bias` 和 `--subagent-logit-bias` | 手动设置特定模型的 logit 偏置（仅限 `llama.cpp`） | @@ -304,3 +351,8 @@ Late 的原生搜索工具会自动遵守你项目的 `.gitignore`，通过排 | `--append-system-prompt "..."` | 在系统 prompt 附加文本（如：额外指令） | | `--enable-images` | 将模型视为支持图像（适用于非 llama.cpp 的服务器） | | `--save-subagent-histories` | 将子智能体的对话记录持久化到磁盘 | +| `--max-stream-retries ` | 每次 LLM 流式调用的最大重试次数，采用叠加抖动的指数退避（默认：10）；`0` 表示禁用流式重试。环境变量：`LATE_MAX_STREAM_RETRIES` | +| `--ask-for-user-approval` | 要求对危险命令进行用户批准（默认值；覆盖 `config.json` 中的 `permission-mode`） | +| `--i-promise-i-have-backups-and-will-not-file-issues` | 运行所有工具而不需要用户确认（覆盖 `config.json` 中的 `permission-mode`） | + +上面的两个权限标志互斥：`--ask-for-user-approval` 和 `--i-promise-i-have-backups-and-will-not-file-issues` 最多只能传入一个。 diff --git a/internal/client/client.go b/internal/client/client.go index 2dd46bd5..bf53def5 100644 --- a/internal/client/client.go +++ b/internal/client/client.go @@ -9,9 +9,12 @@ import ( "io" "net/http" "net/url" + "strconv" "strings" "sync" "time" + "unicode" + "unicode/utf8" ) type Config struct { @@ -192,6 +195,11 @@ func (c *Client) ChatCompletionStream(ctx context.Context, req ChatCompletionReq } scanner := bufio.NewScanner(resp.Body) + // Some providers emit very long SSE lines (e.g. huge tool-call argument + // deltas or inline base64 parts). The default 64 KB scanner limit would + // abort the stream with bufio.ErrTooLong, which callers cannot recover + // from, so raise the cap. + scanner.Buffer(make([]byte, 0, 64*1024), 1<<20) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "data: ") { @@ -224,7 +232,7 @@ func (c *Client) ChatCompletionStream(ctx context.Context, req ChatCompletionReq // or an empty data line, check for read errors and propagate them. if err := scanner.Err(); err != nil { select { - case errCh <- fmt.Errorf("stream interrupted: %w", err): + case errCh <- &StreamInterruptedError{Err: err}: default: } } @@ -287,7 +295,7 @@ func (c *Client) HealthCheck(ctx context.Context) error { } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return fmt.Errorf("status: %d", resp.StatusCode) + return &StatusError{StatusCode: resp.StatusCode, Status: resp.Status} } return nil } @@ -548,7 +556,6 @@ func parsePropsBodyData(body []byte) (int, bool) { return nCtx, vis } - func (c *Client) getBackend() BackendType { c.mu.RLock() defer c.mu.RUnlock() @@ -610,12 +617,179 @@ func (c *Client) marshalFlattened(req ChatCompletionRequest) ([]byte, error) { return json.Marshal(m) } +// StatusError is a non-2xx HTTP response returned by the LLM server. +// It is errors.As-able so callers can classify retryability by status code. +// When the provider's JSON error body includes error.type / error.code, they +// are preserved in Type and Code (the code may be a string or a number) for +// retry classification and incident diagnosis. +type StatusError struct { + StatusCode int + Status string // e.g. "500 Internal Server Error" + // Body is the diagnostic response body: the provider's error.message when + // the body is a JSON API error, otherwise a bounded, sanitized text + // fallback. It is truncated to at most 1024 bytes (rune-safe). + Body string + // RetryAfter is the delay requested by the server via the Retry-After + // header (delta-seconds or HTTP-date form), 0 when absent or invalid. + // The executor honors it as a floor for the retry backoff. + RetryAfter time.Duration + Code any // provider error code (JSON error.code), if provided + Type string // provider error type (JSON error.type), if provided +} + +// Error renders the same messages the previous fmt.Errorf calls produced, +// so logs and tests that match on the text keep working: +// "API error (%d): %s" when a body/message is available, "status: %d" otherwise. +func (e *StatusError) Error() string { + if e.Body != "" { + return fmt.Sprintf("API error (%d): %s", e.StatusCode, e.Body) + } + return fmt.Sprintf("status: %d", e.StatusCode) +} + +// StreamInterruptedError reports a transport failure while reading a +// 200-OK response body mid-stream: connection reset, HTTP/2 RST_STREAM +// or GOAWAY, truncated body. The server already accepted the request, +// so the failure is infrastructure, not the request's content. +type StreamInterruptedError struct { + Err error // underlying transport error (e.g. http2 StreamError) +} + +func (e *StreamInterruptedError) Error() string { + return fmt.Sprintf("stream interrupted: %v", e.Err) +} + +func (e *StreamInterruptedError) Unwrap() error { return e.Err } + +const ( + // maxErrorBodyBytes bounds how much of an error response body is read + // before parsing: a hostile or broken server can send arbitrarily large + // bodies, and slurping one whole could exhaust memory. + maxErrorBodyBytes = 8192 + // maxErrorMessageBytes bounds the diagnostic text stored on StatusError, + // for both the structured JSON message and the sanitized text fallback. + maxErrorMessageBytes = 1024 +) + +// formatError converts a non-2xx response into a *StatusError. The error body +// is read exactly once, bounded by maxErrorBodyBytes: a decoder that consumes +// bytes before failing would lose them, and an unbounded read could exhaust +// memory on a hostile server. When the body is a JSON API error, its message, +// type, and code are preserved; otherwise a bounded, sanitized text fallback +// keeps plain-text and HTML failures diagnosable. A Retry-After header +// (delta-seconds or HTTP-date form) is captured so the retry executor can +// honor the server's requested pacing. func (c *Client) formatError(resp *http.Response) error { + se := &StatusError{ + StatusCode: resp.StatusCode, + Status: resp.Status, + } + // Read the error body once, bounded. Read errors are best-effort: the + // body is treated as empty so the status is still reported. + body, _ := io.ReadAll(io.LimitReader(resp.Body, maxErrorBodyBytes)) var apiErr APIErrorResponse - if err := json.NewDecoder(resp.Body).Decode(&apiErr); err == nil && apiErr.Error.Message != "" { - return fmt.Errorf("API error (%d): %s", resp.StatusCode, apiErr.Error.Message) + if err := json.Unmarshal(body, &apiErr); err == nil { + // Preserve provider error type/code when present; zero values stay unset. + if apiErr.Error.Type != "" { + se.Type = apiErr.Error.Type + } + if apiErr.Error.Code != nil { + se.Code = apiErr.Error.Code + } + if apiErr.Error.Message != "" { + se.Body = truncateErrorText(apiErr.Error.Message, maxErrorMessageBytes) + } else { + // JSON body without a structured message: fall back to the + // sanitized text so something is still diagnosable. + se.Body = sanitizeErrorText(string(body), maxErrorMessageBytes) + } + } else { + // Structured message unavailable: bounded, sanitized text fallback + // so plain-text/HTML failures are still diagnosable. + se.Body = sanitizeErrorText(string(body), maxErrorMessageBytes) + } + if ra := parseRetryAfter(resp.Header.Get("Retry-After")); ra > 0 { + se.RetryAfter = ra + } + return se +} + +// parseRetryAfter parses a Retry-After header value in either delta-seconds +// ("2") or HTTP-date ("Wed, 21 Oct 2015 07:28:00 GMT") form. Empty, invalid, +// and non-positive values yield 0, which callers treat as "no requested +// delay"; HTTP dates are measured against the current time. +func parseRetryAfter(v string) time.Duration { + return parseRetryAfterAt(v, time.Now()) +} + +// parseRetryAfterAt is parseRetryAfter with an injectable clock so the +// HTTP-date form can be tested deterministically. +func parseRetryAfterAt(v string, now time.Time) time.Duration { + v = strings.TrimSpace(v) + if v == "" { + return 0 + } + if secs, err := strconv.Atoi(v); err == nil { + if secs <= 0 { + return 0 + } + return time.Duration(secs) * time.Second } - return fmt.Errorf("status: %d", resp.StatusCode) + if date, err := http.ParseTime(v); err == nil { + // Delay until the requested instant; 0 when it already passed. + if d := date.Sub(now); d > 0 { + return d + } + return 0 + } + return 0 +} + +// sanitizeErrorText turns an arbitrary error body (plain text, HTML, binary +// junk) into a bounded diagnostic string: \r is dropped, control characters +// other than \n and \t are stripped, and runs of whitespace (including +// newlines and tabs) collapse to a single space, so the result is effectively +// single-line. It is trimmed and then truncated to at most limit bytes +// (rune-safe). +func sanitizeErrorText(s string, limit int) string { + var b strings.Builder + b.Grow(len(s)) + lastSpace := false + for _, r := range s { + if r == '\r' { + continue + } + if unicode.IsControl(r) && r != '\n' && r != '\t' { + continue + } + if unicode.IsSpace(r) { + if !lastSpace { + b.WriteRune(' ') + lastSpace = true + } + continue + } + lastSpace = false + b.WriteRune(r) + } + return truncateErrorText(strings.TrimSpace(b.String()), limit) +} + +// truncateErrorText limits s to at most limit bytes without splitting a +// multi-byte rune: when limit falls inside a rune, the cut moves back to the +// start of that rune. +func truncateErrorText(s string, limit int) string { + if limit <= 0 { + return "" + } + if len(s) <= limit { + return s + } + cut := limit + for cut > 0 && !utf8.RuneStart(s[cut]) { + cut-- + } + return s[:cut] } func (c *Client) APIKey() string { @@ -661,5 +835,3 @@ func (c *Client) mergeLogitBias(reqBias map[string]int) map[string]int { defer c.mu.RUnlock() return MergeLogitBiases(c.cfg.LogitBias, reqBias) } - - diff --git a/internal/client/client_test.go b/internal/client/client_test.go index dbb509b0..25aafb55 100644 --- a/internal/client/client_test.go +++ b/internal/client/client_test.go @@ -1,8 +1,13 @@ package client import ( + "bufio" "context" + "encoding/json" + "errors" "fmt" + "io" + "net" "net/http" "net/http/httptest" "strings" @@ -341,6 +346,110 @@ func TestChatCompletionStream_InvalidJSON(t *testing.T) { } } +func TestChatCompletionStream_OversizedSSELine(t *testing.T) { + st := newStreamTest(t) + defer st.Close() + + // Build a single SSE data line whose JSON payload is ~600 KB: a chunk + // with a delta.content string of 600,000 chars. Marshal a + // ChatCompletionChunk so the JSON is guaranteed to be valid. + const bigLen = 600000 + bigContent := strings.Repeat("a", bigLen) + bigChunk := ChatCompletionChunk{ + ID: "c1", + Choices: []ChatCompletionChunkChoice{ + {Delta: ChatMessage{Content: TextContent(bigContent)}}, + }, + } + payload, err := json.Marshal(bigChunk) + if err != nil { + t.Fatalf("failed to marshal oversized chunk: %v", err) + } + + st.Handle(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprintf(w, "data: %s\n", payload) + fmt.Fprint(w, "data: [DONE]\n\n") + })) + + chunks, err := collectStream(t, context.Background(), st.client, defaultRequest()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Exactly one chunk should be delivered — the oversized line must not + // trip bufio.ErrTooLong and abort the stream. + if got := len(chunks); got != 1 { + t.Fatalf("got %d chunks, want 1", got) + } + if len(chunks[0].Choices) == 0 { + t.Fatal("chunk has no choices") + } + if got := chunks[0].Choices[0].Delta.Content.String(); len(got) != bigLen { + t.Errorf("chunk content length = %d, want %d", len(got), bigLen) + } +} + +// TestChatCompletionStream_LineOverScannerCap mirrors +// TestChatCompletionStream_OversizedSSELine but pushes the single data line +// past the client's 1 MB scanner cap (~1.5 MB). The scanner must abort with +// bufio.ErrTooLong and the stream must surface a *StreamInterruptedError on +// the error channel with no chunks delivered. +func TestChatCompletionStream_LineOverScannerCap(t *testing.T) { + st := newStreamTest(t) + defer st.Close() + + // Build a single SSE data line over the 1 MB scanner cap: a chunk with a + // delta.content string of 1,500,000 chars. Marshal a ChatCompletionChunk + // so the JSON is guaranteed to be valid. + const bigLen = 1500000 + bigContent := strings.Repeat("a", bigLen) + bigChunk := ChatCompletionChunk{ + ID: "c1", + Choices: []ChatCompletionChunkChoice{ + {Delta: ChatMessage{Content: TextContent(bigContent)}}, + }, + } + payload, err := json.Marshal(bigChunk) + if err != nil { + t.Fatalf("failed to marshal over-cap chunk: %v", err) + } + if got := len("data: ") + len(payload); got <= 1<<20 { + t.Fatalf("test fixture line length = %d, want > %d (scanner cap)", got, 1<<20) + } + + st.Handle(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprintf(w, "data: %s\n", payload) + fmt.Fprint(w, "data: [DONE]\n\n") + })) + + chunks, err := collectStream(t, context.Background(), st.client, defaultRequest()) + + // The scanner aborts before emitting anything: zero chunks must arrive. + if got := len(chunks); got != 0 { + t.Errorf("got %d chunks, want 0 (the over-cap line must abort the stream)", got) + } + + // The scanner failure must surface as a *StreamInterruptedError wrapping + // bufio.ErrTooLong. + if err == nil { + t.Fatal("expected an error from the error channel, got nil") + } + var sie *StreamInterruptedError + if !errors.As(err, &sie) { + t.Fatalf("error = %v (%T), want errors.As to match *StreamInterruptedError", err, err) + } + if !errors.Is(err, bufio.ErrTooLong) { + t.Errorf("error = %v, want it to wrap bufio.ErrTooLong", err) + } + // The rendered message must stay byte-identical to the previous + // fmt.Errorf("stream interrupted: %w", err). + if got, want := err.Error(), "stream interrupted: bufio.Scanner: token too long"; got != want { + t.Errorf("error message = %q, want %q", got, want) + } +} + func TestChatCompletionStream_ContextCancellation(t *testing.T) { st := newStreamTest(t) defer st.Close() @@ -417,3 +526,134 @@ func TestChatCompletionStream_ContextCancellation(t *testing.T) { t.Fatal("error channel not closed after context cancellation") } } + +func TestStreamInterruptedError(t *testing.T) { + sie := &StreamInterruptedError{Err: errors.New("boom")} + if got := sie.Error(); got != "stream interrupted: boom" { + t.Errorf("Error() = %q, want %q", got, "stream interrupted: boom") + } + + // Unwrap keeps errors.Is working through the chain. + outer := fmt.Errorf("outer: %w", &StreamInterruptedError{Err: io.ErrUnexpectedEOF}) + if !errors.Is(outer, io.ErrUnexpectedEOF) { + t.Errorf("errors.Is(%v, io.ErrUnexpectedEOF) = false, want true", outer) + } + + // errors.As finds the type through a double wrap. + double := fmt.Errorf("level1: %w", fmt.Errorf("level2: %w", &StreamInterruptedError{Err: io.ErrUnexpectedEOF})) + var found *StreamInterruptedError + if !errors.As(double, &found) { + t.Fatalf("errors.As(%v, *StreamInterruptedError) = false, want true", double) + } + if !errors.Is(found.Err, io.ErrUnexpectedEOF) { + t.Errorf("found.Err = %v, want io.ErrUnexpectedEOF", found.Err) + } +} + +// TestChatCompletionStream_MidBodyDisconnectErrorType proves the typed error +// is produced end-to-end: a 200 whose body is truncated mid-stream must reach +// the error channel as a *StreamInterruptedError wrapping the transport cause. +// It mirrors the hijack pattern of the executor's +// TestRunLoopMidBodyDisconnectRetries. +func TestChatCompletionStream_MidBodyDisconnectErrorType(t *testing.T) { + st := newStreamTest(t) + defer st.Close() + + st.Handle(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // DiscoverBackend probes /props and /v1/models before the POST; keep + // those on the plain handler so only the stream request is hijacked. + if r.URL.Path != "/v1/chat/completions" { + w.WriteHeader(http.StatusNotFound) + return + } + + hijacker, ok := w.(http.Hijacker) + if !ok { + t.Error("server ResponseWriter does not support Hijack") + return + } + conn, _, err := hijacker.Hijack() + if err != nil { + t.Errorf("hijack failed: %v", err) + return + } + defer conn.Close() + + // A valid 200 whose declared Content-Length exceeds the bytes sent: + // one complete SSE data line, then a partial line with no newline + // terminator, then FIN. The unterminated line forces the scanner to + // read again, where net/http surfaces the short body as + // io.ErrUnexpectedEOF (verified against this Go version; an RST-style + // close would surface *net.OpError instead and is deliberately + // avoided). The trailing empty line of a normal SSE frame is + // deliberately omitted: the loop treats an empty data line as a clean + // end-of-stream and would miss the transport error entirely. + head := "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: 1000\r\n\r\n" + complete := "data: " + sampleChunkHello + "\n" + partial := "data: " + sampleChunkWorld // no trailing newline + if _, err := conn.Write([]byte(head)); err != nil { + return + } + if _, err := conn.Write([]byte(complete)); err != nil { + return + } + if _, err := conn.Write([]byte(partial)); err != nil { + return + } + // FIN: the client hits EOF before the declared Content-Length. + if tcp, ok := conn.(*net.TCPConn); ok { + tcp.CloseWrite() + } + })) + + outCh, errCh := st.client.ChatCompletionStream(context.Background(), defaultRequest()) + + // Drain chunks concurrently. ScanLines emits an unterminated tail as a + // final token at EOF, so the partial line surfaces as one more chunk and + // the stream goroutine would block forever on the unbuffered out channel + // if the test read chunks synchronously. + var chunks []ChatCompletionChunk + done := make(chan struct{}) + go func() { + defer close(done) + for chunk := range outCh { + chunks = append(chunks, chunk) + } + }() + + // The mid-body transport failure must surface on errCh. + var streamErr error + select { + case streamErr = <-errCh: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for the stream error on errCh") + } + + // outCh closes right after the error is sent. + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("chunk channel did not close after the stream error") + } + + // The complete data line must have been delivered before the disconnect. + if len(chunks) == 0 { + t.Fatal("no chunks delivered before the disconnect, want at least the complete data line") + } + if got := chunks[0].Choices[0].Delta.Content.String(); got != "Hello" { + t.Errorf("first chunk content = %q, want %q", got, "Hello") + } + + var sie *StreamInterruptedError + if !errors.As(streamErr, &sie) { + t.Fatalf("error = %v (%T), want errors.As to match *StreamInterruptedError", streamErr, streamErr) + } + if !errors.Is(streamErr, io.ErrUnexpectedEOF) { + t.Errorf("error = %v, want it to wrap io.ErrUnexpectedEOF", streamErr) + } + // The rendered message must stay byte-identical to the previous + // fmt.Errorf("stream interrupted: %w", err). + if got, want := streamErr.Error(), "stream interrupted: unexpected EOF"; got != want { + t.Errorf("error message = %q, want %q", got, want) + } +} diff --git a/internal/client/status_error_test.go b/internal/client/status_error_test.go new file mode 100644 index 00000000..34f5bd10 --- /dev/null +++ b/internal/client/status_error_test.go @@ -0,0 +1,429 @@ +package client + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestStatusError_ErrorFormat(t *testing.T) { + tests := []struct { + name string + se *StatusError + want string + }{ + { + name: "with body renders legacy API error format", + se: &StatusError{StatusCode: 500, Status: "500 Internal Server Error", Body: "internal error"}, + want: "API error (500): internal error", + }, + { + name: "without body renders legacy status format", + se: &StatusError{StatusCode: 429, Status: "429 Too Many Requests"}, + want: "status: 429", + }, + { + name: "empty body renders legacy status format", + se: &StatusError{StatusCode: 408, Status: "408 Request Timeout", Body: ""}, + want: "status: 408", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.se.Error(); got != tt.want { + t.Errorf("Error() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestStatusError_ErrorsAsThroughWrapChain(t *testing.T) { + se := &StatusError{StatusCode: 503, Status: "503 Service Unavailable", Body: "overloaded"} + + // Mirrors the real chains: executor.go wraps with "stream error: %w" and + // client.go with "stream interrupted: %w". + wrapped := fmt.Errorf("a: %w", fmt.Errorf("stream error: %w", se)) + + var got *StatusError + if !errors.As(wrapped, &got) { + t.Fatal("errors.As failed to recover *StatusError through wrap chain") + } + if got.StatusCode != 503 { + t.Errorf("StatusCode = %d, want 503", got.StatusCode) + } + if got != se { + t.Error("errors.As recovered a different *StatusError instance") + } + + // The legacy message text must survive wrapping unchanged. + want := "a: stream error: API error (503): overloaded" + if msg := wrapped.Error(); msg != want { + t.Errorf("wrapped message = %q, want %q", msg, want) + } +} + +func TestFormatError_ReturnsTypedStatusError(t *testing.T) { + t.Run("API error body", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprint(w, `{"error":{"message":"boom","type":"invalid_request_error","code":"invalid_api_key"}}`) + })) + defer server.Close() + + c := NewClient(Config{BaseURL: server.URL}) + _, err := c.ChatCompletion(context.Background(), ChatCompletionRequest{ + Model: "test-model", + Messages: []ChatMessage{{Role: "user", Content: TextContent("hi")}}, + }) + if err == nil { + t.Fatal("expected error for 500 response, got nil") + } + + var se *StatusError + if !errors.As(err, &se) { + t.Fatalf("error %T (%v) is not a *StatusError", err, err) + } + if se.StatusCode != http.StatusInternalServerError { + t.Errorf("StatusCode = %d, want %d", se.StatusCode, http.StatusInternalServerError) + } + if want := "API error (500): boom"; se.Error() != want { + t.Errorf("Error() = %q, want %q", se.Error(), want) + } + if se.Type != "invalid_request_error" { + t.Errorf("Type = %q, want %q", se.Type, "invalid_request_error") + } + if se.Code != "invalid_api_key" { + t.Errorf("Code = %v, want %q", se.Code, "invalid_api_key") + } + // A short message must be stored verbatim, not truncated. + if want := "boom"; se.Body != want { + t.Errorf("Body = %q, want %q (short message must not be truncated)", se.Body, want) + } + if se.RetryAfter != 0 { + t.Errorf("RetryAfter = %v, want 0 when no Retry-After header is sent", se.RetryAfter) + } + }) + + t.Run("body without type or code fields", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusTooManyRequests) + fmt.Fprint(w, `{"error":{"message":"rate limited"}}`) + })) + defer server.Close() + + c := NewClient(Config{BaseURL: server.URL}) + _, err := c.ChatCompletion(context.Background(), ChatCompletionRequest{ + Model: "test-model", + Messages: []ChatMessage{{Role: "user", Content: TextContent("hi")}}, + }) + if err == nil { + t.Fatal("expected error for 429 response, got nil") + } + + var se *StatusError + if !errors.As(err, &se) { + t.Fatalf("error %T (%v) is not a *StatusError", err, err) + } + if se.StatusCode != http.StatusTooManyRequests { + t.Errorf("StatusCode = %d, want %d", se.StatusCode, http.StatusTooManyRequests) + } + if se.Type != "" { + t.Errorf("Type = %q, want empty", se.Type) + } + if se.Code != nil { + t.Errorf("Code = %v, want nil", se.Code) + } + if want := "rate limited"; se.Body != want { + t.Errorf("Body = %q, want %q", se.Body, want) + } + if want := "API error (429): rate limited"; se.Error() != want { + t.Errorf("Error() = %q, want %q", se.Error(), want) + } + }) + + t.Run("empty body renders legacy status format", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadGateway) + })) + defer server.Close() + + c := NewClient(Config{BaseURL: server.URL}) + _, err := c.ChatCompletion(context.Background(), ChatCompletionRequest{ + Model: "test-model", + Messages: []ChatMessage{{Role: "user", Content: TextContent("hi")}}, + }) + if err == nil { + t.Fatal("expected error for 502 response, got nil") + } + + var se *StatusError + if !errors.As(err, &se) { + t.Fatalf("error %T (%v) is not a *StatusError", err, err) + } + if se.StatusCode != http.StatusBadGateway { + t.Errorf("StatusCode = %d, want %d", se.StatusCode, http.StatusBadGateway) + } + if se.Body != "" { + t.Errorf("Body = %q, want empty", se.Body) + } + if want := "status: 502"; se.Error() != want { + t.Errorf("Error() = %q, want %q", se.Error(), want) + } + }) + + t.Run("plain-text body falls back to sanitized text", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadGateway) + fmt.Fprint(w, "not json") + })) + defer server.Close() + + c := NewClient(Config{BaseURL: server.URL}) + _, err := c.ChatCompletion(context.Background(), ChatCompletionRequest{ + Model: "test-model", + Messages: []ChatMessage{{Role: "user", Content: TextContent("hi")}}, + }) + if err == nil { + t.Fatal("expected error for 502 response, got nil") + } + + var se *StatusError + if !errors.As(err, &se) { + t.Fatalf("error %T (%v) is not a *StatusError", err, err) + } + if want := "not json"; se.Body != want { + t.Errorf("Body = %q, want sanitized fallback %q", se.Body, want) + } + if want := "API error (502): not json"; se.Error() != want { + t.Errorf("Error() = %q, want %q", se.Error(), want) + } + }) +} + +// errorResp builds a bare *http.Response for direct formatError tests. +func errorResp(status int, header map[string]string, body string) *http.Response { + h := http.Header{} + for k, v := range header { + h.Set(k, v) + } + return &http.Response{ + StatusCode: status, + Status: fmt.Sprintf("%d %s", status, http.StatusText(status)), + Header: h, + Body: io.NopCloser(strings.NewReader(body)), + } +} + +// formatErrorStatusError runs formatError on resp and returns the recovered +// *StatusError, failing t if the error is not one. +func formatErrorStatusError(t *testing.T, resp *http.Response) *StatusError { + t.Helper() + c := NewClient(Config{BaseURL: "http://localhost"}) + err := c.formatError(resp) + var se *StatusError + if !errors.As(err, &se) { + t.Fatalf("formatError returned %T (%v), want *StatusError", err, err) + } + return se +} + +func TestFormatError_PlainTextBodyIsSanitized(t *testing.T) { + se := formatErrorStatusError(t, errorResp(http.StatusBadGateway, nil, "502 Bad Gateway\nupstream error")) + + if want := "502 Bad Gateway upstream error"; se.Body != want { + t.Errorf("Body = %q, want collapsed %q", se.Body, want) + } + if want := "API error (502): 502 Bad Gateway upstream error"; se.Error() != want { + t.Errorf("Error() = %q, want %q", se.Error(), want) + } +} + +func TestFormatError_HTMLBodyStripsControlChars(t *testing.T) { + raw := "\r\n502\x00 Bad \t Gateway\r\n" + se := formatErrorStatusError(t, errorResp(http.StatusBadGateway, nil, raw)) + + if strings.ContainsAny(se.Body, "\r\x00\n") { + t.Errorf("Body = %q, want no control characters", se.Body) + } + if want := " 502 Bad Gateway "; se.Body != want { + t.Errorf("Body = %q, want sanitized %q", se.Body, want) + } +} + +func TestFormatError_WhitespaceOnlyBodyKeepsLegacyStatusFormat(t *testing.T) { + se := formatErrorStatusError(t, errorResp(http.StatusBadGateway, nil, " \r\n\t ")) + + if se.Body != "" { + t.Errorf("Body = %q, want empty", se.Body) + } + if want := "status: 502"; se.Error() != want { + t.Errorf("Error() = %q, want %q", se.Error(), want) + } +} + +func TestFormatError_LongJSONMessageTruncatedRuneSafe(t *testing.T) { + t.Run("ascii message truncated to limit", func(t *testing.T) { + msg := strings.Repeat("x", 1500) + se := formatErrorStatusError(t, errorResp(http.StatusInternalServerError, nil, + fmt.Sprintf(`{"error":{"message":%q,"type":"server_error","code":42}}`, msg))) + + if got := len([]rune(se.Body)); got != maxErrorMessageBytes { + t.Errorf("rune count of Body = %d, want %d", got, maxErrorMessageBytes) + } + if got := len(se.Body); got != maxErrorMessageBytes { + t.Errorf("byte length of Body = %d, want %d", got, maxErrorMessageBytes) + } + if want := msg[:maxErrorMessageBytes]; se.Body != want { + t.Errorf("Body prefix mismatch: got %q... want %q...", se.Body[:32], want[:32]) + } + // json.Unmarshal decodes JSON numbers into any as float64. + if se.Type != "server_error" || se.Code != float64(42) { + t.Errorf("Type/Code = %q/%v, want server_error/42 (must survive truncation)", se.Type, se.Code) + } + }) + + t.Run("multibyte message never splits a rune", func(t *testing.T) { + // é is 2 bytes, so the 1024-byte limit would land exactly on a rune + // boundary; € is 3 bytes, so the limit falls mid-rune and the cut must + // walk back to the start of that rune (341 whole runes = 1023 bytes). + msg := strings.Repeat("€", 400) // 1200 bytes + se := formatErrorStatusError(t, errorResp(http.StatusInternalServerError, nil, + fmt.Sprintf(`{"error":{"message":%q}}`, msg))) + + if want := strings.Repeat("€", 341); se.Body != want { + t.Errorf("Body = %d runes / %d bytes, want 341 intact € runes (1023 bytes)", len([]rune(se.Body)), len(se.Body)) + } + }) +} + +func TestFormatError_JSONBodyWithoutMessageFallsBackToRawText(t *testing.T) { + body := `{"error":{"type":"server_error"}}` + se := formatErrorStatusError(t, errorResp(http.StatusInternalServerError, nil, body)) + + // No structured message exists, so the sanitized raw body is surfaced + // instead of collapsing to "status: 500"; the type is still preserved. + if want := "API error (500): " + body; se.Error() != want { + t.Errorf("Error() = %q, want %q", se.Error(), want) + } + if se.Type != "server_error" { + t.Errorf("Type = %q, want %q", se.Type, "server_error") + } +} + +func TestFormatError_OversizedBodyReadIsBounded(t *testing.T) { + // countingReader serves an endless supply of 'a' bytes and records how + // many were consumed, so the read cap can be asserted directly. + cr := &countingReader{} + resp := &http.Response{ + StatusCode: http.StatusBadGateway, + Status: "502 Bad Gateway", + Header: http.Header{}, + Body: io.NopCloser(cr), + } + se := formatErrorStatusError(t, resp) + + if cr.served > maxErrorBodyBytes { + t.Errorf("read %d bytes from the error body, want at most %d", cr.served, maxErrorBodyBytes) + } + if want := strings.Repeat("a", maxErrorMessageBytes); se.Body != want { + t.Errorf("Body = %d bytes, want the first %d bytes of the body", len(se.Body), maxErrorMessageBytes) + } +} + +type countingReader struct { + served int +} + +func (r *countingReader) Read(p []byte) (int, error) { + n := len(p) + if n > 512 { + n = 512 + } + for i := 0; i < n; i++ { + p[i] = 'a' + } + r.served += n + return n, nil +} + +func TestFormatError_CapturesRetryAfterHeader(t *testing.T) { + t.Run("delta-seconds form", func(t *testing.T) { + se := formatErrorStatusError(t, errorResp(http.StatusTooManyRequests, + map[string]string{"Retry-After": "2"}, "rate limited")) + if se.RetryAfter != 2*time.Second { + t.Errorf("RetryAfter = %v, want %v", se.RetryAfter, 2*time.Second) + } + }) + + t.Run("http-date form", func(t *testing.T) { + // A far-future date must yield a positive delay; the exact value is + // clock-dependent and covered deterministically by TestParseRetryAfterAt. + se := formatErrorStatusError(t, errorResp(http.StatusServiceUnavailable, + map[string]string{"Retry-After": "Wed, 01 Jan 2100 00:00:00 GMT"}, "")) + if se.RetryAfter <= 0 { + t.Errorf("RetryAfter = %v, want a positive delay for a future HTTP-date", se.RetryAfter) + } + }) + + t.Run("invalid and absent values are dropped", func(t *testing.T) { + for _, v := range []string{"", "0", "-5", "soon", "2.5"} { + se := formatErrorStatusError(t, errorResp(http.StatusTooManyRequests, + map[string]string{"Retry-After": v}, "")) + if se.RetryAfter != 0 { + t.Errorf("Retry-After %q: RetryAfter = %v, want 0", v, se.RetryAfter) + } + } + se := formatErrorStatusError(t, errorResp(http.StatusTooManyRequests, nil, "")) + if se.RetryAfter != 0 { + t.Errorf("RetryAfter = %v with no Retry-After header, want 0", se.RetryAfter) + } + }) +} + +func TestParseRetryAfterAt(t *testing.T) { + now := time.Date(2015, 10, 21, 7, 28, 0, 0, time.UTC) + tests := []struct { + name string + val string + want time.Duration + }{ + {"empty", "", 0}, + {"delta seconds", "2", 2 * time.Second}, + {"delta seconds with surrounding whitespace", " 3 ", 3 * time.Second}, + {"zero", "0", 0}, + {"negative", "-5", 0}, + {"float is invalid", "2.5", 0}, + {"garbage", "soon", 0}, + {"out of range integer", "99999999999999999999", 0}, + {"future http date", "Wed, 21 Oct 2015 07:28:30 GMT", 30 * time.Second}, + {"http date at now", "Wed, 21 Oct 2015 07:28:00 GMT", 0}, + {"past http date", "Wed, 21 Oct 2015 07:27:00 GMT", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := parseRetryAfterAt(tt.val, now); got != tt.want { + t.Errorf("parseRetryAfterAt(%q, now) = %v, want %v", tt.val, got, tt.want) + } + }) + } +} + +func TestParseRetryAfter_WrapsClock(t *testing.T) { + if got := parseRetryAfter("2"); got != 2*time.Second { + t.Errorf("parseRetryAfter(\"2\") = %v, want %v", got, 2*time.Second) + } + if got := parseRetryAfter(""); got != 0 { + t.Errorf("parseRetryAfter(\"\") = %v, want 0", got) + } + future := time.Now().UTC().Add(45 * time.Second).Format(http.TimeFormat) + if got := parseRetryAfter(future); got <= 0 || got > 45*time.Second { + t.Errorf("parseRetryAfter(future date) = %v, want a positive delay of at most 45s", got) + } +} diff --git a/internal/common/interfaces.go b/internal/common/interfaces.go index f9c1dfca..940027e9 100644 --- a/internal/common/interfaces.go +++ b/internal/common/interfaces.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "late/internal/client" + "time" ) // ToolRunner defines the functional signature for executing a single tool call. @@ -91,6 +92,27 @@ type MessageQueuedEvent struct { func (e MessageQueuedEvent) OrchestratorID() string { return e.ID } +// RetryEvent reports that an LLM stream attempt failed and will be +// automatically retried after Delay. Emitted by the executor retry loop. +type RetryEvent struct { + ID string + Attempt int // 1-based attempt number that just failed + MaxAttempts int // max retries configured (not counting the initial attempt) + Delay time.Duration // backoff before the next attempt + Err error // the underlying stream error +} + +func (e RetryEvent) OrchestratorID() string { return e.ID } + +// RecoveryEvent is sent when a stream attempt that previously failed and was +// being retried finally succeeds — i.e. the retry actually produced a +// response. Emitted by the executor's retry loop exactly once per recovery. +type RecoveryEvent struct { + ID string +} + +func (e RecoveryEvent) OrchestratorID() string { return e.ID } + // PromptRequest defines a generic requirement for user input. type PromptRequest struct { ID string @@ -108,10 +130,12 @@ type InputProvider interface { type contextKey string const ( - InputProviderKey contextKey = "input_provider" - OrchestratorIDKey contextKey = "orchestrator_id" - SkipConfirmationKey contextKey = "skip_confirmation" - ToolApprovalKey contextKey = "tool_approval" + InputProviderKey contextKey = "input_provider" + OrchestratorIDKey contextKey = "orchestrator_id" + SkipConfirmationKey contextKey = "skip_confirmation" + ToolApprovalKey contextKey = "tool_approval" + MaxStreamRetriesKey contextKey = "max_stream_retries" + MaxBadBodyRetriesKey contextKey = "max_bad_body_retries" ) // MainAgentID is the orchestrator ID of the root/main agent. diff --git a/internal/config/config.go b/internal/config/config.go index 1d165193..4b0463fc 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -11,6 +11,15 @@ import ( const DefaultOpenAIBaseURL = "http://localhost:8080" +// Permission modes for supervising potentially dangerous commands. +// The effective mode is resolved by ResolvePermissionMode: +// explicitly set CLI flag > config.json permission-mode entry > +// PermissionModeAskForUserApproval. +const ( + PermissionModeAskForUserApproval = "ask-for-user-approval" + PermissionModeUnsupervised = "i-promise-i-have-backups-and-will-not-file-issues" +) + type EnvLookup func(string) (string, bool) type OpenAISettings struct { @@ -61,6 +70,12 @@ type Config struct { // Enable via config file or the --save-subagent-histories CLI flag. SaveSubagentHistories bool `json:"save_subagent_histories,omitempty"` + // PermissionMode selects how potentially dangerous commands are + // supervised. One of the PermissionMode* constants; empty means the + // default (ask-for-user-approval). Set via config file; the CLI flags + // of the same names override it. + PermissionMode string `json:"permission-mode,omitempty"` + // Legacy subagent fields for backward compatibility SubagentBaseURL string `json:"subagent_base_url,omitempty"` SubagentAPIKey string `json:"subagent_api_key,omitempty"` @@ -241,6 +256,42 @@ func ResolveSaveSubagentHistories(cfg *Config, cliExplicit bool, cliValue bool, return false } +// ResolvePermissionMode returns the effective permission mode. +// Precedence: exactly one explicitly-set CLI flag > config.json +// permission-mode entry > PermissionModeAskForUserApproval. The flags +// are mutually exclusive: setting more than one is an error. An +// unrecognized config.json value yields a warning and falls back to +// the safe default. +func ResolvePermissionMode(cfg *Config, askFlag, unsupervisedFlag bool) (mode string, warning string, err error) { + set := 0 + for _, v := range []bool{askFlag, unsupervisedFlag} { + if v { + set++ + } + } + if set > 1 { + return "", "", fmt.Errorf("permission flags are mutually exclusive; pass at most one of -%s, -%s", + PermissionModeAskForUserApproval, PermissionModeUnsupervised) + } + switch { + case askFlag: + return PermissionModeAskForUserApproval, "", nil + case unsupervisedFlag: + return PermissionModeUnsupervised, "", nil + } + if cfg != nil && cfg.PermissionMode != "" { + switch cfg.PermissionMode { + case PermissionModeAskForUserApproval, PermissionModeUnsupervised: + return cfg.PermissionMode, "", nil + default: + return PermissionModeAskForUserApproval, + fmt.Sprintf("ignoring invalid config.json permission-mode %q; using %q", cfg.PermissionMode, PermissionModeAskForUserApproval), + nil + } + } + return PermissionModeAskForUserApproval, "", nil +} + func nonEmptyEnv(lookup EnvLookup, key string) (string, bool) { if lookup == nil { return "", false diff --git a/internal/config/config_test.go b/internal/config/config_test.go index aa810d16..5d4f8c11 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1,9 +1,11 @@ package config import ( + "encoding/json" "os" "path/filepath" "runtime" + "strings" "testing" ) @@ -701,3 +703,138 @@ func TestSaveConfigAtomicallyReplacesFile(t *testing.T) { } } } + +func TestResolvePermissionMode(t *testing.T) { + tests := []struct { + name string + cfg *Config + askFlag bool + unsupervisedFlag bool + wantMode string + // wantWarning nil: warning must be empty; non-nil: warning must + // contain each substring. + wantWarning []string + wantErr bool + wantErrContains string + }{ + { + name: "no flags, empty config, nil cfg", + cfg: nil, + wantMode: PermissionModeAskForUserApproval, + }, + { + name: "no flags, empty config value", + cfg: &Config{PermissionMode: ""}, + wantMode: PermissionModeAskForUserApproval, + }, + { + name: "ask flag alone", + askFlag: true, + wantMode: PermissionModeAskForUserApproval, + }, + { + name: "unsupervised flag alone", + unsupervisedFlag: true, + wantMode: PermissionModeUnsupervised, + }, + { + name: "ask flag overrides unsupervised config", + cfg: &Config{PermissionMode: PermissionModeUnsupervised}, + askFlag: true, + wantMode: PermissionModeAskForUserApproval, + }, + { + name: "unsupervised flag overrides ask config", + cfg: &Config{PermissionMode: PermissionModeAskForUserApproval}, + unsupervisedFlag: true, + wantMode: PermissionModeUnsupervised, + }, + { + name: "config value: ask-for-user-approval respected", + cfg: &Config{PermissionMode: PermissionModeAskForUserApproval}, + wantMode: PermissionModeAskForUserApproval, + }, + { + name: "config value: i-promise-i-have-backups-and-will-not-file-issues respected", + cfg: &Config{PermissionMode: PermissionModeUnsupervised}, + wantMode: PermissionModeUnsupervised, + }, + { + name: "invalid config value falls back to default with warning", + cfg: &Config{PermissionMode: "yolo"}, + wantMode: PermissionModeAskForUserApproval, + wantWarning: []string{"invalid", "yolo"}, + }, + { + name: "two flags set is an error", + askFlag: true, + unsupervisedFlag: true, + wantErr: true, + wantErrContains: "mutually exclusive", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mode, warning, err := ResolvePermissionMode(tt.cfg, tt.askFlag, tt.unsupervisedFlag) + + if tt.wantErr { + if err == nil { + t.Fatal("ResolvePermissionMode() expected an error, got nil") + } + if tt.wantErrContains != "" && !strings.Contains(err.Error(), tt.wantErrContains) { + t.Fatalf("ResolvePermissionMode() error = %q, want it to contain %q", err.Error(), tt.wantErrContains) + } + return + } + if err != nil { + t.Fatalf("ResolvePermissionMode() error = %v, want nil", err) + } + if mode != tt.wantMode { + t.Fatalf("ResolvePermissionMode() mode = %q, want %q", mode, tt.wantMode) + } + if len(tt.wantWarning) == 0 { + if warning != "" { + t.Fatalf("ResolvePermissionMode() warning = %q, want empty", warning) + } + return + } + if warning == "" { + t.Fatal("ResolvePermissionMode() warning is empty, want a warning") + } + for _, substring := range tt.wantWarning { + if !strings.Contains(warning, substring) { + t.Fatalf("ResolvePermissionMode() warning = %q, want it to contain %q", warning, substring) + } + } + }) + } +} + +func TestConfig_PermissionModeJSONRoundTrip(t *testing.T) { + original := Config{PermissionMode: PermissionModeUnsupervised} + data, err := json.Marshal(original) + if err != nil { + t.Fatalf("json.Marshal() error = %v", err) + } + + var decoded Config + if err := json.Unmarshal(data, &decoded); err != nil { + t.Fatalf("json.Unmarshal() error = %v", err) + } + if decoded.PermissionMode != PermissionModeUnsupervised { + t.Fatalf("PermissionMode after round trip = %q, want %q", decoded.PermissionMode, PermissionModeUnsupervised) + } + + emptyData, err := json.Marshal(Config{}) + if err != nil { + t.Fatalf("json.Marshal() error = %v", err) + } + var raw map[string]any + if err := json.Unmarshal(emptyData, &raw); err != nil { + t.Fatalf("json.Unmarshal() error = %v", err) + } + if _, ok := raw["permission-mode"]; ok { + t.Fatalf("empty config should not marshal a permission-mode key, got %s", emptyData) + } +} diff --git a/internal/executor/executor.go b/internal/executor/executor.go index eb9be73d..2f5b8b81 100644 --- a/internal/executor/executor.go +++ b/internal/executor/executor.go @@ -3,8 +3,10 @@ package executor import ( "context" "encoding/json" + "errors" "fmt" "sync" + "time" "late/internal/client" "late/internal/common" @@ -232,6 +234,9 @@ func ConsumeStream( // RunLoop handles the core, blocking event loop for autonomous agents. // It forces the sequence: inference stream -> verifiable accumulation -> history commit -> safe tool execution. // If the deterministic tool extraction yields zero calls, the loop securely collapses and returns execution control. +// onRetry fires per failed stream attempt that will be retried; onRecover +// fires exactly once per turn whose retries ended in a successful stream +// (i.e. the retry actually produced a response). func RunLoop( ctx context.Context, @@ -241,19 +246,137 @@ func RunLoop( onStartTurn func(), onEndTurn func(), onStreamChunk func(common.StreamResult), + onRetry func(event common.RetryEvent), + onRecover func(), middlewares []common.ToolMiddleware, ) (string, error) { var lastContent string + // Retry budgets for failing LLM stream calls, resolved once per run. + // Two independent tiers: infrastructure failures (transport errors, + // 408/429/5xx) draw from the classic maxRetries budget, while HTTP 400 + // bad-body rejections draw from the much smaller, dedicated badBodyBudget. + maxRetries := maxStreamRetriesFromContext(ctx) + badBodyBudget := maxBadBodyRetriesFromContext(ctx) + for i := 0; maxTurns <= 0 || i < maxTurns; i++ { if onStartTurn != nil { onStartTurn() } - streamCh, errCh := sess.StartStream(ctx, extraBody) - acc, err := ConsumeStream(ctx, streamCh, errCh, onStreamChunk) - if err != nil { - return "", err + // Inner attempt loop around the stream call only: retries never + // consume a turn (the turn counter above is untouched). Each attempt + // starts a fresh stream and ConsumeStream builds a fresh accumulator; + // a failed attempt commits nothing to history. Failures tier into two + // independent retry budgets: infrastructure failures (transport + // errors, 408/429/5xx) share the classic maxRetries budget, while + // HTTP 400 body-parse rejections get their own small dedicated + // badBodyBudget, because strict OpenAI-compatible gateways often fail + // transiently while reading the request body. The two budgets use + // independent counters, so 400 retries never consume infrastructure + // retry budget and vice versa. + var acc *StreamAccumulator + var err error + infraAttempts, badBodyAttempts := 0, 0 + for { + // Pre-attempt guard (retries only): if the context died while we + // were waiting in a previous backoff (both select cases below can + // be ready and the timer may win), do not call StartStream with a + // dead ctx. Handle it as a cancel, not a new attempt. + if infraAttempts+badBodyAttempts > 0 && ctx.Err() != nil { + return "", err + } + + streamCh, errCh := sess.StartStream(ctx, extraBody) + acc, err = ConsumeStream(ctx, streamCh, errCh, onStreamChunk) + if err == nil { + break + } + + // Terminal per tier: budget exhausted for this failure's class or + // a non-retryable failure. Propagates byte-identically to the + // pre-retry behavior. + // + // Server-requested Retry-After: both retry tiers can carry a + // *client.StatusError (429/408/5xx in the infra tier, 400 in the + // bad-body tier), so the error chain is inspected once here and + // the requested delay — 0 when absent or invalid — is combined + // with the local jittered backoff below. effectiveRetryDelay + // guarantees the wait is never shorter than the server asked + // (capped at retryAfterCeiling) and the existing timer select + // keeps it cancelable. + var retryAfter time.Duration + var se *client.StatusError + if errors.As(err, &se) { + retryAfter = se.RetryAfter + } + var delay time.Duration + switch classifyStreamError(err) { + case retryClassNone: + // Non-retryable failure, same as before. + return "", err + case retryClassInfra: + if infraAttempts >= maxRetries { + return "", err + } + infraAttempts++ + delay = effectiveRetryDelay(streamRetryDelay(infraAttempts), retryAfter) + if onRetry != nil { + onRetry(common.RetryEvent{ + ID: common.GetOrchestratorID(ctx), + Attempt: infraAttempts, + MaxAttempts: maxRetries, + // Effective delay: max(local jittered backoff, + // server-requested Retry-After, capped). + Delay: delay, + Err: err, + }) + } + case retryClassBadBody: + if badBodyAttempts >= badBodyBudget { + return "", err + } + badBodyAttempts++ + delay = effectiveRetryDelay(streamRetryDelay(badBodyAttempts), retryAfter) + if onRetry != nil { + onRetry(common.RetryEvent{ + ID: common.GetOrchestratorID(ctx), + Attempt: badBodyAttempts, + MaxAttempts: badBodyBudget, + // Effective delay, same combination as the infra tier. + Delay: delay, + Err: err, + }) + } + } + + timer := time.NewTimer(delay) + select { + case <-timer.C: + // Backoff elapsed: loop around for a fresh StartStream and a + // fresh accumulator via ConsumeStream. + case <-ctx.Done(): + timer.Stop() + // CANCEL SEMANTICS: a stop during the backoff sleep must land + // on the same path as a mid-stream cancel (TUI "Stopped", no + // error box). Returning the underlying stream error is safe + // because BaseOrchestrator's error branch checks ctx.Err() + // and routes canceled runs to the stop path instead of + // emitting StatusEvent{error}. + return "", err + } + } + + // The attempt loop above exits only via break-on-success or an early + // return, so reaching here means an attempt finally produced a + // response. If at least one retry happened in this turn, signal + // recovery exactly once: the turn-start callback fired before the + // retries, so no thinking event will announce it. + if (infraAttempts+badBodyAttempts) > 0 && onRecover != nil { + // The retried attempt actually produced a response: signal + // recovery now (the turn-start callback fired before the + // retries, so no thinking event will announce it). + onRecover() } if acc.FinishReason == "length" { diff --git a/internal/executor/stream_retry.go b/internal/executor/stream_retry.go new file mode 100644 index 00000000..76836555 --- /dev/null +++ b/internal/executor/stream_retry.go @@ -0,0 +1,239 @@ +package executor + +import ( + "bufio" + "context" + "crypto/tls" + "crypto/x509" + "errors" + "io" + "math/rand/v2" + "net" + "net/url" + "strings" + "time" + + "late/internal/client" + "late/internal/common" +) + +const ( + // DefaultMaxStreamRetries is the default retry budget for a failing LLM + // stream call. The interactive default is deliberately small: with full + // jitter over [0, base*2^(n-1)] (base 500ms, capped at 30s) the expected + // total backoff across the whole budget is about 75.75s — long enough to + // ride out transient gateway hiccups, short enough that a hung provider + // does not keep a user waiting for the ~23.8 minutes that 100 retries + // would mean. Overridable via -max-stream-retries / + // LATE_MAX_STREAM_RETRIES; 0 or a negative value disables stream + // retrying entirely. + DefaultMaxStreamRetries = 10 + // DefaultMaxBadBodyRetries is the dedicated retry budget for HTTP 400 + // responses. A 400 "read body failed" from strict OpenAI-compatible + // gateways (e.g. z.ai/GLM) is frequently a transient upstream failure, + // and a handful of quick retries resolves it; genuinely malformed + // requests still fail after this small, bounded budget. It is + // deliberately much smaller than the infrastructure budget + // (DefaultMaxStreamRetries). Not exposed as a CLI flag. + DefaultMaxBadBodyRetries = 3 + // streamRetryBaseDelay is the backoff for the first retry. + streamRetryBaseDelay = 500 * time.Millisecond + // streamRetryMaxDelay caps a single backoff interval. + streamRetryMaxDelay = 30 * time.Second +) + +// streamRetryClass buckets a failed stream attempt into a retry tier: +// retryClassNone fails fast, retryClassInfra draws from the infrastructure +// budget (transport/5xx/429/408), and retryClassBadBody draws from the +// separate, much smaller bad-body budget (HTTP 400 body-parse rejections, +// frequently transient on strict OpenAI-compatible gateways such as +// z.ai/GLM). +type streamRetryClass int + +const ( + // retryClassNone means the error must fail fast: cancellation, permanent + // client errors, and anything unknown. + retryClassNone streamRetryClass = iota + // retryClassInfra covers infrastructure-style failures: network-level + // errors (timeouts, refused/reset connections, mid-body disconnects) and + // transient server responses (408/429/5xx). + retryClassInfra + // retryClassBadBody covers HTTP 400 body-parse rejections, which strict + // OpenAI-compatible gateways often emit transiently. + retryClassBadBody +) + +// maxStreamRetryAttempt clamps the attempt exponent so the +// 1<<(attempt-1) shift can never overflow before the cap is applied. +const maxStreamRetryAttempt = 40 + +// streamRetryDelay returns the exponentially growing, jittered wait before +// retry attempt `attempt` (1-based). The doubling delay is capped at +// streamRetryMaxDelay; full jitter spreads the wait uniformly over [0, cap] +// to avoid synchronized retry storms across agents. +func streamRetryDelay(attempt int) time.Duration { + if attempt < 1 { + attempt = 1 + } + if attempt > maxStreamRetryAttempt { + attempt = maxStreamRetryAttempt + } + backoff := streamRetryBaseDelay * (1 << (attempt - 1)) + if backoff > streamRetryMaxDelay || backoff <= 0 { // <=0 guards shift overflow + backoff = streamRetryMaxDelay + } + return rand.N(backoff) +} + +// retryAfterCeiling caps a server-requested Retry-After wait. Honoring the +// header fully could hang an interactive session for hours on a hostile or +// buggy server; the cap keeps the worst case bounded while still never +// retrying before the requested delay for sane values. The wait remains +// cancelable (stop) regardless. +const retryAfterCeiling = 5 * time.Minute + +// effectiveRetryDelay combines the local jittered backoff with a +// server-requested Retry-After: the result is never shorter than either — +// in particular never earlier than the server asked. RetryAfter of 0 (absent +// or invalid) leaves the local backoff untouched. +func effectiveRetryDelay(local time.Duration, retryAfter time.Duration) time.Duration { + if retryAfter <= 0 { + return local + } + if retryAfter > retryAfterCeiling { + retryAfter = retryAfterCeiling + } + if local > retryAfter { + return local + } + return retryAfter +} + +// classifyStreamError buckets a failed LLM stream attempt into a retry tier. +// retryClassInfra covers infrastructure-style failures that draw from the +// main retry budget: network-level errors (timeouts, refused/reset +// connections, mid-body disconnects), mid-stream transport failures surfaced +// as *client.StreamInterruptedError (HTTP/2 RST_STREAM, GOAWAY, connection +// resets, truncated bodies — the request was accepted with 200 and the body +// then died), and transient server responses (408/429/5xx). +// A *url.Error is infra-tier UNLESS its underlying cause is permanent — +// TLS certificate/trust failures, non-TLS bytes on a TLS connection, or an +// unsupported URL scheme — in which case retrying cannot help and it maps +// to retryClassNone and fails fast (see isPermanentNetworkError). +// The one exception inside *client.StreamInterruptedError is +// bufio.ErrTooLong (the SSE line exceeds the client's scanner cap): that +// failure is deterministic — retrying cannot shrink the line — so it maps to +// retryClassNone and fails fast instead of burning the whole infra budget. +// retryClassBadBody isolates HTTP 400 body-parse rejections, frequently +// transient on strict OpenAI-compatible gateways, into their own tier. +// Everything else — context cancellation, permanent client errors +// (401/403/404), and unknown errors — maps to retryClassNone and fails fast, +// exactly like the pre-retry behavior. +func classifyStreamError(err error) streamRetryClass { + if err == nil { + return retryClassNone + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return retryClassNone + } + var ue *url.Error + if errors.As(err, &ue) { + if isPermanentNetworkError(ue) { + return retryClassNone + } + return retryClassInfra + } + var ne net.Error + if errors.As(err, &ne) && ne.Timeout() { + return retryClassInfra + } + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return retryClassInfra + } + // Mid-stream transport failures: the server accepted the request (200) + // and the body died — HTTP/2 RST_STREAM ("stream error: stream ID N; + // INTERNAL_ERROR"), GOAWAY, connection resets, truncated bodies. These + // are always infrastructure-tier: the attempt commits nothing and a + // fresh stream is a clean retry. + var sie *client.StreamInterruptedError + if errors.As(err, &sie) { + if errors.Is(sie.Err, bufio.ErrTooLong) { + // Deterministic: the SSE line exceeds the client's scanner cap. + // Retrying cannot shrink the line — fail fast instead of burning + // the whole infra budget (~25 min) on a guaranteed failure. + return retryClassNone + } + return retryClassInfra + } + var se *client.StatusError + if errors.As(err, &se) { + if se.StatusCode == 400 { + return retryClassBadBody + } + if se.StatusCode == 408 || se.StatusCode == 429 || se.StatusCode >= 500 { + return retryClassInfra + } + return retryClassNone + } + return retryClassNone +} + +// isPermanentNetworkError reports whether a url.Error wraps a cause that +// retrying cannot fix: TLS certificate failures (untrusted authority, +// hostname mismatch, invalid/expired chain), non-TLS bytes on a TLS +// connection, unsupported URL schemes, and plain-HTTP-on-HTTPS. Everything +// else a url.Error can carry (connection refused/reset, DNS temporary +// failures, timeouts) stays transient. +func isPermanentNetworkError(ue *url.Error) bool { + var authErr x509.UnknownAuthorityError + if errors.As(ue.Err, &authErr) { + return true + } + var hostErr x509.HostnameError + if errors.As(ue.Err, &hostErr) { + return true + } + var certErr x509.CertificateInvalidError + if errors.As(ue.Err, &certErr) { + return true + } + var recordErr tls.RecordHeaderError + if errors.As(ue.Err, &recordErr) { + return true + } + msg := ue.Err.Error() + return strings.HasPrefix(msg, "unsupported protocol scheme") || + strings.HasPrefix(msg, "http: server gave HTTP response to HTTPS client") +} + +// isRetryableStreamError reports whether a failed LLM stream attempt should +// be automatically retried from the infrastructure budget. See +// classifyStreamError for the tiering. +func isRetryableStreamError(err error) bool { + return classifyStreamError(err) == retryClassInfra +} + +// maxStreamRetriesFromContext resolves the retry budget from ctx, falling +// back to DefaultMaxStreamRetries. Negative values mean "disabled". +func maxStreamRetriesFromContext(ctx context.Context) int { + if v, ok := ctx.Value(common.MaxStreamRetriesKey).(int); ok { + if v < 0 { + return 0 + } + return v + } + return DefaultMaxStreamRetries +} + +// maxBadBodyRetriesFromContext resolves the dedicated HTTP 400 retry budget +// from ctx, falling back to DefaultMaxBadBodyRetries. Negative values mean +// "disabled". +func maxBadBodyRetriesFromContext(ctx context.Context) int { + if v, ok := ctx.Value(common.MaxBadBodyRetriesKey).(int); ok { + if v < 0 { + return 0 + } + return v + } + return DefaultMaxBadBodyRetries +} diff --git a/internal/executor/stream_retry_integration_test.go b/internal/executor/stream_retry_integration_test.go new file mode 100644 index 00000000..69fa21c2 --- /dev/null +++ b/internal/executor/stream_retry_integration_test.go @@ -0,0 +1,1071 @@ +package executor + +// Integration tests for the RunLoop stream retry loop (executor.go), driven +// end-to-end against an in-process httptest SSE server: +// +// RunLoop -> session.StartStream -> client.ChatCompletionStream -> httptest +// +// The retry budget is kept small via common.MaxStreamRetriesKey so the +// jittered backoff (full jitter over [0, base*2^(attempt-1)], base 500ms) +// stays well under the test deadline in every interleaving. Assertions only +// pin counts and ordering — never exact timings. + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "late/internal/client" + "late/internal/common" + "late/internal/session" +) + +// SSE chunk payloads used by the success fixture (same shape the client +// tests use): two content deltas, then a terminal chunk with +// finish_reason "stop", then the [DONE] sentinel. +const ( + retryChunkHello = `{"id":"c1","choices":[{"index":0,"delta":{"content":"Hello"}}]}` + retryChunkWorld = `{"id":"c1","choices":[{"index":0,"delta":{"content":" world"}}]}` + retryChunkStop = `{"id":"c1","choices":[{"index":0,"delta":{"content":""},"finish_reason":"stop"}]}` +) + +// successSSEBody renders the complete, well-formed SSE stream served on the +// successful attempt of the retry scenarios. +func successSSEBody() string { + var b strings.Builder + for _, payload := range []string{retryChunkHello, retryChunkWorld, retryChunkStop} { + fmt.Fprintf(&b, "data: %s\n", payload) + } + b.WriteString("data: [DONE]\n") + return b.String() +} + +// errorJSONBody is the JSON error payload served alongside non-200 status +// codes; the client decodes it into client.StatusError.Body. +func errorJSONBody(message string) string { + return fmt.Sprintf(`{"error":{"message":%q,"type":"server_error"}}`, message) +} + +// retryServer wraps an httptest.Server. Only POSTs to */chat/completions are +// routed to the test handler and counted; the client's DiscoverBackend probes +// (GET /props, GET /v1/models) answer 404 immediately and stay uncounted. +type retryServer struct { + server *httptest.Server + posts atomic.Int64 +} + +func newRetryServer(t *testing.T, handlePost func(w http.ResponseWriter, r *http.Request)) *retryServer { + t.Helper() + rs := &retryServer{} + rs.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || !strings.HasSuffix(r.URL.Path, "/chat/completions") { + w.WriteHeader(http.StatusNotFound) + return + } + rs.posts.Add(1) + handlePost(w, r) + })) + t.Cleanup(rs.server.Close) + return rs +} + +func (rs *retryServer) postCount() int { + return int(rs.posts.Load()) +} + +// newRetryTestSession builds an in-memory session bound to the test server. +// An empty HistoryPath keeps every write in memory (saveAndNotify skips +// persistence), so the seeded user message is the only history entry until a +// turn commits its assistant reply. +func newRetryTestSession(t *testing.T, baseURL string) *session.Session { + t.Helper() + c := client.NewClient(client.Config{BaseURL: baseURL}) + return session.New(c, "", []client.ChatMessage{ + {Role: "user", Content: client.TextContent("hello")}, + }, "", false) +} + +// retryCollector returns an onRetry callback that records RetryEvents plus a +// snapshot accessor. RunLoop invokes the callback from its own goroutine and +// the test reads the events after RunLoop returns; the mutex keeps the race +// detector happy regardless of interleaving. +func retryCollector(t *testing.T) (onRetry func(common.RetryEvent), events func() []common.RetryEvent) { + t.Helper() + var mu sync.Mutex + var recorded []common.RetryEvent + return func(ev common.RetryEvent) { + mu.Lock() + defer mu.Unlock() + recorded = append(recorded, ev) + }, func() []common.RetryEvent { + mu.Lock() + defer mu.Unlock() + return append([]common.RetryEvent(nil), recorded...) + } +} + +// recoveryCollector returns an onRecover callback that counts invocations plus +// a snapshot accessor, mirroring retryCollector's goroutine-safety notes: +// RunLoop invokes the callback from its own goroutine and the test reads the +// count after RunLoop returns; the mutex keeps the race detector happy +// regardless of interleaving. +func recoveryCollector(t *testing.T) (onRecover func(), count func() int) { + t.Helper() + var mu sync.Mutex + var calls int + return func() { + mu.Lock() + defer mu.Unlock() + calls++ + }, func() int { + mu.Lock() + defer mu.Unlock() + return calls + } +} + +// runLoopCtx returns a ctx carrying a small retry budget and a generous +// deadline so a bug can never hang a test until the global -timeout. +func runLoopCtx(budget int, timeout time.Duration) (context.Context, context.CancelFunc) { + return context.WithTimeout( + context.WithValue(context.Background(), common.MaxStreamRetriesKey, budget), + timeout, + ) +} + +// serveStatus writes a non-200 JSON error response, which the client turns +// into a *client.StatusError. +func serveStatus(w http.ResponseWriter, code int, message string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + fmt.Fprint(w, errorJSONBody(message)) +} + +// serveStatusWithHeaders is serveStatus with extra response headers (e.g. +// Retry-After) set before the status is written; the client captures them on +// the resulting *client.StatusError. +func serveStatusWithHeaders(w http.ResponseWriter, code int, message string, headers map[string]string) { + for k, v := range headers { + w.Header().Set(k, v) + } + serveStatus(w, code, message) +} + +// serveSSE writes the success fixture as a complete SSE stream. +func serveSSE(w http.ResponseWriter) { + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprint(w, successSSEBody()) +} + +func TestRunLoopRetriesThenSucceeds(t *testing.T) { + var failureServed atomic.Bool + rs := newRetryServer(t, func(w http.ResponseWriter, r *http.Request) { + if !failureServed.CompareAndSwap(false, true) { + // Every attempt after the first: complete SSE stream. + serveSSE(w) + return + } + // First attempt: transient 500 with a JSON error body. + serveStatus(w, http.StatusInternalServerError, "upstream exploded") + }) + + sess := newRetryTestSession(t, rs.server.URL) + onRetry, retryEvents := retryCollector(t) + onRecover, recoveries := recoveryCollector(t) + + ctx, cancel := runLoopCtx(3, 15*time.Second) + defer cancel() + + start := time.Now() + res, err := RunLoop(ctx, sess, 1, nil, nil, nil, nil, onRetry, onRecover, nil) + elapsed := time.Since(start) + + if err != nil { + t.Fatalf("RunLoop returned error after a retryable 500: %v", err) + } + if res != "Hello world" { + t.Errorf("RunLoop result = %q, want %q", res, "Hello world") + } + // One jittered backoff, capped at the 500ms base delay. Generous bound. + if elapsed > 5*time.Second { + t.Errorf("RunLoop took %v with a single retry, want well under that", elapsed) + } + + events := retryEvents() + if len(events) != 1 { + t.Fatalf("got %d RetryEvents, want exactly 1: %+v", len(events), events) + } + ev := events[0] + if ev.Attempt != 1 { + t.Errorf("RetryEvent.Attempt = %d, want 1", ev.Attempt) + } + if ev.MaxAttempts != 3 { + t.Errorf("RetryEvent.MaxAttempts = %d, want 3 (ctx budget)", ev.MaxAttempts) + } + if ev.Delay <= 0 { + t.Errorf("RetryEvent.Delay = %v, want > 0", ev.Delay) + } + if ev.Err == nil { + t.Fatal("RetryEvent.Err is nil, want the underlying stream error") + } + var statusErr *client.StatusError + if !errors.As(ev.Err, &statusErr) { + t.Fatalf("RetryEvent.Err = %v (%T), want it to wrap *client.StatusError", ev.Err, ev.Err) + } + if statusErr.StatusCode != http.StatusInternalServerError { + t.Errorf("RetryEvent.Err status = %d, want 500", statusErr.StatusCode) + } + + // The retried attempt actually produced a response: recovery fires + // exactly once for this turn — the turn-start callback ran before the + // retries, so this is the only signal that the attempt recovered. + if got := recoveries(); got != 1 { + t.Errorf("onRecover fired %d times, want exactly 1 (once per retried turn)", got) + } + + if got := rs.postCount(); got != 2 { + t.Errorf("server got %d POSTs, want 2 (failed attempt + successful retry)", got) + } + + // The failed attempt must not have committed anything; only the + // successful turn appends its assistant message to the seeded history. + if len(sess.History) != 2 { + t.Fatalf("history length = %d, want 2 (seeded user msg + committed assistant msg)", len(sess.History)) + } + last := sess.History[len(sess.History)-1] + if last.Role != "assistant" { + t.Errorf("last history role = %q, want assistant", last.Role) + } + if last.Content.String() != "Hello world" { + t.Errorf("last history content = %q, want %q", last.Content.String(), "Hello world") + } +} + +// TestRunLoopHonorsRetryAfter proves the executor honors a server-requested +// Retry-After end-to-end: a 429 carrying Retry-After: 2 must make RunLoop +// wait at least the requested 2s before retrying — the RetryEvent's Delay is +// the effective (combined) delay and the wall clock confirms the wait really +// happened. Bounded assertions only: no upper timing pin beyond sanity. +func TestRunLoopHonorsRetryAfter(t *testing.T) { + var failureServed atomic.Bool + rs := newRetryServer(t, func(w http.ResponseWriter, r *http.Request) { + if !failureServed.CompareAndSwap(false, true) { + // Second attempt: complete SSE stream. + serveSSE(w) + return + } + // First attempt: transient 429 with a server-requested 2s wait. + serveStatusWithHeaders(w, http.StatusTooManyRequests, "slow down", map[string]string{ + "Retry-After": "2", + }) + }) + + sess := newRetryTestSession(t, rs.server.URL) + onRetry, retryEvents := retryCollector(t) + onRecover, recoveries := recoveryCollector(t) + + ctx, cancel := runLoopCtx(3, 15*time.Second) + defer cancel() + + start := time.Now() + res, err := RunLoop(ctx, sess, 1, nil, nil, nil, nil, onRetry, onRecover, nil) + elapsed := time.Since(start) + + if err != nil { + t.Fatalf("RunLoop returned error after a retryable 429 with Retry-After: %v", err) + } + if res != "Hello world" { + t.Errorf("RunLoop result = %q, want %q", res, "Hello world") + } + + events := retryEvents() + if len(events) != 1 { + t.Fatalf("got %d RetryEvents, want exactly 1: %+v", len(events), events) + } + ev := events[0] + if ev.Attempt != 1 { + t.Errorf("RetryEvent.Attempt = %d, want 1", ev.Attempt) + } + if ev.MaxAttempts != 3 { + t.Errorf("RetryEvent.MaxAttempts = %d, want 3 (ctx budget)", ev.MaxAttempts) + } + if ev.Err == nil { + t.Fatal("RetryEvent.Err is nil, want the underlying stream error") + } + var statusErr *client.StatusError + if !errors.As(ev.Err, &statusErr) { + t.Fatalf("RetryEvent.Err = %v (%T), want it to wrap *client.StatusError", ev.Err, ev.Err) + } + if statusErr.StatusCode != http.StatusTooManyRequests { + t.Errorf("RetryEvent.Err status = %d, want 429", statusErr.StatusCode) + } + if statusErr.RetryAfter != 2*time.Second { + t.Errorf("StatusError.RetryAfter = %v, want 2s (server-requested delay)", statusErr.RetryAfter) + } + + // The retried attempt actually produced a response: recovery fires + // exactly once for this turn, independent of the retry tier. + if got := recoveries(); got != 1 { + t.Errorf("onRecover fired %d times, want exactly 1 (once per retried turn)", got) + } + + // CORE assertion: the effective delay reported (and slept) is never + // shorter than the server-requested 2s — the executor must not retry + // before the requested delay even when the jittered local backoff + // (max 500ms on attempt 1) is smaller. + if ev.Delay < 2*time.Second { + t.Errorf("RetryEvent.Delay = %v, want >= 2s (the server-requested Retry-After)", ev.Delay) + } + if ev.Delay > retryAfterCeiling { + t.Errorf("RetryEvent.Delay = %v, want <= %v (the ceiling)", ev.Delay, retryAfterCeiling) + } + + // The wall clock must reflect the honored wait too: the run cannot + // finish before the requested delay elapsed. time.NewTimer fires no + // earlier than its duration, so this is deterministic, not a flake risk. + if elapsed < 2*time.Second { + t.Errorf("RunLoop finished in %v, want >= 2s (Retry-After honored)", elapsed) + } + // Bounded: a single ~2s wait plus network overhead; 10s is generous. + if elapsed > 10*time.Second { + t.Errorf("RunLoop took %v with a single ~2s wait, want well under that", elapsed) + } + + if got := rs.postCount(); got != 2 { + t.Errorf("server got %d POSTs, want 2 (429 attempt + successful retry)", got) + } + + // The failed attempt must not have committed anything; only the + // successful turn appends its assistant message to the seeded history. + if len(sess.History) != 2 { + t.Fatalf("history length = %d, want 2 (seeded user msg + committed assistant msg)", len(sess.History)) + } + last := sess.History[len(sess.History)-1] + if last.Role != "assistant" { + t.Errorf("last history role = %q, want assistant", last.Role) + } + if last.Content.String() != "Hello world" { + t.Errorf("last history content = %q, want %q", last.Content.String(), "Hello world") + } +} + +func TestRunLoopStopsAfterRetryBudgetExhausted(t *testing.T) { + rs := newRetryServer(t, func(w http.ResponseWriter, r *http.Request) { + serveStatus(w, http.StatusInternalServerError, "still down") + }) + + sess := newRetryTestSession(t, rs.server.URL) + onRetry, retryEvents := retryCollector(t) + onRecover, recoveries := recoveryCollector(t) + + // Budget 2 => initial attempt + 2 retries, then terminal failure. + ctx, cancel := runLoopCtx(2, 15*time.Second) + defer cancel() + + start := time.Now() + _, err := RunLoop(ctx, sess, 1, nil, nil, nil, nil, onRetry, onRecover, nil) + elapsed := time.Since(start) + + if err == nil { + t.Fatal("RunLoop returned nil error, want failure after retry budget exhausted") + } + var statusErr *client.StatusError + if !errors.As(err, &statusErr) || statusErr.StatusCode != http.StatusInternalServerError { + t.Fatalf("RunLoop error = %v, want it to wrap *client.StatusError with 500", err) + } + // Worst-case jitter for two backoffs is 500ms + 1s; 10s is a sanity bound. + if elapsed > 10*time.Second { + t.Errorf("RunLoop took %v to exhaust a budget of 2, want well under that", elapsed) + } + + events := retryEvents() + if len(events) != 2 { + t.Fatalf("got %d RetryEvents, want exactly 2: %+v", len(events), events) + } + for i, ev := range events { + if want := i + 1; ev.Attempt != want { + t.Errorf("events[%d].Attempt = %d, want %d", i, ev.Attempt, want) + } + if ev.MaxAttempts != 2 { + t.Errorf("events[%d].MaxAttempts = %d, want 2", i, ev.MaxAttempts) + } + if ev.Delay <= 0 { + t.Errorf("events[%d].Delay = %v, want > 0", i, ev.Delay) + } + if ev.Err == nil { + t.Errorf("events[%d].Err is nil, want the underlying stream error", i) + } + } + + if got := rs.postCount(); got != 3 { + t.Errorf("server got %d POSTs, want 3 (initial attempt + 2 retries)", got) + } + + // Failed attempts commit nothing to history. + if len(sess.History) != 1 { + t.Errorf("history length = %d, want 1 (only the seeded user msg)", len(sess.History)) + } + + // Exhaustion is not recovery: no attempt ever produced a response. + if got := recoveries(); got != 0 { + t.Errorf("onRecover fired %d times, want 0 (retry exhaustion is not recovery)", got) + } +} + +func TestRunLoopDoesNotRetryNonRetryable(t *testing.T) { + rs := newRetryServer(t, func(w http.ResponseWriter, r *http.Request) { + serveStatus(w, http.StatusUnauthorized, "invalid api key") + }) + + sess := newRetryTestSession(t, rs.server.URL) + onRetry, retryEvents := retryCollector(t) + onRecover, recoveries := recoveryCollector(t) + + // A generous budget that must never be touched by a 401. + ctx, cancel := runLoopCtx(5, 15*time.Second) + defer cancel() + + start := time.Now() + _, err := RunLoop(ctx, sess, 1, nil, nil, nil, nil, onRetry, onRecover, nil) + elapsed := time.Since(start) + + if err == nil { + t.Fatal("RunLoop returned nil error, want the 401 to fail the run") + } + var statusErr *client.StatusError + if !errors.As(err, &statusErr) || statusErr.StatusCode != http.StatusUnauthorized { + t.Fatalf("RunLoop error = %v, want it to wrap *client.StatusError with 401", err) + } + if elapsed > 2*time.Second { + t.Errorf("non-retryable 401 took %v to fail, want a fast failure", elapsed) + } + + if events := retryEvents(); len(events) != 0 { + t.Errorf("got %d RetryEvents, want 0 for a non-retryable error: %+v", len(events), events) + } + // No retries, no recovery: the flag must never fire on a clean failure. + if got := recoveries(); got != 0 { + t.Errorf("onRecover fired %d times, want 0 (no retries happened)", got) + } + if got := rs.postCount(); got != 1 { + t.Errorf("server got %d POSTs, want exactly 1 (no retry after 401)", got) + } + if len(sess.History) != 1 { + t.Errorf("history length = %d, want 1 (nothing committed)", len(sess.History)) + } +} + +func TestRunLoopCancelDuringBackoffStops(t *testing.T) { + rs := newRetryServer(t, func(w http.ResponseWriter, r *http.Request) { + serveStatus(w, http.StatusInternalServerError, "down while user waits") + }) + + sess := newRetryTestSession(t, rs.server.URL) + collect, retryEvents := retryCollector(t) + onRecover, recoveries := recoveryCollector(t) + + ctx, cancel := context.WithCancel( + context.WithValue(context.Background(), common.MaxStreamRetriesKey, 5), + ) + defer cancel() + + // Mirror the TUI stop path: cancel as soon as the first RetryEvent + // arrives, i.e. while RunLoop sits in its ctx-aware backoff sleep. + firstRetry := make(chan struct{}) + onRetry := func(ev common.RetryEvent) { + collect(ev) + select { + case firstRetry <- struct{}{}: + default: + } + } + go func() { + select { + case <-firstRetry: + cancel() + case <-time.After(3 * time.Second): + // No retry ever fired; let the test's own assertions report it. + } + }() + + start := time.Now() + _, err := RunLoop(ctx, sess, 1, nil, nil, nil, nil, onRetry, onRecover, nil) + elapsed := time.Since(start) + + if err == nil { + t.Fatal("RunLoop returned nil error, want the stream error after cancel during backoff") + } + // The cancel lands inside the first (<= 500ms) backoff; anything near the + // bound means the loop kept backing off instead of stopping. + if elapsed > 2*time.Second { + t.Errorf("RunLoop took %v to stop after cancel, want a prompt stop", elapsed) + } + + if events := retryEvents(); len(events) > 1 { + t.Errorf("got %d RetryEvents, want at most 1 (no further retry after cancel): %+v", len(events), events) + } + // Initial attempt + at most one attempt that raced the cancel window. + if got := rs.postCount(); got > 2 { + t.Errorf("server got %d POSTs, want at most 2 after cancel", got) + } + if len(sess.History) != 1 { + t.Errorf("history length = %d, want 1 (a cancelled run commits nothing)", len(sess.History)) + } + // The cancel landed during backoff, so no attempt ever succeeded: no + // recovery signal may fire on a cancelled run. + if got := recoveries(); got != 0 { + t.Errorf("onRecover fired %d times, want 0 (a cancelled run never recovers)", got) + } +} + +func TestRunLoopMidBodyDisconnectRetries(t *testing.T) { + var failureServed atomic.Bool + rs := newRetryServer(t, func(w http.ResponseWriter, r *http.Request) { + if !failureServed.CompareAndSwap(false, true) { + // Second attempt: complete SSE stream. + serveSSE(w) + return + } + // First attempt: a valid 200 whose body is truncated mid-response. + // Hijack the connection, declare a Content-Length larger than the + // bytes actually sent, write one valid partial SSE data line, then + // FIN without the remaining body. net/http surfaces the short body + // as io.ErrUnexpectedEOF on read, which the client wraps into + // "stream interrupted: ..." and isRetryableStreamError treats as + // retryable (verified against this Go version; an RST-style close + // would surface *net.OpError instead and is deliberately avoided). + hijacker, ok := w.(http.Hijacker) + if !ok { + t.Error("server ResponseWriter does not support Hijack") + serveStatus(w, http.StatusInternalServerError, "hijack unsupported") + return + } + conn, _, err := hijacker.Hijack() + if err != nil { + t.Errorf("hijack failed: %v", err) + serveStatus(w, http.StatusInternalServerError, "hijack failed") + return + } + defer conn.Close() + + head := "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: 1000\r\n\r\n" + body := "data: " + `{"choices":[{"delta":{"content":"par"}}]}` + "\n\n" + if _, err := conn.Write([]byte(head)); err != nil { + t.Errorf("writing truncated response head: %v", err) + return + } + if _, err := conn.Write([]byte(body)); err != nil { + t.Errorf("writing truncated response body: %v", err) + return + } + // FIN: the client sees EOF before the declared Content-Length. + if tcp, ok := conn.(*net.TCPConn); ok { + tcp.CloseWrite() + } + }) + + sess := newRetryTestSession(t, rs.server.URL) + onRetry, retryEvents := retryCollector(t) + onRecover, recoveries := recoveryCollector(t) + + ctx, cancel := runLoopCtx(3, 15*time.Second) + defer cancel() + + res, err := RunLoop(ctx, sess, 1, nil, nil, nil, nil, onRetry, onRecover, nil) + + events := retryEvents() + posts := rs.postCount() + + switch { + case err == nil && posts == 2 && len(events) == 1: + // Desired path: the disconnect was retryable and attempt 2 succeeded. + default: + t.Fatalf("unexpected outcome: err=%v, posts=%d, retryEvents=%d, result=%q", err, posts, len(events), res) + } + + if res != "Hello world" { + t.Errorf("RunLoop result = %q, want %q (retry attempt content, not the partial %q)", res, "Hello world", "par") + } + + // The successful retry produced a response: exactly one recovery. + if got := recoveries(); got != 1 { + t.Errorf("onRecover fired %d times, want exactly 1 (once per retried turn)", got) + } + + ev := events[0] + if ev.Attempt != 1 { + t.Errorf("RetryEvent.Attempt = %d, want 1", ev.Attempt) + } + if ev.Delay <= 0 { + t.Errorf("RetryEvent.Delay = %v, want > 0", ev.Delay) + } + if ev.Err == nil { + t.Fatal("RetryEvent.Err is nil, want the disconnect error") + } + // The executor wraps the client's "stream interrupted: unexpected EOF", + // which keeps io.ErrUnexpectedEOF in the chain. + if !errors.Is(ev.Err, io.ErrUnexpectedEOF) { + t.Errorf("RetryEvent.Err = %v, want it to wrap io.ErrUnexpectedEOF", ev.Err) + } + + if got := rs.postCount(); got != 2 { + t.Errorf("server got %d POSTs, want 2 (truncated attempt + successful retry)", got) + } + + if len(sess.History) != 2 { + t.Fatalf("history length = %d, want 2 (seeded user msg + committed assistant msg)", len(sess.History)) + } + last := sess.History[len(sess.History)-1] + if last.Role != "assistant" { + t.Errorf("last history role = %q, want assistant", last.Role) + } + if last.Content.String() != "Hello world" { + t.Errorf("last history content = %q, want %q", last.Content.String(), "Hello world") + } +} + +// TestRunLoopRetriesMidStreamTransportAbort proves the typed mid-stream +// transport failure is retried end-to-end: a 200 whose body dies partway +// surfaces from the client as *client.StreamInterruptedError, +// classifyStreamError maps it to the infrastructure tier, and RunLoop draws +// two infra retries before a complete stream succeeds. The truncated body +// deliberately omits the SSE trailing blank line, so the disconnect is never +// masked as a clean end-of-stream and the retry events are asserted +// unconditionally. +func TestRunLoopRetriesMidStreamTransportAbort(t *testing.T) { + // Declared before newRetryServer so the handler can branch on the + // 1-based POST count (the closure only runs once the server is up). + var rs *retryServer + rs = newRetryServer(t, func(w http.ResponseWriter, r *http.Request) { + // The wrapper counts the POST before handlePost runs, so + // posts.Load() here is the 1-based number of the current request. + if rs.posts.Load() > 2 { + // Third attempt onward: complete SSE stream. + serveOK(w) + return + } + // Attempts 1 and 2: a valid 200 whose body is truncated mid-stream. + // Hijack the connection, declare a Content-Length larger than the + // bytes actually sent, write one complete SSE data line, then an + // unterminated partial line WITHOUT the trailing blank line, then FIN + // without the remaining body. The unterminated tail makes the client's + // bufio.Scanner read again (ScanLines would otherwise only emit a + // final token at EOF), where net/http surfaces the short body as + // io.ErrUnexpectedEOF, which the client wraps into + // *client.StreamInterruptedError (verified against this Go version; an + // RST-style close would surface *net.OpError instead and is + // deliberately avoided). + hijacker, ok := w.(http.Hijacker) + if !ok { + t.Error("server ResponseWriter does not support Hijack") + serveStatus(w, http.StatusInternalServerError, "hijack unsupported") + return + } + conn, _, err := hijacker.Hijack() + if err != nil { + t.Errorf("hijack failed: %v", err) + serveStatus(w, http.StatusInternalServerError, "hijack failed") + return + } + defer conn.Close() + + head := "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: 1000\r\n\r\n" + complete := "data: " + `{"choices":[{"index":0,"delta":{"content":"par"}}]}` + "\n" + partial := "data: " + `{"choices":[{"index":0,"delta":{"content":"ti"}}` // no trailing newline + if _, err := conn.Write([]byte(head)); err != nil { + t.Errorf("writing truncated response head: %v", err) + return + } + if _, err := conn.Write([]byte(complete)); err != nil { + t.Errorf("writing truncated response body: %v", err) + return + } + if _, err := conn.Write([]byte(partial)); err != nil { + t.Errorf("writing unterminated partial line: %v", err) + return + } + // FIN: the client sees EOF before the declared Content-Length. + if tcp, ok := conn.(*net.TCPConn); ok { + tcp.CloseWrite() + } + }) + + sess := newRetryTestSession(t, rs.server.URL) + onRetry, retryEvents := retryCollector(t) + onRecover, recoveries := recoveryCollector(t) + + ctx, cancel := runLoopCtx(3, 15*time.Second) + defer cancel() + + start := time.Now() + res, err := RunLoop(ctx, sess, 1, nil, nil, nil, nil, onRetry, onRecover, nil) + elapsed := time.Since(start) + + if err != nil { + t.Fatalf("RunLoop returned error after two mid-stream aborts: %v", err) + } + if !strings.Contains(res, "ok") { + t.Errorf("RunLoop result = %q, want it to contain %q", res, "ok") + } + // Worst-case jitter for two backoffs is 500ms + 1s; a generous bound like + // the sibling tests (counts are pinned, never exact timings). + if elapsed > 10*time.Second { + t.Errorf("RunLoop took %v with two mid-stream retries, want well under that", elapsed) + } + + // Two retries happened, but both belong to the SAME turn: recovery is a + // once-per-recovered-turn signal, not a per-retry one. + if got := recoveries(); got != 1 { + t.Errorf("onRecover fired %d times, want exactly 1 (once per retried turn, not per retry)", got) + } + + // One retry event per aborted attempt: exactly 2, attempts 1 and 2, drawn + // from the infrastructure budget (MaxAttempts = the ctx budget of 3). + events := retryEvents() + if len(events) != 2 { + t.Fatalf("got %d RetryEvents, want exactly 2 (one per aborted attempt): %+v", len(events), events) + } + for i, ev := range events { + if want := i + 1; ev.Attempt != want { + t.Errorf("events[%d].Attempt = %d, want %d", i, ev.Attempt, want) + } + if ev.MaxAttempts != 3 { + t.Errorf("events[%d].MaxAttempts = %d, want 3 (ctx infra budget)", i, ev.MaxAttempts) + } + if ev.Delay <= 0 { + t.Errorf("events[%d].Delay = %v, want > 0", i, ev.Delay) + } + if ev.Err == nil { + t.Errorf("events[%d].Err is nil, want the mid-stream abort error", i) + continue + } + // Core assertion: the client's typed mid-stream error is produced AND + // retried end-to-end through RunLoop. + var sie *client.StreamInterruptedError + if !errors.As(ev.Err, &sie) { + t.Errorf("events[%d].Err = %v (%T), want errors.As to match *client.StreamInterruptedError", i, ev.Err, ev.Err) + continue + } + if !errors.Is(ev.Err, io.ErrUnexpectedEOF) { + t.Errorf("events[%d].Err = %v, want it to wrap io.ErrUnexpectedEOF", i, ev.Err) + } + } + + // Two aborted attempts + one successful retry. + if got := rs.postCount(); got != 3 { + t.Errorf("server got %d POSTs, want 3 (2 aborted attempts + successful retry)", got) + } + + // Failed attempts commit nothing; only the successful turn appends its + // assistant message to the seeded history. + if len(sess.History) != 2 { + t.Fatalf("history length = %d, want 2 (seeded user msg + committed assistant msg)", len(sess.History)) + } + last := sess.History[len(sess.History)-1] + if last.Role != "assistant" { + t.Errorf("last history role = %q, want assistant", last.Role) + } + if last.Content.String() != "ok" { + t.Errorf("last history content = %q, want %q", last.Content.String(), "ok") + } +} + +// --- HTTP 400 bad-body retry tier --- +// +// RunLoop tiers inner-loop failures into two independent retry budgets: +// infrastructure failures draw from the ctx MaxStreamRetries budget, while +// HTTP 400 body-parse rejections draw from the dedicated, much smaller +// bad-body budget (DefaultMaxBadBodyRetries). The tests below pin that +// tiering end-to-end: a 400 fires exactly badBodyBudget RetryEvents whose +// MaxAttempts is the bad-body budget (never the infra budget), and the whole +// run stays bounded at 1 + DefaultMaxBadBodyRetries requests. + +// badBodyErrorMessage is the error text strict OpenAI-compatible gateways +// (e.g. z.ai/GLM) return while failing to read the request body — the +// transient HTTP 400 the bad-body retry tier exists for. +const badBodyErrorMessage = "The request is invalid: read body failed. Please check the request body, required fields, and request format." + +// retryChunkOK is a minimal success payload: a single delta carrying both the +// content and the terminal finish_reason "stop". +const retryChunkOK = `{"id":"1","choices":[{"index":0,"delta":{"content":"ok"},"finish_reason":"stop"}]}` + +// okSSEBody renders a minimal, well-formed SSE stream: one "ok" content delta +// followed by the [DONE] sentinel. +func okSSEBody() string { + var b strings.Builder + fmt.Fprintf(&b, "data: %s\n", retryChunkOK) + b.WriteString("data: [DONE]\n") + return b.String() +} + +// serveOK writes okSSEBody as a complete SSE stream. +func serveOK(w http.ResponseWriter) { + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprint(w, okSSEBody()) +} + +func TestRunLoopRetries400ThenSucceeds(t *testing.T) { + // Declared before newRetryServer so the handler can branch on the + // 1-based POST count (the closure only runs once the server is up). + var rs *retryServer + rs = newRetryServer(t, func(w http.ResponseWriter, r *http.Request) { + // The wrapper counts the POST before handlePost runs, so + // posts.Load() here is the 1-based number of the current request. + if rs.posts.Load() > 2 { + // Third attempt onward: complete SSE stream. + serveOK(w) + return + } + // First two attempts: transient 400 body-parse rejection. + serveStatus(w, http.StatusBadRequest, badBodyErrorMessage) + }) + + sess := newRetryTestSession(t, rs.server.URL) + onRetry, retryEvents := retryCollector(t) + onRecover, recoveries := recoveryCollector(t) + + // The infra budget must stay untouched by a 400 (those draw from the + // bad-body tier); it is kept small so a tier-wiring regression fails + // fast here instead of grinding through the default 10-retry budget. + ctx, cancel := runLoopCtx(3, 15*time.Second) + defer cancel() + + start := time.Now() + res, err := RunLoop(ctx, sess, 1, nil, nil, nil, nil, onRetry, onRecover, nil) + elapsed := time.Since(start) + + if err != nil { + t.Fatalf("RunLoop returned error after retryable 400s: %v", err) + } + if !strings.Contains(res, "ok") { + t.Errorf("RunLoop result = %q, want it to contain %q", res, "ok") + } + // Worst-case jitter for two backoffs is 500ms + 1s; a generous bound + // like the sibling tests (counts are pinned, never exact timings). + if elapsed > 10*time.Second { + t.Errorf("RunLoop took %v with two bad-body retries, want well under that", elapsed) + } + + // The bad-body tier recovers too: the successful retry produced a + // response, so exactly one recovery for this turn (two 400 retries, + // same turn). + if got := recoveries(); got != 1 { + t.Errorf("onRecover fired %d times, want exactly 1 (once per retried turn)", got) + } + + // One retry event per failed attempt: exactly 2 for the two 400s. Each + // carries the bad-body budget as MaxAttempts — the marker that + // distinguishes this tier from the infrastructure tier. + events := retryEvents() + if len(events) != 2 { + t.Fatalf("got %d RetryEvents, want exactly 2 (one per failed 400): %+v", len(events), events) + } + for i, ev := range events { + if want := i + 1; ev.Attempt != want { + t.Errorf("events[%d].Attempt = %d, want %d", i, ev.Attempt, want) + } + if ev.MaxAttempts != DefaultMaxBadBodyRetries { + t.Errorf("events[%d].MaxAttempts = %d, want %d (bad-body tier, not the ctx infra budget)", i, ev.MaxAttempts, DefaultMaxBadBodyRetries) + } + if ev.Delay <= 0 { + t.Errorf("events[%d].Delay = %v, want > 0", i, ev.Delay) + } + if ev.Err == nil { + t.Errorf("events[%d].Err is nil, want the underlying stream error", i) + continue + } + if !strings.Contains(ev.Err.Error(), "API error (400)") { + t.Errorf("events[%d].Err = %v, want it to contain %q", i, ev.Err, "API error (400)") + } + var statusErr *client.StatusError + if !errors.As(ev.Err, &statusErr) || statusErr.StatusCode != http.StatusBadRequest { + t.Errorf("events[%d].Err = %v, want it to wrap *client.StatusError with 400", i, ev.Err) + } + } + + // Two failed 400 attempts + one successful retry. + if got := rs.postCount(); got != 3 { + t.Errorf("server got %d POSTs, want 3 (2 failed 400s + successful retry)", got) + } + + // Failed attempts commit nothing; only the successful turn appends its + // assistant message to the seeded history. + if len(sess.History) != 2 { + t.Fatalf("history length = %d, want 2 (seeded user msg + committed assistant msg)", len(sess.History)) + } + last := sess.History[len(sess.History)-1] + if last.Role != "assistant" { + t.Errorf("last history role = %q, want assistant", last.Role) + } + if last.Content.String() != "ok" { + t.Errorf("last history content = %q, want %q", last.Content.String(), "ok") + } +} + +func TestRunLoopStopsAfterBadBodyBudgetExhausted(t *testing.T) { + rs := newRetryServer(t, func(w http.ResponseWriter, r *http.Request) { + serveStatus(w, http.StatusBadRequest, badBodyErrorMessage) + }) + + sess := newRetryTestSession(t, rs.server.URL) + onRetry, retryEvents := retryCollector(t) + onRecover, recoveries := recoveryCollector(t) + + // A small infra budget that must never be touched by a 400. If the tier + // wiring were wrong and 400s drew from the infra budget instead, the + // request-count assertion below would fail (or, against the default + // budget of 100, the run would take minutes and hit the 15s deadline). + ctx, cancel := runLoopCtx(2, 15*time.Second) + defer cancel() + + start := time.Now() + _, err := RunLoop(ctx, sess, 1, nil, nil, nil, nil, onRetry, onRecover, nil) + elapsed := time.Since(start) + + if err == nil { + t.Fatal("RunLoop returned nil error, want failure after the bad-body budget exhausted") + } + // The executor wraps the client's StatusError in "stream error: ..." and + // the gateway's message must propagate for diagnostics. + if !strings.Contains(err.Error(), "API error (400)") { + t.Fatalf("RunLoop error = %v, want it to contain %q", err, "API error (400)") + } + if !strings.Contains(err.Error(), "read body failed") { + t.Errorf("RunLoop error = %v, want the provider's message to propagate", err) + } + var statusErr *client.StatusError + if !errors.As(err, &statusErr) || statusErr.StatusCode != http.StatusBadRequest { + t.Fatalf("RunLoop error = %v, want it to wrap *client.StatusError with 400", err) + } + // Worst-case jitter for three backoffs is 500ms + 1s + 2s; 10s is a + // generous sanity bound. + if elapsed > 10*time.Second { + t.Errorf("RunLoop took %v to exhaust the bad-body budget, want well under that", elapsed) + } + + events := retryEvents() + if len(events) != DefaultMaxBadBodyRetries { + t.Fatalf("got %d RetryEvents, want exactly %d: %+v", len(events), DefaultMaxBadBodyRetries, events) + } + for i, ev := range events { + if want := i + 1; ev.Attempt != want { + t.Errorf("events[%d].Attempt = %d, want %d", i, ev.Attempt, want) + } + if ev.MaxAttempts != DefaultMaxBadBodyRetries { + t.Errorf("events[%d].MaxAttempts = %d, want %d (bad-body tier, not the infra budget)", i, ev.MaxAttempts, DefaultMaxBadBodyRetries) + } + if ev.Delay <= 0 { + t.Errorf("events[%d].Delay = %v, want > 0", i, ev.Delay) + } + if ev.Err == nil { + t.Errorf("events[%d].Err is nil, want the underlying stream error", i) + } + } + + // Initial attempt + exactly DefaultMaxBadBodyRetries retries: pins the + // bad-body tier at 3 and proves it does NOT consume the default + // 10-retry infrastructure budget. + want := 1 + DefaultMaxBadBodyRetries + if got := rs.postCount(); got != want { + t.Errorf("server got %d POSTs, want exactly %d (initial attempt + bad-body retries)", got, want) + } + + if len(sess.History) != 1 { + t.Errorf("history length = %d, want 1 (nothing committed)", len(sess.History)) + } + + // Exhaustion is not recovery: no attempt ever produced a response. + if got := recoveries(); got != 0 { + t.Errorf("onRecover fired %d times, want 0 (bad-body exhaustion is not recovery)", got) + } +} + +// toolCallSSEBody renders a complete SSE stream that ends in a tool call +// (finish_reason "tool_calls") instead of a final text response: the turn +// commits an assistant tool-call message and the loop advances to the next +// turn. The named tool is deliberately unregistered, so ExecuteToolCalls +// records an error tool result and the run continues. +func toolCallSSEBody() string { + var b strings.Builder + fmt.Fprintf(&b, "data: %s\n", `{"id":"t1","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"no_such_tool","arguments":"{}"}}]}}]}`) + fmt.Fprintf(&b, "data: %s\n", `{"id":"t1","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]}`) + b.WriteString("data: [DONE]\n") + return b.String() +} + +// serveToolCallSSE writes toolCallSSEBody as a complete SSE stream. +func serveToolCallSSE(w http.ResponseWriter) { + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprint(w, toolCallSSEBody()) +} + +// TestRunLoopRecoveryOncePerTurn proves the recovery signal is scoped to a +// TURN, not a RUN: two consecutive turns that each retry — the first recovers +// into a tool call so the loop advances, the second into the final text +// response — must fire onRecover exactly once per turn, two recoveries total. +// This pins the sequencing the recovery event exists for: the turn-start +// callback fires before the retries, so onRecover is the only per-turn +// "the retry actually produced a response" signal, whether or not the turn +// also ends the run. +func TestRunLoopRecoveryOncePerTurn(t *testing.T) { + // Declared before newRetryServer so the handler can branch on the + // 1-based POST count (the closure only runs once the server is up). + var rs *retryServer + rs = newRetryServer(t, func(w http.ResponseWriter, r *http.Request) { + // The wrapper counts the POST before handlePost runs, so + // posts.Load() here is the 1-based number of the current request. + switch { + case rs.posts.Load() == 1 || rs.posts.Load() == 3: + // Turns 1 and 2 each start with a transient 500. + serveStatus(w, http.StatusInternalServerError, "upstream exploded") + case rs.posts.Load() == 2: + // Turn 1 retries into a tool call: the run continues. + serveToolCallSSE(w) + default: + // Turn 2 retries into the final text response. + serveSSE(w) + } + }) + + sess := newRetryTestSession(t, rs.server.URL) + onRetry, retryEvents := retryCollector(t) + onRecover, recoveries := recoveryCollector(t) + + ctx, cancel := runLoopCtx(3, 15*time.Second) + defer cancel() + + res, err := RunLoop(ctx, sess, 2, nil, nil, nil, nil, onRetry, onRecover, nil) + if err != nil { + t.Fatalf("RunLoop returned error across two retried turns: %v", err) + } + if res != "Hello world" { + t.Errorf("RunLoop result = %q, want %q", res, "Hello world") + } + + if got := rs.postCount(); got != 4 { + t.Errorf("server got %d POSTs, want 4 (500+tool-call turn, 500+success turn)", got) + } + if events := retryEvents(); len(events) != 2 { + t.Errorf("got %d RetryEvents, want exactly 2 (one failed attempt per turn): %+v", len(events), events) + } + // CORE assertion: one recovery PER TURN — turn 1 (retried into a tool + // call) and turn 2 (retried into the final response) each recovered. + if got := recoveries(); got != 2 { + t.Errorf("onRecover fired %d times, want exactly 2 (once per retried turn)", got) + } + + // History: seeded user msg + assistant tool call + tool result + the + // final assistant reply. + if len(sess.History) != 4 { + t.Fatalf("history length = %d, want 4 (seeded user, tool call, tool result, final reply)", len(sess.History)) + } +} diff --git a/internal/executor/stream_retry_test.go b/internal/executor/stream_retry_test.go new file mode 100644 index 00000000..35214ed1 --- /dev/null +++ b/internal/executor/stream_retry_test.go @@ -0,0 +1,551 @@ +package executor + +import ( + "bufio" + "context" + "crypto/x509" + "errors" + "fmt" + "io" + "net" + "net/url" + "testing" + "time" + + "late/internal/client" + "late/internal/common" +) + +// timeoutError is a minimal net.Error implementation that always reports a +// timeout, mirroring what net/http surfaces for stalled connections. +type timeoutError struct{} + +func (timeoutError) Error() string { return "i/o timeout" } +func (timeoutError) Timeout() bool { return true } +func (timeoutError) Temporary() bool { return true } + +func TestStreamRetryDelay(t *testing.T) { + t.Run("attempt 1 is positive and within base delay", func(t *testing.T) { + for i := 0; i < 500; i++ { + d := streamRetryDelay(1) + if d <= 0 { + t.Fatalf("streamRetryDelay(1) = %v, want > 0", d) + } + if d > streamRetryBaseDelay { + t.Fatalf("streamRetryDelay(1) = %v, want <= %v", d, streamRetryBaseDelay) + } + } + }) + + t.Run("large attempts are capped", func(t *testing.T) { + for _, attempt := range []int{40, 100000} { + for i := 0; i < 100; i++ { + d := streamRetryDelay(attempt) + if d < 0 { + t.Fatalf("streamRetryDelay(%d) = %v, want >= 0", attempt, d) + } + if d > streamRetryMaxDelay { + t.Fatalf("streamRetryDelay(%d) = %v, want <= %v", attempt, d, streamRetryMaxDelay) + } + } + } + }) + + t.Run("backoff grows with attempt number", func(t *testing.T) { + const draws = 500 + + // attempt 1 draws uniformly from [0, 500ms], attempt 5 from + // [0, 8s]; the observed maxima must reflect that growth. + maxFirst := time.Duration(0) + for i := 0; i < draws; i++ { + if d := streamRetryDelay(1); d > maxFirst { + maxFirst = d + } + } + maxFifth := time.Duration(0) + for i := 0; i < draws; i++ { + if d := streamRetryDelay(5); d > maxFifth { + maxFifth = d + } + } + if maxFifth <= maxFirst { + t.Fatalf("expected growth between attempts: max over %d draws was %v for attempt 1, %v for attempt 5", draws, maxFirst, maxFifth) + } + }) +} + +func TestEffectiveRetryDelay(t *testing.T) { + tests := []struct { + name string + local time.Duration + retryAfter time.Duration + want time.Duration + }{ + { + // Absent (or client-rejected-invalid) Retry-After: the local + // jittered backoff is used untouched. + name: "Retry-After 0 leaves the local backoff untouched", + local: 500 * time.Millisecond, + retryAfter: 0, + want: 500 * time.Millisecond, + }, + { + // Core owner requirement: never retry before the server asked. + name: "server-requested wait longer than local wins", + local: 500 * time.Millisecond, + retryAfter: 2 * time.Second, + want: 2 * time.Second, + }, + { + // The local backoff still applies when it is the larger of the two. + name: "local backoff longer than the server request wins", + local: 5 * time.Second, + retryAfter: 2 * time.Second, + want: 5 * time.Second, + }, + { + // Hostile/buggy server asking for an absurd wait: capped at the + // ceiling instead of hanging the interactive session. + name: "huge Retry-After is capped at the ceiling", + local: 500 * time.Millisecond, + retryAfter: 10 * time.Minute, + want: retryAfterCeiling, + }, + { + // Defensive: the client maps invalid headers to 0; anything + // non-positive must degrade to the local backoff. + name: "negative Retry-After is treated as absent", + local: 500 * time.Millisecond, + retryAfter: -3 * time.Second, + want: 500 * time.Millisecond, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := effectiveRetryDelay(tt.local, tt.retryAfter); got != tt.want { + t.Errorf("effectiveRetryDelay(%v, %v) = %v, want %v", tt.local, tt.retryAfter, got, tt.want) + } + }) + } +} + +func TestIsRetryableStreamError(t *testing.T) { + urlTimeoutErr := &url.Error{ + Op: "Post", + URL: "https://api/v1/chat/completions", + Err: errors.New("read tcp 1.2.3.4:5->6:7: operation timed out"), + } + urlResetErr := &url.Error{ + Op: "Post", + URL: "https://api/v1/chat/completions", + Err: errors.New("read tcp 1.2.3.4:5->6:7: connection reset by peer"), + } + + tests := []struct { + name string + err error + want bool + }{ + // Retryable: network-level failures. + { + name: "raw url.Error is retryable", + err: urlTimeoutErr, + want: true, + }, + { + name: "executor-wrapped url.Error is retryable", + err: fmt.Errorf("stream error: %w", urlTimeoutErr), + want: true, + }, + { + name: "double-wrapped url.Error is retryable", + err: fmt.Errorf("stream error: %w", fmt.Errorf("stream interrupted: %w", urlResetErr)), + want: true, + }, + { + name: "net.Error timeout wrapped is retryable", + err: fmt.Errorf("stream interrupted: %w", timeoutError{}), + want: true, + }, + { + name: "double-wrapped net.Error timeout is retryable", + err: fmt.Errorf("stream error: %w", fmt.Errorf("stream interrupted: %w", timeoutError{})), + want: true, + }, + { + name: "real DNS timeout wrapped is retryable", + err: fmt.Errorf("stream error: %w", &net.DNSError{Err: "i/o timeout", Name: "api.example.com", IsTimeout: true}), + want: true, + }, + { + name: "io.EOF wrapped is retryable", + err: fmt.Errorf("stream interrupted: %w", io.EOF), + want: true, + }, + { + name: "io.ErrUnexpectedEOF double-wrapped is retryable", + err: fmt.Errorf("stream error: %w", fmt.Errorf("stream interrupted: %w", io.ErrUnexpectedEOF)), + want: true, + }, + + // Retryable: transient server responses. + { + name: "raw 500 is retryable", + err: &client.StatusError{StatusCode: 500, Status: "500 Internal Server Error", Body: "boom"}, + want: true, + }, + { + name: "wrapped 500 is retryable", + err: fmt.Errorf("stream error: %w", &client.StatusError{StatusCode: 500, Status: "500 Internal Server Error", Body: "boom"}), + want: true, + }, + { + name: "double-wrapped 502 is retryable", + err: fmt.Errorf("stream error: %w", fmt.Errorf("stream interrupted: %w", &client.StatusError{StatusCode: 502, Status: "502 Bad Gateway", Body: ""})), + want: true, + }, + { + name: "wrapped 429 is retryable", + err: fmt.Errorf("stream error: %w", &client.StatusError{StatusCode: 429, Status: "429 Too Many Requests"}), + want: true, + }, + { + name: "wrapped 408 is retryable", + err: fmt.Errorf("stream error: %w", &client.StatusError{StatusCode: 408, Status: "408 Request Timeout"}), + want: true, + }, + + // Not retryable: cancellation fails fast even inside the wraps. + { + name: "context.Canceled wrapped is not retryable", + err: fmt.Errorf("stream error: %w", context.Canceled), + want: false, + }, + { + name: "context.DeadlineExceeded wrapped is not retryable", + err: fmt.Errorf("stream error: %w", context.DeadlineExceeded), + want: false, + }, + { + name: "context.Canceled double-wrapped is not retryable", + err: fmt.Errorf("stream error: %w", fmt.Errorf("stream interrupted: %w", context.Canceled)), + want: false, + }, + { + name: "url.Error carrying a canceled context is not retryable", + err: fmt.Errorf("stream error: %w", &url.Error{Op: "Post", URL: "https://api/v1/chat/completions", Err: context.Canceled}), + want: false, + }, + + // Not retryable: permanent server responses. + { + name: "wrapped 400 is not retryable", + err: fmt.Errorf("stream error: %w", &client.StatusError{StatusCode: 400, Status: "400 Bad Request", Body: "invalid model"}), + want: false, + }, + { + name: "wrapped 401 is not retryable", + err: fmt.Errorf("stream error: %w", &client.StatusError{StatusCode: 401, Status: "401 Unauthorized"}), + want: false, + }, + { + name: "wrapped 403 is not retryable", + err: fmt.Errorf("stream error: %w", &client.StatusError{StatusCode: 403, Status: "403 Forbidden"}), + want: false, + }, + { + name: "wrapped 404 is not retryable", + err: fmt.Errorf("stream error: %w", &client.StatusError{StatusCode: 404, Status: "404 Not Found"}), + want: false, + }, + + // Not retryable: anything unknown fails fast, like pre-retry behavior. + { + name: "plain parser error is not retryable", + err: errors.New("some parser error"), + want: false, + }, + { + name: "nil is not retryable", + err: nil, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isRetryableStreamError(tt.err); got != tt.want { + t.Errorf("isRetryableStreamError(%v) = %v, want %v", tt.err, got, tt.want) + } + }) + } +} + +func TestMaxStreamRetriesFromContext(t *testing.T) { + tests := []struct { + name string + ctx context.Context + want int + }{ + { + // Pins the interactive default (owner item 7): 10 retries carry + // about 75.75s of expected total backoff. A deliberate change to + // DefaultMaxStreamRetries must update this literal and its docs. + name: "absent key falls back to the pinned default (10)", + ctx: context.Background(), + want: 10, + }, + { + name: "positive override is honored", + ctx: context.WithValue(context.Background(), common.MaxStreamRetriesKey, 7), + want: 7, + }, + { + name: "zero disables retries", + ctx: context.WithValue(context.Background(), common.MaxStreamRetriesKey, 0), + want: 0, + }, + { + name: "negative value means disabled", + ctx: context.WithValue(context.Background(), common.MaxStreamRetriesKey, -3), + want: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := maxStreamRetriesFromContext(tt.ctx); got != tt.want { + t.Errorf("maxStreamRetriesFromContext() = %d, want %d", got, tt.want) + } + }) + } +} + +func TestClassifyStreamError(t *testing.T) { + tests := []struct { + name string + err error + want streamRetryClass + }{ + // Bad body: 400 gets its own tier. + { + name: "wrapped 400 is bad body", + err: fmt.Errorf("stream error: %w", &client.StatusError{StatusCode: 400, Status: "400 Bad Request", Body: "invalid model"}), + want: retryClassBadBody, + }, + + // Infrastructure: transient server responses. + { + name: "wrapped 408 is infra", + err: fmt.Errorf("stream error: %w", &client.StatusError{StatusCode: 408, Status: "408 Request Timeout"}), + want: retryClassInfra, + }, + { + name: "wrapped 429 is infra", + err: fmt.Errorf("stream error: %w", &client.StatusError{StatusCode: 429, Status: "429 Too Many Requests"}), + want: retryClassInfra, + }, + { + name: "wrapped 500 is infra", + err: fmt.Errorf("stream error: %w", &client.StatusError{StatusCode: 500, Status: "500 Internal Server Error", Body: "boom"}), + want: retryClassInfra, + }, + { + name: "wrapped 503 is infra", + err: fmt.Errorf("stream error: %w", &client.StatusError{StatusCode: 503, Status: "503 Service Unavailable"}), + want: retryClassInfra, + }, + + // None: permanent client responses. + { + name: "wrapped 401 is none", + err: fmt.Errorf("stream error: %w", &client.StatusError{StatusCode: 401, Status: "401 Unauthorized"}), + want: retryClassNone, + }, + { + name: "wrapped 403 is none", + err: fmt.Errorf("stream error: %w", &client.StatusError{StatusCode: 403, Status: "403 Forbidden"}), + want: retryClassNone, + }, + { + name: "wrapped 404 is none", + err: fmt.Errorf("stream error: %w", &client.StatusError{StatusCode: 404, Status: "404 Not Found"}), + want: retryClassNone, + }, + + // None: cancellation fails fast even inside the wraps. + { + name: "plain context.Canceled is none", + err: context.Canceled, + want: retryClassNone, + }, + { + name: "wrapped context.Canceled is none", + err: fmt.Errorf("stream error: %w", context.Canceled), + want: retryClassNone, + }, + { + name: "double-wrapped context.Canceled is none", + err: fmt.Errorf("stream error: %w", fmt.Errorf("stream interrupted: %w", context.Canceled)), + want: retryClassNone, + }, + { + name: "plain context.DeadlineExceeded is none", + err: context.DeadlineExceeded, + want: retryClassNone, + }, + { + name: "double-wrapped context.DeadlineExceeded is none", + err: fmt.Errorf("stream error: %w", fmt.Errorf("stream interrupted: %w", context.DeadlineExceeded)), + want: retryClassNone, + }, + { + name: "url.Error carrying a canceled context is none", + err: fmt.Errorf("stream error: %w", &url.Error{Op: "Post", URL: "https://api/v1/chat/completions", Err: context.Canceled}), + want: retryClassNone, + }, + + // Infrastructure: network-level failures. + { + name: "plain url.Error is infra", + err: &url.Error{Op: "Post", URL: "https://api/v1/chat/completions", Err: errors.New("boom")}, + want: retryClassInfra, + }, + { + name: "wrapped io.EOF is infra", + err: fmt.Errorf("stream error: %w", io.EOF), + want: retryClassInfra, + }, + { + name: "wrapped io.ErrUnexpectedEOF is infra", + err: fmt.Errorf("stream error: %w", io.ErrUnexpectedEOF), + want: retryClassInfra, + }, + + // url.Error: classified by CAUSE, not by the wrapper. Permanent + // causes (TLS trust/hostname, unsupported scheme, HTTP-on-HTTPS) + // fail fast; transient causes (refused, DNS, timeouts) stay infra. + { + // Certificate trust failure is permanent. + name: "url.Error carrying x509.UnknownAuthorityError is none", + err: fmt.Errorf("stream error: %w", &url.Error{Op: "Post", URL: "https://host", Err: x509.UnknownAuthorityError{}}), + want: retryClassNone, + }, + { + name: "url.Error carrying x509.HostnameError is none", + err: &url.Error{Op: "Post", URL: "https://host", Err: x509.HostnameError{Host: "host", Certificate: &x509.Certificate{}}}, + want: retryClassNone, + }, + { + name: "url.Error carrying unsupported protocol scheme is none", + err: &url.Error{Op: "Post", URL: "ftp://host", Err: errors.New(`unsupported protocol scheme "ftp"`)}, + want: retryClassNone, + }, + { + // Still transient — pins the non-overreach of the cause + // classification: plain dial failures must stay infra. + name: "url.Error carrying dial connection refused is infra", + err: &url.Error{Op: "Post", URL: "https://host", Err: &net.OpError{Op: "dial", Err: errors.New("connect: connection refused")}}, + want: retryClassInfra, + }, + { + name: "url.Error carrying temporary DNS failure is infra", + err: &url.Error{Op: "Post", URL: "https://host", Err: errors.New("dial tcp: lookup host: temporary failure in name resolution")}, + want: retryClassInfra, + }, + + // Infrastructure: mid-stream transport failures after a 200 — the + // client surfaces these as *client.StreamInterruptedError. + { + name: "wrapped mid-stream http2 RST_STREAM is infra-retryable", + err: fmt.Errorf("stream error: %w", &client.StreamInterruptedError{Err: errors.New("stream error: stream ID 1; INTERNAL_ERROR; received from peer")}), + want: retryClassInfra, + }, + { + name: "mid-stream wrapper carrying cancellation is not retryable", + err: fmt.Errorf("stream error: %w", &client.StreamInterruptedError{Err: context.Canceled}), + want: retryClassNone, + }, + { + name: "wrapped mid-stream truncated body is infra-retryable", + err: fmt.Errorf("stream error: %w", &client.StreamInterruptedError{Err: io.ErrUnexpectedEOF}), + want: retryClassInfra, + }, + { + name: "wrapped oversized SSE line (bufio.ErrTooLong) fails fast", + err: fmt.Errorf("stream error: %w", &client.StreamInterruptedError{Err: bufio.ErrTooLong}), + want: retryClassNone, + }, + { + // Owner item 2 pin: a non-timeout net.OpError wrapping ECONNRESET + // mid-body is infra-retryable, not unknown/non-retryable. + name: "mid-body ECONNRESET via net.OpError is infra-retryable", + err: fmt.Errorf("stream error: %w", &client.StreamInterruptedError{Err: &net.OpError{Op: "read", Err: errors.New("read: connection reset by peer")}}), + want: retryClassInfra, + }, + + // None: anything unknown fails fast, like pre-retry behavior. + { + name: "nil is none", + err: nil, + want: retryClassNone, + }, + { + name: "generic error is none", + err: fmt.Errorf("stream error: %w", errors.New("x")), + want: retryClassNone, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := classifyStreamError(tt.err); got != tt.want { + t.Errorf("classifyStreamError(%v) = %v, want %v", tt.err, got, tt.want) + } + // isRetryableStreamError is the infrastructure-budget view: it + // must be true exactly for retryClassInfra rows, i.e. false for + // both retryClassNone and retryClassBadBody rows. + if got := isRetryableStreamError(tt.err); got != (tt.want == retryClassInfra) { + t.Errorf("isRetryableStreamError(%v) = %v, want %v (class %v)", tt.err, got, tt.want == retryClassInfra, tt.want) + } + }) + } +} + +func TestMaxBadBodyRetriesFromContext(t *testing.T) { + tests := []struct { + name string + ctx context.Context + want int + }{ + { + name: "absent key falls back to default", + ctx: context.Background(), + want: DefaultMaxBadBodyRetries, + }, + { + name: "positive override is honored", + ctx: context.WithValue(context.Background(), common.MaxBadBodyRetriesKey, 7), + want: 7, + }, + { + name: "zero disables retries", + ctx: context.WithValue(context.Background(), common.MaxBadBodyRetriesKey, 0), + want: 0, + }, + { + name: "negative value means disabled", + ctx: context.WithValue(context.Background(), common.MaxBadBodyRetriesKey, -3), + want: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := maxBadBodyRetriesFromContext(tt.ctx); got != tt.want { + t.Errorf("maxBadBodyRetriesFromContext() = %d, want %d", got, tt.want) + } + }) + } +} diff --git a/internal/git/repo_root_test.go b/internal/git/repo_root_test.go new file mode 100644 index 00000000..8bbe4286 --- /dev/null +++ b/internal/git/repo_root_test.go @@ -0,0 +1,95 @@ +package git + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// initRepoForTest creates a git repository at dir, mirroring the plugin +// package's git-fixture tests (which run `git init` directly and fail loudly). +func initRepoForTest(t *testing.T, dir string) { + t.Helper() + if out, err := exec.Command("git", "init", dir).CombinedOutput(); err != nil { + t.Fatalf("git init %s: %v: %s", dir, err, out) + } +} + +// sameDirForTest reports whether two paths refer to the same directory on +// disk, tolerating symlink-resolved aliases (macOS /tmp -> /private/tmp makes +// `git rev-parse --show-toplevel` report a different spelling than +// t.TempDir()). +func sameDirForTest(t *testing.T, a, b string) bool { + t.Helper() + ai, err := os.Stat(a) + if err != nil { + t.Fatalf("Stat(%s): %v", a, err) + } + bi, err := os.Stat(b) + if err != nil { + t.Fatalf("Stat(%s): %v", b, err) + } + return os.SameFile(ai, bi) +} + +// TestRepoRoot_InsideRepo guards the repo-root resolution: from the repo root +// and from a nested subdirectory alike, RepoRoot must report the repository +// root. +func TestRepoRoot_InsideRepo(t *testing.T) { + repo := t.TempDir() + initRepoForTest(t, repo) + + // From the repo root itself. + root, ok := RepoRoot(repo) + if !ok { + t.Fatalf("RepoRoot(%q) ok = false, want the repo root", repo) + } + if !sameDirForTest(t, root, repo) { + t.Errorf("RepoRoot(%q) = %q, want the repository root", repo, root) + } + + // From a nested subdirectory. + sub := filepath.Join(repo, "internal", "deep") + if err := os.MkdirAll(sub, 0700); err != nil { + t.Fatalf("MkdirAll(%s): %v", sub, err) + } + root, ok = RepoRoot(sub) + if !ok { + t.Fatalf("RepoRoot(%q) ok = false, want the repo root", sub) + } + if !sameDirForTest(t, root, repo) { + t.Errorf("RepoRoot(%q) = %q, want the repository root", sub, root) + } +} + +// TestRepoRoot_OutsideRepo guards the not-a-repo handling: outside a git +// repository (and when git is unavailable) RepoRoot must report false with an +// empty root instead of an error or a bogus path. +func TestRepoRoot_OutsideRepo(t *testing.T) { + plain := t.TempDir() + if root, ok := probeRepoRoot(t, plain); ok && !sameDirForTest(t, root, plain) { + t.Skipf("temp dir %s resolves inside git repo %s; cannot test the not-a-repo case here", plain, root) + } + + root, ok := RepoRoot(plain) + if ok { + t.Errorf("RepoRoot(%q) ok = true (root %q), want false outside a git repository", plain, root) + } + if root != "" { + t.Errorf("RepoRoot(%q) root = %q, want empty outside a git repository", plain, root) + } +} + +// probeRepoRoot exposes the raw repo-root probe so the not-a-repo test can +// detect a temp directory that unexpectedly lives inside a repository. +func probeRepoRoot(t *testing.T, dir string) (string, bool) { + t.Helper() + cmd := exec.Command("git", "-C", dir, "rev-parse", "--show-toplevel") + out, err := cmd.Output() + if err != nil { + return "", false + } + return strings.TrimSpace(string(out)), true +} diff --git a/internal/git/worktree.go b/internal/git/worktree.go index efa694d9..8af3db16 100644 --- a/internal/git/worktree.go +++ b/internal/git/worktree.go @@ -120,6 +120,27 @@ func GetActiveWorktree() (string, error) { return strings.TrimSpace(string(output)), nil } +// RepoRoot returns the root directory of the git repository containing cwd, +// resolved with `git rev-parse --show-toplevel`. The boolean result is false +// when cwd is not inside a git repository — including when git is +// unavailable or exits with an error — so callers can fall back to cwd +// itself. +func RepoRoot(cwd string) (string, bool) { + cmd := exec.Command("git", "rev-parse", "--show-toplevel") + if cwd != "" { + cmd.Dir = cwd + } + output, err := cmd.Output() + if err != nil { + return "", false + } + root := strings.TrimSpace(string(output)) + if root == "" { + return "", false + } + return root, true +} + // CurrentBranch returns the current git branch name at cwd, or "" if not in a git repo. func CurrentBranch(cwd string) string { cmd := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD") diff --git a/internal/orchestrator/base.go b/internal/orchestrator/base.go index 7d6b8052..01c10c4d 100644 --- a/internal/orchestrator/base.go +++ b/internal/orchestrator/base.go @@ -3,6 +3,7 @@ package orchestrator import ( "context" "encoding/base64" + "errors" "fmt" "late/internal/client" "late/internal/common" @@ -271,10 +272,50 @@ func (o *BaseOrchestrator) Execute(text string) (string, error) { Usage: accCopy.Usage, } }, + func(ev common.RetryEvent) { + // The retry starts a fresh stream; reset the shared accumulator so + // the failed attempt's partial deltas do not prefix the retry's + // output in the TUI (the executor's local accumulator is already + // per-attempt; this one is per-turn). + o.mu.Lock() + o.acc.Reset() + o.mu.Unlock() + + ev.ID = o.id // Route to this agent's AppState even if ctx lost the ID + // Non-blocking emit: a slow or stalled TUI must never delay the + // retry backoff loop. The buffered(100) eventCh may be full if the + // consumer lags; dropping a retry notice is acceptable, blocking + // the agent is not. + select { + case o.eventCh <- ev: + default: + } + }, + func() { + // A retried attempt produced a response: emit the dedicated + // recovery event so the UI can toast immediately instead of + // guessing on the next turn's thinking event. Non-blocking: + // a dropped recovery notice is acceptable, blocking the + // agent is not. + select { + case o.eventCh <- common.RecoveryEvent{ID: o.id}: + default: + } + }, o.middlewares, ) if err != nil { + // Canceled runs follow the stop path, not the error path (no error + // box): a stop can surface here as the underlying stream error, e.g. + // when the retry backoff sleep is interrupted by ctx.Done(). We check + // ctx.Err() instead of IsStopRequested() because IsStopRequested() + // consumes the one-shot stopCh token. Emitting "closed" mirrors how a + // mid-stream cancel (nil error) is routed below. + if errors.Is(ctx.Err(), context.Canceled) || errors.Is(err, context.Canceled) { + o.eventCh <- common.StatusEvent{ID: o.id, Status: "closed"} + return res, err + } o.eventCh <- common.StatusEvent{ID: o.id, Status: "error", Error: err} } else { o.eventCh <- common.StatusEvent{ID: o.id, Status: "closed"} @@ -344,6 +385,36 @@ func (o *BaseOrchestrator) run() { Usage: accCopy.Usage, } }, + func(ev common.RetryEvent) { + // The retry starts a fresh stream; reset the shared accumulator + // so the failed attempt's partial deltas do not prefix the + // retry's output in the TUI (the executor's local accumulator + // is already per-attempt; this one is per-turn). + o.mu.Lock() + o.acc.Reset() + o.mu.Unlock() + + ev.ID = o.id // Route to this agent's AppState even if ctx lost the ID + // Non-blocking emit: a slow or stalled TUI must never delay the + // retry backoff loop. The buffered(100) eventCh may be full if + // the consumer lags; dropping a retry notice is acceptable, + // blocking the agent is not. + select { + case o.eventCh <- ev: + default: + } + }, + func() { + // A retried attempt produced a response: emit the dedicated + // recovery event so the UI can toast immediately instead of + // guessing on the next turn's thinking event. Non-blocking: + // a dropped recovery notice is acceptable, blocking the + // agent is not. + select { + case o.eventCh <- common.RecoveryEvent{ID: o.id}: + default: + } + }, o.middlewares, ) @@ -357,6 +428,20 @@ func (o *BaseOrchestrator) run() { o.mu.Unlock() if err != nil { + // Canceled runs follow the stop path, not the error path (no error + // box): a stop can surface here as the underlying stream error, + // e.g. when the retry backoff sleep is interrupted by ctx.Done(). + // We check ctx.Err() instead of IsStopRequested() because + // IsStopRequested() consumes the one-shot stopCh token that the + // StopRequestedEvent emission at the end of run() depends on. + // Emitting "idle" mirrors how a mid-stream cancel (nil error) is + // routed, so the TUI resolves out of its "Stopping..." state; if + // the stopCh token landed, the StopRequestedEvent below still + // turns it into "Stopped". + if errors.Is(ctx.Err(), context.Canceled) || errors.Is(err, context.Canceled) { + o.eventCh <- common.StatusEvent{ID: o.id, Status: "idle"} + break + } // If the error is about unsupported image input, roll back the user message // so it doesn't poison the context for future requests. errStr := err.Error() @@ -368,6 +453,26 @@ func (o *BaseOrchestrator) run() { o.sess.History = o.sess.History[:len(o.sess.History)-1] } o.eventCh <- common.StatusEvent{ID: o.id, Status: "error", Error: fmt.Errorf("image_unsupported")} + } else if isBadRequestStatusError(err) { + // The API rejected the request body even after the executor's bad-body + // retries. Roll the turn back so the session returns to its pre-submit + // state: the user can edit and resend instead of every retry rebuilding + // the same rejected request. Persisted via PopLastUserMessage (unlike + // the image rollback above, this must survive a restart). + var se *client.StatusError + if errors.As(err, &se) { + rolled, saveErr := o.sess.PopLastUserMessage() + msg := fmt.Sprintf("API rejected the request (400) after retries: %s — ", se.Body) + switch { + case rolled && saveErr == nil: + msg += "your last message was rolled back; edit it and resend" + case rolled: + msg += "your last message was rolled back in memory, but saving the rollback to disk failed" + default: + msg += "nothing was rolled back (the turn had no unanswered user message); use /rewind if history needs repair" + } + o.eventCh <- common.StatusEvent{ID: o.id, Status: "error", Error: errors.New(msg)} + } } else { o.eventCh <- common.StatusEvent{ID: o.id, Status: "error", Error: err} } @@ -386,6 +491,13 @@ func (o *BaseOrchestrator) run() { } } +// isBadRequestStatusError reports whether err carries an HTTP 400 from the +// LLM API, even through the executor's "stream error: ..." wrapping. +func isBadRequestStatusError(err error) bool { + var se *client.StatusError + return errors.As(err, &se) && se.StatusCode == http.StatusBadRequest +} + func (o *BaseOrchestrator) Events() <-chan common.Event { return o.eventCh } diff --git a/internal/orchestrator/base_errors_test.go b/internal/orchestrator/base_errors_test.go new file mode 100644 index 00000000..ddeeb69f --- /dev/null +++ b/internal/orchestrator/base_errors_test.go @@ -0,0 +1,56 @@ +package orchestrator + +import ( + "errors" + "fmt" + "late/internal/client" + "testing" +) + +// TestIsBadRequestStatusError verifies detection of terminal HTTP 400s from +// the LLM API, including through the executor's "stream error: ..." wrapping. +func TestIsBadRequestStatusError(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + {name: "nil error", err: nil, want: false}, + {name: "plain error", err: errors.New("connection refused"), want: false}, + { + name: "wrapped 400 StatusError", + err: fmt.Errorf("stream error: %w", &client.StatusError{ + StatusCode: 400, + Status: "400 Bad Request", + Body: "invalid request payload", + }), + want: true, + }, + { + name: "StatusError 500", + err: &client.StatusError{StatusCode: 500, Status: "500 Internal Server Error"}, + want: false, + }, + { + name: "StatusError 401", + err: &client.StatusError{StatusCode: 401, Status: "401 Unauthorized"}, + want: false, + }, + { + name: "double-wrapped 400 StatusError", + err: fmt.Errorf("outer: %w", fmt.Errorf("stream error: %w", &client.StatusError{ + StatusCode: 400, + Body: "messages must alternate", + })), + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isBadRequestStatusError(tt.err); got != tt.want { + t.Errorf("isBadRequestStatusError(%v) = %v, want %v", tt.err, got, tt.want) + } + }) + } +} diff --git a/internal/orchestrator/base_retry_test.go b/internal/orchestrator/base_retry_test.go new file mode 100644 index 00000000..b2b92d6a --- /dev/null +++ b/internal/orchestrator/base_retry_test.go @@ -0,0 +1,337 @@ +package orchestrator + +// Regression tests for the orchestrator-side stream accumulator: the shared, +// per-turn accumulator (BaseOrchestrator.acc, guarded by o.mu) must be reset +// when a stream retry starts. The executor's RunLoop builds a fresh LOCAL +// accumulator per attempt, but the orchestrator appends every onStreamChunk +// delta into o.acc and builds its ContentEvents from it; without a reset on +// the onRetry path, the next ContentEvent carries +// "" and the TUI renders the spliced +// text ("parok..."). +// +// The tests drive the real onRetry callbacks end-to-end: +// +// Execute / run -> executor.RunLoop -> session.StartStream -> +// client.ChatCompletionStream -> in-process httptest SSE server +// +// The first attempt is a valid 200 whose body is truncated mid-response +// (hijacked connection, declared Content-Length larger than the bytes sent, +// one complete SSE data line, then FIN): net/http surfaces the short body as +// io.ErrUnexpectedEOF, which the retry tier classifies as retryable. The +// retry attempt serves a complete stream. Assertions only pin counts and +// content — never exact timings; the retry budget is kept small via +// common.MaxStreamRetriesKey so the jittered backoff (base 500ms) stays well +// under the deadline in every interleaving. + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "late/internal/client" + "late/internal/common" + "late/internal/session" +) + +// retryChunkPartial is the delta streamed by the failed attempt before the +// body dies: the partial content the orchestrator must not splice into the +// retry's output. +const retryChunkPartial = `{"choices":[{"index":0,"delta":{"content":"par"}}]}` + +// newAccumulatorRetryServer serves a truncated mid-stream 200 on the first +// POST to */chat/completions and a complete SSE stream carrying "ok" on every +// later POST. Discovery probes (GET /props, GET /v1/models) answer 404. +func newAccumulatorRetryServer(t *testing.T) *httptest.Server { + t.Helper() + var failureServed atomic.Bool + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || !strings.HasSuffix(r.URL.Path, "/chat/completions") { + w.WriteHeader(http.StatusNotFound) + return + } + if !failureServed.CompareAndSwap(false, true) { + // Every attempt after the first: a complete stream with only "ok". + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprint(w, "data: "+`{"id":"1","choices":[{"index":0,"delta":{"content":"ok"},"finish_reason":"stop"}]}`+"\n") + fmt.Fprint(w, "data: [DONE]\n") + return + } + // First attempt: a valid 200 whose body is truncated mid-response. + // Hijack the connection, declare a Content-Length larger than the + // bytes actually sent, write one valid partial SSE data line, then + // FIN without the remaining body. The client sees EOF before the + // declared Content-Length and the read surfaces io.ErrUnexpectedEOF, + // which the executor's retry tier treats as retryable (same fixture + // as the executor's own mid-body disconnect test). + hijacker, ok := w.(http.Hijacker) + if !ok { + t.Error("server ResponseWriter does not support Hijack") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprint(w, `{"error":{"message":"hijack unsupported","type":"server_error"}}`) + return + } + conn, _, err := hijacker.Hijack() + if err != nil { + t.Errorf("hijack failed: %v", err) + return + } + defer conn.Close() + + head := "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: 1000\r\n\r\n" + body := "data: " + retryChunkPartial + "\n\n" + if _, err := conn.Write([]byte(head)); err != nil { + t.Errorf("writing truncated response head: %v", err) + return + } + if _, err := conn.Write([]byte(body)); err != nil { + t.Errorf("writing truncated response body: %v", err) + return + } + if tcp, ok := conn.(*net.TCPConn); ok { + tcp.CloseWrite() + } + })) + t.Cleanup(ts.Close) + return ts +} + +// newAccumulatorRetryOrchestrator builds a BaseOrchestrator over an in-memory +// session bound to the test server, with a ctx carrying a small retry budget. +func newAccumulatorRetryOrchestrator(t *testing.T, baseURL string) *BaseOrchestrator { + t.Helper() + c := client.NewClient(client.Config{BaseURL: baseURL}) + sess := session.New(c, "", nil, "", false) + o := NewBaseOrchestrator("test-orch", sess, nil, 5) + ctx, cancel := context.WithTimeout( + context.WithValue(context.Background(), common.MaxStreamRetriesKey, 3), + 15*time.Second, + ) + t.Cleanup(cancel) + o.SetContext(ctx) + return o +} + +// collectStreamedContents drains the orchestrator's buffered event channel +// without blocking and returns the streaming (non-Completed) ContentEvent +// contents in emission order plus the RetryEvent and RecoveryEvent counts. +// Safe to call only after the producing goroutine has finished (Execute) or +// after the collector below saw a terminal event (run). +func collectStreamedContents(o *BaseOrchestrator) (contents []string, retries, recoveries int) { + for { + select { + case ev := <-o.Events(): + switch e := ev.(type) { + case common.ContentEvent: + if !e.Completed { + contents = append(contents, e.Content) + } + case common.RetryEvent: + retries++ + case common.RecoveryEvent: + recoveries++ + } + default: + return contents, retries, recoveries + } + } +} + +// TestOnRetryResetsAccumulator covers the Execute path: a stream that dies +// mid-response after emitting "par", then succeeds with "ok" on the retry. +// The failed attempt's partial is streamed live (first ContentEvent "par"), +// but the post-retry ContentEvent must carry only the retry's output ("ok"), +// never the spliced "parok" — the onRetry callback resets the shared +// accumulator before the retry's deltas are appended. +func TestOnRetryResetsAccumulator(t *testing.T) { + ts := newAccumulatorRetryServer(t) + o := newAccumulatorRetryOrchestrator(t, ts.URL) + + res, err := o.Execute("hello") + if err != nil { + t.Fatalf("Execute returned error after a retryable mid-stream disconnect: %v", err) + } + if res != "ok" { + t.Errorf("Execute result = %q, want %q", res, "ok") + } + + contents, retries, recoveries := collectStreamedContents(o) + if retries != 1 { + t.Fatalf("got %d RetryEvents, want exactly 1", retries) + } + // The retry produced a response: the dedicated recovery event must be + // emitted exactly once, so the UI can toast immediately instead of + // guessing on the next turn's thinking event. + if recoveries != 1 { + t.Errorf("got %d RecoveryEvents, want exactly 1 (retry actually succeeded)", recoveries) + } + if len(contents) != 2 { + t.Fatalf("got %d streaming ContentEvents (%q), want exactly 2: the failed attempt's partial and the retry's output", len(contents), contents) + } + if contents[0] != "par" { + t.Errorf("first streaming ContentEvent = %q, want the failed attempt's partial %q", contents[0], "par") + } + if contents[1] != "ok" { + t.Errorf("post-retry streaming ContentEvent = %q, want %q — the retry starts a fresh accumulator, not the spliced %q", contents[1], "ok", "parok") + } +} + +// TestOnRetryResetsAccumulatorRunLoop covers the run path (Submit's +// background loop), whose onRetry callback is a separate closure from the +// Execute one and must reset the shared accumulator the same way. +func TestOnRetryResetsAccumulatorRunLoop(t *testing.T) { + ts := newAccumulatorRetryServer(t) + o := newAccumulatorRetryOrchestrator(t, ts.URL) + + var mu sync.Mutex + var contents []string + retries := 0 + recoveries := 0 + collectorDone := make(chan struct{}) + go func() { + defer close(collectorDone) + for { + ev := <-o.Events() + switch e := ev.(type) { + case common.ContentEvent: + if !e.Completed { + mu.Lock() + contents = append(contents, e.Content) + mu.Unlock() + } + case common.RetryEvent: + mu.Lock() + retries++ + mu.Unlock() + case common.RecoveryEvent: + mu.Lock() + recoveries++ + mu.Unlock() + case common.StatusEvent: + // run() always ends the loop with a terminal status event; + // everything it emits before that has already been read. + if e.Status == "idle" || e.Status == "closed" || e.Status == "error" { + return + } + } + } + }() + + if err := o.Submit("hello", nil); err != nil { + t.Fatalf("Submit returned error: %v", err) + } + select { + case <-collectorDone: + case <-time.After(30 * time.Second): + t.Fatal("run loop did not reach a terminal status within 30s") + } + + mu.Lock() + defer mu.Unlock() + if retries != 1 { + t.Fatalf("got %d RetryEvents, want exactly 1", retries) + } + // The run path's separate onRecover closure must emit the dedicated + // recovery event exactly once when the retried attempt succeeds. + if recoveries != 1 { + t.Errorf("got %d RecoveryEvents, want exactly 1 (retry actually succeeded)", recoveries) + } + if len(contents) != 2 { + t.Fatalf("got %d streaming ContentEvents (%q), want exactly 2: the failed attempt's partial and the retry's output", len(contents), contents) + } + if contents[0] != "par" { + t.Errorf("first streaming ContentEvent = %q, want the failed attempt's partial %q", contents[0], "par") + } + if contents[1] != "ok" { + t.Errorf("post-retry streaming ContentEvent = %q, want %q — the retry starts a fresh accumulator, not the spliced %q", contents[1], "ok", "parok") + } +} + +// --- Recovery must never fire on retry exhaustion --- +// +// There was no orchestrator-level exhaustion fixture, so this small one pins +// the exhaustion side of the recovery contract end-to-end: with the bad-body +// budget drained, the run terminates with a terminal error StatusEvent and +// ZERO RecoveryEvents — exhaustion is not recovery, because no attempt ever +// produced a response. + +// TestBadBodyBudgetExhaustedEmitsNoRecovery drives the Execute path against a +// server that rejects every request with a transient-looking HTTP 400 (the +// bad-body tier) and a ctx with a tiny bad-body budget. The budget drains, +// RunLoop fails, and the orchestrator emits the terminal error StatusEvent — +// without ever emitting a RecoveryEvent. +func TestBadBodyBudgetExhaustedEmitsNoRecovery(t *testing.T) { + var posts atomic.Int64 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || !strings.HasSuffix(r.URL.Path, "/chat/completions") { + w.WriteHeader(http.StatusNotFound) + return + } + posts.Add(1) + // Every attempt: the transient-looking 400 body-parse rejection the + // bad-body retry tier exists for. + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusBadRequest) + fmt.Fprint(w, `{"error":{"message":"The request is invalid: read body failed.","type":"server_error"}}`) + })) + t.Cleanup(ts.Close) + + c := client.NewClient(client.Config{BaseURL: ts.URL}) + sess := session.New(c, "", nil, "", false) + o := NewBaseOrchestrator("test-orch-exhaust", sess, nil, 5) + ctx, cancel := context.WithTimeout( + context.WithValue(context.Background(), common.MaxBadBodyRetriesKey, 1), + 15*time.Second, + ) + t.Cleanup(cancel) + o.SetContext(ctx) + + res, err := o.Execute("hello") + if err == nil { + t.Fatalf("Execute returned %q with nil error, want the bad-body budget exhaustion error", res) + } + var statusErr *client.StatusError + if !errors.As(err, &statusErr) || statusErr.StatusCode != http.StatusBadRequest { + t.Fatalf("Execute error = %v, want it to wrap *client.StatusError with 400", err) + } + + // The budget drained: initial attempt + exactly one retry. + if got := posts.Load(); got != 2 { + t.Errorf("server got %d POSTs, want 2 (initial attempt + one bad-body retry)", got) + } + + // Drain the buffered events: zero RecoveryEvents, and the terminal error + // status must have arrived. All events are emitted synchronously before + // Execute returns, so a non-blocking drain cannot race the producer. + sawErrorStatus := false + recoveries := 0 + for { + select { + case ev := <-o.Events(): + switch e := ev.(type) { + case common.RecoveryEvent: + recoveries++ + case common.StatusEvent: + if e.Status == "error" { + sawErrorStatus = true + } + } + default: + if recoveries != 0 { + t.Errorf("got %d RecoveryEvents, want 0 (exhaustion is not recovery)", recoveries) + } + if !sawErrorStatus { + t.Error("no terminal error StatusEvent, want one after the bad-body budget drained") + } + return + } + } +} diff --git a/internal/plugin/commands_tools_test.go b/internal/plugin/commands_tools_test.go index 3908c733..edb43704 100644 --- a/internal/plugin/commands_tools_test.go +++ b/internal/plugin/commands_tools_test.go @@ -322,7 +322,12 @@ func TestHandleCommand_ConcurrentWithWriters(t *testing.T) { select { case <-done: - case <-time.After(20 * time.Second): + // 60s watchdog: 3x the 20s budget that tripped once under heavy machine + // load (back-to-back -race suites) even though the code under test was + // not deadlocked. On a real nested-RLock regression the loop never + // completes, so the watchdog still fails the test instead of hanging + // the suite. Iterations and assertions unchanged. + case <-time.After(60 * time.Second): t.Fatal("HandleCommand deadlocked against queued writers (nested RLock?)") } } diff --git a/internal/plugin/hooks_unix_test.go b/internal/plugin/hooks_unix_test.go index a803a73d..ca096875 100644 --- a/internal/plugin/hooks_unix_test.go +++ b/internal/plugin/hooks_unix_test.go @@ -22,10 +22,53 @@ func TestRunHook_ProcessGroupKillsChildrenOnCancel(t *testing.T) { body := "sleep 30 &\necho $! > " + pidFile + "\nwait" writeExecutableShell(t, script, body) - ctx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond) + // The hook must stay alive until the script has spawned its child and + // written the pid file, so it is cancelled only once the pid file + // appears — asserting the process-group kill requires the child to + // exist first. The previous fixed 250ms budget for the spawn failed + // deterministically on hosts whose fork/exec latency exceeded it: the + // process group was SIGKILLed before the shell ever executed + // `echo $! > pidFile`, so the pid file never appeared and the test + // failed before the process-group assertion could run. runHook caps + // hook execution at hookTimeout, so extend it for this test (same + // pattern as the hookWaitDelay override below) and poll for the pid + // file instead of budgeting the spawn; on loaded hosts shell startup + // has been observed to take seconds. + oldHookTimeout := hookTimeout + hookTimeout = 60 * time.Second + t.Cleanup(func() { hookTimeout = oldHookTimeout }) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() - _, err := runHook(ctx, pluginDir, "group_child.sh", nil) + hookDone := make(chan error, 1) + go func() { + _, err := runHook(ctx, pluginDir, "group_child.sh", nil) + hookDone <- err + }() + + // Deadline-based poll for the pid file instead of a fixed spawn budget. + // The limit is half the hook window so the poll can never race the + // hook's own timeout. + const pidFileWaitLimit = 30 * time.Second + pidWaitStart := time.Now() + for { + // echo's `>` opens pidFile before writing it, so break only on non-empty content — an empty read in the open->write gap would break pid parsing. + if b, err := os.ReadFile(pidFile); err == nil && strings.TrimSpace(string(b)) != "" { + break + } + if elapsed := time.Since(pidWaitStart); elapsed > pidFileWaitLimit { + cancel() // don't leak the hook if the pid file never appears + t.Fatalf("child pid file %s never appeared within %v", pidFile, elapsed) + } + time.Sleep(25 * time.Millisecond) + } + + // The child now exists; cancelling the hook must kill the whole + // process group (asserted below). + cancel() + + err := <-hookDone if err == nil { t.Fatal("expected hook to fail on context deadline") } diff --git a/internal/plugin/project_test.go b/internal/plugin/project_test.go index 70aecb71..e30d1916 100644 --- a/internal/plugin/project_test.go +++ b/internal/plugin/project_test.go @@ -5,6 +5,7 @@ import ( "path/filepath" "testing" ) + // writeBarePlugin creates a minimal plugin directory with a native-Late // package.json at the given path and returns the directory. It is // intentionally a different name from the rich @@ -463,10 +464,10 @@ func TestHasProjectDir_Methods(t *testing.T) { func TestParseProjectFlag(t *testing.T) { tests := []struct { - name string - args []string - wantProj bool - wantRest string + name string + args []string + wantProj bool + wantRest string }{ { name: "no flag, has source", @@ -531,7 +532,6 @@ func TestParseProjectFlag(t *testing.T) { } } - // --------------------------------------------------------------------------- // PluginPathInDir // --------------------------------------------------------------------------- @@ -686,22 +686,12 @@ func writeSkillPlugin(t *testing.T, dir, pluginName, skillName string) { func TestHandlePluginRemove_PurgesStaleSkillSymlink(t *testing.T) { globalDir := t.TempDir() sourceDir := t.TempDir() - xdgRoot := t.TempDir() - - // Sandbox `lateSkillsDir()` so the test cannot damage the user's real - // `~/.config/late/skills` if our internal invariants drift. Go's - // `os.UserConfigDir()` honors `XDG_CONFIG_HOME` on every Unix-like - // target (Linux, macOS, BSD) per the freedesktop spec; pinning only - // this variable is sufficient and avoids sideways effects on - // `os.UserHomeDir()`/`isSuspiciousPluginPath` that overriding HOME - // would introduce. - t.Setenv("XDG_CONFIG_HOME", xdgRoot) - skillsDir := filepath.Join(xdgRoot, "late", "skills") - if err := os.MkdirAll(skillsDir, 0755); err != nil { - t.Fatalf("mkdir skills dir: %v", err) - } + _, skillsDir := sandboxUserConfig(t) + + // Guard: lateSkillsDir() must resolve inside the sandbox — this is the + // invariant that keeps the test off the developer's real user config. if resolved, _ := lateSkillsDir(); resolved != skillsDir { - t.Fatalf("lateSkillsDir() = %q, want %q (XDG/HOME override misconfigured)", resolved, skillsDir) + t.Fatalf("lateSkillsDir() = %q, want %q (user-config sandbox ineffective)", resolved, skillsDir) } writeSkillPlugin(t, sourceDir, "skills-plugin", "my-skill") @@ -741,14 +731,7 @@ func TestHandlePluginRemove_PreservesSiblingSkillSymlink(t *testing.T) { globalDir := t.TempDir() srcA := t.TempDir() srcB := t.TempDir() - xdgRoot := t.TempDir() - - // Same sandboxing strategy as the single-plugin self-clean test. - t.Setenv("XDG_CONFIG_HOME", xdgRoot) - skillsDir := filepath.Join(xdgRoot, "late", "skills") - if err := os.MkdirAll(skillsDir, 0755); err != nil { - t.Fatalf("mkdir skills dir: %v", err) - } + _, skillsDir := sandboxUserConfig(t) // Two distinct plugins, intentionally both declaring a skill with the // same basename to maximize the chance a bogus "name only" keep key diff --git a/internal/plugin/regression_test.go b/internal/plugin/regression_test.go index 682fc386..c6887fad 100644 --- a/internal/plugin/regression_test.go +++ b/internal/plugin/regression_test.go @@ -66,8 +66,8 @@ func TestRegisterPluginSkills_DoesNotLeakAcrossProjects(t *testing.T) { } func TestRegisterPluginSkills_PreservesSameNamedSkills(t *testing.T) { - configDir, pluginsDir := t.TempDir(), t.TempDir() - t.Setenv("XDG_CONFIG_HOME", configDir) + pluginsDir := t.TempDir() + _, skillsDir := sandboxUserConfig(t) t.Chdir(t.TempDir()) writeDeploySkillPlugin(t, pluginsDir, "alpha") writeDeploySkillPlugin(t, pluginsDir, "beta") @@ -75,7 +75,7 @@ func TestRegisterPluginSkills_PreservesSameNamedSkills(t *testing.T) { if err := pm.Discover(); err != nil { t.Fatal(err) } - if err := pm.RegisterPluginSkills(filepath.Join(configDir, "late", "skills")); err != nil { + if err := pm.RegisterPluginSkills(skillsDir); err != nil { t.Fatal(err) } reg := common.NewToolRegistry() diff --git a/internal/plugin/sandbox_test.go b/internal/plugin/sandbox_test.go new file mode 100644 index 00000000..a2c99977 --- /dev/null +++ b/internal/plugin/sandbox_test.go @@ -0,0 +1,37 @@ +package plugin + +import ( + "os" + "path/filepath" + "testing" +) + +// sandboxUserConfig isolates a test from the developer's real user-level +// late configuration (~/Library/Application Support/late on macOS, +// ~/.config/late on Linux): every path late resolves through +// os.UserConfigDir() (pathutil.LateConfigDir, pathutil.LateSkillsDir, +// pluginStatePath, manager lateSkillsDir) lands inside a throwaway +// directory instead. +// +// It sets BOTH environment variables os.UserConfigDir() consults: +// - HOME governs darwin/ios, where UserConfigDir() always returns +// "$HOME/Library/Application Support" and XDG_CONFIG_HOME is ignored; +// - XDG_CONFIG_HOME governs the remaining Unix targets. +// +// Callers must derive expected paths from the returned skillsDir (or from +// os.UserConfigDir() after this call), never from XDG alone. +func sandboxUserConfig(t *testing.T) (root string, skillsDir string) { + t.Helper() + root = t.TempDir() + t.Setenv("HOME", root) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(root, ".config")) + configBase, err := os.UserConfigDir() + if err != nil { + t.Fatalf("os.UserConfigDir after sandboxing: %v", err) + } + skillsDir = filepath.Join(configBase, "late", "skills") + if err := os.MkdirAll(skillsDir, 0o755); err != nil { + t.Fatalf("mkdir sandboxed skills dir: %v", err) + } + return root, skillsDir +} diff --git a/internal/plugin/security_test.go b/internal/plugin/security_test.go index a802cef8..eb496bc5 100644 --- a/internal/plugin/security_test.go +++ b/internal/plugin/security_test.go @@ -107,7 +107,6 @@ func TestConcurrentSetProjectDirAndRead(t *testing.T) { wg.Wait() } - // --------------------------------------------------------------------------- // Installer (#6) // --------------------------------------------------------------------------- @@ -501,11 +500,12 @@ func TestSavePluginMeta_PersistsEnabledField(t *testing.T) { // global plugin state override file instead (see state.go), and // LoadPluginMeta applies it back on the next discovery. func TestSavePluginMeta_LocalPluginPersistsDisabledAcrossReload(t *testing.T) { - xdgRoot := t.TempDir() - // Sandbox pluginStatePath() (~/.config/late/plugins.json) so this test - // cannot touch the real user's config — see the identical rationale in - // TestHandlePluginRemove_PurgesStaleSkillSymlink. - t.Setenv("XDG_CONFIG_HOME", xdgRoot) + // Sandbox pluginStatePath() (~/.config/late/plugins.json on Linux, + // ~/Library/Application Support/late on macOS) so this test cannot + // touch the real user's config: os.UserConfigDir() ignores + // XDG_CONFIG_HOME on darwin, so sandboxing requires overriding HOME + // too (see sandboxUserConfig). + sandboxUserConfig(t) globalDir := t.TempDir() sourceDir := t.TempDir() @@ -559,8 +559,9 @@ func TestSavePluginMeta_LocalPluginPersistsDisabledAcrossReload(t *testing.T) { // clear its override entry, so relinking a different plugin under the // same name later doesn't silently inherit a stale disabled state. func TestRemovePlugin_ClearsLocalDisabledOverride(t *testing.T) { - xdgRoot := t.TempDir() - t.Setenv("XDG_CONFIG_HOME", xdgRoot) + // Sandboxed user config so the disabled-override state file is a + // throwaway (see sandboxUserConfig for the darwin/XDG rationale). + sandboxUserConfig(t) globalDir := t.TempDir() sourceDir := t.TempDir() @@ -619,5 +620,3 @@ func mkPlugin(t *testing.T, dir, name string) { t.Fatalf("write: %v", err) } } - - diff --git a/internal/session/history_sanitize.go b/internal/session/history_sanitize.go new file mode 100644 index 00000000..c7fed79b --- /dev/null +++ b/internal/session/history_sanitize.go @@ -0,0 +1,103 @@ +package session + +import ( + "late/internal/client" +) + +// interruptedToolResultText is the content synthesized for tool calls whose +// execution was interrupted before a result was recorded (e.g. a crash or +// error between the assistant-history commit and the tool-result writes). +const interruptedToolResultText = "(tool execution was interrupted; no result was recorded)" + +// SanitizeForRequest returns a request-safe copy of msgs for strict +// OpenAI-compatible endpoints: every assistant message carrying tool_calls is +// closed by one tool result per tool_call_id, synthesizing an interrupted- +// result placeholder when history is missing one, and tool calls with an +// empty ID are stripped from the outgoing copy (strict endpoints reject +// ID-less tool calls, and no result can ever correlate to one). It never +// mutates the input. +func SanitizeForRequest(msgs []client.ChatMessage) []client.ChatMessage { + if len(msgs) == 0 { + return msgs + } + + sanitized := make([]client.ChatMessage, 0, len(msgs)+2) + + // pendingIDs holds tool-call IDs awaiting a result, in registration order, + // so synthesized placeholders appear deterministically; pendingSet gives + // O(1) membership tests. Both are emptied in lockstep. + pendingIDs := make([]string, 0, 4) + pendingSet := make(map[string]struct{}, 4) + + // closePending emits one placeholder tool result per still-pending + // tool-call ID, in registration order, and resets the pending state. + closePending := func() { + for _, id := range pendingIDs { + delete(pendingSet, id) + sanitized = append(sanitized, client.ChatMessage{ + Role: "tool", + ToolCallID: id, + Content: client.TextContent(interruptedToolResultText), + }) + } + pendingIDs = pendingIDs[:0] + } + + for _, m := range msgs { + switch m.Role { + case "assistant": + // A new assistant turn implicitly ends the previous tool group + // for strict servers: close it before appending this message. + closePending() + if len(m.ToolCalls) > 0 { + // Strict OpenAI-compatible endpoints reject tool calls with an + // empty ID, and no tool result can ever correlate to one (tool + // results match by ID, and an empty-ID result is already + // dropped as an orphan below), so strip them from the outgoing + // copy only. The filtered slice is freshly allocated: reusing + // m.ToolCalls' backing array would mutate the input. If every + // call had an empty ID the message is sent as a plain + // assistant turn; its Content and ReasoningContent are kept. + filtered := make([]client.ToolCall, 0, len(m.ToolCalls)) + for _, tc := range m.ToolCalls { + if tc.ID != "" { + filtered = append(filtered, tc) + } + } + if len(filtered) == 0 { + filtered = nil + } + m.ToolCalls = filtered + } + sanitized = append(sanitized, m) + for _, tc := range m.ToolCalls { + if _, ok := pendingSet[tc.ID]; ok { + continue + } + pendingIDs = append(pendingIDs, tc.ID) + pendingSet[tc.ID] = struct{}{} + } + case "tool": + if _, ok := pendingSet[m.ToolCallID]; !ok { + // Dangling tool result with no matching assistant tool_call: + // poison for strict endpoints, so drop it. + continue + } + delete(pendingSet, m.ToolCallID) + for i, id := range pendingIDs { + if id == m.ToolCallID { + pendingIDs = append(pendingIDs[:i], pendingIDs[i+1:]...) + break + } + } + sanitized = append(sanitized, m) + default: + // system/user/... turns also terminate an open tool group. + closePending() + sanitized = append(sanitized, m) + } + } + closePending() + + return sanitized +} diff --git a/internal/session/history_sanitize_test.go b/internal/session/history_sanitize_test.go new file mode 100644 index 00000000..41e11e26 --- /dev/null +++ b/internal/session/history_sanitize_test.go @@ -0,0 +1,196 @@ +package session + +import ( + "late/internal/client" + "reflect" + "testing" +) + +// placeholderContent pins the exact text SanitizeForRequest synthesizes for +// interrupted tool calls, asserted literally rather than via the constant. +const placeholderContent = "(tool execution was interrupted; no result was recorded)" + +func userMsg(content string) client.ChatMessage { + return client.ChatMessage{Role: "user", Content: client.TextContent(content)} +} + +func systemMsg(content string) client.ChatMessage { + return client.ChatMessage{Role: "system", Content: client.TextContent(content)} +} + +func toolCall(id, name string) client.ToolCall { + return client.ToolCall{ + ID: id, + Type: "function", + Function: client.FunctionCall{Name: name, Arguments: "{}"}, + } +} + +func assistantWithCalls(calls ...client.ToolCall) client.ChatMessage { + return client.ChatMessage{Role: "assistant", Content: client.TextContent(""), ToolCalls: calls} +} + +func toolResult(id, content string) client.ChatMessage { + return client.ChatMessage{Role: "tool", ToolCallID: id, Content: client.TextContent(content)} +} + +func toolPlaceholder(id string) client.ChatMessage { + return toolResult(id, placeholderContent) +} + +// cloneHistory deep-copies msgs (including ToolCalls and Content.Parts) so +// tests can prove SanitizeForRequest leaves its input untouched. A nil input +// stays nil so the post-call comparison is exact. +func cloneHistory(msgs []client.ChatMessage) []client.ChatMessage { + if msgs == nil { + return nil + } + out := make([]client.ChatMessage, len(msgs)) + for i, m := range msgs { + m.ToolCalls = append([]client.ToolCall(nil), m.ToolCalls...) + m.AttachedFiles = append([]string(nil), m.AttachedFiles...) + m.Content.Parts = append([]client.ContentPart(nil), m.Content.Parts...) + out[i] = m + } + return out +} + +func TestSanitizeForRequest(t *testing.T) { + tests := []struct { + name string + input []client.ChatMessage + want []client.ChatMessage + }{ + { + name: "dangling tool_calls at end of history get placeholders in tool-call ID order", + input: []client.ChatMessage{ + userMsg("list the files"), + assistantWithCalls(toolCall("call_b", "read_file"), toolCall("call_a", "list_dir")), + }, + want: []client.ChatMessage{ + userMsg("list the files"), + assistantWithCalls(toolCall("call_b", "read_file"), toolCall("call_a", "list_dir")), + toolPlaceholder("call_b"), + toolPlaceholder("call_a"), + }, + }, + { + name: "dangling tool_calls group mid-history closes before the following user turn", + input: []client.ChatMessage{ + userMsg("inspect the repo"), + assistantWithCalls(toolCall("call_1", "read_file"), toolCall("call_2", "list_dir")), + userMsg("never mind, continue"), + }, + want: []client.ChatMessage{ + userMsg("inspect the repo"), + assistantWithCalls(toolCall("call_1", "read_file"), toolCall("call_2", "list_dir")), + toolPlaceholder("call_1"), + toolPlaceholder("call_2"), + userMsg("never mind, continue"), + }, + }, + { + name: "partially answered tool group keeps the real result and fills only the gap", + input: []client.ChatMessage{ + assistantWithCalls(toolCall("call_1", "read_file"), toolCall("call_2", "list_dir")), + toolResult("call_1", "file contents"), + userMsg("thanks"), + }, + want: []client.ChatMessage{ + assistantWithCalls(toolCall("call_1", "read_file"), toolCall("call_2", "list_dir")), + toolResult("call_1", "file contents"), + toolPlaceholder("call_2"), + userMsg("thanks"), + }, + }, + { + name: "well-formed history is passed through unchanged", + input: []client.ChatMessage{ + systemMsg("you are a coding agent"), + userMsg("read main.go"), + assistantWithCalls(toolCall("call_1", "read_file"), toolCall("call_2", "list_dir")), + toolResult("call_1", "package main"), + toolResult("call_2", "main.go\nutils.go"), + userMsg("now summarize"), + }, + want: []client.ChatMessage{ + systemMsg("you are a coding agent"), + userMsg("read main.go"), + assistantWithCalls(toolCall("call_1", "read_file"), toolCall("call_2", "list_dir")), + toolResult("call_1", "package main"), + toolResult("call_2", "main.go\nutils.go"), + userMsg("now summarize"), + }, + }, + { + name: "leading orphan tool result with no preceding assistant is dropped", + input: []client.ChatMessage{toolResult("call_orphan", "stale result")}, + want: []client.ChatMessage{}, + }, + { + name: "empty input is returned unchanged", + input: nil, + want: nil, + }, + { + name: "assistant with only empty-ID tool_calls is sent without tool_calls, content preserved", + input: []client.ChatMessage{ + { + Role: "assistant", + Content: client.TextContent("let me look that up"), + ReasoningContent: "reasoning about the request", + ToolCalls: []client.ToolCall{toolCall("", "read_file")}, + }, + }, + want: []client.ChatMessage{ + { + Role: "assistant", + Content: client.TextContent("let me look that up"), + ReasoningContent: "reasoning about the request", + }, + }, + }, + { + name: "assistant with one empty-ID and one real tool_call keeps only the real call and its result", + input: []client.ChatMessage{ + assistantWithCalls(toolCall("", "read_file"), toolCall("call_1", "list_dir")), + toolResult("call_1", "main.go\nutils.go"), + }, + want: []client.ChatMessage{ + assistantWithCalls(toolCall("call_1", "list_dir")), + toolResult("call_1", "main.go\nutils.go"), + }, + }, + { + name: "consecutive assistant tool_call groups close the first before the second", + input: []client.ChatMessage{ + assistantWithCalls(toolCall("call_1", "read_file")), + assistantWithCalls(toolCall("call_2", "list_dir")), + }, + want: []client.ChatMessage{ + assistantWithCalls(toolCall("call_1", "read_file")), + toolPlaceholder("call_1"), + assistantWithCalls(toolCall("call_2", "list_dir")), + toolPlaceholder("call_2"), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + before := cloneHistory(tt.input) + + got := SanitizeForRequest(tt.input) + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("SanitizeForRequest() mismatch\n got: %+v\nwant: %+v", got, tt.want) + } + if !reflect.DeepEqual(tt.input, before) { + t.Errorf("SanitizeForRequest() mutated its input\n got: %+v\nwant: %+v", tt.input, before) + } + if len(got) > 0 && len(tt.input) > 0 && &got[0] == &tt.input[0] { + t.Errorf("SanitizeForRequest() returned a slice aliasing its input") + } + }) + } +} diff --git a/internal/session/models.go b/internal/session/models.go index 03f5cb5f..7bafd3c2 100644 --- a/internal/session/models.go +++ b/internal/session/models.go @@ -22,6 +22,7 @@ type SessionMeta struct { MessageCount int `json:"message_count"` SubagentSeq int `json:"subagent_seq"` SaveSubagentHistories *bool `json:"save_subagent_histories,omitempty"` + WorkingDir string `json:"working_dir,omitempty"` // Absolute path of the project directory where the session was started } // SessionDir returns the directory where session metadata and histories are stored @@ -158,8 +159,12 @@ func ListSessions() ([]SessionMeta, error) { return metas, nil } -// GetLatestSession returns the metadata of the latest session (most recently updated). -// If no sessions exist, it returns nil, nil. +// GetLatestSession returns the metadata of the latest session (most recently +// updated), regardless of which project directory sessions were started in. +// Each sidecar is loaded by its exact enumerated path, so a sidecar that +// disappears or fails to load is merely skipped and the scan never falls back +// to a different session with a matching ID prefix. If no sessions exist, it +// returns nil, nil. func GetLatestSession() (*SessionMeta, error) { sessionsDir, err := SessionDir() if err != nil { @@ -174,23 +179,113 @@ func GetLatestSession() (*SessionMeta, error) { return nil, fmt.Errorf("failed to read sessions directory: %w", err) } - var latestEntry os.DirEntry + var latest *SessionMeta var latestModTime time.Time for _, entry := range entries { - if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".meta.json") { - if info, err := entry.Info(); err == nil { - if latestEntry == nil || info.ModTime().After(latestModTime) { - latestEntry = entry - latestModTime = info.ModTime() - } - } + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".meta.json") { + continue + } + info, err := entry.Info() + if err != nil { + continue + } + meta, err := loadEnumeratedMeta(filepath.Join(sessionsDir, entry.Name())) + if err != nil || meta == nil { + continue + } + if latest == nil || info.ModTime().After(latestModTime) { + latest = meta + latestModTime = info.ModTime() + } + } + return latest, nil +} + +// loadEnumeratedMeta loads one enumerated sidecar by its exact path. It is a +// package-level variable so tests can simulate sidecars that vanish or turn +// unreadable between enumeration (os.ReadDir + entry.Info()) and load — the +// race window that historically produced a (nil, nil) result and a startup +// panic. Production always uses loadMetaFile. +var loadEnumeratedMeta = loadMetaFile + +// GetLatestSessionForDir returns the metadata of the most recently updated +// session that was started in dir, matched against the working_dir recorded +// in each session's metadata. Sessions created before working_dir was +// recorded (empty WorkingDir) are ignored. If no matching session exists, +// it returns nil, nil. +// +// Each enumerated .meta.json sidecar is loaded by its exact path rather than +// through LoadSessionMeta, whose ID-prefix fallback could silently return a +// different session sharing the ID prefix; a sidecar that fails to load or +// yields no metadata is skipped (defensively including a nil meta, although +// loadMetaFile never returns (nil, nil)). +// +// Directory matching takes a lexical fast path (filepath.Clean equality) and +// then compares directory identity with os.Stat + os.SameFile when both paths +// exist, so a session recorded under a real directory is still found when the +// same directory is addressed through a symlink. SameFile also covers +// case-variant aliases on case-insensitive filesystems, so paths are never +// lowercased or case-folded here. When either directory is missing the +// identity check is unavailable and the lexical result stands. +func GetLatestSessionForDir(dir string) (*SessionMeta, error) { + sessionsDir, err := SessionDir() + if err != nil { + return nil, err + } + + entries, err := os.ReadDir(sessionsDir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil } + return nil, fmt.Errorf("failed to read sessions directory: %w", err) } - if latestEntry == nil { - return nil, nil + want := filepath.Clean(dir) + var latest *SessionMeta + var latestModTime time.Time + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".meta.json") { + continue + } + info, err := entry.Info() + if err != nil { + continue + } + // Load the exact enumerated file: no ID-prefix re-resolution. + meta, err := loadEnumeratedMeta(filepath.Join(sessionsDir, entry.Name())) + if err != nil || meta == nil { + continue + } + if meta.WorkingDir == "" || !sameProjectDir(meta.WorkingDir, want) { + continue + } + if latest == nil || info.ModTime().After(latestModTime) { + latest = meta + latestModTime = info.ModTime() + } } + return latest, nil +} - id := strings.TrimSuffix(latestEntry.Name(), ".meta.json") - return LoadSessionMeta(id) +// sameProjectDir reports whether a session's recorded project directory and a +// wanted directory refer to the same directory. The lexical comparison is the +// fast path; when it misses and both paths exist, identity is compared via +// os.SameFile (os.Stat follows symlinks), which also matches case-variant +// aliases on case-insensitive filesystems. If either directory does not +// exist, only the lexical result is available. Paths are never lowercased. +func sameProjectDir(recorded, want string) bool { + recorded = filepath.Clean(recorded) + want = filepath.Clean(want) + if recorded == want { + return true + } + wantInfo, wantErr := os.Stat(want) + recordedInfo, recordedErr := os.Stat(recorded) + if wantErr != nil || recordedErr != nil { + // Missing or inaccessible directory on either side: the identity + // check cannot run, so lexical equality (already ruled out) stands. + return false + } + return os.SameFile(wantInfo, recordedInfo) } diff --git a/internal/session/models_test.go b/internal/session/models_test.go index cbeb9728..fd42defc 100644 --- a/internal/session/models_test.go +++ b/internal/session/models_test.go @@ -2,9 +2,11 @@ package session import ( "errors" + "fmt" "late/internal/client" "os" "path/filepath" + "slices" "testing" "time" ) @@ -293,3 +295,423 @@ func TestLoadSessionMeta_IgnoresSubagentFolders(t *testing.T) { t.Errorf("Expected nil meta for nonexistent session, got %v", notFound) } } + +// newWorkingDirMeta builds a session meta recording dir as its project folder, +// with the surrounding fields modeled on the metas saved by the other tests. +func newWorkingDirMeta(sessionsDir, id, dir string) SessionMeta { + return SessionMeta{ + ID: id, + Title: "Session " + id, + CreatedAt: time.Now().Add(-24 * time.Hour), + LastUpdated: time.Now().Add(-24 * time.Hour), + HistoryPath: filepath.Join(sessionsDir, id+".json"), + LastUserPrompt: "Hello", + MessageCount: 1, + WorkingDir: dir, + } +} + +// setMetaModTimes pins each session's .meta.json file to the given mod time so +// the newest-wins selection in GetLatestSessionForDir is deterministic. +func setMetaModTimes(t *testing.T, sessionsDir string, times map[string]time.Time) { + t.Helper() + for id, modTime := range times { + metaPath := filepath.Join(sessionsDir, id+".meta.json") + if err := os.Chtimes(metaPath, modTime, modTime); err != nil { + t.Fatalf("os.Chtimes(%s) error = %v", metaPath, err) + } + } +} + +func TestGetLatestSessionForDir_ReturnsNewestForDirectory(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "late-latest-for-dir-test-*") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpDir) + + // Mock SessionDir + oldSessionDir := SessionDir + SessionDir = func() (string, error) { + return tmpDir, nil + } + defer func() { SessionDir = oldSessionDir }() + + // Three sessions across two project folders; /proj/a has two candidates. + for _, meta := range []SessionMeta{ + newWorkingDirMeta(tmpDir, "session-20250101-100000", "/proj/a"), + newWorkingDirMeta(tmpDir, "session-20250102-100000", "/proj/a"), + newWorkingDirMeta(tmpDir, "session-20250103-100000", "/proj/b"), + } { + if err := SaveSessionMeta(meta); err != nil { + t.Fatalf("Failed to save meta %s: %v", meta.ID, err) + } + } + + // Enforce deterministic mtime ordering: the newest session overall is the + // /proj/b one, while /proj/a's newest is session-20250102-100000. + base := time.Now().Add(-24 * time.Hour).Truncate(time.Second) + setMetaModTimes(t, tmpDir, map[string]time.Time{ + "session-20250101-100000": base, + "session-20250102-100000": base.Add(1 * time.Hour), + "session-20250103-100000": base.Add(2 * time.Hour), + }) + + latestA, err := GetLatestSessionForDir("/proj/a") + if err != nil { + t.Fatalf("GetLatestSessionForDir(/proj/a): %v", err) + } + if latestA == nil || latestA.ID != "session-20250102-100000" { + t.Fatalf("GetLatestSessionForDir(/proj/a) = %v, want session-20250102-100000 (newest /proj/a session, not the globally newest one)", latestA) + } + + latestB, err := GetLatestSessionForDir("/proj/b") + if err != nil { + t.Fatalf("GetLatestSessionForDir(/proj/b): %v", err) + } + if latestB == nil || latestB.ID != "session-20250103-100000" { + t.Fatalf("GetLatestSessionForDir(/proj/b) = %v, want session-20250103-100000", latestB) + } +} + +func TestGetLatestSessionForDir_NoMatchReturnsNil(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "late-latest-for-dir-test-*") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpDir) + + // Mock SessionDir + oldSessionDir := SessionDir + SessionDir = func() (string, error) { + return tmpDir, nil + } + defer func() { SessionDir = oldSessionDir }() + + if err := SaveSessionMeta(newWorkingDirMeta(tmpDir, "session-20250101-100000", "/proj/a")); err != nil { + t.Fatalf("Failed to save meta: %v", err) + } + + latest, err := GetLatestSessionForDir("/proj/other") + if err != nil { + t.Fatalf("GetLatestSessionForDir(/proj/other): %v", err) + } + if latest != nil { + t.Errorf("Expected nil latest session for a directory with no sessions, got %v", latest) + } +} + +func TestGetLatestSessionForDir_IgnoresSessionsWithoutWorkingDir(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "late-latest-for-dir-test-*") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpDir) + + // Mock SessionDir + oldSessionDir := SessionDir + SessionDir = func() (string, error) { + return tmpDir, nil + } + defer func() { SessionDir = oldSessionDir }() + + // Legacy meta with no working_dir key, hand-crafted like a + // pre-working-dir session sidecar. + legacyID := "session-20250101-legacy" + legacyMetaPath := filepath.Join(tmpDir, legacyID+".meta.json") + legacyJSON := fmt.Sprintf(`{"id":"%s","history_path":"%s"}`, legacyID, filepath.Join(tmpDir, legacyID+".json")) + if err := os.WriteFile(legacyMetaPath, []byte(legacyJSON), 0600); err != nil { + t.Fatalf("WriteFile() error = %v", err) + } + + if err := SaveSessionMeta(newWorkingDirMeta(tmpDir, "session-20250102-100000", "/proj/a")); err != nil { + t.Fatalf("Failed to save meta: %v", err) + } + + // Make the legacy session the globally newest one: it must still never be + // selected because it records no working_dir. + base := time.Now().Add(-24 * time.Hour).Truncate(time.Second) + setMetaModTimes(t, tmpDir, map[string]time.Time{ + legacyID: base.Add(2 * time.Hour), + "session-20250102-100000": base.Add(1 * time.Hour), + }) + + latestLegacy, err := GetLatestSessionForDir("/proj/legacy") + if err != nil { + t.Fatalf("GetLatestSessionForDir(/proj/legacy): %v", err) + } + if latestLegacy != nil { + t.Errorf("Expected nil latest session for a directory with no matching sessions, got %v", latestLegacy) + } + + latestA, err := GetLatestSessionForDir("/proj/a") + if err != nil { + t.Fatalf("GetLatestSessionForDir(/proj/a): %v", err) + } + if latestA == nil || latestA.ID != "session-20250102-100000" { + t.Fatalf("GetLatestSessionForDir(/proj/a) = %v, want session-20250102-100000 (the legacy meta without working_dir must be skipped despite its newer mtime)", latestA) + } +} + +func TestSessionMeta_WorkingDirRoundTrip(t *testing.T) { + tmpDir := t.TempDir() + t.Chdir(tmpDir) + + oldSessionDir := SessionDir + SessionDir = func() (string, error) { return tmpDir, nil } + t.Cleanup(func() { SessionDir = oldSessionDir }) + + // Constructed like TestSessionMetadataRetainsSubagentState: New captures + // the current working directory as the session's project folder. + s := New(nil, filepath.Join(tmpDir, "session-test.json"), nil, "", false) + if err := s.AddUserMessage("Hello"); err != nil { + t.Fatalf("AddUserMessage() error = %v", err) + } + + loaded, err := LoadSessionMeta("session-test") + if err != nil || loaded == nil { + t.Fatalf("LoadSessionMeta() error = %v", err) + } + if loaded.WorkingDir != tmpDir { + t.Errorf("WorkingDir = %q, want %q", loaded.WorkingDir, tmpDir) + } + + // Resume path: the working dir is restored explicitly and must survive the + // next metadata save. + s.SetWorkingDir("/restored/path") + if err := s.AddUserMessage("Hello again"); err != nil { + t.Fatalf("AddUserMessage() after SetWorkingDir error = %v", err) + } + + loaded, err = LoadSessionMeta("session-test") + if err != nil || loaded == nil { + t.Fatalf("LoadSessionMeta() after SetWorkingDir error = %v", err) + } + if loaded.WorkingDir != "/restored/path" { + t.Errorf("WorkingDir after SetWorkingDir = %q, want %q", loaded.WorkingDir, "/restored/path") + } +} + +// TestGetLatestSessionForDir_SkipsVanishedSidecar makes the enumeration-time +// race deterministic. A dangling symlink's lstat (entry.Info) succeeds while +// reading it (loadMetaFile) fails with ENOENT — exactly the "meta file +// disappeared after entry.Info() but before the load" window that used to +// yield a (nil, nil) meta and panic on meta.WorkingDir. The scan must skip +// such a sidecar without panicking and still find the healthy session. +func TestGetLatestSessionForDir_SkipsVanishedSidecar(t *testing.T) { + tmpDir := t.TempDir() + oldSessionDir := SessionDir + SessionDir = func() (string, error) { return tmpDir, nil } + t.Cleanup(func() { SessionDir = oldSessionDir }) + + // Vanished sidecar: lstat succeeds, read fails. + vanishedPath := filepath.Join(tmpDir, "session-20250101-vanished.meta.json") + if err := os.Symlink(filepath.Join(tmpDir, "gone.target"), vanishedPath); err != nil { + t.Skipf("symlinks unavailable on this platform: %v", err) + } + + if err := SaveSessionMeta(newWorkingDirMeta(tmpDir, "session-20250102-100000", "/proj/a")); err != nil { + t.Fatalf("Failed to save meta: %v", err) + } + + latest, err := GetLatestSessionForDir("/proj/a") // must not panic + if err != nil { + t.Fatalf("GetLatestSessionForDir(/proj/a): %v", err) + } + if latest == nil || latest.ID != "session-20250102-100000" { + t.Fatalf("GetLatestSessionForDir(/proj/a) = %v, want session-20250102-100000 (the vanished sidecar must be skipped, not crash the scan)", latest) + } +} + +// TestGetLatestSessionForDir_NilMetaNeverDereferenced injects the historical +// (nil, nil) loader result through the loadEnumeratedMeta seam and asserts +// the scan skips it instead of dereferencing meta.WorkingDir — defense in +// depth beyond loadMetaFile's (meta, error) contract. +func TestGetLatestSessionForDir_NilMetaNeverDereferenced(t *testing.T) { + tmpDir := t.TempDir() + oldSessionDir := SessionDir + SessionDir = func() (string, error) { return tmpDir, nil } + t.Cleanup(func() { SessionDir = oldSessionDir }) + + if err := SaveSessionMeta(newWorkingDirMeta(tmpDir, "session-20250101-100000", "/proj/a")); err != nil { + t.Fatalf("Failed to save meta: %v", err) + } + + oldLoad := loadEnumeratedMeta + loadEnumeratedMeta = func(path string) (*SessionMeta, error) { + return nil, nil // simulate the race result: nothing loaded, no error + } + t.Cleanup(func() { loadEnumeratedMeta = oldLoad }) + + latest, err := GetLatestSessionForDir("/proj/a") // must not panic + if err != nil { + t.Fatalf("GetLatestSessionForDir(/proj/a): %v", err) + } + if latest != nil { + t.Fatalf("GetLatestSessionForDir(/proj/a) = %+v, want nil when every sidecar loads as (nil, nil)", latest) + } +} + +// TestGetLatestSessionForDir_LoadsExactEnumeratedFiles pins the exact-file +// loading rule with prefix-colliding IDs. The vanished +// "session-20250101.meta.json" sidecar's ID is a prefix of the surviving +// "session-20250101-abcdef" and carries the newest mtime: with +// LoadSessionMeta-style prefix fallback the scan would resurrect the abcdef +// meta under the vanished entry's newer mtime and wrongly beat the genuinely +// newest session. The loader must also see the enumerated paths verbatim. +func TestGetLatestSessionForDir_LoadsExactEnumeratedFiles(t *testing.T) { + tmpDir := t.TempDir() + oldSessionDir := SessionDir + SessionDir = func() (string, error) { return tmpDir, nil } + t.Cleanup(func() { SessionDir = oldSessionDir }) + + if err := SaveSessionMeta(newWorkingDirMeta(tmpDir, "session-20250101-abcdef", "/proj/a")); err != nil { + t.Fatalf("Failed to save meta: %v", err) + } + if err := SaveSessionMeta(newWorkingDirMeta(tmpDir, "session-20250102-x", "/proj/a")); err != nil { + t.Fatalf("Failed to save meta: %v", err) + } + // Vanished sidecar whose ID prefix-collides with session-20250101-abcdef. + vanishedPath := filepath.Join(tmpDir, "session-20250101.meta.json") + if err := os.Symlink(filepath.Join(tmpDir, "gone.target"), vanishedPath); err != nil { + t.Skipf("symlinks unavailable on this platform: %v", err) + } + + // Both real sessions are older than the vanished sidecar's (current) + // mtime, so a fallback resurrection would win the newest-wins race. + base := time.Now().Add(-24 * time.Hour).Truncate(time.Second) + setMetaModTimes(t, tmpDir, map[string]time.Time{ + "session-20250101-abcdef": base.Add(1 * time.Hour), + "session-20250102-x": base.Add(2 * time.Hour), + }) + + oldLoad := loadEnumeratedMeta + t.Cleanup(func() { loadEnumeratedMeta = oldLoad }) + var loaded []string + loadEnumeratedMeta = func(path string) (*SessionMeta, error) { + loaded = append(loaded, path) + return loadMetaFile(path) + } + + latest, err := GetLatestSessionForDir("/proj/a") + if err != nil { + t.Fatalf("GetLatestSessionForDir(/proj/a): %v", err) + } + if latest == nil || latest.ID != "session-20250102-x" { + t.Fatalf("GetLatestSessionForDir(/proj/a) = %v, want session-20250102-x (the vanished prefix-colliding sidecar must not resurrect session-20250101-abcdef under its newer mtime)", latest) + } + + wantPaths := []string{ + filepath.Join(tmpDir, "session-20250101-abcdef.meta.json"), + filepath.Join(tmpDir, "session-20250101.meta.json"), + filepath.Join(tmpDir, "session-20250102-x.meta.json"), + } + slices.Sort(loaded) // ReadDir order is already sorted; sort defensively + slices.Sort(wantPaths) + if !slices.Equal(loaded, wantPaths) { + t.Errorf("loader saw paths %v, want the exact enumerated files %v", loaded, wantPaths) + } +} + +// TestGetLatestSession_SkipsVanishedSidecar mirrors the exact-file rule on +// the global --continue path: a vanished sidecar must be skipped, never +// re-resolved through the ID-prefix fallback to a different session. +func TestGetLatestSession_SkipsVanishedSidecar(t *testing.T) { + tmpDir := t.TempDir() + oldSessionDir := SessionDir + SessionDir = func() (string, error) { return tmpDir, nil } + t.Cleanup(func() { SessionDir = oldSessionDir }) + + if err := SaveSessionMeta(newWorkingDirMeta(tmpDir, "session-20250101-abcdef", "/proj/a")); err != nil { + t.Fatalf("Failed to save meta: %v", err) + } + if err := SaveSessionMeta(newWorkingDirMeta(tmpDir, "session-20250102-x", "/proj/b")); err != nil { + t.Fatalf("Failed to save meta: %v", err) + } + vanishedPath := filepath.Join(tmpDir, "session-20250101.meta.json") + if err := os.Symlink(filepath.Join(tmpDir, "gone.target"), vanishedPath); err != nil { + t.Skipf("symlinks unavailable on this platform: %v", err) + } + + // Same mtime setup as the --continue-project variant: the vanished + // sidecar is the newest entry, and its ID prefix-collides with + // session-20250101-abcdef. + base := time.Now().Add(-24 * time.Hour).Truncate(time.Second) + setMetaModTimes(t, tmpDir, map[string]time.Time{ + "session-20250101-abcdef": base.Add(1 * time.Hour), + "session-20250102-x": base.Add(2 * time.Hour), + }) + + latest, err := GetLatestSession() // must not panic + if err != nil { + t.Fatalf("GetLatestSession(): %v", err) + } + if latest == nil || latest.ID != "session-20250102-x" { + t.Fatalf("GetLatestSession() = %v, want session-20250102-x (the vanished sidecar must be skipped, not fall back to session-20250101-abcdef)", latest) + } +} + +// TestGetLatestSessionForDir_MatchesViaSymlinkIdentity covers directory +// identity matching: the session records the real directory while the lookup +// goes through a symlink to it. The lexical fast path misses, but os.SameFile +// must still match. A lookup directory that exists nowhere must return +// (nil, nil) without an error, and a recorded directory that has since been +// deleted must still match through the lexical fast path. +func TestGetLatestSessionForDir_MatchesViaSymlinkIdentity(t *testing.T) { + tmpDir := t.TempDir() + oldSessionDir := SessionDir + SessionDir = func() (string, error) { return tmpDir, nil } + t.Cleanup(func() { SessionDir = oldSessionDir }) + + realDir := filepath.Join(tmpDir, "real-project") + if err := os.MkdirAll(realDir, 0700); err != nil { + t.Fatalf("creating real-project: %v", err) + } + linkDir := filepath.Join(tmpDir, "linked-project") + if err := os.Symlink(realDir, linkDir); err != nil { + t.Skipf("symlinks unavailable on this platform: %v", err) + } + + if err := SaveSessionMeta(newWorkingDirMeta(tmpDir, "session-20250101-100000", realDir)); err != nil { + t.Fatalf("Failed to save meta: %v", err) + } + + latest, err := GetLatestSessionForDir(linkDir) + if err != nil { + t.Fatalf("GetLatestSessionForDir(%s): %v", linkDir, err) + } + if latest == nil || latest.ID != "session-20250101-100000" { + t.Fatalf("GetLatestSessionForDir(%s) = %v, want session-20250101-100000 found through the symlink via os.SameFile", linkDir, latest) + } + + // A directory that exists nowhere: no error, no match. + missing := filepath.Join(tmpDir, "does-not-exist") + latest, err = GetLatestSessionForDir(missing) + if err != nil { + t.Fatalf("GetLatestSessionForDir(%s): %v", missing, err) + } + if latest != nil { + t.Fatalf("GetLatestSessionForDir(%s) = %+v, want nil for a missing directory", missing, latest) + } + + // A recorded project directory that has since been deleted must still be + // matched lexically (identity cannot be checked on a missing directory). + gone := filepath.Join(tmpDir, "gone-project") + if err := os.Mkdir(gone, 0700); err != nil { + t.Fatalf("creating gone-project: %v", err) + } + if err := SaveSessionMeta(newWorkingDirMeta(tmpDir, "session-20250102-100000", gone)); err != nil { + t.Fatalf("Failed to save meta: %v", err) + } + if err := os.Remove(gone); err != nil { + t.Fatalf("removing gone-project: %v", err) + } + + latest, err = GetLatestSessionForDir(gone) + if err != nil { + t.Fatalf("GetLatestSessionForDir(%s): %v", gone, err) + } + if latest == nil || latest.ID != "session-20250102-100000" { + t.Fatalf("GetLatestSessionForDir(%s) = %v, want session-20250102-100000 matched lexically despite the directory being gone", gone, latest) + } +} diff --git a/internal/session/session.go b/internal/session/session.go index 7211f632..93707e41 100644 --- a/internal/session/session.go +++ b/internal/session/session.go @@ -7,6 +7,7 @@ import ( "late/internal/client" "late/internal/common" "late/internal/tool" + "os" "path/filepath" "strings" "sync" @@ -21,14 +22,15 @@ type Session struct { History []client.ChatMessage systemPrompt string useTools bool - skipMetadata bool // when true, no top-level .meta.json sidecar is written (subagents) + skipMetadata bool // when true, no top-level .meta.json sidecar is written (subagents) + workingDir string // absolute path of the directory where the session was started (project folder) subagentSeq int saveSubagentHistories *bool Registry *tool.Registry } func New(c *client.Client, historyPath string, history []client.ChatMessage, systemPrompt string, useTools bool) *Session { - return &Session{ + s := &Session{ client: c, HistoryPath: historyPath, History: history, @@ -36,6 +38,11 @@ func New(c *client.Client, historyPath string, history []client.ChatMessage, sys useTools: useTools, Registry: tool.NewRegistry(), } + // Best-effort capture of the project folder; never fail construction. + if wd, err := os.Getwd(); err == nil { + s.workingDir = wd + } + return s } // NewSubagentSession creates a session for a subagent. History is persisted @@ -59,6 +66,13 @@ func (s *Session) SetSubagentMetadata(seq int, saveHistories *bool) { s.saveSubagentHistories = &value } +// SetWorkingDir overrides the project directory recorded in session +// metadata. Used on resume so a session keeps the directory where it +// was originally started. +func (s *Session) SetWorkingDir(dir string) { + s.workingDir = dir +} + // SubagentSeq returns the next sequence number reserved for a child session. func (s *Session) SubagentSeq() int { return s.subagentSeq @@ -149,6 +163,42 @@ func (s *Session) AddAssistantMessage(content, reasoning string) error { return s.saveAndNotify() } +// PopLastUserMessage removes the trailing user message from history and +// persists the change atomically. The bool reports whether a message was +// removed (false = no-op: empty history or non-user tail). The error is a +// persistence error, returned only when a change was made. +func (s *Session) PopLastUserMessage() (bool, error) { + if len(s.History) == 0 || s.History[len(s.History)-1].Role != "user" { + return false, nil + } + s.History = s.History[:len(s.History)-1] + + // Popping the first-and-only message empties the history. saveAndNotify() + // treats empty history as "nothing to persist" (its empty-guard exists so + // fresh sessions don't create files at startup), which would leave the + // just-popped message stale on disk and let --continue resurrect the + // rejected turn. So when the history file exists on disk, remove it + // instead of saving an empty file. The .meta.json sidecar lives in the + // sessions directory (never next to the history file) and is kept — only + // refreshed — so --continue scoping still finds this session. + if len(s.History) == 0 && s.HistoryPath != "" { + if err := os.Remove(s.HistoryPath); err != nil { + if !os.IsNotExist(err) { + return true, fmt.Errorf("failed to remove emptied history file %s: %w", s.HistoryPath, err) + } + // Nothing persisted yet (history lived only in memory), so there + // is no file or sidecar to update either. + return true, nil + } + if err := s.UpdateSessionMetadata(); err != nil { + return true, err + } + return true, nil + } + + return true, s.saveAndNotify() +} + // AppendToLastMessage appends content to the last message (continuation). func (s *Session) AppendToLastMessage(content, reasoning string) error { if len(s.History) == 0 { @@ -197,7 +247,12 @@ func (s *Session) StartStream(ctx context.Context, extraBody map[string]any) (<- if s.systemPrompt != "" { messages = append(messages, client.ChatMessage{Role: "system", Content: client.TextContent(s.systemPrompt)}) } - messages = append(messages, s.History...) + // Sanitize per request: a history interrupted mid-tool-run (crash, fatal + // stream error) can end with assistant tool_calls that never got results, + // which strict OpenAI-compatible endpoints reject with HTTP 400. The + // sanitizer repairs the copy sent to the API; the saved history is + // intentionally left untouched. + messages = append(messages, SanitizeForRequest(s.History)...) req := client.ChatCompletionRequest{ Messages: messages, @@ -218,6 +273,17 @@ func (s *Session) StartStream(ctx context.Context, extraBody map[string]any) (<- select { case chunk, ok := <-streamOut: if !ok { + // The client closes errCh before out (LIFO defers). A + // random select win here must not swallow a mid-stream + // failure: drain the terminal error before returning, + // or ConsumeStream would treat the attempt as a clean, + // partial success and commit a truncated turn. + if err, ok := <-streamErr; ok && err != nil { + select { + case errCh <- err: + case <-ctx.Done(): + } + } return } var content, reasoning, finishReason string @@ -325,6 +391,7 @@ func (s *Session) GenerateSessionMeta() SessionMeta { MessageCount: len(s.History), SubagentSeq: s.subagentSeq, SaveSubagentHistories: s.saveSubagentHistories, + WorkingDir: s.workingDir, } } diff --git a/internal/session/session_pop_test.go b/internal/session/session_pop_test.go new file mode 100644 index 00000000..c35e1c67 --- /dev/null +++ b/internal/session/session_pop_test.go @@ -0,0 +1,191 @@ +package session + +import ( + "bytes" + "os" + "path/filepath" + "testing" +) + +// TestPopLastUserMessage covers the rollback primitive used by the +// orchestrator when the API terminally rejects a turn with HTTP 400: the +// trailing user message must be removed from history and the removal must be +// persisted, while non-user or empty tails must be left untouched. +func TestPopLastUserMessage(t *testing.T) { + t.Run("pops trailing user message and persists the removal", func(t *testing.T) { + tmpDir := t.TempDir() + oldSessionDir := SessionDir + SessionDir = func() (string, error) { return tmpDir, nil } + t.Cleanup(func() { SessionDir = oldSessionDir }) + + historyPath := filepath.Join(tmpDir, "session-pop.json") + s := New(nil, historyPath, nil, "sp", true) + if err := s.AddUserMessage("first"); err != nil { + t.Fatalf("AddUserMessage returned error: %v", err) + } + if err := s.AddAssistantMessage("reply", ""); err != nil { + t.Fatalf("AddAssistantMessage returned error: %v", err) + } + if err := s.AddUserMessage("second"); err != nil { + t.Fatalf("AddUserMessage returned error: %v", err) + } + + rolled, err := s.PopLastUserMessage() + if err != nil { + t.Fatalf("PopLastUserMessage returned error: %v", err) + } + if !rolled { + t.Errorf("PopLastUserMessage reported rolled=false, want true") + } + + // In-memory rollback: the assistant reply is now the tail. + if len(s.History) != 2 { + t.Fatalf("len(s.History) = %d, want 2", len(s.History)) + } + if got := s.History[len(s.History)-1].Content.String(); got != "reply" { + t.Errorf("last in-memory message = %q, want %q", got, "reply") + } + + // Persisted rollback: the file on disk must match the trimmed history. + loaded, err := LoadHistory(historyPath) + if err != nil { + t.Fatalf("LoadHistory returned error: %v", err) + } + if len(loaded) != 2 { + t.Fatalf("persisted history has %d messages, want 2", len(loaded)) + } + if got := loaded[len(loaded)-1].Content.String(); got != "reply" { + t.Errorf("last persisted message = %q, want %q", got, "reply") + } + }) + + t.Run("no-op when history ends with assistant", func(t *testing.T) { + tmpDir := t.TempDir() + oldSessionDir := SessionDir + SessionDir = func() (string, error) { return tmpDir, nil } + t.Cleanup(func() { SessionDir = oldSessionDir }) + + historyPath := filepath.Join(tmpDir, "session-pop-assistant.json") + s := New(nil, historyPath, nil, "sp", true) + if err := s.AddUserMessage("only"); err != nil { + t.Fatalf("AddUserMessage returned error: %v", err) + } + if err := s.AddAssistantMessage("reply", ""); err != nil { + t.Fatalf("AddAssistantMessage returned error: %v", err) + } + + before, err := os.ReadFile(historyPath) + if err != nil { + t.Fatalf("reading persisted history before pop: %v", err) + } + + rolled, err := s.PopLastUserMessage() + if err != nil { + t.Fatalf("PopLastUserMessage returned error: %v", err) + } + if rolled { + t.Errorf("PopLastUserMessage reported rolled=true, want false (tail is not a user message)") + } + + if len(s.History) != 2 { + t.Errorf("len(s.History) = %d, want 2 (unchanged)", len(s.History)) + } + after, err := os.ReadFile(historyPath) + if err != nil { + t.Fatalf("reading persisted history after pop: %v", err) + } + if !bytes.Equal(before, after) { + t.Errorf("history file changed on no-op pop:\nbefore: %s\nafter: %s", before, after) + } + }) + + t.Run("no-op on empty history", func(t *testing.T) { + tmpDir := t.TempDir() + oldSessionDir := SessionDir + SessionDir = func() (string, error) { return tmpDir, nil } + t.Cleanup(func() { SessionDir = oldSessionDir }) + + historyPath := filepath.Join(tmpDir, "session-pop-empty.json") + s := New(nil, historyPath, nil, "sp", true) + + rolled, err := s.PopLastUserMessage() + if err != nil { + t.Fatalf("PopLastUserMessage returned error: %v", err) + } + if rolled { + t.Errorf("PopLastUserMessage reported rolled=true, want false on empty history") + } + if len(s.History) != 0 { + t.Errorf("len(s.History) = %d, want 0", len(s.History)) + } + if _, err := os.Stat(historyPath); !os.IsNotExist(err) { + t.Errorf("expected no history file to be created, stat err=%v", err) + } + }) + + // The PR's motivating scenario: a terminal 400 on the FIRST turn of a + // session. saveAndNotify() skips persistence for empty history (its + // empty-guard exists so fresh sessions don't create files), so popping + // the only message must remove the stale history file instead — + // otherwise --continue would resurrect the rejected turn. + t.Run("pop to empty with existing file removes stale history and keeps sidecar", func(t *testing.T) { + tmpDir := t.TempDir() + oldSessionDir := SessionDir + SessionDir = func() (string, error) { return tmpDir, nil } + t.Cleanup(func() { SessionDir = oldSessionDir }) + + historyPath := filepath.Join(tmpDir, "session-pop-empty-file.json") + s := New(nil, historyPath, nil, "sp", true) + if err := s.AddUserMessage("first and only"); err != nil { + t.Fatalf("AddUserMessage returned error: %v", err) + } + + // Precondition: the single user message was persisted with its sidecar. + if _, err := os.Stat(historyPath); err != nil { + t.Fatalf("history file missing before pop: %v", err) + } + metaPath := filepath.Join(tmpDir, "session-pop-empty-file.meta.json") + if _, err := os.Stat(metaPath); err != nil { + t.Fatalf("meta sidecar missing before pop: %v", err) + } + + rolled, err := s.PopLastUserMessage() + if err != nil { + t.Fatalf("PopLastUserMessage returned error: %v", err) + } + if !rolled { + t.Errorf("PopLastUserMessage reported rolled=false, want true") + } + + // In-memory rollback. + if len(s.History) != 0 { + t.Errorf("len(s.History) = %d, want 0", len(s.History)) + } + + // The stale history file must be gone: reloading from disk yields no + // messages, so the rejected turn cannot resurrect via --continue. + loaded, err := LoadHistory(historyPath) + if err != nil { + t.Fatalf("LoadHistory returned error: %v", err) + } + if len(loaded) != 0 { + t.Errorf("reloaded history has %d messages, want 0 (rejected message resurrected)", len(loaded)) + } + + // The sidecar is kept so --continue scoping still finds the session, + // with message_count refreshed to the current (empty) count. + if _, err := os.Stat(metaPath); err != nil { + t.Errorf("meta sidecar missing after pop: %v", err) + } + meta, err := LoadSessionMeta("session-pop-empty-file") + if err != nil { + t.Fatalf("LoadSessionMeta returned error: %v", err) + } + if meta == nil { + t.Fatalf("LoadSessionMeta found no sidecar for session-pop-empty-file") + } + if meta.MessageCount != 0 { + t.Errorf("meta.MessageCount = %d, want 0", meta.MessageCount) + } + }) +} diff --git a/internal/session/session_startstream_test.go b/internal/session/session_startstream_test.go new file mode 100644 index 00000000..e5c57d48 --- /dev/null +++ b/internal/session/session_startstream_test.go @@ -0,0 +1,122 @@ +package session + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "late/internal/client" +) + +// TestStartStream_SanitizesInterruptedToolRun drives StartStream end-to-end +// against a fake OpenAI-compatible server (the client.Config{BaseURL} approach +// used by the client package's own stream tests) and asserts that a history +// interrupted mid-tool-run is repaired in the outgoing request messages while +// s.History itself is left untouched. +func TestStartStream_SanitizesInterruptedToolRun(t *testing.T) { + // requestBodies carries the decoded-time JSON body of each + // /chat/completions request the fake server receives. + requestBodies := make(chan []byte, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Answer non-200 so the client's backend discovery probe stays + // "unknown" and does not change request behavior. + if !strings.HasSuffix(r.URL.Path, "/chat/completions") { + http.NotFound(w, r) + return + } + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "read error", http.StatusBadRequest) + return + } + requestBodies <- body + + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprint(w, "data: {\"id\":\"c1\",\"choices\":[{\"delta\":{\"content\":\"ok\"},\"finish_reason\":\"stop\"}]}\n") + fmt.Fprint(w, "data: [DONE]\n") + })) + defer server.Close() + + history := []client.ChatMessage{ + {Role: "user", Content: client.TextContent("run the tools")}, + {Role: "assistant", ToolCalls: []client.ToolCall{ + {ID: "call_1", Type: "function", Function: client.FunctionCall{Name: "read_file", Arguments: "{}"}}, + {ID: "call_2", Type: "function", Function: client.FunctionCall{Name: "list_dir", Arguments: "{}"}}, + }}, + {Role: "tool", ToolCallID: "call_1", Content: client.TextContent("first result")}, + } + // Deep snapshot of the saved history, to prove StartStream does not + // mutate it (the sanitizer must only repair the request copy). + snapshot, err := json.Marshal(history) + if err != nil { + t.Fatalf("marshal history snapshot: %v", err) + } + + c := client.NewClient(client.Config{BaseURL: server.URL, Model: "test-model"}) + s := New(c, "", history, "", false) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + outCh, errCh := s.StartStream(ctx, nil) + for range outCh { + } + if err, ok := <-errCh; ok && err != nil { + t.Fatalf("unexpected stream error: %v", err) + } + + var body []byte + select { + case body = <-requestBodies: + case <-time.After(5 * time.Second): + t.Fatal("fake server never received the chat completion request") + } + + var payload struct { + Messages []client.ChatMessage `json:"messages"` + } + if err := json.Unmarshal(body, &payload); err != nil { + t.Fatalf("decode captured request: %v\nbody: %s", err, body) + } + + // Expect: user, assistant(2 tool_calls), tool(call_1), tool(call_2 placeholder). + if len(payload.Messages) != 4 { + t.Fatalf("outgoing messages = %d, want 4:\n%s", len(payload.Messages), body) + } + if got := payload.Messages[1].Role; got != "assistant" || len(payload.Messages[1].ToolCalls) != 2 { + t.Fatalf("outgoing messages[1] = role %q with %d tool_calls, want assistant with 2", + got, len(payload.Messages[1].ToolCalls)) + } + wantIDs := [2]string{"call_1", "call_2"} + for i, wantID := range wantIDs { + m := payload.Messages[2+i] + if m.Role != "tool" { + t.Fatalf("outgoing messages[%d].Role = %q, want %q", 2+i, m.Role, "tool") + } + if m.ToolCallID != wantID { + t.Fatalf("outgoing messages[%d].ToolCallID = %q, want %q", 2+i, m.ToolCallID, wantID) + } + } + if got := payload.Messages[3].Content.String(); got != interruptedToolResultText { + t.Fatalf("synthesized tool result content = %q, want %q", got, interruptedToolResultText) + } + + // The saved history must remain untouched. + after, err := json.Marshal(s.History) + if err != nil { + t.Fatalf("marshal history after StartStream: %v", err) + } + if !bytes.Equal(snapshot, after) { + t.Fatalf("s.History was mutated by StartStream:\nbefore: %s\nafter: %s", snapshot, after) + } + if len(s.History) != 3 { + t.Fatalf("len(s.History) = %d, want 3 (no synthesized tool result persisted)", len(s.History)) + } +} diff --git a/internal/session/ttystyle.go b/internal/session/ttystyle.go index 0675760f..75f3fba3 100644 --- a/internal/session/ttystyle.go +++ b/internal/session/ttystyle.go @@ -52,10 +52,17 @@ func formatSessionDisplayVerbose(meta SessionMeta) string { lines := []string{ colorID(fmt.Sprintf("ID: %s", strings.TrimSuffix(meta.ID, ".json"))), fmt.Sprintf(" Title: %s", meta.Title), + } + // Project directory is only shown for sessions that recorded it + // (legacy sessions have an empty WorkingDir). + if meta.WorkingDir != "" { + lines = append(lines, fmt.Sprintf(" Project: %s", meta.WorkingDir)) + } + lines = append(lines, fmt.Sprintf(" Created: %s", meta.CreatedAt.Format("2006-01-02 15:04:05")), fmt.Sprintf(" Updated: %s", meta.LastUpdated.Format("2006-01-02 15:04:05")), fmt.Sprintf(" Msg #: %d", meta.MessageCount), - } + ) if meta.LastUserPrompt != "" { last := truncateUTF8(meta.LastUserPrompt, 50) lines = append(lines, fmt.Sprintf(" Last: %s", last)) diff --git a/internal/session/ttystyle_test.go b/internal/session/ttystyle_test.go new file mode 100644 index 00000000..d5497cd8 --- /dev/null +++ b/internal/session/ttystyle_test.go @@ -0,0 +1,43 @@ +package session + +import ( + "strings" + "testing" + "time" +) + +// TestFormatSessionDisplay_ShowsProjectDir verifies that the verbose session +// display shows the project directory when recorded, and omits the Project +// line entirely for legacy sessions with an empty WorkingDir. +func TestFormatSessionDisplay_ShowsProjectDir(t *testing.T) { + base := SessionMeta{ + ID: "session-20250101-120000", + Title: "Test", + CreatedAt: time.Date(2025, 1, 1, 12, 0, 0, 0, time.UTC), + LastUpdated: time.Date(2025, 1, 1, 12, 5, 0, 0, time.UTC), + HistoryPath: "/tmp/late-sessions/session-20250101-120000.json", + WorkingDir: "/tmp/proj-a", + } + + t.Run("with project dir", func(t *testing.T) { + out := FormatSessionDisplay(base, true) + if !strings.Contains(out, "/tmp/proj-a") { + t.Errorf("expected output to contain project dir %q, got:\n%s", base.WorkingDir, out) + } + if !strings.Contains(out, " Project: /tmp/proj-a") { + t.Errorf("expected aligned 'Project:' line in output, got:\n%s", out) + } + }) + + t.Run("without project dir", func(t *testing.T) { + legacy := base + legacy.WorkingDir = "" + out := FormatSessionDisplay(legacy, true) + if strings.Contains(out, "Project:") { + t.Errorf("expected no 'Project:' label for legacy session with empty WorkingDir, got:\n%s", out) + } + if !strings.Contains(out, "Test") { + t.Errorf("expected output to still render the session title, got:\n%s", out) + } + }) +} diff --git a/internal/tool/ast/policy.go b/internal/tool/ast/policy.go index 6602495e..2d9eeb3e 100644 --- a/internal/tool/ast/policy.go +++ b/internal/tool/ast/policy.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" ) + // tier2Commands is the set of commands that have mandatory subcommands. // The AST adapters should emit compound command keys (e.g. "git log", "go mod") // for these commands to maintain fine-grained allow-list granularity. diff --git a/internal/tool/ast/policy_test.go b/internal/tool/ast/policy_test.go index b279287f..1e4ee907 100644 --- a/internal/tool/ast/policy_test.go +++ b/internal/tool/ast/policy_test.go @@ -61,11 +61,11 @@ func TestPolicyEngine_Decide_SoftSignals(t *testing.T) { func TestPolicyEngine_Decide_DeletionOverridesAllowlist(t *testing.T) { pe := &PolicyEngine{ AllowedCommands: map[string]map[string]bool{ - "rm": {"-rf": true}, + "rm": {"-rf": true}, "remove-item": {"-recurse": true}, }, } - + commands := []string{"rm", "rmdir", "unlink", "remove-item", "del", "erase", "rd", "ri"} for _, cmd := range commands { t.Run(cmd, func(t *testing.T) { @@ -76,7 +76,7 @@ func TestPolicyEngine_Decide_DeletionOverridesAllowlist(t *testing.T) { } else if cmd == "remove-item" { ir.CommandArgs = map[string][]string{"remove-item": {"-recurse"}} } - + d := pe.Decide(ir) if !d.NeedsConfirmation { t.Errorf("expected NeedsConfirmation for %v even if allowlisted", cmd) diff --git a/internal/tool/ast/snapshot_test.go b/internal/tool/ast/snapshot_test.go index 80083114..d7fed8dc 100644 --- a/internal/tool/ast/snapshot_test.go +++ b/internal/tool/ast/snapshot_test.go @@ -20,7 +20,7 @@ var unixCorpus = []snapshotEntry{ {"ls -rt", false, false}, {"date", false, false}, {"echo 'hello world'", false, false}, - {"echo $HOME", false, false}, // expansion + {"echo $HOME", false, false}, // expansion {"cd /tmp", true, true}, // cd blocked {"ls > out.txt", true, true}, // redirect blocked {"echo foo >> bar.txt", true, true}, // redirect blocked diff --git a/internal/tool/implementations.go b/internal/tool/implementations.go index 876b0c87..53243805 100644 --- a/internal/tool/implementations.go +++ b/internal/tool/implementations.go @@ -411,7 +411,7 @@ func (t ShellTool) Execute(ctx context.Context, args json.RawMessage) (string, e if orchestratorID := common.GetOrchestratorID(ctx); strings.Contains(strings.ToLower(orchestratorID), "coder") { sandwich = "\n\n=========================================\nSYSTEM DIRECTIVE:\nYou just encountered an error. If fixing this requires modifying components or architecture you were not explicitly instructed to edit, YOU MUST ABORT AND RETURN TO THE MAIN AGENT.\n=========================================" } - + if exitErr, ok := err.(*exec.ExitError); ok { return fmt.Sprintf("Command failed with exit code %d\n%s%s", exitErr.ExitCode(), finalOutput, sandwich), nil } diff --git a/internal/tool/implementations_test.go b/internal/tool/implementations_test.go index 9190033a..fb8bfbf4 100644 --- a/internal/tool/implementations_test.go +++ b/internal/tool/implementations_test.go @@ -803,4 +803,3 @@ func TestBashTool_WrapError(t *testing.T) { t.Errorf("Expected wrapped non-cd error to match %q, got %q", expectedOtherStr, wrappedOtherErr.Error()) } } - diff --git a/internal/tool/search_test.go b/internal/tool/search_test.go index 99095c50..dedd23d9 100644 --- a/internal/tool/search_test.go +++ b/internal/tool/search_test.go @@ -417,5 +417,3 @@ func TestFindFilesTool_GlobstarIntegration(t *testing.T) { t.Errorf("expected no-match with recursive hint, got:\n%s", resZzz) } } - - diff --git a/internal/tui/agent_type_test.go b/internal/tui/agent_type_test.go new file mode 100644 index 00000000..539f8443 --- /dev/null +++ b/internal/tui/agent_type_test.go @@ -0,0 +1,73 @@ +package tui + +import ( + "regexp" + "strings" + "testing" +) + +var ansiRe = regexp.MustCompile(`\x1b\[[0-9;]*m`) + +func stripANSI(s string) string { return ansiRe.ReplaceAllString(s, "") } + +func TestAgentTypeFromID(t *testing.T) { + cases := map[string]string{ + "main": "orchestrator", + "researcher-subagent-0": "researcher", + "coder-subagent-12": "coder", + "planner-subagent-3": "planner", // future category: new JSON config, no code change + "mock": "mock", + "": "", + } + for id, want := range cases { + if got := agentTypeFromID(id); got != want { + t.Errorf("agentTypeFromID(%q) = %q, want %q", id, got, want) + } + } +} + +func TestAgentTypeColorsAreDistinctAndStable(t *testing.T) { + orch := agentTypeStyle("orchestrator").Render("orchestrator") + res := agentTypeStyle("researcher").Render("researcher") + code := agentTypeStyle("coder").Render("coder") + if orch == res || res == code || orch == code { + t.Fatalf("orchestrator, researcher and coder must use different bright colors") + } + if !strings.Contains(stripANSI(orch), "orchestrator") { + t.Fatalf("label text lost: %q", orch) + } + if first, second := agentTypeStyle("future-type").Render("x"), agentTypeStyle("future-type").Render("x"); first != second { + t.Fatalf("future categories must get a stable color") + } +} + +func TestStatusBarShowsAgentTypeBeforeBranch(t *testing.T) { + model := NewModel(&mockOrchestrator{}, nil, nil) + model.Focused = &focusTestOrchestrator{id: "main"} + model.Width = 120 + model.ShowCWD = true + model.CWD = "/tmp/repo" + model.GitBranch = "feat/test" + bar := stripANSI(model.statusBarView()) + if !strings.Contains(bar, "orchestrator") { + t.Fatalf("status bar must show the agent category, got: %q", bar) + } + if strings.Index(bar, "orchestrator") > strings.Index(bar, "feat/test") { + t.Fatalf("agent category must appear before the git branch, got: %q", bar) + } + if !strings.Contains(bar, "feat/test") { + t.Fatalf("branch missing entirely: %q", bar) + } +} + +func TestStatusBarShowsAgentTypeWhenShowCWDDisabled(t *testing.T) { + model := NewModel(&mockOrchestrator{}, nil, nil) + model.Focused = &focusTestOrchestrator{id: "researcher-subagent-0"} + model.Width = 120 + model.ShowCWD = false + model.GitBranch = "" + bar := stripANSI(model.statusBarView()) + if !strings.Contains(bar, "researcher") { + t.Fatalf("agent category must be visible even with ShowCWD=false, got: %q", bar) + } +} diff --git a/internal/tui/retry_test.go b/internal/tui/retry_test.go new file mode 100644 index 00000000..f19eff57 --- /dev/null +++ b/internal/tui/retry_test.go @@ -0,0 +1,538 @@ +package tui + +import ( + "errors" + "fmt" + "strings" + "testing" + "time" + + tea "charm.land/bubbletea/v2" + "late/internal/client" + "late/internal/common" +) + +// TestRetryEventKeepsAgentThinking covers the RetryEvent dispatch: the status +// bar announces the retry, the agent stays thinking (spinner keeps running), +// the failed attempt's partial output and render caches are dropped, and a +// pinned error is left untouched until a successful turn clears it. +func TestRetryEventKeepsAgentThinking(t *testing.T) { + m, s := newViewportBenchmarkModel(nil) + + sentinel := errors.New("previous failure") + s.Error = sentinel + s.State = StateStreaming + s.StreamingState = common.ContentEvent{ID: m.Focused.ID(), Content: "partial attempt output"} + s.StreamingStyledCache = "styled cache" + s.StreamingChunkCount = 7 + + updated, _ := m.Update(OrchestratorEventMsg{Event: common.RetryEvent{ + ID: m.Focused.ID(), + Attempt: 2, + MaxAttempts: 5, + Delay: 1500 * time.Millisecond, + Err: errors.New("connection reset by peer"), + }}) + *m = updated.(Model) + s = m.GetAgentState(m.Focused.ID()) + + if !strings.Contains(s.StatusText, "retry 2/5") { + t.Fatalf("StatusText = %q, want it to contain retry 2/5", s.StatusText) + } + if !strings.Contains(s.StatusText, "after 1.5s backoff") { + t.Fatalf("StatusText = %q, want the delay truncated to 1.5s in the past-tense backoff phrasing", s.StatusText) + } + if strings.Contains(s.StatusText, "retrying in") { + t.Fatalf("StatusText = %q, must not imply a live countdown (the line is rendered once and never updated)", s.StatusText) + } + if s.State != StateThinking { + t.Fatalf("State = %v, want StateThinking", s.State) + } + if s.StreamingStyledCache != "" || s.StreamingChunkCount != 0 { + t.Fatalf("streaming render cache not cleared (cache=%q, chunks=%d)", s.StreamingStyledCache, s.StreamingChunkCount) + } + if s.StreamingState.Content != "" { + t.Fatalf("failed attempt's partial text survived: %q", s.StreamingState.Content) + } + if s.RetryVerb != retryVerbConnectionLost { + t.Fatalf("RetryVerb = %q, want %q after an infra failure", s.RetryVerb, retryVerbConnectionLost) + } + if s.Error == nil || s.Error != sentinel { + t.Fatalf("Error = %v, want the sentinel %v to survive the retry", s.Error, sentinel) + } +} + +// TestRetryEventHTTP400NamesTheRejection covers failure-class honesty: when the +// underlying stream error is an HTTP 400 from the API (the request body was +// rejected, not the connection lost), the status bar says so instead of +// claiming the connection dropped. +func TestRetryEventHTTP400NamesTheRejection(t *testing.T) { + m, _ := newViewportBenchmarkModel(nil) + + updated, _ := m.Update(OrchestratorEventMsg{Event: common.RetryEvent{ + ID: m.Focused.ID(), + Attempt: 1, + MaxAttempts: 3, + Delay: 750 * time.Millisecond, + Err: fmt.Errorf("stream error: %w", &client.StatusError{StatusCode: 400, Body: "read body failed"}), + }}) + *m = updated.(Model) + s := m.GetAgentState(m.Focused.ID()) + + if !strings.Contains(s.StatusText, "request rejected by the API") { + t.Fatalf("StatusText = %q, want it to name the API rejection", s.StatusText) + } + if strings.Contains(s.StatusText, "connection lost") { + t.Fatalf("StatusText = %q, must not claim a lost connection for an HTTP 400", s.StatusText) + } + if !strings.Contains(s.StatusText, "retry 1/3 after 700ms backoff") { + t.Fatalf("StatusText = %q, want it to contain retry 1/3 after 700ms backoff", s.StatusText) + } + if strings.Contains(s.StatusText, "attempt 1/3") { + t.Fatalf("StatusText = %q, must not use the old attempt-in-parens countdown format", s.StatusText) + } +} + +// TestThinkingClearsRetryVerbSilently covers the thinking branch after recovery +// moved to the dedicated RecoveryEvent: a new turn still clears a pinned error +// box, and the retry verb is only silently cleared here as a safety net for a +// dropped RecoveryEvent — the "thinking" status must NOT fire any toast +// (recovery is announced immediately by the RecoveryEvent, if it arrives). +func TestThinkingClearsRetryVerbSilently(t *testing.T) { + m, s := newViewportBenchmarkModel(nil) + + updated, _ := m.Update(OrchestratorEventMsg{Event: common.RetryEvent{ + ID: m.Focused.ID(), + Attempt: 1, + MaxAttempts: 3, + Delay: 750 * time.Millisecond, + Err: errors.New("connection reset by peer"), + }}) + *m = updated.(Model) + s = m.GetAgentState(m.Focused.ID()) + + if s.RetryVerb != retryVerbConnectionLost { + t.Fatalf("RetryVerb = %q, want %q after an infra failure", s.RetryVerb, retryVerbConnectionLost) + } + + s.Error = errors.New("stale failure") + m.ToastMessage = "" + m.ToastWarning = true + + updated, cmd := m.Update(OrchestratorEventMsg{Event: common.StatusEvent{ID: m.Focused.ID(), Status: "thinking"}}) + *m = updated.(Model) + s = m.GetAgentState(m.Focused.ID()) + + if s.Error != nil { + t.Fatalf("Error = %v, want nil once a new turn starts", s.Error) + } + if s.RetryVerb != "" { + t.Fatalf("RetryVerb = %q, want it silently cleared on the new turn", s.RetryVerb) + } + + // Run any returned command and feed its messages back through Update, + // exactly as Bubble Tea would, so a stale recovery toast cannot hide + // behind a deferred command. The frame tick only coalesces presentation. + if cmd != nil { + msg := cmd() + if batch, ok := msg.(tea.BatchMsg); ok { + for _, child := range batch { + childMsg := child() + if _, isFrame := childMsg.(transcriptFrameMsg); isFrame { + continue + } + updated, _ := m.Update(childMsg) + *m = updated.(Model) + } + } else { + updated, _ := m.Update(msg) + *m = updated.(Model) + } + } + + if m.ToastMessage != "" { + t.Fatalf("ToastMessage = %q, want no recovery toast from the thinking status", m.ToastMessage) + } +} + +// TestRecoveryEventToastsImmediately covers the dedicated RecoveryEvent +// sequence: the orchestrator emits it exactly once when the retried attempt +// actually produces a response. The toast ("connection restored") fires +// immediately — not speculatively on the next turn's "thinking" status — and +// only once: neither the final response's content events nor the next turn's +// thinking status may repeat it. +func TestRecoveryEventToastsImmediately(t *testing.T) { + m, s := newViewportBenchmarkModel(nil) + + updated, _ := m.Update(OrchestratorEventMsg{Event: common.RetryEvent{ + ID: m.Focused.ID(), + Attempt: 1, + MaxAttempts: 3, + Delay: 750 * time.Millisecond, + Err: errors.New("connection reset by peer"), + }}) + *m = updated.(Model) + s = m.GetAgentState(m.Focused.ID()) + + if s.RetryVerb != retryVerbConnectionLost { + t.Fatalf("RetryVerb = %q, want %q after an infra failure", s.RetryVerb, retryVerbConnectionLost) + } + + m.ToastMessage = "" + updated, cmd := m.Update(OrchestratorEventMsg{Event: common.RecoveryEvent{ID: m.Focused.ID()}}) + *m = updated.(Model) + s = m.GetAgentState(m.Focused.ID()) + + if s.StatusText != "connection restored — streaming response" { + t.Fatalf("StatusText = %q, want %q", s.StatusText, "connection restored — streaming response") + } + if s.RetryVerb != "" { + t.Fatalf("RetryVerb = %q, want it cleared once recovery is announced", s.RetryVerb) + } + if cmd == nil { + t.Fatal("expected a command delivering the restored toast") + } + + // Run the returned command and feed every produced message back through + // Update, exactly as Bubble Tea would. The frame tick message is skipped: + // it only coalesces presentation. + msg := cmd() + if batch, ok := msg.(tea.BatchMsg); ok { + for _, child := range batch { + childMsg := child() + if _, isFrame := childMsg.(transcriptFrameMsg); isFrame { + continue + } + updated, _ := m.Update(childMsg) + *m = updated.(Model) + } + } else { + updated, _ := m.Update(msg) + *m = updated.(Model) + } + + if m.ToastMessage != "connection restored" { + t.Fatalf("ToastMessage = %q, want %q", m.ToastMessage, "connection restored") + } + if m.ToastWarning { + t.Fatal("restored toast must be success-style, not warning") + } + if m.ToastExpireTime <= time.Now().UnixMilli() { + t.Fatalf("ToastExpireTime = %d, want a future expiry (~3s)", m.ToastExpireTime) + } + + // The retried attempt streams its final response: no second toast. + updated, _ = m.Update(OrchestratorEventMsg{Event: common.ContentEvent{ + ID: m.Focused.ID(), + Content: "final response after the retry", + Completed: true, + }}) + *m = updated.(Model) + + if m.ToastMessage != "connection restored" { + t.Fatalf("ToastMessage = %q after the final response, want it unchanged (no second toast)", m.ToastMessage) + } + + // The next turn's thinking status must not repeat the recovery toast. + updated, cmd = m.Update(OrchestratorEventMsg{Event: common.StatusEvent{ID: m.Focused.ID(), Status: "thinking"}}) + *m = updated.(Model) + + // Pump any returned command; the frame tick only coalesces presentation. + if cmd != nil { + msg := cmd() + if batch, ok := msg.(tea.BatchMsg); ok { + for _, child := range batch { + childMsg := child() + if _, isFrame := childMsg.(transcriptFrameMsg); isFrame { + continue + } + updated, _ := m.Update(childMsg) + *m = updated.(Model) + } + } else { + updated, _ := m.Update(msg) + *m = updated.(Model) + } + } + if m.ToastMessage != "connection restored" { + t.Fatalf("ToastMessage = %q after the next turn's thinking, want it unchanged (no second toast)", m.ToastMessage) + } +} + +// TestRecoveryEventToastMatchesFailureClass covers the 400-class recovery: when +// the retried failure was an HTTP 400 from the API (the request body was +// rejected, not the connection lost), the immediate recovery toast must +// announce the request was accepted after the retry instead of claiming the +// connection was restored. +func TestRecoveryEventToastMatchesFailureClass(t *testing.T) { + m, _ := newViewportBenchmarkModel(nil) + + updated, _ := m.Update(OrchestratorEventMsg{Event: common.RetryEvent{ + ID: m.Focused.ID(), + Attempt: 1, + MaxAttempts: 3, + Delay: 750 * time.Millisecond, + Err: fmt.Errorf("stream error: %w", &client.StatusError{StatusCode: 400, Body: "read body failed"}), + }}) + *m = updated.(Model) + s := m.GetAgentState(m.Focused.ID()) + + if s.RetryVerb != retryVerbRejectedByAPI { + t.Fatalf("RetryVerb = %q, want %q after an HTTP 400", s.RetryVerb, retryVerbRejectedByAPI) + } + + m.ToastMessage = "" + updated, cmd := m.Update(OrchestratorEventMsg{Event: common.RecoveryEvent{ID: m.Focused.ID()}}) + *m = updated.(Model) + s = m.GetAgentState(m.Focused.ID()) + + if s.StatusText != "request accepted after retry — streaming response" { + t.Fatalf("StatusText = %q, want %q", s.StatusText, "request accepted after retry — streaming response") + } + if s.RetryVerb != "" { + t.Fatalf("RetryVerb = %q, want it cleared once recovery is announced", s.RetryVerb) + } + if cmd == nil { + t.Fatal("expected a command delivering the recovered toast") + } + + // Run the returned command and feed every produced message back through + // Update, exactly as Bubble Tea would. The frame tick message is skipped: + // it only coalesces presentation. + msg := cmd() + if batch, ok := msg.(tea.BatchMsg); ok { + for _, child := range batch { + childMsg := child() + if _, isFrame := childMsg.(transcriptFrameMsg); isFrame { + continue + } + updated, _ := m.Update(childMsg) + *m = updated.(Model) + } + } else { + updated, _ := m.Update(msg) + *m = updated.(Model) + } + + if !strings.Contains(m.ToastMessage, "request accepted after retry") { + t.Fatalf("ToastMessage = %q, want it to mention the accepted retry", m.ToastMessage) + } + if strings.Contains(m.ToastMessage, "connection restored") { + t.Fatalf("ToastMessage = %q, must not claim the connection was restored for an HTTP 400", m.ToastMessage) + } +} + +// TestErrorStatusClearsStaleRetryVerb covers the stale-verb bug: a 400 that +// exhausts the bad-body budget ends the turn in an error box, but RetryVerb +// used to survive the error, so the resubmitted turn's first "thinking" fired +// a bogus "request accepted after retry" toast for a request that was never +// accepted. The error branch must clear the stored verb. +func TestErrorStatusClearsStaleRetryVerb(t *testing.T) { + m, _ := newViewportBenchmarkModel(nil) + + updated, _ := m.Update(OrchestratorEventMsg{Event: common.RetryEvent{ + ID: m.Focused.ID(), + Attempt: 3, + MaxAttempts: 3, + Delay: 750 * time.Millisecond, + Err: fmt.Errorf("stream error: %w", &client.StatusError{StatusCode: 400, Body: "read body failed"}), + }}) + *m = updated.(Model) + s := m.GetAgentState(m.Focused.ID()) + + if s.RetryVerb != retryVerbRejectedByAPI { + t.Fatalf("RetryVerb = %q, want %q after an HTTP 400", s.RetryVerb, retryVerbRejectedByAPI) + } + + updated, _ = m.Update(OrchestratorEventMsg{Event: common.StatusEvent{ + ID: m.Focused.ID(), + Status: "error", + Error: errors.New("request rejected by the API after retries"), + }}) + *m = updated.(Model) + s = m.GetAgentState(m.Focused.ID()) + + if s.RetryVerb != "" { + t.Fatalf("RetryVerb = %q, want it cleared when the turn ends in error", s.RetryVerb) + } + + m.ToastMessage = "" + updated, cmd := m.Update(OrchestratorEventMsg{Event: common.StatusEvent{ID: m.Focused.ID(), Status: "thinking"}}) + *m = updated.(Model) + s = m.GetAgentState(m.Focused.ID()) + + if s.RetryVerb != "" { + t.Fatalf("RetryVerb = %q, want it to stay clear on the next turn", s.RetryVerb) + } + + // Run any returned command and feed its messages back through Update, + // exactly as Bubble Tea would, so a stale recovery toast cannot hide + // behind a deferred command. The frame tick only coalesces presentation. + if cmd != nil { + msg := cmd() + if batch, ok := msg.(tea.BatchMsg); ok { + for _, child := range batch { + childMsg := child() + if _, isFrame := childMsg.(transcriptFrameMsg); isFrame { + continue + } + updated, _ := m.Update(childMsg) + *m = updated.(Model) + } + } else { + updated, _ := m.Update(msg) + *m = updated.(Model) + } + } + + if m.ToastMessage != "" { + t.Fatalf("ToastMessage = %q, want no stale recovery toast after an errored turn", m.ToastMessage) + } + if strings.Contains(m.ToastMessage, "request accepted after retry") { + t.Fatalf("ToastMessage = %q, must not claim the request was accepted", m.ToastMessage) + } +} + +// TestStopRequestedClearsStaleRetryVerb covers the stop path: a user stop +// during retry backoff ends the turn, and the stored verb must not survive +// into the next turn's "thinking" as a recovery toast. +func TestStopRequestedClearsStaleRetryVerb(t *testing.T) { + m, _ := newViewportBenchmarkModel(nil) + + updated, _ := m.Update(OrchestratorEventMsg{Event: common.RetryEvent{ + ID: m.Focused.ID(), + Attempt: 1, + MaxAttempts: 3, + Delay: 750 * time.Millisecond, + Err: fmt.Errorf("stream error: %w", &client.StatusError{StatusCode: 400, Body: "read body failed"}), + }}) + *m = updated.(Model) + s := m.GetAgentState(m.Focused.ID()) + + if s.RetryVerb != retryVerbRejectedByAPI { + t.Fatalf("RetryVerb = %q, want %q after an HTTP 400", s.RetryVerb, retryVerbRejectedByAPI) + } + + updated, _ = m.Update(OrchestratorEventMsg{Event: common.StopRequestedEvent{ID: m.Focused.ID()}}) + *m = updated.(Model) + s = m.GetAgentState(m.Focused.ID()) + + if s.RetryVerb != "" { + t.Fatalf("RetryVerb = %q, want it cleared when the user stops the turn", s.RetryVerb) + } + + m.ToastMessage = "" + updated, cmd := m.Update(OrchestratorEventMsg{Event: common.StatusEvent{ID: m.Focused.ID(), Status: "thinking"}}) + *m = updated.(Model) + s = m.GetAgentState(m.Focused.ID()) + + if s.RetryVerb != "" { + t.Fatalf("RetryVerb = %q, want it to stay clear on the next turn", s.RetryVerb) + } + + // Run any returned command and feed its messages back through Update, + // exactly as Bubble Tea would, so a stale recovery toast cannot hide + // behind a deferred command. The frame tick only coalesces presentation. + if cmd != nil { + msg := cmd() + if batch, ok := msg.(tea.BatchMsg); ok { + for _, child := range batch { + childMsg := child() + if _, isFrame := childMsg.(transcriptFrameMsg); isFrame { + continue + } + updated, _ := m.Update(childMsg) + *m = updated.(Model) + } + } else { + updated, _ := m.Update(msg) + *m = updated.(Model) + } + } + + if m.ToastMessage != "" { + t.Fatalf("ToastMessage = %q, want no stale recovery toast after a stop", m.ToastMessage) + } + if strings.Contains(m.ToastMessage, "request accepted after retry") { + t.Fatalf("ToastMessage = %q, must not claim the request was accepted", m.ToastMessage) + } +} + +// TestThinkingWithoutRetryNoToast: a plain new turn (no retry in flight) +// still clears a pinned error box but must not fire the restored toast. +func TestThinkingWithoutRetryNoToast(t *testing.T) { + m, s := newViewportBenchmarkModel(nil) + + s.Error = errors.New("stale failure") + s.RetryVerb = "" + m.ToastMessage = "" + + updated, _ := m.Update(OrchestratorEventMsg{Event: common.StatusEvent{ID: m.Focused.ID(), Status: "thinking"}}) + *m = updated.(Model) + s = m.GetAgentState(m.Focused.ID()) + + if s.Error != nil { + t.Fatalf("Error = %v, want nil once a new turn starts", s.Error) + } + if s.RetryVerb != "" { + t.Fatal("RetryVerb must stay clear") + } + if m.ToastMessage != "" { + t.Fatalf("ToastMessage = %q, want no toast without a retry", m.ToastMessage) + } +} + +// TestRecoveryEventWithoutRetryVerbKeepsStatusAccurate covers the documented +// race: a user stop (or an error) can clear RetryVerb while the retried +// attempt is still in flight, and the orchestrator's RecoveryEvent then +// arrives with no retry verb stored. The branch must not panic, must not +// toast a recovery nobody was waiting for, and must keep the status line +// accurate ("streaming response"). +func TestRecoveryEventWithoutRetryVerbKeepsStatusAccurate(t *testing.T) { + m, s := newViewportBenchmarkModel(nil) + + // A stop raced the recovery: the verb was already cleared (the user + // stopped during backoff, StopRequestedEvent reset it), yet the retried + // attempt went on to succeed and the orchestrator still emits recovery. + s.RetryVerb = "" + s.State = StateThinking + m.ToastMessage = "" + + updated, cmd := m.Update(OrchestratorEventMsg{Event: common.RecoveryEvent{ID: m.Focused.ID()}}) + *m = updated.(Model) + s = m.GetAgentState(m.Focused.ID()) + + if s.StatusText != "streaming response" { + t.Fatalf("StatusText = %q, want %q", s.StatusText, "streaming response") + } + if s.RetryVerb != "" { + t.Fatalf("RetryVerb = %q, want it to stay clear", s.RetryVerb) + } + + // Run any returned command and feed its messages back through Update, + // exactly as Bubble Tea would, so a stray recovery toast cannot hide + // behind a deferred command. The frame tick only coalesces presentation. + if cmd != nil { + msg := cmd() + if batch, ok := msg.(tea.BatchMsg); ok { + for _, child := range batch { + childMsg := child() + if _, isFrame := childMsg.(transcriptFrameMsg); isFrame { + continue + } + updated, _ := m.Update(childMsg) + *m = updated.(Model) + } + } else { + updated, _ := m.Update(msg) + *m = updated.(Model) + } + } + + if m.ToastMessage != "" { + t.Fatalf("ToastMessage = %q, want no recovery toast", m.ToastMessage) + } + if m.ToastWarning { + t.Fatal("no toast of any kind must fire for a stop-raced recovery") + } +} diff --git a/internal/tui/state.go b/internal/tui/state.go index 616fe4d0..b606c1b9 100644 --- a/internal/tui/state.go +++ b/internal/tui/state.go @@ -88,6 +88,14 @@ type RenderBlock struct { EndLine int } +// Retry failure-class verbs, stored in AppState.RetryVerb by the +// common.RetryEvent branch (the same verb the retry status line computes) so +// the recovery toast can match the class of the failure that was retried. +const ( + retryVerbConnectionLost = "connection lost" + retryVerbRejectedByAPI = "request rejected by the API" +) + // RewindEntry represents a user message that can be rewound to. type RewindEntry struct { Index int @@ -134,6 +142,14 @@ type AppState struct { ContextWarningShown bool // Whether the preflight context warning has been shown for the current input Error error + + // RetryVerb records the failure class of the retry an agent is in: + // retryVerbConnectionLost ("connection lost") for infra failures or + // retryVerbRejectedByAPI ("request rejected by the API") for HTTP 400s. + // It is set by common.RetryEvent and cleared by the next "thinking" + // status as a silent safety net; recovery is announced separately by + // the dedicated RecoveryEvent. Empty means the agent is not retrying. + RetryVerb string } type Model struct { diff --git a/internal/tui/update.go b/internal/tui/update.go index 69789299..03e0d861 100644 --- a/internal/tui/update.go +++ b/internal/tui/update.go @@ -2,8 +2,10 @@ package tui import ( "context" + "errors" "fmt" "late/internal/assets" + "late/internal/client" "late/internal/common" "late/internal/config" "late/internal/git" @@ -1488,6 +1490,11 @@ func (m Model) updateChat(msg tea.Msg) (Model, tea.Cmd) { case OrchestratorEventMsg: s := m.GetAgentState(msg.Event.OrchestratorID()) + // restoredToast delivers the recovery toast through the existing + // ToastMsg handler when the RecoveryEvent branch below reports that + // the retried attempt actually produced a response; it is returned + // after the event switch below. + var restoredToast tea.Cmd switch event := msg.Event.(type) { case common.ContentEvent: @@ -1530,6 +1537,16 @@ func (m Model) updateChat(msg tea.Msg) (Model, tea.Cmd) { if s.State != StateConfirmTool { s.State = StateThinking } + // The agent is productive again: clear any error box pinned + // by a previous failure. Recovery is announced by the + // dedicated RecoveryEvent branch below (the orchestrator + // emits it when the retried attempt actually succeeds), so + // clearing the retry verb here is only a silent safety net + // for a dropped RecoveryEvent — no toast. + if s.Error != nil { + s.Error = nil + } + s.RetryVerb = "" s.StatusText = "Working..." s.StreamingState = common.ContentEvent{ID: event.ID} // Clear streaming render cache for new turn @@ -1539,6 +1556,9 @@ func (m Model) updateChat(msg tea.Msg) (Model, tea.Cmd) { s.State = StateIdle s.StatusText = "Closed" s.Closed = true + // A turn that ended closed must not produce a recovery + // toast on the next turn. + s.RetryVerb = "" // If the focused agent closed, switch back to parent (if any) or root if event.ID == m.Focused.ID() && s.State == StateIdle { if m.Focused.Parent() != nil { @@ -1557,6 +1577,9 @@ func (m Model) updateChat(msg tea.Msg) (Model, tea.Cmd) { s.StatusText = fmt.Sprintf("Error: %v", event.Error) s.Error = event.Error } + // A turn that ended in error must not produce a recovery + // toast on the next turn. + s.RetryVerb = "" // We don't clear rendered history so user can see what happened default: s.State = StateIdle @@ -1567,6 +1590,65 @@ func (m Model) updateChat(msg tea.Msg) (Model, tea.Cmd) { if event.ID == m.Focused.ID() { m.updateViewport() } + case common.RetryEvent: + // A stream attempt failed and the executor is retrying after + // event.Delay. The agent stays busy (the spinner keeps running) + // and the failed attempt's partial output is dropped so it does + // not linger in the transcript. The pinned error box is left + // alone: it clears when the next successful turn starts + // (the "thinking" branch above), and recovery is announced by + // the dedicated RecoveryEvent branch below. + s.Transcript.generation++ + s.Transcript.busy = false + s.State = StateThinking + // The failure class decides the verb: an HTTP 400 is the API + // rejecting the request body, not a lost connection. + retryVerb := retryVerbConnectionLost + var retryStatusErr *client.StatusError + if errors.As(event.Err, &retryStatusErr) && retryStatusErr.StatusCode == http.StatusBadRequest { + retryVerb = retryVerbRejectedByAPI + } + // The status line is rendered once and never refreshed, so the + // wording deliberately uses the past tense: "after Xs backoff" + // is accurate once the wait completes, whereas "retrying in Xs" + // would imply a live countdown that never ticks and could linger + // on screen while the next attempt already streams. + s.StatusText = fmt.Sprintf("%s — retry %d/%d after %s backoff", retryVerb, event.Attempt, event.MaxAttempts, event.Delay.Truncate(100*time.Millisecond)) + s.StreamingState = common.ContentEvent{ID: event.ID} + // Clear streaming render cache for the failed attempt + s.StreamingStyledCache = "" + s.StreamingChunkCount = 0 + s.RetryVerb = retryVerb + if event.ID == m.Focused.ID() { + m.updateViewport() + } + case common.RecoveryEvent: + // The retried attempt actually produced a response — announce it + // immediately instead of guessing on the next turn's thinking + // event (which may never come before content streams). + s.Transcript.generation++ + s.State = StateThinking + if s.RetryVerb != "" { + if s.RetryVerb == retryVerbRejectedByAPI { + s.StatusText = "request accepted after retry — streaming response" + restoredToast = func() tea.Msg { + return ToastMsg{Text: "request accepted after retry"} + } + } else { + s.StatusText = "connection restored — streaming response" + restoredToast = func() tea.Msg { + return ToastMsg{Text: "connection restored"} + } + } + s.RetryVerb = "" + } else { + // Recovery for an agent whose retry verb was already cleared + // (e.g. a stop raced the recovery): keep the status accurate. + s.StatusText = "streaming response" + } + if event.ID == m.Focused.ID() { + m.updateViewport() + } case common.ChildAddedEvent: s.StatusText = "Subagent spawned" m.updateViewport() @@ -1576,6 +1658,9 @@ func (m Model) updateChat(msg tea.Msg) (Model, tea.Cmd) { s.PendingStop = false s.State = StateIdle s.StatusText = "Stopped" + // A turn that ended in a user stop must not produce a recovery + // toast on the next turn. + s.RetryVerb = "" s.RenderedHistory = nil s.StreamingStyledCache = "" s.StreamingChunkCount = 0 @@ -1588,6 +1673,10 @@ func (m Model) updateChat(msg tea.Msg) (Model, tea.Cmd) { } } + if restoredToast != nil { + return m, restoredToast + } + case ConfirmRequestMsg: s := m.GetAgentState(msg.OrchestratorID) s.State = StateConfirmTool diff --git a/internal/tui/view.go b/internal/tui/view.go index d4e6826a..611be5f2 100644 --- a/internal/tui/view.go +++ b/internal/tui/view.go @@ -2,6 +2,7 @@ package tui import ( "fmt" + "hash/fnv" "image/color" "math" "os" @@ -360,7 +361,7 @@ func (m *Model) renderMinimalEqualizerAt(now time.Time) string { // Gentle incommensurate harmonic (golden ratio 1.618) creates organic, non-repeating crests // Low amplitude ensures it never causes erratic snap or jitter - w2 := 0.35 * math.Sin(t*0.93 + float64(i)*0.55 + 1.2) + w2 := 0.35 * math.Sin(t*0.93+float64(i)*0.55+1.2) // Breathing envelope gives gentle natural cadence swell := 0.88 + 0.20*math.Sin(t*0.38+float64(i)*0.25) @@ -504,6 +505,53 @@ func (m *Model) renderScannerTrackAt(symbol string, symbolColor color.Color, now return sb.String() } +// agentTypeFromID derives a display category from an orchestrator ID: +// "main" is the root orchestrator; subagents are minted as +// "-subagent-" (BaseOrchestrator.NextChildID), so the category +// is the prefix before "-subagent-". Any other id (future categories, +// test doubles) falls back to the id itself. Returns "" for "". +func agentTypeFromID(id string) string { + if id == "" { + return "" + } + if id == common.MainAgentID { + return "orchestrator" + } + if i := strings.Index(id, "-subagent-"); i > 0 { + return id[:i] + } + return id +} + +// agentTypeColors assigns one bright color per agent category so the +// status bar makes the current agent unmistakable at a glance. +var agentTypeColors = map[string]color.Color{ + "orchestrator": lipgloss.BrightGreen, + "researcher": lipgloss.BrightCyan, + "coder": lipgloss.BrightMagenta, +} + +// futureAgentTypeColors gives categories without a dedicated entry a +// stable bright color (FNV-1a of the category name), so agent types +// added later as new subagent JSON configs remain visible and +// consistently colored across renders and restarts. +var futureAgentTypeColors = []color.Color{ + lipgloss.BrightYellow, + lipgloss.BrightBlue, + lipgloss.BrightWhite, +} + +// agentTypeStyle returns the bold bright-colored style for a category. +func agentTypeStyle(agentType string) lipgloss.Style { + color, ok := agentTypeColors[agentType] + if !ok { + h := fnv.New32a() + h.Write([]byte(agentType)) + color = futureAgentTypeColors[h.Sum32()%uint32(len(futureAgentTypeColors))] + } + return lipgloss.NewStyle().Bold(true).Foreground(color) +} + func (m *Model) statusBarView() string { w := max(m.Width, 1) @@ -567,6 +615,13 @@ func (m *Model) statusBarView() string { } leftItems = append(leftItems, statePart) + // Agent category of the focused agent (bright, one color per type), + // shown before the branch/CWD context. Always visible: agent identity + // is not CWD context, so -show-cwd=false must not hide it. + if agentType := agentTypeFromID(m.Focused.ID()); agentType != "" { + leftItems = append(leftItems, agentTypeStyle(agentType).Render(agentType)) + } + // Branch or CWD (whisper-muted, unobtrusive context) if m.ShowCWD { if m.GitBranch != "" { From 2605313017866cefc33a3cd84fd1606634556fdc Mon Sep 17 00:00:00 2001 From: Emasoft <713559+Emasoft@users.noreply.github.com> Date: Sat, 19 Sep 2026 00:40:40 +0200 Subject: [PATCH 2/4] fix: --max-stream-retries=0 now disables all retry tiers The flag sets only the infrastructure tier's budget; the bad-body tier resolved its own default budget of 3, so HTTP 400 responses were still retried 3 times despite the documented "0 or negative disables retries entirely" contract (owner review, comment 5736731620). A global disable (flag/env 0 or negative) now silences both tiers at the budget-resolution site; an explicit bad-body budget still applies whenever the global budget is positive. Covered by TestRunLoopGlobalDisableAlsoSilencesBadBodyTier (0 and -1: exactly one POST, zero retry events, no backoff) while the positive-budget tests pin that nothing else changed. Docs updated (en + zh-CN). --- docs/architecture.md | 2 +- docs/architecture.zh-CN.md | 2 +- internal/executor/executor.go | 9 +++ .../executor/stream_retry_integration_test.go | 65 +++++++++++++++++++ 4 files changed, 76 insertions(+), 2 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index a3e8c83c..766155f1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -125,7 +125,7 @@ Late wraps each LLM stream call in two independent retry tiers, each with its ow - **Infrastructure tier:** Covers transport errors (connection refused/reset, timeouts, mid-body disconnects) and HTTP 408/429/5xx. Budgeted via `-max-stream-retries` / `LATE_MAX_STREAM_RETRIES` (default: 10; `0` or a negative value disables stream retrying). Precedence is CLI flag > environment variable > built-in default. - **Mid-stream interruptions:** Mid-body transport failures arrive after the server has already accepted the request (HTTP 200) — HTTP/2 RST_STREAM / INTERNAL_ERROR (classic error text: `stream error: stream ID N; INTERNAL_ERROR; received from peer`), GOAWAY, connection resets, and truncated bodies — so the client wraps them in a typed `StreamInterruptedError`, which is retried from the infrastructure budget. The single exception is an SSE line exceeding the 1 MB scanner cap (`bufio.ErrTooLong`), which fails fast because retrying cannot shrink the line. -- **Bad-body tier:** Covers HTTP 400 only, with a dedicated small budget (3 attempts, `DefaultMaxBadBodyRetries`; not yet flag-configurable). Strict OpenAI-compatible gateways (e.g. z.ai/GLM) frequently fail transiently while reading the request body ("read body failed"), which a few quick retries resolve; genuinely malformed requests still terminate after this small bounded budget. +- **Bad-body tier:** Covers HTTP 400 only, with a dedicated small budget (3 attempts, `DefaultMaxBadBodyRetries`; not yet flag-configurable). Strict OpenAI-compatible gateways (e.g. z.ai/GLM) frequently fail transiently while reading the request body ("read body failed"), which a few quick retries resolve; genuinely malformed requests still terminate after this small bounded budget. A global disable (`-max-stream-retries 0` or negative) silences both tiers — including this one — so a globally disabled run never retries HTTP 400s either. - **Backoff:** Exponential — 500 ms base doubling per attempt, capped at 30 s, with full jitter (uniform over `[0, cap]`). A server `Retry-After` header is honored as a backoff floor — the combined wait is never shorter than the server requested — capped at 5 minutes so a hostile or buggy server cannot hang an interactive session; the wait remains cancelable throughout. - **Fail-fast errors:** Errors that retrying cannot help are never retried: TLS certificate/trust failures (untrusted authority, hostname mismatch, invalid or expired chains), non-TLS bytes on a TLS connection, unsupported URL schemes, HTTP-on-HTTPS, context cancellation, and permanent client errors (401/403/404). Unknown errors also fail fast, exactly like the pre-retry behavior. - **Independent counters:** 400 retries never consume the infrastructure budget, and vice versa. diff --git a/docs/architecture.zh-CN.md b/docs/architecture.zh-CN.md index dec47701..3ecac942 100644 --- a/docs/architecture.zh-CN.md +++ b/docs/architecture.zh-CN.md @@ -125,7 +125,7 @@ Late 将每次 LLM 流式调用包裹在两个相互独立的重试层级中， - **基础设施层（Infrastructure tier）：** 覆盖传输类错误（连接被拒绝/重置、超时、响应体中途断开）以及 HTTP 408/429/5xx。预算通过 `-max-stream-retries` / `LATE_MAX_STREAM_RETRIES` 控制（默认 10；`0` 或负值会完全禁用流式重试）。优先级为 CLI 标志 > 环境变量 > 内置默认值。 - **流式中断：** 响应体中途的传输故障发生在服务器已经接受请求（HTTP 200）之后——包括 HTTP/2 RST_STREAM / INTERNAL_ERROR（典型报错文本：`stream error: stream ID N; INTERNAL_ERROR; received from peer`）、GOAWAY、连接重置以及响应体被截断——因此客户端会将其包装为类型化的 `StreamInterruptedError`，并从基础设施层预算中进行重试。唯一的例外是超出 1 MB 扫描器上限（`bufio.ErrTooLong`）的 SSE 行——它会快速失败，因为重试无法缩短该行。 -- **无效请求体层（Bad-body tier）：** 仅覆盖 HTTP 400，使用一个专用的小预算（3 次，常量 `DefaultMaxBadBodyRetries`；暂不支持通过命令行标志配置）。严格的 OpenAI 兼容网关（如 z.ai/GLM）在读取请求体时经常发生瞬时故障（"read body failed"），少量快速重试即可解决；而真正格式错误的请求仍会在这一小额有界预算耗尽后终止。 +- **无效请求体层（Bad-body tier）：** 仅覆盖 HTTP 400，使用一个专用的小预算（3 次，常量 `DefaultMaxBadBodyRetries`；暂不支持通过命令行标志配置）。严格的 OpenAI 兼容网关（如 z.ai/GLM）在读取请求体时经常发生瞬时故障（"read body failed"），少量快速重试即可解决；而真正格式错误的请求仍会在这一小额有界预算耗尽后终止。全局禁用（`-max-stream-retries 0` 或负值）会同时关闭两个层级——包括本层级——因此全局禁用后的运行也不会再重试 HTTP 400。 - **退避（Backoff）：** 指数退避——以 500 ms 为基数逐次翻倍，上限 30 s，并叠加完全抖动（full jitter，在 `[0, cap]` 区间内均匀取值）。服务器返回的 `Retry-After` 会被作为退避下限遵守——合并后的等待时间绝不短于服务器要求的时长——并设有 5 分钟上限，以防止恶意或有缺陷的服务器挂起交互会话；等待过程始终可以取消。 - **快速失败（Fail-fast）：** 重试无法解决的错误绝不会重试：TLS 证书/信任故障（不受信任的颁发机构、主机名不匹配、无效或过期的证书链）、TLS 连接上的非 TLS 字节、不支持的 URL scheme、HTTPS 端点上的纯 HTTP、上下文取消，以及永久性的客户端错误（401/403/404）。未知错误同样会快速失败，与引入重试之前的行为完全一致。 - **计数器相互独立：** 400 的重试不会消耗基础设施层的预算，反之亦然。 diff --git a/internal/executor/executor.go b/internal/executor/executor.go index 2f5b8b81..f06bfb9c 100644 --- a/internal/executor/executor.go +++ b/internal/executor/executor.go @@ -259,6 +259,15 @@ func RunLoop( maxRetries := maxStreamRetriesFromContext(ctx) badBodyBudget := maxBadBodyRetriesFromContext(ctx) + // A global disable (--max-stream-retries=0 / negative, or the ctx key) + // must silence BOTH tiers: the bad-body tier has its own default + // budget, which would otherwise keep retrying HTTP 400s despite the + // advertised "retries disabled" contract. An explicit bad-body budget + // still applies whenever the global budget is positive. + if maxRetries <= 0 { + badBodyBudget = 0 + } + for i := 0; maxTurns <= 0 || i < maxTurns; i++ { if onStartTurn != nil { onStartTurn() diff --git a/internal/executor/stream_retry_integration_test.go b/internal/executor/stream_retry_integration_test.go index 69fa21c2..41865646 100644 --- a/internal/executor/stream_retry_integration_test.go +++ b/internal/executor/stream_retry_integration_test.go @@ -989,6 +989,71 @@ func TestRunLoopStopsAfterBadBodyBudgetExhausted(t *testing.T) { } } +// TestRunLoopGlobalDisableAlsoSilencesBadBodyTier pins the global-disable +// contract end-to-end: a global disable (--max-stream-retries=0, negative, +// or the ctx key) must silence BOTH retry tiers. Without the budget clamp at +// the resolution site in RunLoop, the bad-body tier would fall back to its +// own default budget (DefaultMaxBadBodyRetries) and keep retrying HTTP 400s +// despite the advertised "retries disabled" contract. With the disable in +// effect, an always-400 server sees exactly one POST — the initial attempt, +// zero retries, zero backoff sleeps — and the run fails with the terminal +// 400. The positive-budget contrast is pinned by +// TestRunLoopStopsAfterBadBodyBudgetExhausted above: with MaxStreamRetriesKey +// = 2 the bad-body tier still uses its own budget (4 POSTs). +func TestRunLoopGlobalDisableAlsoSilencesBadBodyTier(t *testing.T) { + for _, tc := range []struct { + name string + budget int + }{ + {name: "zero_budget", budget: 0}, + {name: "negative_budget", budget: -1}, + } { + t.Run(tc.name, func(t *testing.T) { + rs := newRetryServer(t, func(w http.ResponseWriter, r *http.Request) { + serveStatus(w, http.StatusBadRequest, badBodyErrorMessage) + }) + + sess := newRetryTestSession(t, rs.server.URL) + onRetry, retryEvents := retryCollector(t) + + // runLoopCtx passes the budget through as the MaxStreamRetriesKey + // ctx value and adds only a generous deadline; with both tiers + // disabled there are no backoff sleeps, so the deadline is inert. + // It is kept (rather than a bare context.WithValue) so a bug can + // never hang the test until the global -timeout, same rationale + // as every sibling test in this file. + ctx, cancel := runLoopCtx(tc.budget, 15*time.Second) + defer cancel() + + start := time.Now() + _, err := RunLoop(ctx, sess, 1, nil, nil, nil, nil, onRetry, nil, nil) + elapsed := time.Since(start) + + if err == nil { + t.Fatal("RunLoop returned nil error, want the terminal 400 despite retries being disabled") + } + var statusErr *client.StatusError + if !errors.As(err, &statusErr) || statusErr.StatusCode != http.StatusBadRequest { + t.Fatalf("RunLoop error = %v, want it to wrap *client.StatusError with 400", err) + } + + // Exactly one POST: the initial attempt. Zero retries from either + // tier — the global disable silences the bad-body tier too. + if got := rs.postCount(); got != 1 { + t.Errorf("server got %d POSTs, want exactly 1 (initial attempt, zero retries)", got) + } + if events := retryEvents(); len(events) != 0 { + t.Errorf("got %d RetryEvents, want 0 (retries disabled): %+v", len(events), events) + } + + // No backoff sleeps: the terminal 400 must surface immediately. + if elapsed >= 2*time.Second { + t.Errorf("RunLoop took %v, want well under 2s (no backoff sleeps when retries are disabled)", elapsed) + } + }) + } +} + // toolCallSSEBody renders a complete SSE stream that ends in a tool call // (finish_reason "tool_calls") instead of a final text response: the turn // commits an assistant tool-call message and the loop advances to the next From a3414b4fbcbf97cf0a7dd83261e203f09ef2c7c6 Mon Sep 17 00:00:00 2001 From: Emasoft <713559+Emasoft@users.noreply.github.com> Date: Sat, 19 Sep 2026 00:40:47 +0200 Subject: [PATCH 3/4] chore: resolve golangci-lint findings Two ineffectual assignments in retry tests (ineffassign); test semantics unchanged. --- internal/tui/retry_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/tui/retry_test.go b/internal/tui/retry_test.go index f19eff57..f425a28c 100644 --- a/internal/tui/retry_test.go +++ b/internal/tui/retry_test.go @@ -99,7 +99,7 @@ func TestRetryEventHTTP400NamesTheRejection(t *testing.T) { // dropped RecoveryEvent — the "thinking" status must NOT fire any toast // (recovery is announced immediately by the RecoveryEvent, if it arrives). func TestThinkingClearsRetryVerbSilently(t *testing.T) { - m, s := newViewportBenchmarkModel(nil) + m, _ := newViewportBenchmarkModel(nil) updated, _ := m.Update(OrchestratorEventMsg{Event: common.RetryEvent{ ID: m.Focused.ID(), @@ -109,7 +109,7 @@ func TestThinkingClearsRetryVerbSilently(t *testing.T) { Err: errors.New("connection reset by peer"), }}) *m = updated.(Model) - s = m.GetAgentState(m.Focused.ID()) + s := m.GetAgentState(m.Focused.ID()) if s.RetryVerb != retryVerbConnectionLost { t.Fatalf("RetryVerb = %q, want %q after an infra failure", s.RetryVerb, retryVerbConnectionLost) @@ -162,7 +162,7 @@ func TestThinkingClearsRetryVerbSilently(t *testing.T) { // only once: neither the final response's content events nor the next turn's // thinking status may repeat it. func TestRecoveryEventToastsImmediately(t *testing.T) { - m, s := newViewportBenchmarkModel(nil) + m, _ := newViewportBenchmarkModel(nil) updated, _ := m.Update(OrchestratorEventMsg{Event: common.RetryEvent{ ID: m.Focused.ID(), @@ -172,7 +172,7 @@ func TestRecoveryEventToastsImmediately(t *testing.T) { Err: errors.New("connection reset by peer"), }}) *m = updated.(Model) - s = m.GetAgentState(m.Focused.ID()) + s := m.GetAgentState(m.Focused.ID()) if s.RetryVerb != retryVerbConnectionLost { t.Fatalf("RetryVerb = %q, want %q after an infra failure", s.RetryVerb, retryVerbConnectionLost) From e656df379d75b7ac0b376969808fa2e48205ee3e Mon Sep 17 00:00:00 2001 From: ml Date: Tue, 22 Sep 2026 19:07:17 +0200 Subject: [PATCH 4/4] refactor: remove duplicated agent type --- internal/tui/agent_type_test.go | 73 --------------------------------- internal/tui/view.go | 57 +------------------------ 2 files changed, 1 insertion(+), 129 deletions(-) delete mode 100644 internal/tui/agent_type_test.go diff --git a/internal/tui/agent_type_test.go b/internal/tui/agent_type_test.go deleted file mode 100644 index 539f8443..00000000 --- a/internal/tui/agent_type_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package tui - -import ( - "regexp" - "strings" - "testing" -) - -var ansiRe = regexp.MustCompile(`\x1b\[[0-9;]*m`) - -func stripANSI(s string) string { return ansiRe.ReplaceAllString(s, "") } - -func TestAgentTypeFromID(t *testing.T) { - cases := map[string]string{ - "main": "orchestrator", - "researcher-subagent-0": "researcher", - "coder-subagent-12": "coder", - "planner-subagent-3": "planner", // future category: new JSON config, no code change - "mock": "mock", - "": "", - } - for id, want := range cases { - if got := agentTypeFromID(id); got != want { - t.Errorf("agentTypeFromID(%q) = %q, want %q", id, got, want) - } - } -} - -func TestAgentTypeColorsAreDistinctAndStable(t *testing.T) { - orch := agentTypeStyle("orchestrator").Render("orchestrator") - res := agentTypeStyle("researcher").Render("researcher") - code := agentTypeStyle("coder").Render("coder") - if orch == res || res == code || orch == code { - t.Fatalf("orchestrator, researcher and coder must use different bright colors") - } - if !strings.Contains(stripANSI(orch), "orchestrator") { - t.Fatalf("label text lost: %q", orch) - } - if first, second := agentTypeStyle("future-type").Render("x"), agentTypeStyle("future-type").Render("x"); first != second { - t.Fatalf("future categories must get a stable color") - } -} - -func TestStatusBarShowsAgentTypeBeforeBranch(t *testing.T) { - model := NewModel(&mockOrchestrator{}, nil, nil) - model.Focused = &focusTestOrchestrator{id: "main"} - model.Width = 120 - model.ShowCWD = true - model.CWD = "/tmp/repo" - model.GitBranch = "feat/test" - bar := stripANSI(model.statusBarView()) - if !strings.Contains(bar, "orchestrator") { - t.Fatalf("status bar must show the agent category, got: %q", bar) - } - if strings.Index(bar, "orchestrator") > strings.Index(bar, "feat/test") { - t.Fatalf("agent category must appear before the git branch, got: %q", bar) - } - if !strings.Contains(bar, "feat/test") { - t.Fatalf("branch missing entirely: %q", bar) - } -} - -func TestStatusBarShowsAgentTypeWhenShowCWDDisabled(t *testing.T) { - model := NewModel(&mockOrchestrator{}, nil, nil) - model.Focused = &focusTestOrchestrator{id: "researcher-subagent-0"} - model.Width = 120 - model.ShowCWD = false - model.GitBranch = "" - bar := stripANSI(model.statusBarView()) - if !strings.Contains(bar, "researcher") { - t.Fatalf("agent category must be visible even with ShowCWD=false, got: %q", bar) - } -} diff --git a/internal/tui/view.go b/internal/tui/view.go index 611be5f2..d4e6826a 100644 --- a/internal/tui/view.go +++ b/internal/tui/view.go @@ -2,7 +2,6 @@ package tui import ( "fmt" - "hash/fnv" "image/color" "math" "os" @@ -361,7 +360,7 @@ func (m *Model) renderMinimalEqualizerAt(now time.Time) string { // Gentle incommensurate harmonic (golden ratio 1.618) creates organic, non-repeating crests // Low amplitude ensures it never causes erratic snap or jitter - w2 := 0.35 * math.Sin(t*0.93+float64(i)*0.55+1.2) + w2 := 0.35 * math.Sin(t*0.93 + float64(i)*0.55 + 1.2) // Breathing envelope gives gentle natural cadence swell := 0.88 + 0.20*math.Sin(t*0.38+float64(i)*0.25) @@ -505,53 +504,6 @@ func (m *Model) renderScannerTrackAt(symbol string, symbolColor color.Color, now return sb.String() } -// agentTypeFromID derives a display category from an orchestrator ID: -// "main" is the root orchestrator; subagents are minted as -// "-subagent-" (BaseOrchestrator.NextChildID), so the category -// is the prefix before "-subagent-". Any other id (future categories, -// test doubles) falls back to the id itself. Returns "" for "". -func agentTypeFromID(id string) string { - if id == "" { - return "" - } - if id == common.MainAgentID { - return "orchestrator" - } - if i := strings.Index(id, "-subagent-"); i > 0 { - return id[:i] - } - return id -} - -// agentTypeColors assigns one bright color per agent category so the -// status bar makes the current agent unmistakable at a glance. -var agentTypeColors = map[string]color.Color{ - "orchestrator": lipgloss.BrightGreen, - "researcher": lipgloss.BrightCyan, - "coder": lipgloss.BrightMagenta, -} - -// futureAgentTypeColors gives categories without a dedicated entry a -// stable bright color (FNV-1a of the category name), so agent types -// added later as new subagent JSON configs remain visible and -// consistently colored across renders and restarts. -var futureAgentTypeColors = []color.Color{ - lipgloss.BrightYellow, - lipgloss.BrightBlue, - lipgloss.BrightWhite, -} - -// agentTypeStyle returns the bold bright-colored style for a category. -func agentTypeStyle(agentType string) lipgloss.Style { - color, ok := agentTypeColors[agentType] - if !ok { - h := fnv.New32a() - h.Write([]byte(agentType)) - color = futureAgentTypeColors[h.Sum32()%uint32(len(futureAgentTypeColors))] - } - return lipgloss.NewStyle().Bold(true).Foreground(color) -} - func (m *Model) statusBarView() string { w := max(m.Width, 1) @@ -615,13 +567,6 @@ func (m *Model) statusBarView() string { } leftItems = append(leftItems, statePart) - // Agent category of the focused agent (bright, one color per type), - // shown before the branch/CWD context. Always visible: agent identity - // is not CWD context, so -show-cwd=false must not hide it. - if agentType := agentTypeFromID(m.Focused.ID()); agentType != "" { - leftItems = append(leftItems, agentTypeStyle(agentType).Render(agentType)) - } - // Branch or CWD (whisper-muted, unobtrusive context) if m.ShowCWD { if m.GitBranch != "" {