diff --git a/README.md b/README.md index 7a52314..3552c81 100644 --- a/README.md +++ b/README.md @@ -18,11 +18,11 @@ The mock provider is local and repeatable, so it lets you see the report before ## What it checks -The quick check tests the provider connection and streaming, and then checks basic and nested tool calls. It also tests multiple calls and recovery after a tool error, so you can see where common failures begin. +The quick check tests the provider connection and streaming, and then checks basic and nested tool calls. It also tests enum choices, multiple calls, recovery after a tool error, a file edit in a temporary project, and a restricted file listing. -Add `--harness opencode` to run a read only fixture through the installed OpenCode command. ModelStackCheck creates a temporary project, reads one fixture file, and removes the project afterward. +Add `--harness opencode` to run the same fixture task through the provider and the installed OpenCode command. ModelStackCheck compares the results, creates a temporary project, reads one fixture file, and removes the project afterward. Use `--timeout` for provider requests and `--harness-timeout` for the OpenCode task. -Add `--profile context` to test retrieval at about 8K and 32K tokens. These sizes are estimates based on four characters per token, and the large request may use more tokens with a hosted provider. +Add `--profile context` to test retrieval at about 16K and 32K tokens. It repeats each check three times, and these sizes are estimates based on four characters per token. The large request may use more tokens with a hosted provider. Add `--profile vision` to send a generated red image and check whether the model identifies it. Add `--profile full` to run both context and image checks. @@ -30,6 +30,15 @@ Add `--profile vision` to send a generated red image and check whether the model modelstackcheck doctor --harness opencode --profile full ``` +Use the built in mock scenarios to see how ModelStackCheck reports a broken tool call, an OpenCode parser failure, a timeout, or a context limit without relying on a live model. + +```sh +modelstackcheck doctor --provider mock --mock-scenario malformed-tool +modelstackcheck doctor --provider mock --harness opencode --mock-scenario harness-parser +modelstackcheck doctor --provider mock --harness opencode --mock-scenario harness-timeout --harness-timeout 300ms +modelstackcheck doctor --provider mock --profile context --mock-scenario context-degradation +``` + ModelStackCheck sends prompts to the provider you select, but it does not save them. Review reports before sharing, and remember that a successful probe covers only the tested request. ## Connect a provider diff --git a/cmd/modelstackcheck/main.go b/cmd/modelstackcheck/main.go index 867c6ed..345eb0b 100644 --- a/cmd/modelstackcheck/main.go +++ b/cmd/modelstackcheck/main.go @@ -60,9 +60,11 @@ Doctor options --endpoint provider address --harness auto or opencode --profile quick, context, vision, or full + --mock-scenario normal, tool-failure, malformed-tool, slow, rate-limit, server-error, harness-parser, harness-timeout, or context-degradation --format text, markdown, or json --output save a report to a file --timeout maximum time for each request + --harness-timeout maximum time allowed for the OpenCode fixture OpenAI compatible providers read OPENAI_API_KEY, OPENAI_BASE_URL, and OPENAI_MODEL. Reports stay on this computer unless you save or share them.`) @@ -78,7 +80,9 @@ func runDoctor(args []string, out, errOut *os.File) int { format := fs.String("format", "text", "text, markdown, or json") output := fs.String("output", "", "report output path") timeout := fs.Duration("timeout", 30*time.Second, "request timeout") + harnessTimeout := fs.Duration("harness-timeout", 5*time.Minute, "OpenCode fixture timeout") profile := fs.String("profile", "quick", "quick, context, vision, or full") + mockScenario := fs.String("mock-scenario", "normal", "deterministic mock provider behavior") if err := fs.Parse(args); err != nil { return doctor.ExitConfiguration } @@ -90,13 +94,16 @@ func runDoctor(args []string, out, errOut *os.File) int { fmt.Fprintln(errOut, "timeout must be greater than zero") return doctor.ExitConfiguration } + if *harnessTimeout <= 0 { + fmt.Fprintln(errOut, "harness-timeout must be greater than zero") + return doctor.ExitConfiguration + } if *format != "text" && *format != "markdown" && *format != "json" { fmt.Fprintln(errOut, "format must be text, markdown, or json") return doctor.ExitConfiguration } - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) - defer cancel() - report, err := doctor.Run(ctx, doctor.Options{Provider: *providerName, Model: *model, Endpoint: *endpoint, Harness: *harness, Profile: *profile, Timeout: *timeout}) + ctx := context.Background() + report, err := doctor.Run(ctx, doctor.Options{Provider: *providerName, Model: *model, Endpoint: *endpoint, Harness: *harness, Profile: *profile, MockScenario: *mockScenario, Timeout: *timeout, HarnessTimeout: *harnessTimeout}) if err != nil { fmt.Fprintln(errOut, doctor.Redact(err.Error())) return doctor.ExitConfiguration diff --git a/internal/doctor/doctor_test.go b/internal/doctor/doctor_test.go index 6ceb21e..129604c 100644 --- a/internal/doctor/doctor_test.go +++ b/internal/doctor/doctor_test.go @@ -6,15 +6,33 @@ import ( "encoding/base64" "encoding/json" "image/png" + "net" "net/http" "net/http/httptest" "os" "path/filepath" + "strconv" "strings" "testing" "time" ) +func newMockClient(t *testing.T, scenario string) *apiClient { + t.Helper() + server := httptest.NewServer(mockHandler(scenario)) + t.Cleanup(server.Close) + return &apiClient{base: server.URL + "/v1", model: "modelstackcheck-mock", http: server.Client()} +} + +func diagnosisCode(report *Report, code string) *Diagnosis { + for index := range report.Diagnoses { + if report.Diagnoses[index].Code == code { + return &report.Diagnoses[index] + } + } + return nil +} + func TestMockFullProfilePassesContextAndVision(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -22,7 +40,7 @@ func TestMockFullProfilePassesContextAndVision(t *testing.T) { if err != nil { t.Fatal(err) } - for _, id := range []string{"context.8k", "context.32k", "vision.image_input"} { + for _, id := range []string{"context.16k", "context.32k", "vision.image_input"} { found := false for _, check := range report.Checks { if check.ID == id && check.Status == "pass" { @@ -35,6 +53,131 @@ func TestMockFullProfilePassesContextAndVision(t *testing.T) { } } +func TestContextDegradationRequiresRepeatedSizeDifference(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + report, err := Run(ctx, Options{Provider: "mock", Profile: "context", MockScenario: "context-degradation", Timeout: 3 * time.Second}) + if err != nil { + t.Fatal(err) + } + if got := diagnosisCode(report, "MODEL_CONTEXT_DEGRADATION"); got == nil || got.Confidence != "high" { + t.Fatalf("expected high confidence context diagnosis, got %#v", report.Diagnoses) + } + if report.Checks[9].Passed != 3 || report.Checks[10].Passed != 0 { + t.Fatalf("expected consistent 16K pass and 32K failure, got %+v %+v", report.Checks[9], report.Checks[10]) + } +} + +func TestMockRunDiagnosesHarnessParserAfterDirectPass(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + report, err := Run(ctx, Options{Provider: "mock", Harness: "opencode", MockScenario: "harness-parser", Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + if diagnosisCode(report, "HARNESS_TOOL_PARSER") == nil { + t.Fatalf("expected differential harness diagnosis, got %+v", report.Diagnoses) + } + if report.Checks[9].Status != "pass" || report.Checks[10].Status != "fail" { + t.Fatalf("expected direct pass and harness failure, got %+v %+v", report.Checks[9], report.Checks[10]) + } +} + +func TestDiagnosisDistinguishesDirectModelAndHarnessFailures(t *testing.T) { + baseline := Check{ID: "harness.baseline", Status: "pass"} + parser := diagnose([]Check{baseline, {ID: "harness.opencode", Status: "fail", Error: "OpenCode could not complete the fixture"}}) + if len(parser) != 1 || parser[0].Code != "HARNESS_TOOL_PARSER" { + t.Fatalf("unexpected parser diagnosis: %+v", parser) + } + timeout := diagnose([]Check{baseline, {ID: "harness.opencode", Status: "fail", Error: "OpenCode exceeded the configured harness timeout"}}) + if len(timeout) != 1 || timeout[0].Code != "HARNESS_TIMEOUT" { + t.Fatalf("unexpected timeout diagnosis: %+v", timeout) + } + missing := diagnose([]Check{baseline, {ID: "harness.opencode", Status: "fail", Error: "OpenCode CLI was not found on PATH"}}) + if len(missing) != 1 || missing[0].Code != "HARNESS_NOT_FOUND" { + t.Fatalf("unexpected missing harness diagnosis: %+v", missing) + } + directFailure := diagnose([]Check{{ID: "tool.basic", Status: "fail"}, {ID: "harness.baseline", Status: "fail"}, {ID: "harness.opencode", Status: "fail"}}) + if diagnosisCode(&Report{Diagnoses: directFailure}, "MODEL_SCHEMA_FAILURE") == nil { + t.Fatalf("expected model schema diagnosis, got %+v", directFailure) + } +} + +func TestOpenCodeVerificationRequiresFinalAssistantText(t *testing.T) { + output := []byte("{\"type\":\"tool_use\",\"part\":{\"type\":\"tool\",\"state\":{\"output\":\"MDOC_OPENCODE_CANARY_91C4\"}}}\n" + + "{\"type\":\"text\",\"part\":{\"type\":\"text\",\"text\":\"The fixture tool result could not be parsed.\"}}\n") + if strings.Contains(openCodeFinalText(output), "MDOC_OPENCODE_CANARY_91C4") { + t.Fatal("tool output was mistaken for a successful assistant response") + } + output = []byte("{\"type\":\"text\",\"part\":{\"type\":\"text\",\"text\":\"MDOC_OPENCODE_CANARY_91C4\"}}\n") + if !strings.Contains(openCodeFinalText(output), "MDOC_OPENCODE_CANARY_91C4") { + t.Fatal("final assistant text did not preserve the expected marker") + } +} + +func TestReportsShowHarnessTimeoutAndRepeatedContextResults(t *testing.T) { + report := &Report{HarnessTimeoutMS: 300, Checks: []Check{{Name: "Context", Status: "pass", Attempts: 3, Passed: 2}}} + for name, rendered := range map[string]string{"text": Text(report), "markdown": Markdown(report)} { + if !strings.Contains(rendered, "300 ms") || !strings.Contains(rendered, "2 of 3") { + t.Errorf("%s report omitted timeout or attempt evidence: %s", name, rendered) + } + } +} + +func TestUnreachableProviderUsesConnectionDiagnosis(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + endpoint := "http://" + listener.Addr().String() + "/v1" + listener.Close() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + report, err := Run(ctx, Options{Provider: "openai", Endpoint: endpoint, Model: "test", Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + if diagnosisCode(report, "PROVIDER_CONNECTION") == nil { + t.Fatalf("expected provider connection diagnosis, got %+v", report.Diagnoses) + } +} + +func TestMalformedDirectToolCallUsesSchemaDiagnosis(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + report, err := Run(ctx, Options{Provider: "mock", MockScenario: "malformed-tool", Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + if diagnosisCode(report, "MODEL_SCHEMA_FAILURE") == nil { + t.Fatalf("expected model schema diagnosis, got %+v", report.Diagnoses) + } +} + +func TestMockProviderStatusAndSlowModes(t *testing.T) { + for _, test := range []struct { + scenario string + status int + }{{"rate-limit", http.StatusTooManyRequests}, {"server-error", http.StatusInternalServerError}} { + client := newMockClient(t, test.scenario) + _, err := client.chat(context.Background(), []message{{Role: "user", Content: "hello"}}, nil, false) + if err == nil || !strings.Contains(err.Error(), "HTTP "+strconv.Itoa(test.status)) { + t.Errorf("expected HTTP %d to be reported as an error, got %v", test.status, err) + } + } + client := newMockClient(t, "slow") + client.http.Timeout = time.Second + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + started := time.Now() + if _, err := client.chat(ctx, []message{{Role: "user", Content: "hello"}}, nil, false); err == nil { + t.Fatal("slow mode should observe request cancellation") + } + if elapsed := time.Since(started); elapsed > time.Second { + t.Fatalf("slow mode ignored cancellation for %v", elapsed) + } +} + func TestSyntheticVisionImageIsRed(t *testing.T) { data, err := base64.StdEncoding.DecodeString(syntheticRedPNG()) if err != nil { @@ -101,8 +244,19 @@ func TestMockProviderDoctorPasses(t *testing.T) { if report.ExitCode() != ExitOK { t.Fatalf("mock diagnostics were not green: %s", Text(report)) } - if len(report.Checks) != 6 { - t.Fatalf("got %d checks, expected six", len(report.Checks)) + if len(report.Checks) != 9 { + t.Fatalf("got %d checks, expected nine", len(report.Checks)) + } + for _, id := range []string{"tool.enum", "tool.file_edit", "tool.shell"} { + found := false + for _, check := range report.Checks { + if check.ID == id && check.Status == "pass" { + found = true + } + } + if !found { + t.Fatalf("missing passing MVP probe %q", id) + } } } diff --git a/internal/doctor/engine.go b/internal/doctor/engine.go index 7351f34..0408e3d 100644 --- a/internal/doctor/engine.go +++ b/internal/doctor/engine.go @@ -40,9 +40,23 @@ func Run(ctx context.Context, options Options) (*Report, error) { if options.Provider != "auto" && options.Provider != "ollama" && options.Provider != "openai" && options.Provider != "mock" { return nil, errors.New("provider must be auto, ollama, openai, or mock") } + switch options.MockScenario { + case "", "normal", "tool-failure", "malformed-tool", "slow", "rate-limit", "server-error", "harness-parser", "harness-timeout", "context-degradation": + default: + return nil, errors.New("unsupported mock scenario") + } if options.Timeout == 0 { options.Timeout = 30 * time.Second } + if options.Timeout < 0 { + return nil, errors.New("provider timeout must be greater than zero") + } + if options.HarnessTimeout == 0 { + options.HarnessTimeout = 5 * time.Minute + } + if options.HarnessTimeout < time.Millisecond { + return nil, errors.New("harness timeout must be at least one millisecond") + } client, provider, model, cleanup, err := selectProvider(options) if err != nil { return nil, err @@ -161,6 +175,28 @@ func Run(ctx context.Context, options Options) (*Report, error) { } return "Returned valid calls for read_file and search.", nil }) + runProbe("tool.enum", "Enum selection", func(c context.Context) (string, error) { + resp, err := client.chat(c, []message{{Role: "user", Content: "Choose a mode for inspecting this project. Use set_mode with mode read."}}, makeTools("set_mode"), false) + if err != nil { + return "", err + } + calls := resp.Choices[0].Message.ToolCalls + if len(calls) != 1 || calls[0].Function.Name != "set_mode" || !callIsValid(calls[0]) { + return "", errors.New("model did not select a valid enum tool call") + } + var args map[string]any + _ = json.Unmarshal([]byte(calls[0].Function.Arguments), &args) + if args["mode"] != "read" { + return "", errors.New("model selected a value outside the requested enum option") + } + return "Selected the supported read value from the mode enum.", nil + }) + runProbe("tool.file_edit", "Safe file edit", func(c context.Context) (string, error) { + return editFixtureWithModel(c, client) + }) + runProbe("tool.shell", "Restricted shell task", func(c context.Context) (string, error) { + return listFixtureWithModel(c, client) + }) runProbe("tool.failure_recovery", "Tool error recovery", func(c context.Context) (string, error) { first, err := client.chat(c, []message{{Role: "user", Content: "Try reading missing.txt with read_file."}}, makeTools("read_file", "list_files"), false) if err != nil { @@ -184,21 +220,37 @@ func Run(ctx context.Context, options Options) (*Report, error) { return "Responded after receiving a simulated missing file error.", nil }) if options.Profile == "context" || options.Profile == "full" { - for _, size := range []int{8 << 10, 32 << 10} { + for _, size := range []int{16 << 10, 32 << 10} { size := size - runProbe(fmt.Sprintf("context.%dk", size/1024), fmt.Sprintf("Context retrieval at about %dK tokens", size/1024), func(c context.Context) (string, error) { + startedContext := time.Now() + passed, failures := 0, 0 + var lastErr error + for attempt := 0; attempt < 3; attempt++ { + probeCtx, cancel := context.WithTimeout(ctx, options.Timeout) const canary = "MDOC_CONTEXT_CANARY_7F3A" - filler := strings.Repeat("oak river cloud meadow ", size*4/22) + filler := strings.Repeat("oak river cloud meadow ", size*4/len("oak river cloud meadow ")) prompt := fmt.Sprintf("Read this note and return only the code after CODE.\n%s\nCODE %s", filler, canary) - resp, err := client.chat(c, []message{{Role: "user", Content: prompt}}, nil, false) - if err != nil { - return "", err + resp, err := client.chat(probeCtx, []message{{Role: "user", Content: prompt}}, nil, false) + cancel() + if err == nil && !strings.Contains(textContent(resp.Choices[0].Message.Content), canary) { + err = errors.New("model did not retrieve the code placed at the end of the context") } - if !strings.Contains(textContent(resp.Choices[0].Message.Content), canary) { - return "", errors.New("model did not retrieve the code placed at the end of the context") + if err != nil { + failures++ + lastErr = err + } else { + passed++ } - return fmt.Sprintf("Retrieved the final marker from about %d tokens of generated context, estimated at four characters per token.", size/1024), nil - }) + } + check := Check{ID: fmt.Sprintf("context.%dk", size/1024), Name: fmt.Sprintf("Context retrieval at about %dK tokens", size/1024), Status: "pass", Attempts: 3, Passed: passed, DurationMS: time.Since(startedContext).Milliseconds()} + if failures > 0 { + check.Status = "fail" + check.Error = Redact(lastErr.Error()) + check.Evidence = fmt.Sprintf("Retrieved the final marker in %d of 3 attempts. Context size is estimated at four characters per token.", passed) + } else { + check.Evidence = "Retrieved the final marker in all 3 attempts. Context size is estimated at four characters per token." + } + report.Checks = append(report.Checks, check) } } if options.Profile == "vision" || options.Profile == "full" { @@ -215,21 +267,51 @@ func Run(ctx context.Context, options Options) (*Report, error) { return "Identified the red pixel in a synthetic one pixel image.", nil }) } + simulatedHarnessFailure := false if options.Harness == "opencode" { + report.HarnessTimeoutMS = options.HarnessTimeout.Milliseconds() + startedBaseline := time.Now() + baselineEvidence, baselineErr := directOpenCodeBaseline(ctx, client, options.Timeout) + baseline := Check{ID: "harness.baseline", Name: "Direct fixture baseline", Status: "pass", DurationMS: time.Since(startedBaseline).Milliseconds(), Evidence: baselineEvidence} + if baselineErr != nil { + baseline.Status = "fail" + baseline.Error = Redact(baselineErr.Error()) + baseline.Evidence = "The same fixture task did not complete through the provider directly." + } + report.Checks = append(report.Checks, baseline) startedHarness := time.Now() - evidence, err := runOpenCode(ctx, client, model, options.Timeout) + var evidence string + var err error + if provider == "mock" && options.MockScenario == "harness-parser" && baseline.Status == "pass" { + simulatedHarnessFailure = true + evidence = "The mock scenario injected a harness parser failure after the direct baseline passed." + err = errors.New("simulated OpenCode tool parser failure") + } else { + evidence, err = runOpenCode(ctx, client, model, options.HarnessTimeout) + } check := Check{ID: "harness.opencode", Name: "OpenCode execution", Status: "pass", DurationMS: time.Since(startedHarness).Milliseconds(), Evidence: evidence} if err != nil { check.Status = "fail" check.Error = Redact(err.Error()) - check.Evidence = "The controlled fixture was not completed through OpenCode." + if check.Evidence == "" { + check.Evidence = "The controlled fixture was not completed through OpenCode." + } + if errors.Is(err, context.DeadlineExceeded) || ctx.Err() == context.DeadlineExceeded { + check.Error = "OpenCode exceeded the configured harness timeout." + } } report.Checks = append(report.Checks, check) } report.Diagnoses = diagnose(report.Checks) - if report.Harness != "" { + if simulatedHarnessFailure { + report.Notes = append(report.Notes, "The parser finding came from a deterministic mock scenario. OpenCode was not launched.") + } + if report.Harness != "" && options.Harness == "opencode" && !simulatedHarnessFailure { report.Notes = append(report.Notes, "The OpenCode check uses an isolated temporary project and configuration, and grants read access only to its fixture.") } + if report.Harness != "" && options.Harness == "auto" { + report.Notes = append(report.Notes, "OpenCode was detected but not tested. Pass --harness opencode to run the fixture.") + } if report.Harness == "" && options.Harness == "opencode" { report.Notes = append(report.Notes, "OpenCode was requested but was not detected.") } @@ -250,7 +332,7 @@ func selectProvider(options Options) (*apiClient, string, string, func(), error) name := strings.ToLower(options.Provider) base, key, model := options.Endpoint, os.Getenv("OPENAI_API_KEY"), options.Model if name == "mock" { - server := httptest.NewServer(mockHandler()) + server := httptest.NewServer(mockHandler(options.MockScenario)) if model == "" { model = "modelstackcheck-mock" } @@ -334,11 +416,41 @@ func diagnose(checks []Check) []Diagnosis { if failed["tool.failure_recovery"] { out = append(out, Diagnosis{Code: "MODEL_TOOL_SELECTION", Confidence: "medium", Summary: "The model did not recover cleanly after a tool error.", Evidence: []string{"The simulated missing file error was not handled as expected."}, Recommendation: "Use clearer tool error messages or a model with stronger tool recovery behavior."}) } - if failed["harness.opencode"] { + if failed["tool.enum"] { + out = append(out, Diagnosis{Code: "MODEL_ENUM_FAILURE", Confidence: "medium", Summary: "The model did not select a supported value from the tool schema.", Evidence: []string{"The enum selection probe failed."}, Recommendation: "Use a model that follows constrained tool arguments reliably or simplify the enum schema."}) + } + if failed["tool.file_edit"] { + out = append(out, Diagnosis{Code: "MODEL_FILE_EDIT_FAILURE", Confidence: "medium", Summary: "The model could not produce and apply the requested edit in a temporary project.", Evidence: []string{"The file edit did not match the expected fixture change."}, Recommendation: "Check the model's edit tool format and repeat the task on a temporary project."}) + } + if failed["tool.shell"] { + out = append(out, Diagnosis{Code: "MODEL_SHELL_FAILURE", Confidence: "medium", Summary: "The model did not request the permitted shell operation correctly.", Evidence: []string{"The restricted shell probe failed within its temporary fixture."}, Recommendation: "Check shell tool argument formatting and keep commands restricted to a reviewed allowlist."}) + } + checksByID := map[string]Check{} + for _, check := range checks { + checksByID[check.ID] = check + } + if failed["harness.opencode"] && checksByID["harness.baseline"].Status == "pass" { + if strings.Contains(checksByID["harness.opencode"].Error, "harness timeout") { + out = append(out, Diagnosis{Code: "HARNESS_TIMEOUT", Confidence: "high", Summary: "The provider completed the fixture directly, but OpenCode exceeded its timeout.", Evidence: []string{"The direct fixture baseline passed and the OpenCode fixture timed out."}, Recommendation: "Raise the OpenCode provider timeout or use a faster model, then repeat the harness check."}) + } else if strings.Contains(checksByID["harness.opencode"].Error, "OpenCode CLI was not found") { + out = append(out, Diagnosis{Code: "HARNESS_NOT_FOUND", Confidence: "high", Summary: "The provider passed the direct fixture, but the OpenCode command is not installed or available on PATH.", Evidence: []string{"The direct fixture baseline passed and OpenCode could not be launched."}, Recommendation: "Install OpenCode or add its command to PATH, then repeat the harness check."}) + } else { + out = append(out, Diagnosis{Code: "HARNESS_TOOL_PARSER", Confidence: "medium", Summary: "The provider completed the fixture directly, but OpenCode could not complete the same tool task.", Evidence: []string{"The direct fixture baseline passed and the OpenCode fixture failed."}, Recommendation: "Check OpenCode tool call parsing and provider compatibility, then repeat the same fixture through both paths."}) + } + } else if failed["harness.opencode"] { out = append(out, Diagnosis{Code: "HARNESS_EXECUTION", Confidence: "high", Summary: "The controlled tool call did not complete through OpenCode.", Evidence: []string{"The check used a temporary project and a provider fixture."}, Recommendation: "Compare the OpenCode model and provider settings with the direct provider results, then repeat the harness check."}) } - if failed["context.8k"] || failed["context.32k"] { - out = append(out, Diagnosis{Code: "CONTEXT_RETRIEVAL", Confidence: "medium", Summary: "The model did not retrieve a marker placed at the end of a long prompt.", Evidence: []string{"The context probe failed at one or more requested sizes."}, Recommendation: "Lower the prompt size or raise the provider context limit, then repeat the same retrieval check."}) + if failed["context.16k"] && failed["context.32k"] { + out = append(out, Diagnosis{Code: "CONTEXT_RETRIEVAL", Confidence: "medium", Summary: "The model did not reliably retrieve a marker placed at the end of a long prompt.", Evidence: []string{"The 16K and 32K context probes failed."}, Recommendation: "Lower the prompt size or raise the provider context limit, then repeat the same retrieval check."}) + } else if failed["context.32k"] && !failed["context.16k"] { + c16, c32 := checksByID["context.16k"], checksByID["context.32k"] + confidence := "medium" + if c16.Attempts >= 3 && c16.Passed == c16.Attempts && c32.Attempts >= 3 && c32.Passed == 0 { + confidence = "high" + } + out = append(out, Diagnosis{Code: "MODEL_CONTEXT_DEGRADATION", Confidence: confidence, Summary: "The model retrieved the marker at 16K but not reliably at 32K.", Evidence: []string{fmt.Sprintf("16K retrieval passed %d of %d attempts.", c16.Passed, c16.Attempts), fmt.Sprintf("32K retrieval passed %d of %d attempts.", c32.Passed, c32.Attempts)}, Recommendation: "Set the context limit near the largest size that passed consistently, then confirm it with the task that first failed."}) + } else if failed["context.16k"] { + out = append(out, Diagnosis{Code: "CONTEXT_RETRIEVAL", Confidence: "medium", Summary: "The model did not reliably retrieve a marker at 16K context.", Evidence: []string{"The 16K context probe failed."}, Recommendation: "Lower the prompt size or check the provider context limit, then repeat the retrieval check."}) } if failed["vision.image_input"] { out = append(out, Diagnosis{Code: "PROVIDER_VISION", Confidence: "medium", Summary: "The model did not identify the image sent with the request.", Evidence: []string{"The synthetic image recognition probe failed."}, Recommendation: "Choose a vision enabled model and confirm the provider supports image input."}) @@ -380,13 +492,17 @@ func contains(items []string, target string) bool { return false } -func ServeMock(address string) error { - server := &http.Server{Addr: address, Handler: mockHandler(), ReadHeaderTimeout: 5 * time.Second} +func ServeMock(address string, scenarios ...string) error { + scenario := "normal" + if len(scenarios) > 0 { + scenario = scenarios[0] + } + server := &http.Server{Addr: address, Handler: mockHandler(scenario), ReadHeaderTimeout: 5 * time.Second} fmt.Printf("Mock provider listening at http://%s/v1\n", address) return server.ListenAndServe() } -func mockHandler() http.Handler { +func mockHandler(scenario string) http.Handler { mux := http.NewServeMux() mux.HandleFunc("GET /v1/models", func(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(map[string]any{"data": []map[string]string{{"id": "modelstackcheck-mock", "object": "model"}}}) @@ -397,6 +513,23 @@ func mockHandler() http.Handler { http.Error(w, "invalid request", http.StatusBadRequest) return } + if scenario == "rate-limit" { + http.Error(w, "rate limit", http.StatusTooManyRequests) + return + } + if scenario == "server-error" { + http.Error(w, "provider error", http.StatusInternalServerError) + return + } + if scenario == "slow" || (scenario == "harness-timeout" && hasOpenCodeToolResult(request.Messages)) { + timer := time.NewTimer(2 * time.Second) + defer timer.Stop() + select { + case <-timer.C: + case <-r.Context().Done(): + return + } + } if request.Stream { w.Header().Set("Content-Type", "text/event-stream") w.WriteHeader(http.StatusOK) @@ -404,6 +537,15 @@ func mockHandler() http.Handler { return } answer := mockAnswer(request) + if scenario == "malformed-tool" && len(request.Tools) > 0 { + answer = message{Role: "assistant", ToolCalls: []toolCall{{ID: "malformed", Type: "function", Function: calledFunction{Name: request.Tools[0].Function.Name, Arguments: "{"}}}} + } + if scenario == "tool-failure" && len(request.Tools) > 0 { + answer = message{Role: "assistant", Content: "I cannot call a tool."} + } + if scenario == "context-degradation" && strings.Contains(strings.ToLower(textContent(request.Messages[len(request.Messages)-1].Content)), "mdoc_context_canary_7f3a") && len(textContent(request.Messages[len(request.Messages)-1].Content)) > 100000 { + answer = message{Role: "assistant", Content: "I cannot locate the marker."} + } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": answer}}}) }) @@ -470,6 +612,12 @@ func mockAnswer(req chatRequest) message { calls = append(calls, toolCall{ID: fmt.Sprintf("call_%d", len(calls)+1), Type: "function", Function: calledFunction{Name: name, Arguments: args}}) } switch { + case strings.Contains(text, "choose a mode"): + add("set_mode", `{"mode":"read"}`) + case strings.Contains(text, "change hello()"): + add("edit_file", `{"path":"main.py","old":"return \"hello\"","new":"return \"Hello, world!\""}`) + case strings.Contains(text, "list files in the directory"): + add("run_command", `{"command":"ls"}`) case strings.Contains(text, "inspect_profile") || strings.Contains(text, "profile name ada"): add("inspect_profile", `{"profile":{"name":"Ada","settings":{"theme":"dark"}}}`) case strings.Contains(text, "both tools") || strings.Contains(text, "readme.md and search"): @@ -505,6 +653,88 @@ func mockAnswer(req chatRequest) message { return message{Role: "assistant", ToolCalls: calls, Content: "All requested checks completed."} } +func editFixtureWithModel(ctx context.Context, client *apiClient) (string, error) { + root, err := os.MkdirTemp("", "modelstackcheck-edit-") + if err != nil { + return "", errors.New("could not create the file edit fixture") + } + defer os.RemoveAll(root) + path := filepath.Join(root, "main.py") + const original = "def hello():\n return \"hello\"\n" + if err := os.WriteFile(path, []byte(original), 0600); err != nil { + return "", errors.New("could not create the file edit fixture") + } + resp, err := client.chat(ctx, []message{{Role: "user", Content: "Change hello() so it returns \"Hello, world!\". Use edit_file on main.py and replace only the return value."}}, makeTools("edit_file"), false) + if err != nil { + return "", err + } + calls := resp.Choices[0].Message.ToolCalls + if len(calls) != 1 || calls[0].Function.Name != "edit_file" || !callIsValid(calls[0]) { + return "", errors.New("model did not return a valid file edit") + } + var args struct { + Path string `json:"path"` + Old string `json:"old"` + New string `json:"new"` + } + if err := json.Unmarshal([]byte(calls[0].Function.Arguments), &args); err != nil || args.Path != "main.py" || args.Old != `return "hello"` || args.New != `return "Hello, world!"` { + return "", errors.New("model returned an unsafe or incorrect file edit") + } + current, err := os.ReadFile(path) + if err != nil || !bytes.Contains(current, []byte(args.Old)) { + return "", errors.New("file edit did not match the fixture content") + } + updated := bytes.Replace(current, []byte(args.Old), []byte(args.New), 1) + if err := os.WriteFile(path, updated, 0600); err != nil { + return "", errors.New("could not apply the fixture file edit") + } + verified, err := os.ReadFile(path) + if err != nil || !bytes.Contains(verified, []byte(`return "Hello, world!"`)) || bytes.Contains(verified, []byte(args.Old)) { + return "", errors.New("edited fixture did not match the requested result") + } + return "Updated and verified main.py inside a temporary fixture project.", nil +} + +func listFixtureWithModel(ctx context.Context, client *apiClient) (string, error) { + root, err := os.MkdirTemp("", "modelstackcheck-shell-") + if err != nil { + return "", errors.New("could not create the shell fixture") + } + defer os.RemoveAll(root) + for _, name := range []string{"README.md", "main.py"} { + if err := os.WriteFile(filepath.Join(root, name), nil, 0600); err != nil { + return "", errors.New("could not create the shell fixture") + } + } + resp, err := client.chat(ctx, []message{{Role: "user", Content: "List files in the directory by calling run_command with the permitted command ls."}}, makeTools("run_command"), false) + if err != nil { + return "", err + } + calls := resp.Choices[0].Message.ToolCalls + if len(calls) != 1 || calls[0].Function.Name != "run_command" || !callIsValid(calls[0]) { + return "", errors.New("model did not request the permitted file listing command") + } + var args map[string]any + _ = json.Unmarshal([]byte(calls[0].Function.Arguments), &args) + if args["command"] != "ls" { + return "", errors.New("model requested a command outside the safe fixture allowlist") + } + var output []byte + if runtime.GOOS == "windows" { + output, err = exec.CommandContext(ctx, "cmd.exe", "/d", "/c", "dir", "/b", root).Output() + } else { + output, err = exec.CommandContext(ctx, "ls", "-1", root).Output() + } + if err != nil { + return "", errors.New("could not list fixture files") + } + entries := strings.FieldsFunc(string(output), func(r rune) bool { return r == '\n' || r == '\r' }) + if len(entries) != 2 || !contains(entries, "README.md") || !contains(entries, "main.py") { + return "", errors.New("restricted file listing returned an unexpected result") + } + return "Requested the allowlisted ls operation and verified the two fixture files inside the temporary project.", nil +} + func textContent(value any) string { switch content := value.(type) { case string: @@ -536,6 +766,53 @@ func syntheticRedPNG() string { return base64.StdEncoding.EncodeToString(data.Bytes()) } +func hasOpenCodeToolResult(messages []message) bool { + for _, msg := range messages { + content := textContent(msg.Content) + if msg.Role == "tool" && strings.Contains(content, "MDOC_OPENCODE_CANARY") && !isDirectFixtureBaseline(messages) { + return true + } + } + return false +} + +func isDirectFixtureBaseline(messages []message) bool { + for _, msg := range messages { + if msg.Role == "system" && msg.Content == "ModelStackCheck direct fixture baseline" { + return true + } + } + return false +} + +func directOpenCodeBaseline(ctx context.Context, client *apiClient, timeout time.Duration) (string, error) { + const prompt = "Read fixture.txt and return the exact marker that it contains." + probeCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + baseline := message{Role: "system", Content: "ModelStackCheck direct fixture baseline"} + first, err := client.chat(probeCtx, []message{baseline, {Role: "user", Content: prompt}}, makeTools("read_file"), false) + if err != nil { + return "", err + } + calls := first.Choices[0].Message.ToolCalls + if len(calls) != 1 || calls[0].Function.Name != "read_file" || !callIsValid(calls[0]) { + return "", errors.New("provider did not make a valid direct fixture read call") + } + var args map[string]any + if err := json.Unmarshal([]byte(calls[0].Function.Arguments), &args); err != nil || args["path"] != "fixture.txt" { + return "", errors.New("provider requested the wrong direct fixture path") + } + const canary = "MDOC_OPENCODE_CANARY_91C4" + second, err := client.chat(probeCtx, []message{baseline, {Role: "user", Content: prompt}, {Role: "assistant", ToolCalls: calls}, {Role: "tool", ToolCallID: calls[0].ID, Content: canary}}, nil, false) + if err != nil { + return "", err + } + if !strings.Contains(textContent(second.Choices[0].Message.Content), canary) { + return "", errors.New("provider did not return the direct fixture marker") + } + return "The provider called read_file correctly and returned the fixture marker after the tool result.", nil +} + func runOpenCode(ctx context.Context, client *apiClient, model string, timeout time.Duration) (string, error) { path, err := exec.LookPath("opencode") if err != nil { @@ -576,22 +853,39 @@ func runOpenCode(ctx context.Context, client *apiClient, model string, timeout t } probeCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - cmd := exec.CommandContext(probeCtx, path, "run", "--pure", "--agent", "plan", "--model", "modelstackcheck/"+providerModel, "--format", "json", "--dir", project, "Read fixture.txt and return the exact marker that it contains. Include MDOC_OPENCODE_CANARY in your tool call plan.") + cmd := exec.CommandContext(probeCtx, path, "run", "--pure", "--agent", "plan", "--model", "modelstackcheck/"+providerModel, "--format", "json", "--dir", project, "Read fixture.txt and return the exact marker that it contains.") cmd.Dir = project cmd.Env = isolatedEnv(os.Environ(), map[string]string{"OPENCODE_CONFIG": configPath, "OPENCODE_CONFIG_DIR": root, "OPENCODE_DISABLE_MODELS_FETCH": "1", "OPENCODE_DISABLE_DEFAULT_PLUGINS": "1", "HOME": root, "USERPROFILE": root, "XDG_CONFIG_HOME": root, "XDG_DATA_HOME": root, "APPDATA": root}) output, err := cmd.CombinedOutput() if err != nil { if probeCtx.Err() != nil { - return "", errors.New("OpenCode did not finish the read only fixture before the timeout") + return "", errors.New("OpenCode exceeded the configured harness timeout") } return "", errors.New("OpenCode could not complete the read only fixture") } - if !strings.Contains(string(output), canary) { + if !strings.Contains(openCodeFinalText(output), canary) { return "", errors.New("OpenCode did not return the fixture marker") } return "OpenCode read the temporary fixture through the selected provider and returned its marker.", nil } +func openCodeFinalText(output []byte) string { + var text strings.Builder + for _, line := range strings.Split(string(output), "\n") { + var event struct { + Type string `json:"type"` + Part struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"part"` + } + if json.Unmarshal([]byte(line), &event) == nil && event.Type == "text" && event.Part.Type == "text" { + text.WriteString(event.Part.Text) + } + } + return text.String() +} + func isolatedEnv(environment []string, replacements map[string]string) []string { out := make([]string, 0, len(environment)+len(replacements)) for _, item := range environment { diff --git a/internal/doctor/provider.go b/internal/doctor/provider.go index c6dd6fd..6a3352d 100644 --- a/internal/doctor/provider.go +++ b/internal/doctor/provider.go @@ -152,6 +152,9 @@ func makeTools(names ...string) []toolSpec { "search": {Name: "search", Description: "Search text in project files", Parameters: objectSchema(map[string]any{"query": map[string]any{"type": "string"}}, "query")}, "inspect_profile": {Name: "inspect_profile", Description: "Inspect a profile and its settings", Parameters: objectSchema(map[string]any{"profile": objectSchema(map[string]any{"name": map[string]any{"type": "string"}, "settings": objectSchema(map[string]any{"theme": map[string]any{"type": "string"}}, "theme")}, "name", "settings")}, "profile")}, "list_files": {Name: "list_files", Description: "List available project files", Parameters: objectSchema(map[string]any{"path": map[string]any{"type": "string"}}, "path")}, + "set_mode": {Name: "set_mode", Description: "Choose a supported operation mode", Parameters: objectSchema(map[string]any{"mode": map[string]any{"type": "string", "enum": []string{"read", "write", "append"}}}, "mode")}, + "edit_file": {Name: "edit_file", Description: "Replace exact text in a fixture file", Parameters: objectSchema(map[string]any{"path": map[string]any{"type": "string"}, "old": map[string]any{"type": "string"}, "new": map[string]any{"type": "string"}}, "path", "old", "new")}, + "run_command": {Name: "run_command", Description: "List fixture files using the one permitted command", Parameters: objectSchema(map[string]any{"command": map[string]any{"type": "string", "enum": []string{"ls"}}}, "command")}, } out := make([]toolSpec, 0, len(names)) for _, name := range names { diff --git a/internal/doctor/report.go b/internal/doctor/report.go index aedd4db..454778d 100644 --- a/internal/doctor/report.go +++ b/internal/doctor/report.go @@ -38,6 +38,9 @@ func Text(report *Report) string { if report.Harness != "" { fmt.Fprintf(&b, "Harness %s\n", Redact(report.Harness)) } + if report.HarnessTimeoutMS > 0 { + fmt.Fprintf(&b, "Harness timeout %d ms\n", report.HarnessTimeoutMS) + } fmt.Fprintf(&b, "System %s %s\n\nCHECKS\n", report.System, report.Architecture) for _, check := range report.Checks { mark := "✓" @@ -51,6 +54,9 @@ func Text(report *Report) string { if check.DurationMS > 0 { fmt.Fprintf(&b, " in %d ms", check.DurationMS) } + if check.Attempts > 0 { + fmt.Fprintf(&b, " (%d of %d attempts passed)", check.Passed, check.Attempts) + } b.WriteByte('\n') if check.Error != "" { fmt.Fprintf(&b, " %s\n", Redact(check.Error)) @@ -82,6 +88,9 @@ func Markdown(report *Report) string { if report.Harness != "" { fmt.Fprintf(&b, "Harness %s \n", Redact(report.Harness)) } + if report.HarnessTimeoutMS > 0 { + fmt.Fprintf(&b, "Harness timeout %d ms \n", report.HarnessTimeoutMS) + } fmt.Fprintf(&b, "System %s %s\n\n## Checks\n\n", report.System, report.Architecture) for _, check := range report.Checks { fmt.Fprintf(&b, "### %s\n\nStatus %s\n\n", check.Name, strings.ToUpper(check.Status)) @@ -94,6 +103,9 @@ func Markdown(report *Report) string { if check.DurationMS > 0 { fmt.Fprintf(&b, "Observed duration %d ms\n\n", check.DurationMS) } + if check.Attempts > 0 { + fmt.Fprintf(&b, "Passed attempts %d of %d\n\n", check.Passed, check.Attempts) + } } if len(report.Diagnoses) > 0 { b.WriteString("## Diagnosis\n\n") diff --git a/internal/doctor/types.go b/internal/doctor/types.go index 7b15b26..2b35368 100644 --- a/internal/doctor/types.go +++ b/internal/doctor/types.go @@ -16,12 +16,14 @@ const ( ) type Options struct { - Provider string - Model string - Endpoint string - Harness string - Profile string - Timeout time.Duration + Provider string + Model string + Endpoint string + Harness string + Profile string + MockScenario string + Timeout time.Duration + HarnessTimeout time.Duration } type Check struct { @@ -31,6 +33,8 @@ type Check struct { DurationMS int64 `json:"latency_ms"` Evidence string `json:"evidence,omitempty"` Error string `json:"error,omitempty"` + Attempts int `json:"attempts,omitempty"` + Passed int `json:"passed,omitempty"` } type Diagnosis struct { @@ -42,17 +46,18 @@ type Diagnosis struct { } type Report struct { - SchemaVersion string `json:"schema_version"` - GeneratedAt time.Time `json:"generated_at"` - Provider string `json:"provider"` - Endpoint string `json:"endpoint,omitempty"` - Model string `json:"model,omitempty"` - Harness string `json:"harness,omitempty"` - System string `json:"system"` - Architecture string `json:"architecture"` - Checks []Check `json:"results"` - Diagnoses []Diagnosis `json:"diagnoses"` - Notes []string `json:"notes,omitempty"` + SchemaVersion string `json:"schema_version"` + GeneratedAt time.Time `json:"generated_at"` + Provider string `json:"provider"` + Endpoint string `json:"endpoint,omitempty"` + Model string `json:"model,omitempty"` + Harness string `json:"harness,omitempty"` + HarnessTimeoutMS int64 `json:"harness_timeout_ms,omitempty"` + System string `json:"system"` + Architecture string `json:"architecture"` + Checks []Check `json:"results"` + Diagnoses []Diagnosis `json:"diagnoses"` + Notes []string `json:"notes,omitempty"` } func (r *Report) ExitCode() int {