Complete the MVP diagnostics

README.md

@@ -18,18 +18,27 @@ The mock provider is local and repeatable, so it lets you see the report before
 
 ## What it checks
 
-The quick check tests the provider connection and streaming, and then checks basic and nested tool calls. It also tests multiple calls and recovery after a tool error, so you can see where common failures begin.
+The quick check tests the provider connection and streaming, and then checks basic and nested tool calls. It also tests enum choices, multiple calls, recovery after a tool error, a file edit in a temporary project, and a restricted file listing.
 
-Add `--harness opencode` to run a read only fixture through the installed OpenCode command. ModelStackCheck creates a temporary project, reads one fixture file, and removes the project afterward.
+Add `--harness opencode` to run the same fixture task through the provider and the installed OpenCode command. ModelStackCheck compares the results, creates a temporary project, reads one fixture file, and removes the project afterward. Use `--timeout` for provider requests and `--harness-timeout` for the OpenCode task.
 
-Add `--profile context` to test retrieval at about 8K and 32K tokens. These sizes are estimates based on four characters per token, and the large request may use more tokens with a hosted provider.
+Add `--profile context` to test retrieval at about 16K and 32K tokens. It repeats each check three times, and these sizes are estimates based on four characters per token. The large request may use more tokens with a hosted provider.
 
 Add `--profile vision` to send a generated red image and check whether the model identifies it. Add `--profile full` to run both context and image checks.
 
 ```sh
 modelstackcheck doctor --harness opencode --profile full
 ```
 
+Use the built in mock scenarios to see how ModelStackCheck reports a broken tool call, an OpenCode parser failure, a timeout, or a context limit without relying on a live model.
+
+```sh
+modelstackcheck doctor --provider mock --mock-scenario malformed-tool
+modelstackcheck doctor --provider mock --harness opencode --mock-scenario harness-parser
+modelstackcheck doctor --provider mock --harness opencode --mock-scenario harness-timeout --harness-timeout 300ms
+modelstackcheck doctor --provider mock --profile context --mock-scenario context-degradation
+```
+
 ModelStackCheck sends prompts to the provider you select, but it does not save them. Review reports before sharing, and remember that a successful probe covers only the tested request.
 
 ## Connect a provider

cmd/modelstackcheck/main.go

@@ -60,9 +60,11 @@ Doctor options
   --endpoint provider address
   --harness auto or opencode
   --profile quick, context, vision, or full
+  --mock-scenario normal, tool-failure, malformed-tool, slow, rate-limit, server-error, harness-parser, harness-timeout, or context-degradation
   --format text, markdown, or json
   --output save a report to a file
   --timeout maximum time for each request
+  --harness-timeout maximum time allowed for the OpenCode fixture
 
 OpenAI compatible providers read OPENAI_API_KEY, OPENAI_BASE_URL, and OPENAI_MODEL.
 Reports stay on this computer unless you save or share them.`)
@@ -78,7 +80,9 @@ func runDoctor(args []string, out, errOut *os.File) int {
 	format := fs.String("format", "text", "text, markdown, or json")
 	output := fs.String("output", "", "report output path")
 	timeout := fs.Duration("timeout", 30*time.Second, "request timeout")
+	harnessTimeout := fs.Duration("harness-timeout", 5*time.Minute, "OpenCode fixture timeout")
 	profile := fs.String("profile", "quick", "quick, context, vision, or full")
+	mockScenario := fs.String("mock-scenario", "normal", "deterministic mock provider behavior")
 	if err := fs.Parse(args); err != nil {
 		return doctor.ExitConfiguration
 	}
@@ -90,13 +94,16 @@ func runDoctor(args []string, out, errOut *os.File) int {
 		fmt.Fprintln(errOut, "timeout must be greater than zero")
 		return doctor.ExitConfiguration
 	}
+	if *harnessTimeout <= 0 {
+		fmt.Fprintln(errOut, "harness-timeout must be greater than zero")
+		return doctor.ExitConfiguration
+	}
 	if *format != "text" && *format != "markdown" && *format != "json" {
 		fmt.Fprintln(errOut, "format must be text, markdown, or json")
 		return doctor.ExitConfiguration
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
-	defer cancel()
-	report, err := doctor.Run(ctx, doctor.Options{Provider: *providerName, Model: *model, Endpoint: *endpoint, Harness: *harness, Profile: *profile, Timeout: *timeout})
+	ctx := context.Background()
+	report, err := doctor.Run(ctx, doctor.Options{Provider: *providerName, Model: *model, Endpoint: *endpoint, Harness: *harness, Profile: *profile, MockScenario: *mockScenario, Timeout: *timeout, HarnessTimeout: *harnessTimeout})
 	if err != nil {
 		fmt.Fprintln(errOut, doctor.Redact(err.Error()))
 		return doctor.ExitConfiguration

internal/doctor/doctor_test.go

@@ -6,23 +6,41 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"image/png"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
 )
 
+func newMockClient(t *testing.T, scenario string) *apiClient {
+	t.Helper()
+	server := httptest.NewServer(mockHandler(scenario))
+	t.Cleanup(server.Close)
+	return &apiClient{base: server.URL + "/v1", model: "modelstackcheck-mock", http: server.Client()}
+}
+
+func diagnosisCode(report *Report, code string) *Diagnosis {
+	for index := range report.Diagnoses {
+		if report.Diagnoses[index].Code == code {
+			return &report.Diagnoses[index]
+		}
+	}
+	return nil
+}
+
 func TestMockFullProfilePassesContextAndVision(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 	report, err := Run(ctx, Options{Provider: "mock", Profile: "full", Timeout: 5 * time.Second})
 	if err != nil {
 		t.Fatal(err)
 	}
-	for _, id := range []string{"context.8k", "context.32k", "vision.image_input"} {
+	for _, id := range []string{"context.16k", "context.32k", "vision.image_input"} {
 		found := false
 		for _, check := range report.Checks {
 			if check.ID == id && check.Status == "pass" {
@@ -35,6 +53,131 @@ func TestMockFullProfilePassesContextAndVision(t *testing.T) {
 	}
 }
 
+func TestContextDegradationRequiresRepeatedSizeDifference(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+	report, err := Run(ctx, Options{Provider: "mock", Profile: "context", MockScenario: "context-degradation", Timeout: 3 * time.Second})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := diagnosisCode(report, "MODEL_CONTEXT_DEGRADATION"); got == nil || got.Confidence != "high" {
+		t.Fatalf("expected high confidence context diagnosis, got %#v", report.Diagnoses)
+	}
+	if report.Checks[9].Passed != 3 || report.Checks[10].Passed != 0 {
+		t.Fatalf("expected consistent 16K pass and 32K failure, got %+v %+v", report.Checks[9], report.Checks[10])
+	}
+}
+
+func TestMockRunDiagnosesHarnessParserAfterDirectPass(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	report, err := Run(ctx, Options{Provider: "mock", Harness: "opencode", MockScenario: "harness-parser", Timeout: time.Second})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diagnosisCode(report, "HARNESS_TOOL_PARSER") == nil {
+		t.Fatalf("expected differential harness diagnosis, got %+v", report.Diagnoses)
+	}
+	if report.Checks[9].Status != "pass" || report.Checks[10].Status != "fail" {
+		t.Fatalf("expected direct pass and harness failure, got %+v %+v", report.Checks[9], report.Checks[10])
+	}
+}
+
+func TestDiagnosisDistinguishesDirectModelAndHarnessFailures(t *testing.T) {
+	baseline := Check{ID: "harness.baseline", Status: "pass"}
+	parser := diagnose([]Check{baseline, {ID: "harness.opencode", Status: "fail", Error: "OpenCode could not complete the fixture"}})
+	if len(parser) != 1 || parser[0].Code != "HARNESS_TOOL_PARSER" {
+		t.Fatalf("unexpected parser diagnosis: %+v", parser)
+	}
+	timeout := diagnose([]Check{baseline, {ID: "harness.opencode", Status: "fail", Error: "OpenCode exceeded the configured harness timeout"}})
+	if len(timeout) != 1 || timeout[0].Code != "HARNESS_TIMEOUT" {
+		t.Fatalf("unexpected timeout diagnosis: %+v", timeout)
+	}
+	missing := diagnose([]Check{baseline, {ID: "harness.opencode", Status: "fail", Error: "OpenCode CLI was not found on PATH"}})
+	if len(missing) != 1 || missing[0].Code != "HARNESS_NOT_FOUND" {
+		t.Fatalf("unexpected missing harness diagnosis: %+v", missing)
+	}
+	directFailure := diagnose([]Check{{ID: "tool.basic", Status: "fail"}, {ID: "harness.baseline", Status: "fail"}, {ID: "harness.opencode", Status: "fail"}})
+	if diagnosisCode(&Report{Diagnoses: directFailure}, "MODEL_SCHEMA_FAILURE") == nil {
+		t.Fatalf("expected model schema diagnosis, got %+v", directFailure)
+	}
+}
+
+func TestOpenCodeVerificationRequiresFinalAssistantText(t *testing.T) {
+	output := []byte("{\"type\":\"tool_use\",\"part\":{\"type\":\"tool\",\"state\":{\"output\":\"MDOC_OPENCODE_CANARY_91C4\"}}}\n" +
+		"{\"type\":\"text\",\"part\":{\"type\":\"text\",\"text\":\"The fixture tool result could not be parsed.\"}}\n")
+	if strings.Contains(openCodeFinalText(output), "MDOC_OPENCODE_CANARY_91C4") {
+		t.Fatal("tool output was mistaken for a successful assistant response")
+	}
+	output = []byte("{\"type\":\"text\",\"part\":{\"type\":\"text\",\"text\":\"MDOC_OPENCODE_CANARY_91C4\"}}\n")
+	if !strings.Contains(openCodeFinalText(output), "MDOC_OPENCODE_CANARY_91C4") {
+		t.Fatal("final assistant text did not preserve the expected marker")
+	}
+}
+
+func TestReportsShowHarnessTimeoutAndRepeatedContextResults(t *testing.T) {
+	report := &Report{HarnessTimeoutMS: 300, Checks: []Check{{Name: "Context", Status: "pass", Attempts: 3, Passed: 2}}}
+	for name, rendered := range map[string]string{"text": Text(report), "markdown": Markdown(report)} {
+		if !strings.Contains(rendered, "300 ms") || !strings.Contains(rendered, "2 of 3") {
+			t.Errorf("%s report omitted timeout or attempt evidence: %s", name, rendered)
+		}
+	}
+}
+
+func TestUnreachableProviderUsesConnectionDiagnosis(t *testing.T) {
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	endpoint := "http://" + listener.Addr().String() + "/v1"
+	listener.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	report, err := Run(ctx, Options{Provider: "openai", Endpoint: endpoint, Model: "test", Timeout: time.Second})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diagnosisCode(report, "PROVIDER_CONNECTION") == nil {
+		t.Fatalf("expected provider connection diagnosis, got %+v", report.Diagnoses)
+	}
+}
+
+func TestMalformedDirectToolCallUsesSchemaDiagnosis(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	report, err := Run(ctx, Options{Provider: "mock", MockScenario: "malformed-tool", Timeout: time.Second})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diagnosisCode(report, "MODEL_SCHEMA_FAILURE") == nil {
+		t.Fatalf("expected model schema diagnosis, got %+v", report.Diagnoses)
+	}
+}
+
+func TestMockProviderStatusAndSlowModes(t *testing.T) {
+	for _, test := range []struct {
+		scenario string
+		status   int
+	}{{"rate-limit", http.StatusTooManyRequests}, {"server-error", http.StatusInternalServerError}} {
+		client := newMockClient(t, test.scenario)
+		_, err := client.chat(context.Background(), []message{{Role: "user", Content: "hello"}}, nil, false)
+		if err == nil || !strings.Contains(err.Error(), "HTTP "+strconv.Itoa(test.status)) {
+			t.Errorf("expected HTTP %d to be reported as an error, got %v", test.status, err)
+		}
+	}
+	client := newMockClient(t, "slow")
+	client.http.Timeout = time.Second
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	started := time.Now()
+	if _, err := client.chat(ctx, []message{{Role: "user", Content: "hello"}}, nil, false); err == nil {
+		t.Fatal("slow mode should observe request cancellation")
+	}
+	if elapsed := time.Since(started); elapsed > time.Second {
+		t.Fatalf("slow mode ignored cancellation for %v", elapsed)
+	}
+}
+
 func TestSyntheticVisionImageIsRed(t *testing.T) {
 	data, err := base64.StdEncoding.DecodeString(syntheticRedPNG())
 	if err != nil {
@@ -101,8 +244,19 @@ func TestMockProviderDoctorPasses(t *testing.T) {
 	if report.ExitCode() != ExitOK {
 		t.Fatalf("mock diagnostics were not green: %s", Text(report))
 	}
-	if len(report.Checks) != 6 {
-		t.Fatalf("got %d checks, expected six", len(report.Checks))
+	if len(report.Checks) != 9 {
+		t.Fatalf("got %d checks, expected nine", len(report.Checks))
+	}
+	for _, id := range []string{"tool.enum", "tool.file_edit", "tool.shell"} {
+		found := false
+		for _, check := range report.Checks {
+			if check.ID == id && check.Status == "pass" {
+				found = true
+			}
+		}
+		if !found {
+			t.Fatalf("missing passing MVP probe %q", id)
+		}
 	}
 }