diff --git a/cmd/aviator/edit.go b/cmd/aviator/edit.go index 52b0907..a14b988 100644 --- a/cmd/aviator/edit.go +++ b/cmd/aviator/edit.go @@ -15,14 +15,16 @@ var editFlags struct { CriteriaFile string ExpectedVersion int Preview string + EvidenceSource string JSON bool } var editCmd = &cobra.Command{ Use: "edit ", - Short: "Update a review or runbook session's intent, acceptance criteria, or preview config (e.g. aviator edit r/123)", - Long: "Update a session's intent, replace its acceptance criteria, or switch the\n" + - "preview config its runs use (--preview), in any combination.\n" + + Short: "Update a review or runbook session's intent, acceptance criteria, preview config, or evidence source (e.g. aviator edit r/123)", + Long: "Update a session's intent, replace its acceptance criteria, switch the\n" + + "preview config its runs use (--preview), or change where its evidence\n" + + "comes from (--evidence-source aviator|external), in any combination.\n" + "\n" + "A preview config choice sticks until changed. Going back to the repo\n" + "default means passing that config's name.\n" + @@ -43,8 +45,11 @@ var editCmd = &cobra.Command{ if err != nil { return err } - if editFlags.Intent == "" && len(criteria) == 0 && editFlags.Preview == "" { - return errors.New("pass --intent, --criteria (or --criteria-file), or --preview") + if editFlags.Intent == "" && len(criteria) == 0 && editFlags.Preview == "" && editFlags.EvidenceSource == "" { + return errors.New("pass --intent, --criteria (or --criteria-file), --preview, or --evidence-source") + } + if err := checkEvidenceSource(editFlags.EvidenceSource); err != nil { + return err } if len(criteria) > 0 && !cmd.Flags().Changed("expected-version") { return errors.New("--expected-version is required when replacing criteria") @@ -59,6 +64,7 @@ var editCmd = &cobra.Command{ AcceptanceCriteria: criteria, ExpectedVersion: editFlags.ExpectedVersion, PreviewConfigName: editFlags.Preview, + EvidenceSource: editFlags.EvidenceSource, }) if err != nil { return err @@ -73,6 +79,9 @@ var editCmd = &cobra.Command{ if name := deref(resp.PreviewConfigName); editFlags.Preview != "" && name != "" { fmt.Printf(" Preview config: %s\n", name) } + if editFlags.EvidenceSource != "" { + fmt.Printf(" Evidence source: %s\n", resp.EvidenceSource) + } if len(criteria) > 0 { fmt.Printf(" %s\n", colors.Faint("Verify with: aviator verify "+id)) } @@ -88,6 +97,8 @@ func init() { "session version you expect to be editing (required with --criteria; guards against stale edits)") f.StringVar(&editFlags.Preview, "preview", "", "preview config the session's runs use from now on, instead of the repo default; sticks until changed") + f.StringVar(&editFlags.EvidenceSource, "evidence-source", "", + "where runtime evidence comes from: aviator, or external for evidence you submit yourself") f.BoolVar(&editFlags.JSON, "json", false, "print the result as a single JSON object instead of the human summary") } @@ -96,6 +107,7 @@ type editJSON struct { Intent string `json:"intent"` Version *int `json:"version"` PreviewConfigName *string `json:"preview_config_name"` + EvidenceSource string `json:"evidence_source"` } func newEditJSON(resp *api.VerifySession) editJSON { @@ -104,5 +116,6 @@ func newEditJSON(resp *api.VerifySession) editJSON { Intent: resp.Intent, Version: resp.Version, PreviewConfigName: resp.PreviewConfigName, + EvidenceSource: resp.EvidenceSource, } } diff --git a/cmd/aviator/edit_test.go b/cmd/aviator/edit_test.go index 7625710..d5a1f93 100644 --- a/cmd/aviator/edit_test.go +++ b/cmd/aviator/edit_test.go @@ -22,5 +22,6 @@ func TestEditJSON(t *testing.T) { "intent": "new intent", "version": float64(5), "preview_config_name": "docs", + "evidence_source": "", }) } diff --git a/cmd/aviator/evidence_submit.go b/cmd/aviator/evidence_submit.go new file mode 100644 index 0000000..5efe100 --- /dev/null +++ b/cmd/aviator/evidence_submit.go @@ -0,0 +1,332 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + "emperror.dev/errors" + "github.com/aviator-co/aviator-cli/internal/api" + "github.com/aviator-co/aviator-cli/internal/utils/colors" + "github.com/spf13/cobra" + "go.yaml.in/yaml/v3" +) + +var evidenceSubmitFlags struct { + Commit string + JSON bool +} + +const docsEvidenceSubmit = "https://docs.aviator.co/verify/how-to-guides/submitting-evidence-from-your-own-run" + +var evidenceSubmitCmd = &cobra.Command{ + Use: "submit ", + Short: "Submit scenarios and evidence from your own run of the change (e.g. aviator evidence submit r/123 evidence.yaml)", + Long: "Upload the scenarios you ran yourself and the evidence each step\n" + + "produced, for a session created with --evidence-source external.\n" + + "Aviator judges that evidence instead of collecting its own.\n" + + "\n" + + "The manifest is YAML or JSON. Each scenario has a summary and steps;\n" + + "each step may list evidence files with a type (screenshot, dom_snapshot,\n" + + "console_log, api_response, network_request, client_storage) and a label\n" + + "saying what the file shows. Paths are relative to the manifest. A\n" + + "scenario may pin the criteria it covers by stable key or invariant id\n" + + "(from aviator show); the rest are mapped when a run starts.\n" + + "\n" + + "Submitting replaces whatever was submitted before and starts no run:\n" + + "follow it with `aviator verify r/`. The evidence is bound to\n" + + "the full commit SHA it was captured against (`commit` in the manifest\n" + + "or --commit); a run at another commit judges from code instead.\n" + + "\n" + + "Manifest reference: " + docsEvidenceSubmit, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + runbookNumber, err := parseRunbookID(args[0]) + if err != nil { + return err + } + loaded, err := loadEvidenceManifest(args[1], evidenceSubmitFlags.Commit) + if err != nil { + return err + } + + client, err := api.NewClient() + if err != nil { + return err + } + ctx := cmd.Context() + submission, err := client.SubmitEvidenceManifest(ctx, runbookNumber, loaded.manifest) + if err != nil { + return err + } + + var ids []int + var failed []string + for i, scenario := range submission.Scenarios { + for j, step := range scenario.Steps { + for k, slot := range step.Evidence { + path, ok := loaded.pathAt(i, j, k) + if !ok { + return errors.Errorf("server returned an upload slot the manifest did not declare (scenario %d, step %d, evidence %d)", i+1, j+1, k+1) + } + if err := client.UploadEvidenceFile(ctx, slot, path); err != nil { + failed = append(failed, err.Error()) + continue + } + ids = append(ids, slot.ID) + } + } + } + var completion *api.CompleteUploadsResponse + if len(ids) > 0 { + completion, err = client.CompleteEvidenceUploads(ctx, runbookNumber, ids) + if err != nil { + return err + } + } + + id := formatRunbookID(submission.RunbookNumber) + if evidenceSubmitFlags.JSON { + return printJSON(newEvidenceSubmitJSON(id, submission, completion, failed)) + } + fmt.Printf("%s Submitted %d scenario(s) for %s at %s\n", + colors.Success("✓"), len(submission.Scenarios), id, shortSHA(submission.CommitSHA)) + for i, scenario := range submission.Scenarios { + fmt.Printf(" %s (scenario %d, %d evidence file(s))\n", + loaded.manifest.Scenarios[i].Summary, scenario.ID, countSlots(scenario)) + } + if completion != nil { + fmt.Printf(" Uploaded: %d", len(completion.UploadedEvidenceIDs)) + if n := len(completion.MissingEvidenceIDs); n > 0 { + fmt.Printf(", not in storage: %d", n) + } + if n := len(completion.RejectedEvidenceIDs); n > 0 { + fmt.Printf(", rejected as empty or over 10 MB: %d", n) + } + fmt.Println() + } + for _, msg := range failed { + fmt.Printf(" %s %s\n", colors.Warning("failed:"), msg) + } + if len(failed) > 0 || (completion != nil && (len(completion.MissingEvidenceIDs) > 0 || len(completion.RejectedEvidenceIDs) > 0)) { + return errors.New("some evidence did not upload; fix the files and submit the manifest again") + } + fmt.Printf(" %s\n", colors.Faint("Judge it with: aviator verify "+id)) + return nil + }, +} + +func init() { + f := evidenceSubmitCmd.Flags() + f.StringVar(&evidenceSubmitFlags.Commit, "commit", "", + "full commit SHA the evidence was captured against (overrides the manifest's commit)") + f.BoolVar(&evidenceSubmitFlags.JSON, "json", false, + "print the result as a single JSON object instead of the human summary") + evidenceCmd.AddCommand(evidenceSubmitCmd) +} + +// manifestFile is the on-disk manifest shape: the API shape plus a file path +// per evidence entry, which the CLI turns into a content type and size. +type manifestFile struct { + Commit string `yaml:"commit" json:"commit"` + Scenarios []manifestScenario `yaml:"scenarios" json:"scenarios"` +} + +type manifestScenario struct { + Summary string `yaml:"summary" json:"summary"` + Steps []manifestStep `yaml:"steps" json:"steps"` + Criteria []manifestCriterion `yaml:"criteria" json:"criteria"` +} + +type manifestCriterion struct { + StableKey *string `yaml:"stable_key" json:"stable_key"` + BaselineInvariantID *int `yaml:"baseline_invariant_id" json:"baseline_invariant_id"` +} + +type manifestStep struct { + Text string `yaml:"text" json:"text"` + Evidence []manifestEvidence `yaml:"evidence" json:"evidence"` +} + +type manifestEvidence struct { + Type string `yaml:"type" json:"type"` + File string `yaml:"file" json:"file"` + Label string `yaml:"label" json:"label"` +} + +// loadedManifest is the API manifest plus the local path behind each upload +// slot, indexed the way the server answers (scenario, step, evidence). +type loadedManifest struct { + manifest api.EvidenceManifest + paths [][][]string +} + +func (m *loadedManifest) pathAt(scenario, step, evidence int) (string, bool) { + if scenario >= len(m.paths) || step >= len(m.paths[scenario]) || evidence >= len(m.paths[scenario][step]) { + return "", false + } + return m.paths[scenario][step][evidence], true +} + +// contentTypes maps evidence file extensions to the content type the server +// accepts for them; screenshots are images, everything else is text or JSON. +var contentTypes = map[string]string{ + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".webp": "image/webp", + ".json": "application/json", + ".txt": "text/plain", + ".log": "text/plain", + ".html": "text/html", + ".htm": "text/html", +} + +func loadEvidenceManifest(path, commit string) (*loadedManifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, errors.Wrapf(err, "failed to read %s", path) + } + var file manifestFile + if strings.HasSuffix(strings.ToLower(path), ".json") { + err = json.Unmarshal(data, &file) + } else { + err = yaml.Unmarshal(data, &file) + } + if err != nil { + return nil, errors.Wrapf(err, "failed to parse %s", path) + } + if commit == "" { + commit = file.Commit + } + commit = strings.ToLower(strings.TrimSpace(commit)) + if commit == "" { + return nil, errors.New("no commit: set `commit` in the manifest or pass --commit with the full SHA the evidence was captured against") + } + if !fullSHA.MatchString(commit) { + return nil, errors.Errorf("commit %q must be the full 40-character SHA", commit) + } + if len(file.Scenarios) == 0 { + return nil, errors.New("the manifest lists no scenarios") + } + + base := filepath.Dir(path) + loaded := &loadedManifest{manifest: api.EvidenceManifest{CommitSHA: commit}} + for si, scenario := range file.Scenarios { + if strings.TrimSpace(scenario.Summary) == "" { + return nil, errors.Errorf("scenario %d needs a summary", si+1) + } + apiScenario := api.ManifestScenario{Summary: scenario.Summary} + for ci, criterion := range scenario.Criteria { + if (criterion.StableKey == nil) == (criterion.BaselineInvariantID == nil) { + return nil, errors.Errorf("scenario %d criteria %d needs exactly one of stable_key or baseline_invariant_id", si+1, ci+1) + } + apiScenario.Criteria = append(apiScenario.Criteria, api.ManifestCriterion{ + StableKey: criterion.StableKey, + BaselineInvariantID: criterion.BaselineInvariantID, + }) + } + var scenarioPaths [][]string + for ti, step := range scenario.Steps { + if strings.TrimSpace(step.Text) == "" { + return nil, errors.Errorf("scenario %d step %d needs text", si+1, ti+1) + } + apiStep := api.ManifestStep{Text: step.Text} + var stepPaths []string + for ei, item := range step.Evidence { + where := fmt.Sprintf("scenario %d step %d evidence %d", si+1, ti+1, ei+1) + if item.File == "" { + return nil, errors.Errorf("%s needs a file", where) + } + if strings.TrimSpace(item.Label) == "" { + return nil, errors.Errorf("%s needs a label saying what the file shows", where) + } + contentType, ok := contentTypes[strings.ToLower(filepath.Ext(item.File))] + if !ok { + return nil, errors.Errorf("%s: unsupported file type %q (use png, jpg, webp, json, txt, log, or html)", where, filepath.Ext(item.File)) + } + full := item.File + if !filepath.IsAbs(full) { + full = filepath.Join(base, full) + } + info, err := os.Stat(full) + if err != nil { + return nil, errors.Wrapf(err, "%s: cannot read %s", where, item.File) + } + if info.Size() == 0 { + return nil, errors.Errorf("%s: %s is empty", where, item.File) + } + apiStep.Evidence = append(apiStep.Evidence, api.ManifestEvidence{ + Type: item.Type, + Label: item.Label, + ContentType: contentType, + SizeBytes: info.Size(), + }) + stepPaths = append(stepPaths, full) + } + apiScenario.Steps = append(apiScenario.Steps, apiStep) + scenarioPaths = append(scenarioPaths, stepPaths) + } + if len(apiScenario.Steps) == 0 { + return nil, errors.Errorf("scenario %d needs at least one step", si+1) + } + loaded.manifest.Scenarios = append(loaded.manifest.Scenarios, apiScenario) + loaded.paths = append(loaded.paths, scenarioPaths) + } + return loaded, nil +} + +var fullSHA = regexp.MustCompile(`^[0-9a-f]{40}$`) + +func countSlots(scenario api.SubmittedScenario) int { + n := 0 + for _, step := range scenario.Steps { + n += len(step.Evidence) + } + return n +} + +type evidenceSubmitJSON struct { + ID string `json:"id"` + CommitSHA string `json:"commit_sha"` + ScenarioIDs []int `json:"scenario_ids"` + ScenarioRunIDs []int `json:"scenario_run_ids"` + UploadedEvidenceIDs []int `json:"uploaded_evidence_ids"` + MissingEvidenceIDs []int `json:"missing_evidence_ids"` + RejectedEvidenceIDs []int `json:"rejected_evidence_ids"` + CompletedScenarioRunIDs []int `json:"completed_scenario_run_ids"` + UploadErrors []string `json:"upload_errors"` +} + +func newEvidenceSubmitJSON( + id string, submission *api.EvidenceSubmission, completion *api.CompleteUploadsResponse, failed []string, +) evidenceSubmitJSON { + out := evidenceSubmitJSON{ + ID: id, + CommitSHA: submission.CommitSHA, + ScenarioIDs: []int{}, + ScenarioRunIDs: []int{}, + UploadedEvidenceIDs: []int{}, + MissingEvidenceIDs: []int{}, + RejectedEvidenceIDs: []int{}, + CompletedScenarioRunIDs: []int{}, + UploadErrors: failed, + } + if out.UploadErrors == nil { + out.UploadErrors = []string{} + } + for _, s := range submission.Scenarios { + out.ScenarioIDs = append(out.ScenarioIDs, s.ID) + out.ScenarioRunIDs = append(out.ScenarioRunIDs, s.ScenarioRunID) + } + if completion != nil { + out.UploadedEvidenceIDs = append(out.UploadedEvidenceIDs, completion.UploadedEvidenceIDs...) + out.MissingEvidenceIDs = append(out.MissingEvidenceIDs, completion.MissingEvidenceIDs...) + out.RejectedEvidenceIDs = append(out.RejectedEvidenceIDs, completion.RejectedEvidenceIDs...) + out.CompletedScenarioRunIDs = append(out.CompletedScenarioRunIDs, completion.CompletedScenarioRunIDs...) + } + return out +} diff --git a/cmd/aviator/evidence_submit_test.go b/cmd/aviator/evidence_submit_test.go new file mode 100644 index 0000000..cf875b7 --- /dev/null +++ b/cmd/aviator/evidence_submit_test.go @@ -0,0 +1,133 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/aviator-co/aviator-cli/internal/api" +) + +func writeManifestFixture(t *testing.T, manifest string) string { + t.Helper() + dir := t.TempDir() + for name, body := range map[string]string{ + "shots/home.png": "\x89PNG", + "api/flags.json": `{"banner": false}`, + } { + full := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + path := filepath.Join(dir, "evidence.yaml") + if err := os.WriteFile(path, []byte(manifest), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +const goodManifest = ` +commit: 4f2c9e1a7b3d5c6e8f9a0b1c2d3e4f5a6b7c8d9e +scenarios: + - summary: Toggle the flag and reload + criteria: + - stable_key: ac-1 + steps: + - text: Open the home page + - text: Capture it without the banner + evidence: + - type: screenshot + file: shots/home.png + label: Home page, no banner + - type: api_response + file: ./api/flags.json + label: GET /api/flags shows banner=false +` + +func TestLoadEvidenceManifest(t *testing.T) { + path := writeManifestFixture(t, goodManifest) + + loaded, err := loadEvidenceManifest(path, "") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + m := loaded.manifest + if m.CommitSHA != "4f2c9e1a7b3d5c6e8f9a0b1c2d3e4f5a6b7c8d9e" { + t.Errorf("commit = %q", m.CommitSHA) + } + if len(m.Scenarios) != 1 || len(m.Scenarios[0].Steps) != 2 { + t.Fatalf("manifest = %+v", m) + } + if key := m.Scenarios[0].Criteria[0].StableKey; key == nil || *key != "ac-1" { + t.Errorf("criteria = %+v", m.Scenarios[0].Criteria) + } + evidence := m.Scenarios[0].Steps[1].Evidence + want := []api.ManifestEvidence{ + {Type: "screenshot", Label: "Home page, no banner", ContentType: "image/png", SizeBytes: 4}, + {Type: "api_response", Label: "GET /api/flags shows banner=false", ContentType: "application/json", SizeBytes: 17}, + } + for i, w := range want { + if evidence[i] != w { + t.Errorf("evidence[%d] = %+v, want %+v", i, evidence[i], w) + } + } + if p, ok := loaded.pathAt(0, 1, 1); !ok || !strings.HasSuffix(p, filepath.Join("api", "flags.json")) { + t.Errorf("pathAt(0,1,1) = %q, %v", p, ok) + } + if _, ok := loaded.pathAt(0, 0, 0); ok { + t.Errorf("a step without evidence has no paths") + } +} + +func TestLoadEvidenceManifestCommitFlagWins(t *testing.T) { + path := writeManifestFixture(t, goodManifest) + loaded, err := loadEvidenceManifest(path, "9a01bb7c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if loaded.manifest.CommitSHA != "9a01bb7c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a" { + t.Errorf("commit = %q", loaded.manifest.CommitSHA) + } +} + +func TestLoadEvidenceManifestRejectsBadEntries(t *testing.T) { + cases := map[string]string{ + "missing file": strings.Replace(goodManifest, "shots/home.png", "shots/nope.png", 1), + "bad extension": strings.Replace(goodManifest, "api/flags.json", "api/flags.csv", 1), + "no label": strings.Replace(goodManifest, "label: Home page, no banner", "label: ''", 1), + "no scenarios": "commit: 4f2c9e1a7b3d5c6e8f9a0b1c2d3e4f5a6b7c8d9e\nscenarios: []\n", + "no commit": strings.Replace(goodManifest, "commit: 4f2c9e1a7b3d5c6e8f9a0b1c2d3e4f5a6b7c8d9e\n", "", 1), + "short commit": strings.Replace(goodManifest, "commit: 4f2c9e1a7b3d5c6e8f9a0b1c2d3e4f5a6b7c8d9e", "commit: 4f2c9e1", 1), + } + for name, manifest := range cases { + path := writeManifestFixture(t, manifest) + if _, err := loadEvidenceManifest(path, ""); err == nil { + t.Errorf("%s: expected an error", name) + } + } +} + +func TestEvidenceSubmitJSON(t *testing.T) { + got := decodeJSON(t, newEvidenceSubmitJSON("r/123", &api.EvidenceSubmission{ + CommitSHA: "4f2c9e1a", + Scenarios: []api.SubmittedScenario{{ID: 7, ScenarioRunID: 70}}, + }, &api.CompleteUploadsResponse{ + UploadedEvidenceIDs: []int{1, 2}, + MissingEvidenceIDs: []int{}, + CompletedScenarioRunIDs: []int{70}, + }, nil)) + if got["id"] != "r/123" || got["commit_sha"] != "4f2c9e1a" { + t.Errorf("top level = %v", got) + } + if ids := got["uploaded_evidence_ids"].([]any); len(ids) != 2 { + t.Errorf("uploaded = %v", ids) + } + if errs := got["upload_errors"].([]any); len(errs) != 0 { + t.Errorf("upload_errors = %v", errs) + } +} diff --git a/cmd/aviator/helpers.go b/cmd/aviator/helpers.go index c1439e7..26eb0f8 100644 --- a/cmd/aviator/helpers.go +++ b/cmd/aviator/helpers.go @@ -170,3 +170,19 @@ func warnPreviewNotApplied(sent string, got *string) { fmt.Fprintf(os.Stderr, "%s %s\n", colors.Warning("warning:"), msg) } } + +const ( + evidenceSourceAviator = "aviator" + evidenceSourceExternal = "external" + scenarioSourceExternal = "external" +) + +// checkEvidenceSource validates an --evidence-source value; empty means unset. +func checkEvidenceSource(value string) error { + switch value { + case "", evidenceSourceAviator, evidenceSourceExternal: + return nil + } + return errors.Errorf("--evidence-source must be %s or %s, got %q", + evidenceSourceAviator, evidenceSourceExternal, value) +} diff --git a/cmd/aviator/scenarios.go b/cmd/aviator/scenarios.go index ad05b2f..d754818 100644 --- a/cmd/aviator/scenarios.go +++ b/cmd/aviator/scenarios.go @@ -72,7 +72,15 @@ func formatScenarios(s *api.VerifyScenarios) string { if sr.Reused { reused = ", reused from an earlier run" } - fmt.Fprintf(&b, " %s [%s, %d tool calls%s]\n", sr.Scenario.Summary, status, sr.ToolCallCount, reused) + work := fmt.Sprintf("%d tool calls", sr.ToolCallCount) + if sr.Source == scenarioSourceExternal { + work = "submitted by " + deref(sr.SubmittedBy) + if commit := deref(sr.CommitSHA); commit != "" { + work += " for " + shortSHA(commit) + } + reused = "" + } + fmt.Fprintf(&b, " %s [%s, %s%s]\n", sr.Scenario.Summary, status, work, reused) if reason := deref(sr.FailureReason); reason != "" { fmt.Fprintf(&b, " %s\n", reason) } @@ -116,6 +124,9 @@ type scenarioJSON struct { TerminationReason *string `json:"termination_reason"` FailureReason *string `json:"failure_reason"` ToolCallCount int `json:"tool_call_count"` + Source string `json:"source"` + SubmittedBy *string `json:"submitted_by"` + CommitSHA *string `json:"commit_sha"` Reused bool `json:"reused"` StartedAt *string `json:"started_at"` CompletedAt *string `json:"completed_at"` @@ -154,6 +165,9 @@ func newScenariosJSON(s *api.VerifyScenarios) scenariosJSON { TerminationReason: sr.TerminationReason, FailureReason: sr.FailureReason, ToolCallCount: sr.ToolCallCount, + Source: sr.Source, + SubmittedBy: sr.SubmittedBy, + CommitSHA: sr.CommitSHA, Reused: sr.Reused, StartedAt: sr.StartedAt, CompletedAt: sr.CompletedAt, diff --git a/cmd/aviator/verify.go b/cmd/aviator/verify.go index 79c3f70..1d21f2d 100644 --- a/cmd/aviator/verify.go +++ b/cmd/aviator/verify.go @@ -24,6 +24,7 @@ var verifyFlags struct { Force bool RegenerateScenarios string Preview string + EvidenceSource string JSON bool } @@ -31,7 +32,7 @@ var verifyFlags struct { // session, so trigger mode rejects them by name. var verifySubmitOnlyFlags = []string{ "repo", "intent", "criteria", "criteria-file", - "working-branch", "target-branch", "spec", "scenario-guidance", + "working-branch", "target-branch", "spec", "scenario-guidance", "evidence-source", } var verifyTriggerOnlyFlags = []string{"evaluator-only", "force", "regenerate-scenarios"} @@ -71,6 +72,11 @@ var verifyCmd = &cobra.Command{ "collected instead of collecting it again. Evaluator-only is refused once\n" + "criteria have been added or reworded, since those need a full run.\n" + "\n" + + "--evidence-source external makes this a session Aviator never collects\n" + + "evidence for: run the change yourself, submit what you saw with\n" + + "`aviator evidence submit`, and a run judges it. Nothing is generated\n" + + "or driven in a preview on such a session.\n" + + "\n" + "--preview names one of the repo's preview configs for the session's runs\n" + "to use instead of the repo default, in either mode. The choice sticks:\n" + "later runs on the session, including automatic ones, keep using it until\n" + @@ -130,6 +136,9 @@ func runVerifySubmit(cmd *cobra.Command) error { if cmd.Flags().Changed("scenario-guidance") && guidance == "" { return errors.New("--scenario-guidance can't be blank") } + if err := checkEvidenceSource(verifyFlags.EvidenceSource); err != nil { + return err + } var spec *api.SpecFile if verifyFlags.Spec != "" { @@ -153,6 +162,7 @@ func runVerifySubmit(cmd *cobra.Command) error { SpecFile: spec, PreviewConfigName: verifyFlags.Preview, ScenarioGuidance: guidance, + EvidenceSource: verifyFlags.EvidenceSource, }) if err != nil { return err @@ -179,6 +189,10 @@ func runVerifySubmit(cmd *cobra.Command) error { if name := deref(resp.PreviewConfigName); name != "" { fmt.Printf(" Preview config: %s\n", name) } + if resp.EvidenceSource == evidenceSourceExternal { + fmt.Printf(" Evidence source: external\n") + fmt.Printf(" %s\n", colors.Faint("Submit what you saw with: aviator evidence submit "+formatRunbookID(resp.RunbookNumber)+" ")) + } return nil } @@ -259,6 +273,8 @@ func init() { "start a fresh full run even if an equivalent run already exists (trigger mode only)") f.StringVar(&verifyFlags.Preview, "preview", "", "preview config the session's runs use instead of the repo default; sticks until changed") + f.StringVar(&verifyFlags.EvidenceSource, "evidence-source", "", + "where runtime evidence comes from: aviator (default) or external, for evidence you submit yourself (submit mode only)") f.StringVar(&verifyFlags.RegenerateScenarios, "regenerate-scenarios", "", "rewrite the scenarios before running, with feedback on what to change (trigger mode only)") f.BoolVar(&verifyFlags.JSON, "json", false, "print the result as a single JSON object instead of the human summary") @@ -270,6 +286,7 @@ type verifySubmitJSON struct { TargetBranch string `json:"target_branch"` CriteriaCount int `json:"criteria_count"` PreviewConfigName *string `json:"preview_config_name"` + EvidenceSource string `json:"evidence_source"` } func newVerifySubmitJSON(resp *api.SubmitVerifyResponse) verifySubmitJSON { @@ -279,6 +296,7 @@ func newVerifySubmitJSON(resp *api.SubmitVerifyResponse) verifySubmitJSON { TargetBranch: resp.TargetBranch, CriteriaCount: len(resp.AcceptanceCriteria), PreviewConfigName: resp.PreviewConfigName, + EvidenceSource: resp.EvidenceSource, } } diff --git a/cmd/aviator/verify_test.go b/cmd/aviator/verify_test.go index a1aadd3..3a25e9a 100644 --- a/cmd/aviator/verify_test.go +++ b/cmd/aviator/verify_test.go @@ -52,6 +52,7 @@ func TestVerifySubmitJSON(t *testing.T) { "target_branch": "main", "criteria_count": float64(2), "preview_config_name": "docs", + "evidence_source": "", }) } @@ -69,6 +70,7 @@ func TestVerifySubmitJSONKeepsEmptyFields(t *testing.T) { "target_branch": "", "criteria_count": float64(0), "preview_config_name": nil, + "evidence_source": "", }) } diff --git a/internal/api/verify.go b/internal/api/verify.go index e1681ba..701ce32 100644 --- a/internal/api/verify.go +++ b/internal/api/verify.go @@ -12,6 +12,9 @@ type SubmitVerifyRequest struct { SpecFile *SpecFile `json:"spec_file,omitempty"` PreviewConfigName string `json:"preview_config_name,omitempty"` ScenarioGuidance string `json:"scenario_guidance,omitempty"` + // EvidenceSource is "aviator" (the default) or "external": on an external + // session Aviator never collects; scenarios and evidence are submitted. + EvidenceSource string `json:"evidence_source,omitempty"` } // SubmitVerifyResponse is the response from POST /api/v1/verify. @@ -22,6 +25,7 @@ type SubmitVerifyResponse struct { TargetBranch string `json:"target_branch"` AcceptanceCriteria []string `json:"acceptance_criteria"` PreviewConfigName *string `json:"preview_config_name"` + EvidenceSource string `json:"evidence_source"` } // SubmitVerify creates a verify submission. diff --git a/internal/api/verify_edit.go b/internal/api/verify_edit.go index 40d0e5b..aa3f448 100644 --- a/internal/api/verify_edit.go +++ b/internal/api/verify_edit.go @@ -12,6 +12,7 @@ type EditVerifyRequest struct { AcceptanceCriteria []string `json:"acceptance_criteria,omitempty"` ExpectedVersion int `json:"expected_version,omitempty"` PreviewConfigName string `json:"preview_config_name,omitempty"` + EvidenceSource string `json:"evidence_source,omitempty"` } // VerifySession is the session shape returned by PATCH /api/v1/verify/. @@ -25,6 +26,7 @@ type VerifySession struct { Intent string `json:"intent"` Version *int `json:"version"` PreviewConfigName *string `json:"preview_config_name"` + EvidenceSource string `json:"evidence_source"` } // EditVerify updates a session's intent and/or replaces its acceptance criteria. diff --git a/internal/api/verify_evidence_submit.go b/internal/api/verify_evidence_submit.go new file mode 100644 index 0000000..5389ad6 --- /dev/null +++ b/internal/api/verify_evidence_submit.go @@ -0,0 +1,147 @@ +package api + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + + "emperror.dev/errors" +) + +// EvidenceManifest is the body for POST /api/v1/verify//evidence: the +// scenarios someone ran outside Aviator and the evidence each step produced. +// Files are declared by type, content type and size; the server answers with +// one upload URL per declared file. +type EvidenceManifest struct { + CommitSHA string `json:"commit_sha"` + Scenarios []ManifestScenario `json:"scenarios"` +} + +// ManifestScenario is one scenario in a manifest. +type ManifestScenario struct { + Summary string `json:"summary"` + Steps []ManifestStep `json:"steps"` + Criteria []ManifestCriterion `json:"criteria,omitempty"` +} + +// ManifestStep is one step of a manifest scenario. +type ManifestStep struct { + Text string `json:"text"` + Evidence []ManifestEvidence `json:"evidence,omitempty"` +} + +// ManifestEvidence declares one evidence file to upload. +type ManifestEvidence struct { + Type string `json:"type"` + Label string `json:"label"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` +} + +// ManifestCriterion pins a scenario to a criterion by the handles +// `aviator show` prints. +type ManifestCriterion struct { + StableKey *string `json:"stable_key,omitempty"` + BaselineInvariantID *int `json:"baseline_invariant_id,omitempty"` +} + +// EvidenceSubmission is the response from POST /api/v1/verify//evidence. +type EvidenceSubmission struct { + RunbookNumber int `json:"runbook_number"` + CommitSHA string `json:"commit_sha"` + UploadURLTTLSeconds int `json:"upload_url_ttl_seconds"` + Scenarios []SubmittedScenario `json:"scenarios"` +} + +// SubmittedScenario is one scenario the server created from a manifest. +type SubmittedScenario struct { + ID int `json:"id"` + ScenarioRunID int `json:"scenario_run_id"` + Steps []SubmittedStep `json:"steps"` +} + +// SubmittedStep is one step of a submitted scenario, with an upload slot per +// declared evidence file, in manifest order. +type SubmittedStep struct { + ID int `json:"id"` + Evidence []UploadSlot `json:"evidence"` +} + +// UploadSlot is where one declared evidence file is uploaded to. +type UploadSlot struct { + ID int `json:"id"` + UploadURL string `json:"upload_url"` + ContentType string `json:"content_type"` +} + +// CompleteUploadsResponse is the response from +// POST /api/v1/verify//evidence/complete. +type CompleteUploadsResponse struct { + RunbookNumber int `json:"runbook_number"` + UploadedEvidenceIDs []int `json:"uploaded_evidence_ids"` + MissingEvidenceIDs []int `json:"missing_evidence_ids"` + RejectedEvidenceIDs []int `json:"rejected_evidence_ids"` + CompletedScenarioRunIDs []int `json:"completed_scenario_run_ids"` +} + +// SubmitEvidenceManifest replaces a session's submitted scenarios with the +// manifest and returns the upload slots for its evidence files. +func (c *Client) SubmitEvidenceManifest( + ctx context.Context, runbookNumber int, manifest EvidenceManifest, +) (*EvidenceSubmission, error) { + path := fmt.Sprintf("/api/v1/verify/%d/evidence", runbookNumber) + var out EvidenceSubmission + if err := c.postJSON(ctx, path, manifest, &out); err != nil { + return nil, err + } + return &out, nil +} + +// CompleteEvidenceUploads tells the server which evidence files were uploaded +// so it can check storage and complete the scenario runs they belong to. +func (c *Client) CompleteEvidenceUploads( + ctx context.Context, runbookNumber int, evidenceIDs []int, +) (*CompleteUploadsResponse, error) { + path := fmt.Sprintf("/api/v1/verify/%d/evidence/complete", runbookNumber) + body := struct { + EvidenceIDs []int `json:"evidence_ids"` + }{EvidenceIDs: evidenceIDs} + var out CompleteUploadsResponse + if err := c.postJSON(ctx, path, body, &out); err != nil { + return nil, err + } + return &out, nil +} + +// UploadEvidenceFile PUTs one file to a signed upload URL. The URL authorizes +// the upload, so no bearer token is sent; the content type must match what the +// manifest declared or storage rejects the signature. +func (c *Client) UploadEvidenceFile(ctx context.Context, slot UploadSlot, path string) error { + f, err := os.Open(path) + if err != nil { + return errors.Wrapf(err, "failed to open %s", path) + } + defer func() { _ = f.Close() }() + info, err := f.Stat() + if err != nil { + return errors.Wrapf(err, "failed to stat %s", path) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPut, slot.UploadURL, f) + if err != nil { + return errors.Wrap(err, "failed to build upload request") + } + req.ContentLength = info.Size() + req.Header.Set("Content-Type", slot.ContentType) + resp, err := c.http.Do(req) + if err != nil { + return errors.Wrapf(err, "upload of %s failed", path) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + data, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return errors.Errorf("upload of %s failed (%d): %s", path, resp.StatusCode, string(data)) + } + return nil +} diff --git a/internal/api/verify_evidence_submit_test.go b/internal/api/verify_evidence_submit_test.go new file mode 100644 index 0000000..09a25c5 --- /dev/null +++ b/internal/api/verify_evidence_submit_test.go @@ -0,0 +1,130 @@ +package api + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" +) + +func TestSubmitEvidenceManifest(t *testing.T) { + var got EvidenceManifest + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/v1/verify/123/evidence" { + t.Errorf("%s %s", r.Method, r.URL.Path) + } + if err := json.NewDecoder(r.Body).Decode(&got); err != nil { + t.Errorf("decode: %v", err) + } + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"runbook_number":123,"commit_sha":"4f2c9e1a","upload_url_ttl_seconds":900,` + + `"scenarios":[{"id":7,"scenario_run_id":70,"steps":[{"id":1,"evidence":[]},` + + `{"id":2,"evidence":[{"id":900,"upload_url":"https://bucket/900","content_type":"image/png"}]}]}]}`)) + })) + defer srv.Close() + + key := "ac-1" + resp, err := newTestClient(srv).SubmitEvidenceManifest(context.Background(), 123, EvidenceManifest{ + CommitSHA: "4f2c9e1a", + Scenarios: []ManifestScenario{{ + Summary: "toggle the flag", + Criteria: []ManifestCriterion{{StableKey: &key}}, + Steps: []ManifestStep{ + {Text: "open"}, + {Text: "capture", Evidence: []ManifestEvidence{{Type: "screenshot", Label: "home", ContentType: "image/png", SizeBytes: 4}}}, + }, + }}, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.CommitSHA != "4f2c9e1a" || len(got.Scenarios) != 1 || got.Scenarios[0].Criteria[0].StableKey == nil { + t.Errorf("sent = %+v", got) + } + if resp.Scenarios[0].ScenarioRunID != 70 || resp.Scenarios[0].Steps[1].Evidence[0].ID != 900 { + t.Errorf("resp = %+v", resp) + } +} + +func TestCompleteEvidenceUploads(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/verify/123/evidence/complete" { + t.Errorf("path = %q", r.URL.Path) + } + body, _ := io.ReadAll(r.Body) + if string(body) != `{"evidence_ids":[900,901]}` { + t.Errorf("body = %s", body) + } + _, _ = w.Write([]byte(`{"runbook_number":123,"uploaded_evidence_ids":[900],"missing_evidence_ids":[901],"completed_scenario_run_ids":[]}`)) + })) + defer srv.Close() + + resp, err := newTestClient(srv).CompleteEvidenceUploads(context.Background(), 123, []int{900, 901}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(resp.UploadedEvidenceIDs) != 1 || len(resp.MissingEvidenceIDs) != 1 { + t.Errorf("resp = %+v", resp) + } +} + +func TestUploadEvidenceFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "home.png") + if err := os.WriteFile(path, []byte("\x89PNG"), 0o600); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPut || r.Header.Get("Content-Type") != "image/png" { + t.Errorf("%s %s content-type=%q", r.Method, r.URL.Path, r.Header.Get("Content-Type")) + } + if r.Header.Get("Authorization") != "" { + t.Errorf("bearer token sent to storage") + } + body, _ := io.ReadAll(r.Body) + if string(body) != "\x89PNG" || r.ContentLength != 4 { + t.Errorf("body = %q, length = %d", body, r.ContentLength) + } + })) + defer srv.Close() + + err := newTestClient(srv).UploadEvidenceFile(context.Background(), UploadSlot{ + ID: 900, UploadURL: srv.URL + "/900", ContentType: "image/png", + }, path) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestUploadEvidenceFileReportsStorageRefusal(t *testing.T) { + path := filepath.Join(t.TempDir(), "home.png") + if err := os.WriteFile(path, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte("SignatureDoesNotMatch")) + })) + defer srv.Close() + + err := newTestClient(srv).UploadEvidenceFile(context.Background(), UploadSlot{ + ID: 900, UploadURL: srv.URL + "/900", ContentType: "image/png", + }, path) + if err == nil || !contains(err.Error(), "SignatureDoesNotMatch") { + t.Errorf("err = %v", err) + } +} + +func contains(s, sub string) bool { + return len(sub) == 0 || (len(s) >= len(sub) && func() bool { + for i := 0; i+len(sub) <= len(s); i++ { + if s[i:i+len(sub)] == sub { + return true + } + } + return false + }()) +} diff --git a/internal/api/verify_scenarios.go b/internal/api/verify_scenarios.go index c9f4558..00d3530 100644 --- a/internal/api/verify_scenarios.go +++ b/internal/api/verify_scenarios.go @@ -19,16 +19,21 @@ type VerifyScenarios struct { // ScenarioRun is one scenario's execution within a verification run. Reused // runs were carried over from an earlier verification run. type ScenarioRun struct { - ID int `json:"id"` - Reused bool `json:"reused"` - Status string `json:"status"` - TerminationReason *string `json:"termination_reason"` - FailureReason *string `json:"failure_reason"` - ToolCallCount int `json:"tool_call_count"` - StartedAt *string `json:"started_at"` - CompletedAt *string `json:"completed_at"` - Scenario Scenario `json:"scenario"` - Evidence []Evidence `json:"evidence"` + ID int `json:"id"` + Reused bool `json:"reused"` + Status string `json:"status"` + TerminationReason *string `json:"termination_reason"` + FailureReason *string `json:"failure_reason"` + ToolCallCount int `json:"tool_call_count"` + // Source is "collector" when Aviator captured the evidence, "external" + // when someone submitted it from their own run. + Source string `json:"source"` + CommitSHA *string `json:"commit_sha"` + SubmittedBy *string `json:"submitted_by"` + StartedAt *string `json:"started_at"` + CompletedAt *string `json:"completed_at"` + Scenario Scenario `json:"scenario"` + Evidence []Evidence `json:"evidence"` } // Scenario is what a scenario run set out to exercise.