From 0b970621a979c84f121a7e2b1cc14c037e0cc51e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=83=9E=E3=82=A4=E3=83=8E=E3=83=AB?= <97069334+MY-RV@users.noreply.github.com> Date: Sun, 20 Sep 2026 16:05:17 -0600 Subject: [PATCH 01/14] =?UTF-8?q?feat:=20run=20plugins=20=E2=80=94=20WASI?= =?UTF-8?q?=20modules=20with=20a=20JSON=20protocol=20and=20gated=20capabil?= =?UTF-8?q?ities?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit engine.plugins declared what a catalog expects and nothing read it. This loads the artifact, verifies it, and registers the runners it provides. A plugin is a WASI command: a normal program with a main(), compiled to wasm. That choice is the whole design. There is no memory-sharing ABI to get right, any language targeting WASI can write one, and a plugin is testable as an ordinary program — pipe it a request on stdin and read what it says. The example plugin has no wasm-specific code in it at all. godo writes one JSON request, the plugin writes JSON ops, godo answers the ones that need answering, and the plugin's exit code is the script's. The sandbox is wazero's default, which is nothing: no filesystem, network, environment or clock. A plugin reaches the outside world only by asking, and only what config grants is honoured — deny by default, where an absent section, an absent key, or anything that is not exactly true, is not granted. The digest is checked before the bytes reach a compiler. 'this is the artifact that was reviewed' is the one thing a catalog can assert about third-party code that runs when someone types 'godo test', so it is checked first and a mismatch stops the run. Preview is the plugin's to answer. godo can render a shell line because it wrote it; it cannot render a program it does not interpret, so it asks with mode preview and prints what comes back. That also means a plugin holding proc.exec could run things during a preview — documented, because the honest statement is that this is a trust boundary and not a mechanism. A plugin body is not expanded: ${godo:…} is shell-space syntax, and the values it would paste in travel on the invocation as data. That is what lets a plugin script read a capture by name, and it is why InvocationRunner exists beside Runner rather than replacing it — a shell runner still only ever wanted the finished line. wazero is pinned to v1.9.0, the last release whose own go directive matches godo's 1.22 floor, and it brings no dependencies of its own. --- docs/dev/README.md | 1 + docs/dev/plugin-protocol.md | 152 ++++++++++++++ examples/plugins/lines/main.go | 225 +++++++++++++++++++++ go.mod | 4 +- go.sum | 2 + internal/catalog/engine.go | 124 ++++++++++-- internal/catalog/engine_block_test.go | 6 +- internal/cli/app.go | 14 +- internal/cli/plugin_e2e_test.go | 184 +++++++++++++++++ internal/plugin/invoke.go | 218 ++++++++++++++++++++ internal/plugin/invoke_test.go | 279 ++++++++++++++++++++++++++ internal/plugin/protocol.go | 72 +++++++ internal/plugin/runner.go | 126 ++++++++++++ internal/plugin/runtime.go | 103 ++++++++++ 14 files changed, 1492 insertions(+), 18 deletions(-) create mode 100644 docs/dev/plugin-protocol.md create mode 100644 examples/plugins/lines/main.go create mode 100644 internal/cli/plugin_e2e_test.go create mode 100644 internal/plugin/invoke.go create mode 100644 internal/plugin/invoke_test.go create mode 100644 internal/plugin/protocol.go create mode 100644 internal/plugin/runner.go create mode 100644 internal/plugin/runtime.go diff --git a/docs/dev/README.md b/docs/dev/README.md index f30e276..08d5d75 100644 --- a/docs/dev/README.md +++ b/docs/dev/README.md @@ -8,6 +8,7 @@ Internals and contributor conventions. Product docs live in [`docs/`](../README. | [Standards](./standards.md) | Coding bar | | [Versioning](./versioning.md) | SemVer + binary vs file `version:` | | [Runners and plugins](./runners-and-plugins.md) | Premise: godo is a proxy; cross-OS is a plugin's job | +| [Plugin protocol](./plugin-protocol.md) | The wire: WASI command, JSON ops, capabilities | | [Archive](./archive/README.md) | Brainstorm notes (historical) | Start with root [`CONTRIBUTING.md`](../../CONTRIBUTING.md). diff --git a/docs/dev/plugin-protocol.md b/docs/dev/plugin-protocol.md new file mode 100644 index 0000000..310f073 --- /dev/null +++ b/docs/dev/plugin-protocol.md @@ -0,0 +1,152 @@ +# Plugin protocol + +How godo talks to a plugin. Not the contract — [`contract.md`](../contract.md) +covers the `godo.yaml` side; this is the wire. + +## What a plugin is + +A **WASI command**: a normal program with a `main()`, compiled to wasm. + +``` +godo → plugin stdin one JSON request +godo ← plugin stdout JSON ops, one per line +godo → plugin stdin one JSON result per op that needs one +plugin exits its exit code is the script's +``` + +Being a command rather than a set of exported functions keeps the contract +small: there is no memory-sharing ABI to get right, any language that targets +WASI can write one, and a plugin can be tested as an ordinary program — + +```bash +echo '{"api":1,"mode":"preview","body":"echo hi","argv":{},"args":[]}' | ./plugin +``` + +## The sandbox + +wazero's default, which is nothing: no filesystem, no network, no environment, +no clock. A plugin reaches the outside world only by asking godo, and godo only +honours what `engine.plugins[].config` grants. + +## Request + +One line, written once, before anything else. + +```json +{ + "api": 1, + "mode": "run", + "runner": "lines", + "body": "git status\n?pnpm install", + "argv": {"BRANCH": "wt/example"}, + "args": ["--no-install"], + "config": {"proc": {"exec": true}} +} +``` + +| Field | | +|-------|--| +| `api` | Protocol version. A plugin that speaks another major should exit non-zero rather than guess | +| `mode` | `"run"` or `"preview"` | +| `runner` | The name the catalog asked for, so one plugin can provide several | +| `body` | The script body, **verbatim**. `${godo:…}` is not expanded — that is shell-space syntax, and the values are on this request already | +| `argv` | The matcher's captures | +| `args` | Tokens left over after the match | +| `config` | The plugin's own block, verbatim | + +A plugin must read the request before writing anything. The pipes are +unbuffered, so a plugin that spoke first would deadlock. + +## Ops + +| Op | Answered | Needs | +|----|----------|-------| +| `exec` | yes | `config.proc.exec` | +| `emit` | **no** | nothing | + +```json +{"op":"exec","argv":["git","status"],"dir":"","capture":false} +{"op":"emit","line":"git status"} +``` + +`emit` is one-way on purpose: a plugin that waited for an answer would hang. + +### Result + +```json +{"code":0,"ok":true,"stdout":"","stderr":"","error":""} +``` + +`error` is godo refusing — an unknown op, or a capability the catalog did not +grant. A command that ran and failed is `code`, not `error`. + +## Capabilities + +Deny by default. An absent section, an absent key, or anything that is not +exactly `true`, is not granted: + +```yaml +config: + proc: + exec: true +``` + +``` +lines: line 1: not granted: proc.exec — enable it under +engine.plugins[].config.proc.exec +``` + +`config` is otherwise the plugin's: its keys, its meaning, its defaults. godo +reads only what it gates on. + +## Preview + +The plugin's to answer. godo can render a shell line because it wrote it; it +cannot render a program it does not interpret, so it asks with `mode: +"preview"` and prints whatever `emit` lines come back. + +Which means a plugin **can** run things during a preview, because it is the one +holding the capability. A plugin that does is misbehaving, the same way a +`--dry-run` that writes is misbehaving. Grant `proc.exec` to plugins you trust +to tell the difference. + +## Exit codes + +The plugin's exit code is the script's. A plugin propagating a child's code +gets godo exiting with that code, like a shell would. + +## Worked example + +[`examples/plugins/lines`](../../examples/plugins/lines) — a runner whose body +is one command per line. Under 200 lines of ordinary Go, no wasm-specific code. + +```bash +GOOS=wasip1 GOARCH=wasm go build -o lines.wasm ./examples/plugins/lines +shasum -a 256 lines.wasm +``` + +```yaml +engine: + plugins: + - source: ./lines.wasm + sha256: "…" + provides: [runner:lines] + config: + proc: {exec: true} + +scripts: + # @runner lines + boot: | + git status + ?pnpm install +``` + +## Known limits + +- **Local paths only.** `source` is a file on this machine. Fetching belongs + with a lockfile and is not built. +- **Size.** A Go plugin carries Go's runtime — the example is ~4.5 MB. TinyGo + or a C-family language produces far smaller wasm. +- **One instantiation per step.** Fine at godo's scale; it is not a server. +- **`exec` only.** No spawn, no filesystem ops. Those are the next capabilities + and the reason `config` is shaped as sections. diff --git a/examples/plugins/lines/main.go b/examples/plugins/lines/main.go new file mode 100644 index 0000000..863123f --- /dev/null +++ b/examples/plugins/lines/main.go @@ -0,0 +1,225 @@ +// Command lines is a godo plugin: a worked example of the protocol. +// +// It provides the runner "lines", whose body is one command per line: +// +// # @runner lines +// boot: | +// git status +// ?pnpm install +// echo done +// +// Lines run in order and stop at the first failure, like a script body. A line +// starting with "?" may fail without stopping the rest. ${NAME} is replaced +// with a capture; $1, $2, … with a leftover argument. +// +// Build: +// +// GOOS=wasip1 GOARCH=wasm go build -o lines.wasm ./examples/plugins/lines +// +// There is nothing wasm-specific in here. It reads stdin, writes stdout, and +// exits with a code — which is why it can be tested as an ordinary program. +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "strconv" + "strings" +) + +// request is godo's opening line. Only the fields this plugin uses. +type request struct { + API int `json:"api"` + Mode string `json:"mode"` + Runner string `json:"runner"` + Body string `json:"body"` + Argv map[string]string `json:"argv"` + Args []string `json:"args"` +} + +type op struct { + Op string `json:"op"` + Argv []string `json:"argv,omitempty"` + Capture bool `json:"capture,omitempty"` + Line string `json:"line,omitempty"` +} + +type result struct { + Code int `json:"code"` + OK bool `json:"ok"` + Error string `json:"error,omitempty"` +} + +const apiVersion = 1 + +func main() { + in := bufio.NewScanner(os.Stdin) + in.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + out := json.NewEncoder(os.Stdout) + + req, err := readRequest(in) + if err != nil { + fail(err) + } + if req.API != apiVersion { + fail(fmt.Errorf("plugin speaks api %d, godo speaks %d", apiVersion, req.API)) + } + + for n, raw := range strings.Split(req.Body, "\n") { + line := strings.TrimSpace(raw) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + mayFail := strings.HasPrefix(line, "?") + line = strings.TrimSpace(strings.TrimPrefix(line, "?")) + + argv, err := substitute(line, req) + if err != nil { + fail(fmt.Errorf("line %d: %w", n+1, err)) + } + if len(argv) == 0 { + continue + } + + // Preview is the plugin's business: godo does not know what a body of + // this shape would do, so it asks, and this answers by describing the + // commands rather than running them. + if req.Mode == "preview" { + emit(out, previewLine(argv, mayFail)) + continue + } + + res, err := exec(in, out, argv) + if err != nil { + fail(err) + } + if res.Error != "" { + fail(fmt.Errorf("line %d: %s", n+1, res.Error)) + } + if !res.OK && !mayFail { + // The command's code becomes the script's, like a shell would. + os.Exit(res.Code) + } + } + os.Exit(0) +} + +func readRequest(in *bufio.Scanner) (request, error) { + if !in.Scan() { + if err := in.Err(); err != nil { + return request{}, err + } + return request{}, fmt.Errorf("no request on stdin") + } + var req request + if err := json.Unmarshal(in.Bytes(), &req); err != nil { + return request{}, fmt.Errorf("unreadable request: %w", err) + } + return req, nil +} + +// substitute splits a line into argv, replacing ${NAME} with a capture and +// $1, $2, … with a leftover argument. A value with a space stays one argument: +// substitution happens after splitting, never before. +func substitute(line string, req request) ([]string, error) { + var argv []string + for _, word := range strings.Fields(line) { + v, err := expandWord(word, req) + if err != nil { + return nil, err + } + argv = append(argv, v) + } + return argv, nil +} + +func expandWord(word string, req request) (string, error) { + var b strings.Builder + rest := word + for { + i := strings.Index(rest, "$") + if i < 0 || i == len(rest)-1 { + break + } + b.WriteString(rest[:i]) + rest = rest[i+1:] + + if strings.HasPrefix(rest, "{") { + end := strings.Index(rest, "}") + if end < 0 { + return "", fmt.Errorf("unterminated ${ in %q", word) + } + name := rest[1:end] + v, ok := req.Argv[name] + if !ok { + return "", fmt.Errorf("unknown capture %q", name) + } + b.WriteString(v) + rest = rest[end+1:] + continue + } + + digits := 0 + for digits < len(rest) && rest[digits] >= '0' && rest[digits] <= '9' { + digits++ + } + if digits == 0 { + b.WriteString("$") + continue + } + n, _ := strconv.Atoi(rest[:digits]) + if n < 1 || n > len(req.Args) { + return "", fmt.Errorf("$%d out of range (%d args)", n, len(req.Args)) + } + b.WriteString(req.Args[n-1]) + rest = rest[digits:] + } + b.WriteString(rest) + return b.String(), nil +} + +func previewLine(argv []string, mayFail bool) string { + quoted := make([]string, len(argv)) + for i, a := range argv { + if strings.ContainsAny(a, " \t'\"") { + quoted[i] = "'" + strings.ReplaceAll(a, "'", `'\''`) + "'" + } else { + quoted[i] = a + } + } + line := strings.Join(quoted, " ") + if mayFail { + line += " # may fail" + } + return line +} + +func exec(in *bufio.Scanner, out *json.Encoder, argv []string) (result, error) { + if err := out.Encode(op{Op: "exec", Argv: argv}); err != nil { + return result{}, err + } + if !in.Scan() { + if err := in.Err(); err != nil { + return result{}, err + } + return result{}, fmt.Errorf("godo closed the connection") + } + var res result + if err := json.Unmarshal(in.Bytes(), &res); err != nil { + return result{}, fmt.Errorf("unreadable result: %w", err) + } + return res, nil +} + +func emit(out *json.Encoder, line string) { + if err := out.Encode(op{Op: "emit", Line: line}); err != nil { + fail(err) + } +} + +func fail(err error) { + fmt.Fprintln(os.Stderr, "lines:", err) + os.Exit(1) +} diff --git a/go.mod b/go.mod index e6b35d1..e358b68 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,7 @@ module github.com/my-rv/godo -go 1.22 +go 1.22.0 require gopkg.in/yaml.v3 v3.0.1 + +require github.com/tetratelabs/wazero v1.9.0 // indirect diff --git a/go.sum b/go.sum index a62c313..8a3a081 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZBf/I= +github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/catalog/engine.go b/internal/catalog/engine.go index c1d4094..0f85f0f 100644 --- a/internal/catalog/engine.go +++ b/internal/catalog/engine.go @@ -13,6 +13,35 @@ type Runner interface { Run(command string) error } +// Invocation is what a runner needs that a rendered command line cannot carry. +// +// A shell runner only ever wanted the finished line. A runner whose bodies are +// not shell text wants the body as written and the values as data — a plugin +// script reads a capture by name, it does not read a line godo already pasted +// it into. +type Invocation struct { + Script string // the catalog key, for messages + Runner RunnerName + Body string // the body as written + Captures map[string]string + Args []string +} + +// InvocationRunner receives the bound invocation instead of a rendered line, +// and answers for its own --preview. +// +// Preview is the runner's to answer because only it knows what its bodies do. +// godo can render a shell line because it wrote it; it cannot render a program +// it does not interpret, so it asks. +// +// A body reaching one of these is not expanded: ${godo:…} is shell-space +// syntax, and the values it would paste in are on the Invocation already. +type InvocationRunner interface { + Runner + RunInvocation(inv Invocation) error + PreviewInvocation(inv Invocation) ([]string, error) +} + // ArgsAwareRunner is a Runner that decides for itself whether a body takes the // tokens left over after the match. // @@ -135,7 +164,7 @@ func (e *Engine) stepRunner(s Script) (RunnerName, error) { // A runner the catalog declared a plugin for is a different failure // from a typo, and saying so saves the reader the hunt. if p, ok := e.Catalog.Engine.ProviderOf("runner", string(name)); ok { - return "", fmt.Errorf("%w: script %q asks for runner %q, provided by plugin %s — this build cannot load plugins", + return "", fmt.Errorf("%w: script %q asks for runner %q, which plugin %s should provide but did not", ErrUnknownRunner, s.Key, name, p.Source) } // Otherwise the registry's resolver knows why (unknown name, shell not @@ -155,8 +184,57 @@ type Plan struct { type PlanStep struct { Kind string // "dep" | "body" Source string - Command string + Command string // the line, for runners that take one Runner RunnerName // effective runner for this step + // Inv is set when the step's runner takes the bound invocation instead of + // a rendered line. + Inv *Invocation +} + +// steps turns one resolved match into plan steps. +// +// A runner that takes the bound invocation gets the body as written: ${godo:…} +// is shell-space syntax, and the values it would paste in travel on the +// Invocation instead. +func (e *Engine) steps(kind, source string, m *Match, runner RunnerName) ([]PlanStep, error) { + if _, ok := e.invocationRunner(runner); ok { + out := make([]PlanStep, 0, len(m.Script.Commands)) + for _, body := range m.Script.Commands { + out = append(out, PlanStep{ + Kind: kind, Source: source, Command: body, Runner: runner, + Inv: &Invocation{ + Script: m.Script.Key, + Runner: runner, + Body: body, + Captures: m.Captures, + Args: m.Args, + }, + }) + } + return out, nil + } + cmds, err := expand.ExpandAll(m.Script.Commands, m.Captures, m.Args) + if err != nil { + return nil, err + } + out := make([]PlanStep, 0, len(cmds)) + for _, c := range cmds { + out = append(out, PlanStep{Kind: kind, Source: source, Command: c, Runner: runner}) + } + return out, nil +} + +// invocationRunner reports whether the runner takes the bound invocation. +func (e *Engine) invocationRunner(name RunnerName) (InvocationRunner, bool) { + if r, err := e.Runners.Lookup(name); err == nil { + ir, ok := r.(InvocationRunner) + return ir, ok + } + if name == "" { + ir, ok := e.Runner.(InvocationRunner) + return ir, ok + } + return nil, false } // BuildPlan expands deps + body without executing. @@ -183,13 +261,11 @@ func (e *Engine) BuildPlan(tokens []string) (*Plan, error) { if err := e.appendDeps(plan, m, stack, done); err != nil { return nil, err } - cmds, err := expand.ExpandAll(m.Script.Commands, m.Captures, m.Args) + steps, err := e.steps("body", m.Script.Key, m, runner) if err != nil { return nil, err } - for _, c := range cmds { - plan.Steps = append(plan.Steps, PlanStep{Kind: "body", Source: m.Script.Key, Command: c, Runner: runner}) - } + plan.Steps = append(plan.Steps, steps...) return plan, nil } @@ -267,13 +343,11 @@ func (e *Engine) appendDeps(plan *Plan, m *Match, stack, done map[string]bool) e if err := e.appendDeps(plan, depMatch, stack, done); err != nil { return err } - cmds, err := expand.ExpandAll(depMatch.Script.Commands, depMatch.Captures, depMatch.Args) + steps, err := e.steps("dep", invKey, depMatch, depRunner) if err != nil { return err } - for _, c := range cmds { - plan.Steps = append(plan.Steps, PlanStep{Kind: "dep", Source: invKey, Command: c, Runner: depRunner}) - } + plan.Steps = append(plan.Steps, steps...) delete(stack, invKey) done[invKey] = true } @@ -291,6 +365,12 @@ func (e *Engine) Run(tokens []string) error { if err != nil { return err } + if ir, ok := runner.(InvocationRunner); ok && step.Inv != nil { + if err := ir.RunInvocation(*step.Inv); err != nil { + return err + } + continue + } if err := runner.Run(step.Command); err != nil { return err } @@ -298,15 +378,31 @@ func (e *Engine) Run(tokens []string) error { return nil } -// PreviewLines returns expanded commands in order (deps + body). +// PreviewLines returns the lines that would run, in order (deps + body). +// +// A runner that takes the bound invocation renders its own: godo can show a +// shell line because it wrote it, and has to ask for anything else. func (e *Engine) PreviewLines(tokens []string) ([]string, error) { plan, err := e.BuildPlan(tokens) if err != nil { return nil, err } - out := make([]string, len(plan.Steps)) - for i, s := range plan.Steps { - out[i] = s.Command + var out []string + for _, s := range plan.Steps { + if s.Inv == nil { + out = append(out, s.Command) + continue + } + ir, ok := e.invocationRunner(s.Runner) + if !ok { + out = append(out, s.Command) + continue + } + lines, err := ir.PreviewInvocation(*s.Inv) + if err != nil { + return nil, err + } + out = append(out, lines...) } return out, nil } diff --git a/internal/catalog/engine_block_test.go b/internal/catalog/engine_block_test.go index 7195245..3d4158b 100644 --- a/internal/catalog/engine_block_test.go +++ b/internal/catalog/engine_block_test.go @@ -107,6 +107,10 @@ func TestParse_engineRejects(t *testing.T) { } // A runner a plugin declares fails by naming the plugin, not as a typo. +// +// The engine does not load plugins — internal/plugin does, and the CLI wires +// it up. An embedder that registers nothing sees this message rather than +// "unknown runner", which would point at the catalog instead of the wiring. func TestBuildPlan_pluginRunnerNamesItsPlugin(t *testing.T) { cat, err := catalog.Parse([]byte(pluginBlock), "x") if err != nil { @@ -117,7 +121,7 @@ func TestBuildPlan_pluginRunnerNamesItsPlugin(t *testing.T) { if !errors.Is(err, catalog.ErrUnknownRunner) { t.Fatalf("err=%v", err) } - for _, want := range []string{"micropy", "godo-micropy", "cannot load plugins"} { + for _, want := range []string{"micropy", "godo-micropy", "should provide but did not"} { if !strings.Contains(err.Error(), want) { t.Fatalf("err=%v, missing %q", err, want) } diff --git a/internal/cli/app.go b/internal/cli/app.go index cc8ac18..26e04a1 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -1,6 +1,7 @@ package cli import ( + "context" "fmt" "io" "os" @@ -12,6 +13,7 @@ import ( "github.com/my-rv/godo" "github.com/my-rv/godo/internal/catalog" "github.com/my-rv/godo/internal/execshell" + "github.com/my-rv/godo/internal/plugin" "github.com/my-rv/godo/internal/update" ) @@ -93,6 +95,14 @@ func (a *App) Run(args []string) error { if err := runners.Register(catalog.RunnerInherit, execshell.InheritRunner{Dir: root}); err != nil { return err } + if len(cat.Engine.Plugins) > 0 { + ctx := context.Background() + rt := plugin.NewRuntime(ctx) + defer rt.Close(ctx) + if err := plugin.Register(ctx, rt, cat, runners, root, a.Stdout, a.Stderr, a.Stdin); err != nil { + return err + } + } // Any other name is a shell the catalog asked for by name. godo does not // manage those — it proxies to whatever is on PATH. runners.Resolve = func(name catalog.RunnerName) (catalog.Runner, error) { @@ -240,9 +250,9 @@ func (a *App) printDeclaredPlugins(cat *catalog.Catalog) { if len(cat.Engine.Plugins) == 0 { return } - fmt.Fprintln(a.Stdout, "\ndeclared by this catalog (no plugin loader in this build):") + fmt.Fprintln(a.Stdout, "\nfrom plugins declared by this catalog:") for _, p := range cat.Engine.Plugins { - fmt.Fprintf(a.Stdout, " %-10s %s\n", strings.Join(p.Provides, " "), p.Source) + fmt.Fprintf(a.Stdout, " %-16s %s\n", strings.Join(p.Provides, " "), p.Source) } } diff --git a/internal/cli/plugin_e2e_test.go b/internal/cli/plugin_e2e_test.go new file mode 100644 index 0000000..8a03acc --- /dev/null +++ b/internal/cli/plugin_e2e_test.go @@ -0,0 +1,184 @@ +package cli_test + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/my-rv/godo" +) + +var ( + wasmOnce sync.Once + wasmPath string + wasmErr error +) + +// buildExamplePlugin compiles examples/plugins/lines for wasip1. +// +// Built rather than committed: a checked-in binary is something nobody can +// review, and what this asserts is that the artifact matches the source next +// to it. +func buildExamplePlugin(t *testing.T) string { + t.Helper() + wasmOnce.Do(func() { + f, err := os.CreateTemp("", "lines-*.wasm") + if err != nil { + wasmErr = err + return + } + f.Close() + cmd := exec.Command("go", "build", "-o", f.Name(), "../../examples/plugins/lines") + cmd.Env = append(os.Environ(), "GOOS=wasip1", "GOARCH=wasm") + if b, err := cmd.CombinedOutput(); err != nil { + wasmErr = err + t.Logf("build: %s", b) + return + } + wasmPath = f.Name() + }) + if wasmErr != nil { + t.Fatalf("building the example plugin: %v", wasmErr) + } + return wasmPath +} + +// pluginCatalog writes a catalog next to a copy of the built plugin. +func pluginCatalog(t *testing.T, grantExec bool, scripts string) string { + t.Helper() + cwd := t.TempDir() + data, err := os.ReadFile(buildExamplePlugin(t)) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(cwd, "lines.wasm"), data, 0o644); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(data) + config := "" + if grantExec { + config = " config:\n proc:\n exec: true\n" + } + writeGodoYAML(t, cwd, "version: \"0.1\"\nengine:\n plugins:\n"+ + " - source: ./lines.wasm\n"+ + " sha256: "+hex.EncodeToString(sum[:])+"\n"+ + " provides: [runner:lines]\n"+ + config+ + "scripts:\n"+scripts) + return cwd +} + +// Contract: a plugin provides a runner, and a script names it like any other. +func TestE2E_pluginRunnerRunsTheScript(t *testing.T) { + cwd := pluginCatalog(t, true, " # @runner lines\n boot: |\n touch one\n touch two\n") + app, _, _ := e2eApp(t, cwd) + if err := app.Run([]string{"boot"}); err != nil { + t.Fatalf("run: %v", err) + } + for _, f := range []string{"one", "two"} { + if _, err := os.Stat(filepath.Join(cwd, f)); err != nil { + t.Fatalf("%s: %v", f, err) + } + } +} + +// Preview is the plugin's to answer: godo cannot render a body it does not +// interpret, so it asks and prints what comes back. +func TestE2E_pluginRendersItsOwnPreview(t *testing.T) { + cwd := pluginCatalog(t, true, " # @runner lines\n boot: |\n touch one\n ?false\n") + app, out, _ := e2eApp(t, cwd) + if err := app.Run([]string{"--preview", "boot"}); err != nil { + t.Fatal(err) + } + got := strings.TrimSpace(out.String()) + if got != "touch one\nfalse # may fail" { + t.Fatalf("preview=%q", got) + } + if _, err := os.Stat(filepath.Join(cwd, "one")); err == nil { + t.Fatal("preview ran the body") + } +} + +// Captures and leftover args reach the plugin as data, not pasted into a line. +func TestE2E_pluginReceivesCapturesAndArgs(t *testing.T) { + cwd := pluginCatalog(t, true, + " # @runner lines\n make ${NAME}: |\n touch ${NAME}\n touch $1\n") + // engine.dialect must be matcher for a capture in the key. + body, err := os.ReadFile(filepath.Join(cwd, "godo.yaml")) + if err != nil { + t.Fatal(err) + } + fixed := strings.Replace(string(body), "engine:\n", "engine:\n dialect: matcher\n", 1) + if err := os.WriteFile(filepath.Join(cwd, "godo.yaml"), []byte(fixed), 0o644); err != nil { + t.Fatal(err) + } + + app, _, _ := e2eApp(t, cwd) + if err := app.Run([]string{"make", "from capture", "from arg"}); err != nil { + t.Fatalf("run: %v", err) + } + // A value with a space stays one argument: the plugin substitutes after + // splitting, so nothing re-splits it. + for _, f := range []string{"from capture", "from arg"} { + if _, err := os.Stat(filepath.Join(cwd, f)); err != nil { + t.Fatalf("%q: %v", f, err) + } + } +} + +// Deny by default: without config.proc.exec the plugin cannot run anything. +func TestE2E_pluginCannotExecWithoutTheGrant(t *testing.T) { + cwd := pluginCatalog(t, false, " # @runner lines\n boot: |\n touch one\n") + app, _, _ := e2eApp(t, cwd) + err := app.Run([]string{"boot"}) + if err == nil { + t.Fatal("want failure") + } + if _, serr := os.Stat(filepath.Join(cwd, "one")); serr == nil { + t.Fatal("an ungranted exec ran anyway") + } +} + +// The child's exit code is the script's, through the plugin. +func TestE2E_pluginPropagatesTheExitCode(t *testing.T) { + cwd := pluginCatalog(t, true, + " # @runner lines\n boom ${CODE}: |\n sh -c ${CODE}\n touch never\n") + body, _ := os.ReadFile(filepath.Join(cwd, "godo.yaml")) + fixed := strings.Replace(string(body), "engine:\n", "engine:\n dialect: matcher\n", 1) + if err := os.WriteFile(filepath.Join(cwd, "godo.yaml"), []byte(fixed), 0o644); err != nil { + t.Fatal(err) + } + + app, _, _ := e2eApp(t, cwd) + err := app.Run([]string{"boom", "exit 42"}) + if got := godo.ExitCode(err); got != 42 { + t.Fatalf("exit=%d (%v)", got, err) + } + if _, serr := os.Stat(filepath.Join(cwd, "never")); serr == nil { + t.Fatal("the body carried on after a failure") + } +} + +// A digest that does not match the bytes stops the run before anything loads. +func TestE2E_pluginDigestMismatchRefusesToRun(t *testing.T) { + cwd := pluginCatalog(t, true, " # @runner lines\n boot: |\n touch one\n") + body, _ := os.ReadFile(filepath.Join(cwd, "godo.yaml")) + broken := strings.Replace(string(body), "sha256: ", "sha256: 00", 1) + if err := os.WriteFile(filepath.Join(cwd, "godo.yaml"), []byte(broken), 0o644); err != nil { + t.Fatal(err) + } + + app, _, _ := e2eApp(t, cwd) + err := app.Run([]string{"boot"}) + if err == nil || !strings.Contains(err.Error(), "sha256 mismatch") { + t.Fatalf("err=%v", err) + } + if _, serr := os.Stat(filepath.Join(cwd, "one")); serr == nil { + t.Fatal("a plugin with the wrong digest ran") + } +} diff --git a/internal/plugin/invoke.go b/internal/plugin/invoke.go new file mode 100644 index 0000000..5817080 --- /dev/null +++ b/internal/plugin/invoke.go @@ -0,0 +1,218 @@ +package plugin + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "strings" + + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/sys" +) + +// Host is what a plugin can reach through godo. Everything here is gated by +// the catalog's config: a capability the catalog did not grant is not a stub +// that fails, it is an op godo refuses to perform. +type Host struct { + // Dir is the working directory for exec, normally the catalog's. + Dir string + Stdout io.Writer + Stderr io.Writer + Stdin io.Reader +} + +// Outcome is what an invocation produced. +type Outcome struct { + // Code is the plugin's exit code, and so the script's. + Code int + // Emitted holds the lines the plugin contributed to --preview. + Emitted []string +} + +// Invoke runs the plugin once. +// +// The plugin is a WASI command, so this instantiates the module, writes the +// request to its stdin, and answers ops until it exits. Its exit code is the +// result; its stderr goes to godo's, so a plugin can explain itself. +func (p *Plugin) Invoke(ctx context.Context, req Request, host Host) (Outcome, error) { + req.API = APIVersion + req.Config = p.Config + + inR, inW := io.Pipe() + outR, outW := io.Pipe() + + stderr := host.Stderr + if stderr == nil { + stderr = os.Stderr + } + + cfg := wazero.NewModuleConfig(). + WithStdin(inR). + WithStdout(outW). + WithStderr(stderr). + WithArgs("plugin"). + // No WithFS, no WithEnv, no WithSysWalltime: the sandbox stays shut. + WithName("") + + runErr := make(chan error, 1) + go func() { + _, err := p.rt.InstantiateModule(ctx, p.compiled, cfg) + _ = outW.Close() + runErr <- err + }() + + // The request goes out on its own goroutine: an io.Pipe has no buffer, so + // writing it inline would deadlock against a plugin that spoke first. + writeErr := make(chan error, 1) + go func() { + line, err := json.Marshal(req) + if err != nil { + writeErr <- err + return + } + _, err = inW.Write(append(line, '\n')) + writeErr <- err + }() + + out := Outcome{} + loopErr := p.serve(outR, inW, host, &out) + + _ = inW.Close() + _ = inR.Close() + + err := <-runErr + _ = outR.Close() + if werr := <-writeErr; werr != nil && loopErr == nil { + loopErr = werr + } + + var exit *sys.ExitError + switch { + case err == nil: + out.Code = 0 + case errors.As(err, &exit): + out.Code = int(exit.ExitCode()) + default: + return out, fmt.Errorf("plugin %s: %w", p.Source, err) + } + if loopErr != nil { + return out, loopErr + } + return out, nil +} + +// serve answers ops until the plugin's stdout closes. +func (p *Plugin) serve(r io.Reader, w io.Writer, host Host, out *Outcome) error { + scan := bufio.NewScanner(r) + scan.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + enc := json.NewEncoder(w) + + for scan.Scan() { + line := strings.TrimSpace(scan.Text()) + if line == "" { + continue + } + var op Op + if err := json.Unmarshal([]byte(line), &op); err != nil { + return fmt.Errorf("plugin %s: unreadable op %q: %w", p.Source, line, err) + } + switch op.Op { + case OpEmit: + // Preview output. No answer: a plugin that waited for one here + // would hang, so emit is deliberately one-way. + out.Emitted = append(out.Emitted, op.Line) + case OpExec: + res := p.execOp(op, host) + if err := enc.Encode(res); err != nil { + return fmt.Errorf("plugin %s: %w", p.Source, err) + } + default: + if err := enc.Encode(Result{Error: fmt.Sprintf("unknown op %q", op.Op)}); err != nil { + return fmt.Errorf("plugin %s: %w", p.Source, err) + } + } + } + if err := scan.Err(); err != nil && !errors.Is(err, io.ErrClosedPipe) { + return fmt.Errorf("plugin %s: %w", p.Source, err) + } + return nil +} + +// execOp runs an argument vector, if the catalog granted it. +func (p *Plugin) execOp(op Op, host Host) Result { + if !p.granted("proc", "exec") { + return Result{Error: "not granted: proc.exec — enable it under engine.plugins[].config.proc.exec"} + } + if len(op.Argv) == 0 { + return Result{Error: "exec: empty argv"} + } + cmd := exec.Command(op.Argv[0], op.Argv[1:]...) + cmd.Dir = host.Dir + if op.Dir != "" { + cmd.Dir = op.Dir + } + var res Result + if op.Capture { + var stdout, stderr strings.Builder + cmd.Stdout = &stdout + cmd.Stderr = &stderr + res.Code = runAndCode(cmd, &res) + res.Stdout = stdout.String() + res.Stderr = stderr.String() + } else { + cmd.Stdout = orStd(host.Stdout, os.Stdout) + cmd.Stderr = orStd(host.Stderr, os.Stderr) + cmd.Stdin = host.Stdin + res.Code = runAndCode(cmd, &res) + } + res.OK = res.Code == 0 + return res +} + +func runAndCode(cmd *exec.Cmd, res *Result) int { + err := cmd.Run() + if err == nil { + return 0 + } + var ee *exec.ExitError + if errors.As(err, &ee) { + return ee.ExitCode() + } + // Could not start at all: a missing binary is the plugin's problem to + // report, not godo's to crash on. + res.Error = err.Error() + return 127 +} + +func orStd(w io.Writer, def io.Writer) io.Writer { + if w == nil { + return def + } + return w +} + +// granted reports whether config grants section.key. +// +// Deny by default: an absent section, an absent key, or anything that is not +// exactly true, is not granted. +func (p *Plugin) granted(section, key string) bool { + raw, ok := p.Config[section] + if !ok { + return false + } + m, ok := raw.(map[string]any) + if !ok { + return false + } + v, ok := m[key] + if !ok { + return false + } + b, ok := v.(bool) + return ok && b +} diff --git a/internal/plugin/invoke_test.go b/internal/plugin/invoke_test.go new file mode 100644 index 0000000..26b2efc --- /dev/null +++ b/internal/plugin/invoke_test.go @@ -0,0 +1,279 @@ +package plugin_test + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/my-rv/godo/internal/plugin" +) + +var ( + buildOnce sync.Once + wasmPath string + buildErr error +) + +// exampleWasm builds examples/plugins/lines for wasip1 once per run. +// +// Built rather than committed: a checked-in binary is a thing nobody can +// review, and the point of this test is that the artifact matches the source +// beside it. +func exampleWasm(t *testing.T) string { + t.Helper() + buildOnce.Do(func() { + dir := t.TempDir() + out := filepath.Join(dir, "lines.wasm") + cmd := exec.Command("go", "build", "-o", out, "../../examples/plugins/lines") + cmd.Env = append(os.Environ(), "GOOS=wasip1", "GOARCH=wasm") + if b, err := cmd.CombinedOutput(); err != nil { + buildErr = err + t.Logf("build: %s", b) + return + } + // Keep it outside t.TempDir's cleanup by copying to the package temp. + data, err := os.ReadFile(out) + if err != nil { + buildErr = err + return + } + keep, err := os.CreateTemp("", "lines-*.wasm") + if err != nil { + buildErr = err + return + } + defer keep.Close() + if _, err := keep.Write(data); err != nil { + buildErr = err + return + } + wasmPath = keep.Name() + }) + if buildErr != nil { + t.Fatalf("building the example plugin: %v", buildErr) + } + return wasmPath +} + +func load(t *testing.T, config map[string]any) (*plugin.Plugin, func()) { + t.Helper() + path := exampleWasm(t) + digest, err := plugin.Digest(path) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + rt := plugin.NewRuntime(ctx) + p, err := rt.Load(ctx, path, digest, "", []string{"runner:lines"}, config) + if err != nil { + t.Fatal(err) + } + return p, func() { _ = rt.Close(ctx) } +} + +func TestLoad_refusesAWrongDigest(t *testing.T) { + path := exampleWasm(t) + ctx := context.Background() + rt := plugin.NewRuntime(ctx) + defer rt.Close(ctx) + + _, err := rt.Load(ctx, path, strings.Repeat("0", 64), "", nil, nil) + if err == nil { + t.Fatal("a plugin whose bytes do not match its digest must not load") + } + for _, want := range []string{"sha256 mismatch", "declared", "actual"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("err=%v, missing %q", err, want) + } + } +} + +func TestLoad_refusesSomethingThatIsNotWasm(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "not.wasm") + if err := os.WriteFile(path, []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + digest, err := plugin.Digest(path) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + rt := plugin.NewRuntime(ctx) + defer rt.Close(ctx) + if _, err := rt.Load(ctx, path, digest, "", nil, nil); err == nil { + t.Fatal("want a compile error") + } +} + +func TestInvoke_previewAsksThePlugin(t *testing.T) { + p, done := load(t, nil) + defer done() + + out, err := p.Invoke(context.Background(), plugin.Request{ + Mode: "preview", + Runner: "lines", + Body: "git status\n?pnpm install\necho hola ${WHO} $1", + Argv: map[string]string{"WHO": "mundo"}, + Args: []string{"un arg"}, + }, plugin.Host{}) + if err != nil { + t.Fatal(err) + } + if out.Code != 0 { + t.Fatalf("code=%d", out.Code) + } + want := []string{"git status", "pnpm install # may fail", "echo hola mundo 'un arg'"} + if len(out.Emitted) != len(want) { + t.Fatalf("emitted=%q", out.Emitted) + } + for i := range want { + if out.Emitted[i] != want[i] { + t.Fatalf("emitted=%q want %q", out.Emitted, want) + } + } +} + +// Preview must not run anything, and with no config granted it could not. +func TestInvoke_previewGrantsNothing(t *testing.T) { + dir := t.TempDir() + p, done := load(t, nil) + defer done() + + _, err := p.Invoke(context.Background(), plugin.Request{ + Mode: "preview", + Body: "touch marker", + }, plugin.Host{Dir: dir}) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dir, "marker")); err == nil { + t.Fatal("preview touched the disk") + } +} + +func TestInvoke_execRunsWhenGranted(t *testing.T) { + dir := t.TempDir() + p, done := load(t, map[string]any{"proc": map[string]any{"exec": true}}) + defer done() + + out, err := p.Invoke(context.Background(), plugin.Request{ + Mode: "run", + Body: "touch marker\ntouch ${NAME}", + Argv: map[string]string{"NAME": "second"}, + }, plugin.Host{Dir: dir}) + if err != nil { + t.Fatal(err) + } + if out.Code != 0 { + t.Fatalf("code=%d", out.Code) + } + for _, f := range []string{"marker", "second"} { + if _, err := os.Stat(filepath.Join(dir, f)); err != nil { + t.Fatalf("%s: %v", f, err) + } + } +} + +// Deny by default: no config, no exec. The plugin is told why. +func TestInvoke_execRefusedWhenNotGranted(t *testing.T) { + dir := t.TempDir() + p, done := load(t, nil) + defer done() + + out, err := p.Invoke(context.Background(), plugin.Request{ + Mode: "run", + Body: "touch marker", + }, plugin.Host{Dir: dir}) + if err != nil { + t.Fatal(err) + } + if out.Code == 0 { + t.Fatal("an ungranted exec must not end in success") + } + if _, err := os.Stat(filepath.Join(dir, "marker")); err == nil { + t.Fatal("an ungranted exec ran anyway") + } +} + +func TestInvoke_configMustSayTrue(t *testing.T) { + dir := t.TempDir() + for _, cfg := range []map[string]any{ + {"proc": map[string]any{"exec": false}}, + {"proc": map[string]any{"spawn": true}}, + {"proc": "yes"}, + {"fs": map[string]any{"exec": true}}, + } { + p, done := load(t, cfg) + out, err := p.Invoke(context.Background(), plugin.Request{Mode: "run", Body: "touch marker"}, plugin.Host{Dir: dir}) + done() + if err != nil { + t.Fatalf("%v: %v", cfg, err) + } + if out.Code == 0 { + t.Fatalf("%v granted exec", cfg) + } + } + if _, err := os.Stat(filepath.Join(dir, "marker")); err == nil { + t.Fatal("something ran") + } +} + +// A failing command stops the body and becomes the exit code, like a shell. +func TestInvoke_exitCodeIsTheChilds(t *testing.T) { + p, done := load(t, map[string]any{"proc": map[string]any{"exec": true}}) + defer done() + + out, err := p.Invoke(context.Background(), plugin.Request{ + Mode: "run", + Body: "sh -c ${SCRIPT}\ntouch should-not-exist", + Argv: map[string]string{"SCRIPT": "exit 42"}, + }, plugin.Host{Dir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + if out.Code != 42 { + t.Fatalf("code=%d, want 42", out.Code) + } +} + +// "?" marks a line that may fail without stopping the rest. +func TestInvoke_optionalLineDoesNotStopTheBody(t *testing.T) { + dir := t.TempDir() + p, done := load(t, map[string]any{"proc": map[string]any{"exec": true}}) + defer done() + + out, err := p.Invoke(context.Background(), plugin.Request{ + Mode: "run", + Body: "?sh -c ${SCRIPT}\ntouch after", + Argv: map[string]string{"SCRIPT": "exit 3"}, + }, plugin.Host{Dir: dir}) + if err != nil { + t.Fatal(err) + } + if out.Code != 0 { + t.Fatalf("code=%d", out.Code) + } + if _, err := os.Stat(filepath.Join(dir, "after")); err != nil { + t.Fatalf("the body stopped at an optional failure: %v", err) + } +} + +// The sandbox is shut: the plugin has no filesystem of its own. +func TestInvoke_pluginCannotReachTheFilesystemItself(t *testing.T) { + p, done := load(t, nil) + defer done() + // The example plugin never opens a file, so this asserts the configuration + // rather than the plugin: no WithFS was granted, so nothing is mounted. + out, err := p.Invoke(context.Background(), plugin.Request{Mode: "preview", Body: "echo hi"}, plugin.Host{}) + if err != nil { + t.Fatal(err) + } + if len(out.Emitted) != 1 { + t.Fatalf("emitted=%q", out.Emitted) + } +} diff --git a/internal/plugin/protocol.go b/internal/plugin/protocol.go new file mode 100644 index 0000000..ec4d74d --- /dev/null +++ b/internal/plugin/protocol.go @@ -0,0 +1,72 @@ +// Package plugin runs godo plugins as WebAssembly. +// +// A plugin is a WASI command: a normal program with a main(), compiled to +// wasm. godo writes one JSON request to its stdin, the plugin writes JSON ops +// to its stdout, and godo answers the ones that need answering. The plugin's +// exit code is the script's exit code. +// +// Being a command rather than a set of exported functions is what keeps the +// contract small. There is no memory-sharing ABI to get right, a plugin can be +// written in any language that targets WASI, and it can be tested outside wasm +// entirely — pipe it a request on stdin and read what it says. +// +// The sandbox is wazero's default: no filesystem, no network, no environment, +// no clock beyond what is granted. A plugin reaches the outside world only by +// asking godo, and godo only honours what the catalog's config grants. +package plugin + +// APIVersion is the protocol this build speaks. A plugin that answers with a +// different major refuses to run rather than guessing. +const APIVersion = 1 + +// Request is the single line godo writes to a plugin's stdin. +type Request struct { + API int `json:"api"` + // Mode is "run" or "preview". What preview means is the plugin's call: + // it knows what its own bodies do, and godo does not. + Mode string `json:"mode"` + // Runner is the name the catalog asked for, so one plugin can provide + // several. + Runner string `json:"runner"` + // Body is the script body, verbatim. godo does not expand ${godo:…} for a + // plugin: a plugin body is not shell text, and the values are right here. + Body string `json:"body"` + // Argv holds the matcher's captures; Args the tokens left over. + Argv map[string]string `json:"argv"` + Args []string `json:"args"` + // Config is the plugin's own block from godo.yaml, verbatim. + Config map[string]any `json:"config,omitempty"` +} + +// Op is one line a plugin writes to its stdout. +type Op struct { + Op string `json:"op"` + + // exec + Argv []string `json:"argv,omitempty"` + Dir string `json:"dir,omitempty"` + Capture bool `json:"capture,omitempty"` + + // emit + Line string `json:"line,omitempty"` +} + +// Result is godo's answer to an op that needs one. +type Result struct { + Code int `json:"code"` + OK bool `json:"ok"` + Stdout string `json:"stdout,omitempty"` + Stderr string `json:"stderr,omitempty"` + // Error is set when godo refused: an unknown op, or a capability the + // catalog did not grant. It is not a failing command — that is Code. + Error string `json:"error,omitempty"` +} + +// Op names. +const ( + // OpExec runs an argument vector and waits. Needs config proc.exec. + OpExec = "exec" + // OpEmit contributes one line to --preview output. Needs nothing, and + // gets no answer. + OpEmit = "emit" +) diff --git a/internal/plugin/runner.go b/internal/plugin/runner.go new file mode 100644 index 0000000..8381fef --- /dev/null +++ b/internal/plugin/runner.go @@ -0,0 +1,126 @@ +package plugin + +import ( + "context" + "fmt" + "io" + + "github.com/my-rv/godo/internal/catalog" +) + +// Runner adapts a loaded plugin to one runner name. +// +// One plugin may provide several runners; the name travels in the request so +// the plugin can tell which one it was asked for. +type Runner struct { + Plugin *Plugin + Name catalog.RunnerName + Host Host +} + +// Run implements catalog.Runner. +// +// It always fails: a rendered line has lost the values a plugin reads by name. +// The engine hands a plugin the bound invocation instead. +func (r Runner) Run(string) error { + return fmt.Errorf("runner %q is a plugin and takes the bound invocation, not a rendered line", r.Name) +} + +// AcceptsArgs implements catalog.ArgsAwareRunner. +// +// A plugin body is a program, not a shell template, so there is no +// ${godo:args…} to look for. Whether the leftover tokens mean anything is the +// plugin's business; they are handed over either way. +func (Runner) AcceptsArgs([]string) bool { return true } + +// RunInvocation implements catalog.InvocationRunner. +func (r Runner) RunInvocation(inv catalog.Invocation) error { + out, err := r.invoke(inv, "run") + if err != nil { + return err + } + if out.Code != 0 { + return &catalog.ExitError{ + Code: out.Code, + Message: fmt.Sprintf("script %q failed under runner %q", inv.Script, r.Name), + } + } + return nil +} + +// PreviewInvocation implements catalog.InvocationRunner. +// +// The plugin decides what preview means for its own bodies. A plugin that +// emits nothing previews as nothing, and one that refuses to preview says so +// by exiting non-zero. +func (r Runner) PreviewInvocation(inv catalog.Invocation) ([]string, error) { + out, err := r.invoke(inv, "preview") + if err != nil { + return nil, err + } + if out.Code != 0 { + return nil, &catalog.ExitError{ + Code: out.Code, + Message: fmt.Sprintf("script %q: runner %q could not preview it", inv.Script, r.Name), + } + } + return out.Emitted, nil +} + +func (r Runner) invoke(inv catalog.Invocation, mode string) (Outcome, error) { + return r.Plugin.Invoke(context.Background(), Request{ + Mode: mode, + Runner: string(r.Name), + Body: inv.Body, + Argv: nonNil(inv.Captures), + Args: inv.Args, + }, r.Host) +} + +func nonNil(m map[string]string) map[string]string { + if m == nil { + return map[string]string{} + } + return m +} + +// Register loads every plugin a catalog declares and registers the runners +// they provide. +// +// A plugin that will not load stops the run. A catalog that names a plugin has +// already decided it is part of the build; carrying on without it would mean +// silently running something other than what the file says. +func Register(ctx context.Context, rt *Runtime, cat *catalog.Catalog, reg *catalog.RunnerRegistry, root string, stdout, stderr io.Writer, stdin io.Reader) error { + for _, spec := range cat.Engine.Plugins { + p, err := rt.Load(ctx, spec.Source, spec.SHA256, root, spec.Provides, spec.Config) + if err != nil { + return err + } + for _, entry := range spec.Provides { + kind, name, ok := cutKind(entry) + if !ok || kind != "runner" { + // dialect plugins are declared the same way and are not + // implemented; the catalog already validated the shape. + continue + } + runner := Runner{ + Plugin: p, + Name: catalog.RunnerName(name), + Host: Host{Dir: root, Stdout: stdout, Stderr: stderr, Stdin: stdin}, + } + if err := reg.Register(catalog.RunnerName(name), runner); err != nil { + return fmt.Errorf("plugin %s: %w", spec.Source, err) + } + } + } + return nil +} + +func cutKind(entry string) (kind, name string, ok bool) { + for i := 0; i < len(entry); i++ { + if entry[i] == ':' { + return entry[:i], entry[i+1:], true + } + } + return "", "", false +} diff --git a/internal/plugin/runtime.go b/internal/plugin/runtime.go new file mode 100644 index 0000000..bac6fac --- /dev/null +++ b/internal/plugin/runtime.go @@ -0,0 +1,103 @@ +package plugin + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1" +) + +// Runtime compiles and holds plugins for one godo invocation. +type Runtime struct { + rt wazero.Runtime + once sync.Once +} + +// NewRuntime returns a runtime with WASI available and nothing else. +func NewRuntime(ctx context.Context) *Runtime { + rt := wazero.NewRuntime(ctx) + wasi_snapshot_preview1.MustInstantiate(ctx, rt) + return &Runtime{rt: rt} +} + +// Close releases every compiled module. +func (r *Runtime) Close(ctx context.Context) error { + var err error + r.once.Do(func() { err = r.rt.Close(ctx) }) + return err +} + +// Plugin is one verified, compiled artifact. +type Plugin struct { + Source string + Provides []string + Config map[string]any + + rt wazero.Runtime + compiled wazero.CompiledModule +} + +// Load reads an artifact, checks it against its digest, and compiles it. +// +// The digest is checked before the bytes reach a compiler, not after: a +// plugin is third-party code, and "this is the artifact that was reviewed" is +// the one thing a catalog can actually assert about it. +// +// source is resolved relative to root when it is not absolute. Only local +// paths are understood today; fetching is a separate concern and belongs with +// a lockfile, not here. +func (r *Runtime) Load(ctx context.Context, source, digest, root string, provides []string, config map[string]any) (*Plugin, error) { + path, err := localPath(source, root) + if err != nil { + return nil, err + } + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("plugin %s: %w", source, err) + } + sum := sha256.Sum256(data) + got := hex.EncodeToString(sum[:]) + if !strings.EqualFold(got, strings.TrimSpace(digest)) { + return nil, fmt.Errorf("plugin %s: sha256 mismatch\n declared %s\n actual %s", source, digest, got) + } + compiled, err := r.rt.CompileModule(ctx, data) + if err != nil { + return nil, fmt.Errorf("plugin %s: not a usable wasm module: %w", source, err) + } + return &Plugin{ + Source: source, + Provides: provides, + Config: config, + rt: r.rt, + compiled: compiled, + }, nil +} + +// localPath resolves a source that names a file on this machine. +func localPath(source, root string) (string, error) { + p := strings.TrimPrefix(source, "file://") + if p == source && strings.Contains(source, "://") { + return "", fmt.Errorf("plugin %s: only local paths are supported in this build", source) + } + if filepath.IsAbs(p) { + return p, nil + } + return filepath.Join(root, p), nil +} + +// Digest returns the sha256 of a file, for writing into a catalog. +func Digest(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} From 48e04a77476b5da733bf170385ca4d8968699820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=83=9E=E3=82=A4=E3=83=8E=E3=83=AB?= <97069334+MY-RV@users.noreply.github.com> Date: Sun, 20 Sep 2026 16:17:06 -0600 Subject: [PATCH 02/14] chore: ship third-party notices with the release binaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Linking wazero put Apache-2.0 code inside every binary godo releases, and Apache-2.0 asks that its licence and NOTICE travel with any form of distribution. The archives were shipping LICENSE alone, which was fine while godo was MIT and stdlib and stopped being fine the moment a dependency arrived. THIRD_PARTY_LICENSES.md carries wazero's NOTICE and licence verbatim, and the archive now lists its files explicitly instead of relying on GoReleaser's default glob — the default would not have picked this up, and a default that silently decides what ships with a binary is not something to lean on for a licence obligation. Nothing about the licence of godo itself changes: it stays MIT. --- .goreleaser.yaml | 7 ++ THIRD_PARTY_LICENSES.md | 236 ++++++++++++++++++++++++++++++++++++ docs/dev/plugin-protocol.md | 16 +++ 3 files changed, 259 insertions(+) create mode 100644 THIRD_PARTY_LICENSES.md diff --git a/.goreleaser.yaml b/.goreleaser.yaml index dc9a45b..3f1f2c7 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -28,6 +28,13 @@ builds: archives: - id: archive + # Default files are LICENSE*/README*/CHANGELOG*. The binaries statically + # link Apache-2.0 code (wazero), whose attribution has to ship with them. + files: + - LICENSE + - THIRD_PARTY_LICENSES.md + - README.md + - CHANGELOG.md formats: [tar.gz] format_overrides: - goos: windows diff --git a/THIRD_PARTY_LICENSES.md b/THIRD_PARTY_LICENSES.md new file mode 100644 index 0000000..2c9bd49 --- /dev/null +++ b/THIRD_PARTY_LICENSES.md @@ -0,0 +1,236 @@ +# Third-party licenses + +GoDo is MIT (see [LICENSE](./LICENSE)). The binaries shipped on GitHub Releases +statically link the libraries below, so their notices travel with them. + +Source dependencies are listed in [go.mod](./go.mod). + +--- + +## github.com/tetratelabs/wazero + +WebAssembly runtime. Used to run plugins. + +Licensed under the Apache License, Version 2.0. + +### NOTICE + +``` +wazero +Copyright 2020-2023 wazero authors +``` + +### License + +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020-2023 wazero authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` + +--- + +## gopkg.in/yaml.v3 + +YAML parser. Used to read `godo.yaml`. + +Licensed under the MIT License, with portions under the Apache License, +Version 2.0. See the module's `LICENSE` and `NOTICE` files. diff --git a/docs/dev/plugin-protocol.md b/docs/dev/plugin-protocol.md index 310f073..536624c 100644 --- a/docs/dev/plugin-protocol.md +++ b/docs/dev/plugin-protocol.md @@ -141,6 +141,22 @@ scripts: ?pnpm install ``` +## Licensing + +A plugin is a separate artifact from a separate repository, and godo does not +ship one. So godo's own licensing is unaffected by what a plugin contains — +but the repository that *does* ship one carries whatever is inside it. + +A repository distributing a `.wasm` is distributing everything compiled into +it. Whatever the interpreter or runtime inside it is licensed under, its notice +travels with the artifact. Check the build's own license inventory rather than +assuming, and ship the notices next to the `.wasm`, not only in the source +tree — the artifact is what people download. + +godo's own third-party notices are in +[THIRD_PARTY_LICENSES.md](../../THIRD_PARTY_LICENSES.md) and ship inside the +release archives. + ## Known limits - **Local paths only.** `source` is a file on this machine. Fetching belongs From 3d4ea48b736fd6c2d4b44667d10d447697722cd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=83=9E=E3=82=A4=E3=83=8E=E3=83=AB?= <97069334+MY-RV@users.noreply.github.com> Date: Sun, 20 Sep 2026 16:25:22 -0600 Subject: [PATCH 03/14] docs: say what third-party notices do and do not cover Adds the two sentences that were missing from the notices file: every licence in it is permissive, so none asks godo to change its own; and none of them reaches into a plugin, because a plugin is a separate artifact from a separate repository that godo does not ship. What a plugin carries is that repository's inventory to publish next to its own artifact. --- THIRD_PARTY_LICENSES.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/THIRD_PARTY_LICENSES.md b/THIRD_PARTY_LICENSES.md index 2c9bd49..a963c92 100644 --- a/THIRD_PARTY_LICENSES.md +++ b/THIRD_PARTY_LICENSES.md @@ -5,6 +5,11 @@ statically link the libraries below, so their notices travel with them. Source dependencies are listed in [go.mod](./go.mod). +Every licence here is permissive. None of them asks GoDo to change its own, and +none of them reaches into a plugin: a plugin is a separate artifact from a +separate repository, and GoDo does not ship one. What a plugin contains is +that repository's inventory to publish, next to its own artifact. + --- ## github.com/tetratelabs/wazero From d1d3016a50bffa589e6478bfbd196a262f897e45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=83=9E=E3=82=A4=E3=83=8E=E3=83=AB?= <97069334+MY-RV@users.noreply.github.com> Date: Sun, 20 Sep 2026 17:56:12 -0600 Subject: [PATCH 04/14] feat!: preview prints the body, and the sandbox grants only what config names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two things that were the same mistake. --preview used to start the plugin in a 'preview mode' and print what it chose to emit, which meant a plugin holding proc.exec could run commands during a preview and godo could only ask it not to. And the sandbox was described as shut without a way to open it, so a MicroPython body could not read the file next to the godo.yaml it belongs to. --preview now prints the body and does not start the plugin at all. A plugin body is a program, and the only faithful answer to 'what will this do' without running it is the program itself. Anything else is a guess, and a guess is useful but deserves its own flag rather than quietly borrowing this one — that is a later --predict. The mode field and the emit op are removed rather than left unused; plugins ignore fields they do not know, so --predict can add them back without breaking anything written today. The sandbox still starts shut — measured from inside a plugin with an empty config, the filesystem is ENOENT, the environment is empty, the clock is frozen at 2022, and there is no socket module. config now names what comes back: proc.exec for the op, fs.mount for a filesystem, time.wall for the real clock. fs.mount is a bool, not a path. What gets mounted is the directory the godo.yaml lives in, as the guest's root, so a script reaches the catalog it belongs to and nothing above it. A catalog that chose its own mount point could ask for / and the grant would mean nothing. go.mod moves to 1.26.0, the oldest Go release still receiving security fixes. The 1.22 it declared has been out of support for a while, and wazero v1.12 — the only version with the WebAssembly exception handling that a C interpreter compiled for WASI needs — made that visible rather than caused it. --- THIRD_PARTY_LICENSES.md | 10 +++ docs/dev/plugin-protocol.md | 51 +++++++++------ examples/plugins/lines/main.go | 35 +---------- go.mod | 7 ++- go.sum | 4 ++ internal/cli/plugin_e2e_test.go | 23 +++++-- internal/plugin/invoke.go | 39 +++++++++--- internal/plugin/invoke_test.go | 108 +++++++++++++++----------------- internal/plugin/protocol.go | 14 ++--- internal/plugin/runner.go | 39 ++++-------- internal/plugin/runtime.go | 10 ++- 11 files changed, 180 insertions(+), 160 deletions(-) diff --git a/THIRD_PARTY_LICENSES.md b/THIRD_PARTY_LICENSES.md index a963c92..233b77a 100644 --- a/THIRD_PARTY_LICENSES.md +++ b/THIRD_PARTY_LICENSES.md @@ -233,6 +233,16 @@ Copyright 2020-2023 wazero authors --- +## golang.org/x/sys + +Low-level OS primitives. Pulled in by wazero. + +Licensed under the BSD 3-Clause License, `Copyright (c) 2009 The Go Authors`. +Same terms as the Go standard library, whose runtime is already part of every +Go binary. + +--- + ## gopkg.in/yaml.v3 YAML parser. Used to read `godo.yaml`. diff --git a/docs/dev/plugin-protocol.md b/docs/dev/plugin-protocol.md index 536624c..b2696ad 100644 --- a/docs/dev/plugin-protocol.md +++ b/docs/dev/plugin-protocol.md @@ -14,6 +14,8 @@ godo → plugin stdin one JSON result per op that needs one plugin exits its exit code is the script's ``` +`--preview` never starts a plugin. It prints the body. + Being a command rather than a set of exported functions keeps the contract small: there is no memory-sharing ABI to get right, any language that targets WASI can write one, and a plugin can be tested as an ordinary program — @@ -24,9 +26,28 @@ echo '{"api":1,"mode":"preview","body":"echo hi","argv":{},"args":[]}' | ./plugi ## The sandbox -wazero's default, which is nothing: no filesystem, no network, no environment, -no clock. A plugin reaches the outside world only by asking godo, and godo only -honours what `engine.plugins[].config` grants. +It starts shut, and nothing here disables anything — nothing is granted. With +an empty `config`, measured from inside a plugin: + +``` +os.listdir(".") OSError [Errno 44] ENOENT +open("/etc/passwd") OSError [Errno 44] ENOENT +os.getenv("HOME") None +time.time() 1640995200.0 (frozen) +import socket the module is not there +``` + +`engine.plugins[].config` adds back exactly what it names, and nothing else. + +| Grant | Gives | +|-------|-------| +| `proc.exec` | the `exec` op | +| `fs.mount` | the catalog's own directory, as the guest's root | +| `time.wall` | the real clock instead of a frozen one | + +`fs.mount` is a bool, not a path: what gets mounted is the directory the +`godo.yaml` lives in, never somewhere the file names. A catalog that chose its +own mount point could ask for `/`, and the grant would mean nothing. ## Request @@ -35,7 +56,6 @@ One line, written once, before anything else. ```json { "api": 1, - "mode": "run", "runner": "lines", "body": "git status\n?pnpm install", "argv": {"BRANCH": "wt/example"}, @@ -47,7 +67,6 @@ One line, written once, before anything else. | Field | | |-------|--| | `api` | Protocol version. A plugin that speaks another major should exit non-zero rather than guess | -| `mode` | `"run"` or `"preview"` | | `runner` | The name the catalog asked for, so one plugin can provide several | | `body` | The script body, **verbatim**. `${godo:…}` is not expanded — that is shell-space syntax, and the values are on this request already | | `argv` | The matcher's captures | @@ -62,15 +81,11 @@ unbuffered, so a plugin that spoke first would deadlock. | Op | Answered | Needs | |----|----------|-------| | `exec` | yes | `config.proc.exec` | -| `emit` | **no** | nothing | ```json {"op":"exec","argv":["git","status"],"dir":"","capture":false} -{"op":"emit","line":"git status"} ``` -`emit` is one-way on purpose: a plugin that waited for an answer would hang. - ### Result ```json @@ -101,14 +116,14 @@ reads only what it gates on. ## Preview -The plugin's to answer. godo can render a shell line because it wrote it; it -cannot render a program it does not interpret, so it asks with `mode: -"preview"` and prints whatever `emit` lines come back. +`godo --preview` prints the body and **does not start the plugin**. Nothing is +compiled, nothing is instantiated, no grant is needed. -Which means a plugin **can** run things during a preview, because it is the one -holding the capability. A plugin that does is misbehaving, the same way a -`--dry-run` that writes is misbehaving. Grant `proc.exec` to plugins you trust -to tell the difference. +A plugin body is a program. The only faithful answer to "what will this do" +without running it is the program itself; anything else is a guess. A guess is +useful, but it deserves its own flag rather than quietly borrowing this one — +that is a later `--predict`, which will add back a mode field and an output op. +Plugins ignore fields they do not know, so nothing written today breaks then. ## Exit codes @@ -164,5 +179,5 @@ release archives. - **Size.** A Go plugin carries Go's runtime — the example is ~4.5 MB. TinyGo or a C-family language produces far smaller wasm. - **One instantiation per step.** Fine at godo's scale; it is not a server. -- **`exec` only.** No spawn, no filesystem ops. Those are the next capabilities - and the reason `config` is shaped as sections. +- **`exec` only.** No spawn. Filesystem access is the mount, not an op. + `config` is shaped as sections so more can be added without moving anything. diff --git a/examples/plugins/lines/main.go b/examples/plugins/lines/main.go index 863123f..039b682 100644 --- a/examples/plugins/lines/main.go +++ b/examples/plugins/lines/main.go @@ -12,6 +12,9 @@ // starting with "?" may fail without stopping the rest. ${NAME} is replaced // with a capture; $1, $2, … with a leftover argument. // +// There is no preview branch: godo prints the body itself and never starts a +// plugin for --preview. +// // Build: // // GOOS=wasip1 GOARCH=wasm go build -o lines.wasm ./examples/plugins/lines @@ -32,7 +35,6 @@ import ( // request is godo's opening line. Only the fields this plugin uses. type request struct { API int `json:"api"` - Mode string `json:"mode"` Runner string `json:"runner"` Body string `json:"body"` Argv map[string]string `json:"argv"` @@ -43,7 +45,6 @@ type op struct { Op string `json:"op"` Argv []string `json:"argv,omitempty"` Capture bool `json:"capture,omitempty"` - Line string `json:"line,omitempty"` } type result struct { @@ -83,14 +84,6 @@ func main() { continue } - // Preview is the plugin's business: godo does not know what a body of - // this shape would do, so it asks, and this answers by describing the - // commands rather than running them. - if req.Mode == "preview" { - emit(out, previewLine(argv, mayFail)) - continue - } - res, err := exec(in, out, argv) if err != nil { fail(err) @@ -180,22 +173,6 @@ func expandWord(word string, req request) (string, error) { return b.String(), nil } -func previewLine(argv []string, mayFail bool) string { - quoted := make([]string, len(argv)) - for i, a := range argv { - if strings.ContainsAny(a, " \t'\"") { - quoted[i] = "'" + strings.ReplaceAll(a, "'", `'\''`) + "'" - } else { - quoted[i] = a - } - } - line := strings.Join(quoted, " ") - if mayFail { - line += " # may fail" - } - return line -} - func exec(in *bufio.Scanner, out *json.Encoder, argv []string) (result, error) { if err := out.Encode(op{Op: "exec", Argv: argv}); err != nil { return result{}, err @@ -213,12 +190,6 @@ func exec(in *bufio.Scanner, out *json.Encoder, argv []string) (result, error) { return res, nil } -func emit(out *json.Encoder, line string) { - if err := out.Encode(op{Op: "emit", Line: line}); err != nil { - fail(err) - } -} - func fail(err error) { fmt.Fprintln(os.Stderr, "lines:", err) os.Exit(1) diff --git a/go.mod b/go.mod index e358b68..1354bf4 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,10 @@ module github.com/my-rv/godo -go 1.22.0 +go 1.26.0 require gopkg.in/yaml.v3 v3.0.1 -require github.com/tetratelabs/wazero v1.9.0 // indirect +require ( + github.com/tetratelabs/wazero v1.12.0 // indirect + golang.org/x/sys v0.44.0 // indirect +) diff --git a/go.sum b/go.sum index 8a3a081..d385a7e 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZBf/I= github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM= +github.com/tetratelabs/wazero v1.12.0 h1:DuWcpNu/FzgEXgGBDp8J1Spc+CWOvvtvVyjKlaZopYU= +github.com/tetratelabs/wazero v1.12.0/go.mod h1:LvKtzl2RqO4gyF27BiXU+nKAjcV8f38U+kP/q2vgxh0= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/cli/plugin_e2e_test.go b/internal/cli/plugin_e2e_test.go index 8a03acc..0dc3c97 100644 --- a/internal/cli/plugin_e2e_test.go +++ b/internal/cli/plugin_e2e_test.go @@ -87,16 +87,19 @@ func TestE2E_pluginRunnerRunsTheScript(t *testing.T) { } } -// Preview is the plugin's to answer: godo cannot render a body it does not -// interpret, so it asks and prints what comes back. -func TestE2E_pluginRendersItsOwnPreview(t *testing.T) { +// --preview prints the body and never starts the plugin. +// +// A plugin body is a program; the only faithful answer to "what will this do" +// without running it is the program itself. Guessing the flow is a separate +// flag's job, not this one's. +func TestE2E_previewPrintsTheBodyAndStartsNothing(t *testing.T) { cwd := pluginCatalog(t, true, " # @runner lines\n boot: |\n touch one\n ?false\n") app, out, _ := e2eApp(t, cwd) if err := app.Run([]string{"--preview", "boot"}); err != nil { t.Fatal(err) } got := strings.TrimSpace(out.String()) - if got != "touch one\nfalse # may fail" { + if got != "touch one\n?false" { t.Fatalf("preview=%q", got) } if _, err := os.Stat(filepath.Join(cwd, "one")); err == nil { @@ -104,6 +107,18 @@ func TestE2E_pluginRendersItsOwnPreview(t *testing.T) { } } +// Preview works with nothing granted at all: it never reaches the sandbox. +func TestE2E_previewNeedsNoGrants(t *testing.T) { + cwd := pluginCatalog(t, false, " # @runner lines\n boot: |\n touch one\n") + app, out, _ := e2eApp(t, cwd) + if err := app.Run([]string{"--preview", "boot"}); err != nil { + t.Fatal(err) + } + if strings.TrimSpace(out.String()) != "touch one" { + t.Fatalf("preview=%q", out.String()) + } +} + // Captures and leftover args reach the plugin as data, not pasted into a line. func TestE2E_pluginReceivesCapturesAndArgs(t *testing.T) { cwd := pluginCatalog(t, true, diff --git a/internal/plugin/invoke.go b/internal/plugin/invoke.go index 5817080..4997960 100644 --- a/internal/plugin/invoke.go +++ b/internal/plugin/invoke.go @@ -30,8 +30,6 @@ type Host struct { type Outcome struct { // Code is the plugin's exit code, and so the script's. Code int - // Emitted holds the lines the plugin contributed to --preview. - Emitted []string } // Invoke runs the plugin once. @@ -51,13 +49,23 @@ func (p *Plugin) Invoke(ctx context.Context, req Request, host Host) (Outcome, e stderr = os.Stderr } + // The sandbox starts shut: no filesystem, no environment, no network, and + // a frozen clock. Nothing is disabled here — nothing is granted. What the + // catalog's config asks for is added back below, and only that. cfg := wazero.NewModuleConfig(). WithStdin(inR). WithStdout(outW). WithStderr(stderr). WithArgs("plugin"). - // No WithFS, no WithEnv, no WithSysWalltime: the sandbox stays shut. WithName("") + if dir := p.mountDir(host.Dir); dir != "" { + // Scoped to one directory, which becomes the guest's root. A script + // can reach the catalog it belongs to and nothing above it. + cfg = cfg.WithFSConfig(wazero.NewFSConfig().WithDirMount(dir, "/")) + } + if p.granted("time", "wall") { + cfg = cfg.WithSysWalltime().WithSysNanotime() + } runErr := make(chan error, 1) go func() { @@ -80,7 +88,7 @@ func (p *Plugin) Invoke(ctx context.Context, req Request, host Host) (Outcome, e }() out := Outcome{} - loopErr := p.serve(outR, inW, host, &out) + loopErr := p.serve(outR, inW, host) _ = inW.Close() _ = inR.Close() @@ -107,7 +115,7 @@ func (p *Plugin) Invoke(ctx context.Context, req Request, host Host) (Outcome, e } // serve answers ops until the plugin's stdout closes. -func (p *Plugin) serve(r io.Reader, w io.Writer, host Host, out *Outcome) error { +func (p *Plugin) serve(r io.Reader, w io.Writer, host Host) error { scan := bufio.NewScanner(r) scan.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) enc := json.NewEncoder(w) @@ -122,10 +130,6 @@ func (p *Plugin) serve(r io.Reader, w io.Writer, host Host, out *Outcome) error return fmt.Errorf("plugin %s: unreadable op %q: %w", p.Source, line, err) } switch op.Op { - case OpEmit: - // Preview output. No answer: a plugin that waited for one here - // would hang, so emit is deliberately one-way. - out.Emitted = append(out.Emitted, op.Line) case OpExec: res := p.execOp(op, host) if err := enc.Encode(res); err != nil { @@ -196,6 +200,23 @@ func orStd(w io.Writer, def io.Writer) io.Writer { return w } +// mountDir returns the directory to mount, or "" for no filesystem. +// +// config.fs.mount is a bool rather than a path: what gets mounted is the +// catalog's own directory, never somewhere the catalog names. A file that +// picks its own mount point could ask for "/" and the grant would mean +// nothing. +func (p *Plugin) mountDir(catalogDir string) string { return p.MountedDir(catalogDir) } + +// MountedDir reports which directory this plugin would be given, or "" for +// none. Exported so a caller can show the grant without invoking anything. +func (p *Plugin) MountedDir(catalogDir string) string { + if catalogDir == "" || !p.granted("fs", "mount") { + return "" + } + return catalogDir +} + // granted reports whether config grants section.key. // // Deny by default: an absent section, an absent key, or anything that is not diff --git a/internal/plugin/invoke_test.go b/internal/plugin/invoke_test.go index 26b2efc..9312aa1 100644 --- a/internal/plugin/invoke_test.go +++ b/internal/plugin/invoke_test.go @@ -110,59 +110,12 @@ func TestLoad_refusesSomethingThatIsNotWasm(t *testing.T) { } } -func TestInvoke_previewAsksThePlugin(t *testing.T) { - p, done := load(t, nil) - defer done() - - out, err := p.Invoke(context.Background(), plugin.Request{ - Mode: "preview", - Runner: "lines", - Body: "git status\n?pnpm install\necho hola ${WHO} $1", - Argv: map[string]string{"WHO": "mundo"}, - Args: []string{"un arg"}, - }, plugin.Host{}) - if err != nil { - t.Fatal(err) - } - if out.Code != 0 { - t.Fatalf("code=%d", out.Code) - } - want := []string{"git status", "pnpm install # may fail", "echo hola mundo 'un arg'"} - if len(out.Emitted) != len(want) { - t.Fatalf("emitted=%q", out.Emitted) - } - for i := range want { - if out.Emitted[i] != want[i] { - t.Fatalf("emitted=%q want %q", out.Emitted, want) - } - } -} - -// Preview must not run anything, and with no config granted it could not. -func TestInvoke_previewGrantsNothing(t *testing.T) { - dir := t.TempDir() - p, done := load(t, nil) - defer done() - - _, err := p.Invoke(context.Background(), plugin.Request{ - Mode: "preview", - Body: "touch marker", - }, plugin.Host{Dir: dir}) - if err != nil { - t.Fatal(err) - } - if _, err := os.Stat(filepath.Join(dir, "marker")); err == nil { - t.Fatal("preview touched the disk") - } -} - func TestInvoke_execRunsWhenGranted(t *testing.T) { dir := t.TempDir() p, done := load(t, map[string]any{"proc": map[string]any{"exec": true}}) defer done() out, err := p.Invoke(context.Background(), plugin.Request{ - Mode: "run", Body: "touch marker\ntouch ${NAME}", Argv: map[string]string{"NAME": "second"}, }, plugin.Host{Dir: dir}) @@ -186,7 +139,6 @@ func TestInvoke_execRefusedWhenNotGranted(t *testing.T) { defer done() out, err := p.Invoke(context.Background(), plugin.Request{ - Mode: "run", Body: "touch marker", }, plugin.Host{Dir: dir}) if err != nil { @@ -209,7 +161,7 @@ func TestInvoke_configMustSayTrue(t *testing.T) { {"fs": map[string]any{"exec": true}}, } { p, done := load(t, cfg) - out, err := p.Invoke(context.Background(), plugin.Request{Mode: "run", Body: "touch marker"}, plugin.Host{Dir: dir}) + out, err := p.Invoke(context.Background(), plugin.Request{Body: "touch marker"}, plugin.Host{Dir: dir}) done() if err != nil { t.Fatalf("%v: %v", cfg, err) @@ -229,7 +181,6 @@ func TestInvoke_exitCodeIsTheChilds(t *testing.T) { defer done() out, err := p.Invoke(context.Background(), plugin.Request{ - Mode: "run", Body: "sh -c ${SCRIPT}\ntouch should-not-exist", Argv: map[string]string{"SCRIPT": "exit 42"}, }, plugin.Host{Dir: t.TempDir()}) @@ -248,7 +199,6 @@ func TestInvoke_optionalLineDoesNotStopTheBody(t *testing.T) { defer done() out, err := p.Invoke(context.Background(), plugin.Request{ - Mode: "run", Body: "?sh -c ${SCRIPT}\ntouch after", Argv: map[string]string{"SCRIPT": "exit 3"}, }, plugin.Host{Dir: dir}) @@ -263,17 +213,57 @@ func TestInvoke_optionalLineDoesNotStopTheBody(t *testing.T) { } } -// The sandbox is shut: the plugin has no filesystem of its own. -func TestInvoke_pluginCannotReachTheFilesystemItself(t *testing.T) { - p, done := load(t, nil) +// The sandbox starts shut: with nothing granted the plugin has no filesystem. +func TestInvoke_noFilesystemUnlessGranted(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "secreto.txt"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + p, done := load(t, map[string]any{"proc": map[string]any{"exec": true}}) defer done() - // The example plugin never opens a file, so this asserts the configuration - // rather than the plugin: no WithFS was granted, so nothing is mounted. - out, err := p.Invoke(context.Background(), plugin.Request{Mode: "preview", Body: "echo hi"}, plugin.Host{}) + + // "ls" here runs on the host through exec, which is granted; what is not + // granted is the guest seeing the directory itself. + out, err := p.Invoke(context.Background(), plugin.Request{Body: "true"}, plugin.Host{Dir: dir}) if err != nil { t.Fatal(err) } - if len(out.Emitted) != 1 { - t.Fatalf("emitted=%q", out.Emitted) + if out.Code != 0 { + t.Fatalf("code=%d", out.Code) + } +} + +// config.fs.mount opens exactly one directory: the catalog's, as the guest root. +func TestInvoke_mountIsGatedAndScoped(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "presente.txt"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + cfg map[string]any + mount bool + }{ + {"sin config", nil, false}, + {"mount false", map[string]any{"fs": map[string]any{"mount": false}}, false}, + {"mount true", map[string]any{"fs": map[string]any{"mount": true}}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := map[string]any{"proc": map[string]any{"exec": true}} + for k, v := range tc.cfg { + cfg[k] = v + } + p, done := load(t, cfg) + defer done() + // The example plugin does not read files, so this asserts the + // wiring: a mount that is not granted is simply not configured. + got := p.MountedDir(dir) + if tc.mount && got != dir { + t.Fatalf("granted mount did not resolve: %q", got) + } + if !tc.mount && got != "" { + t.Fatalf("mount granted without config: %q", got) + } + }) } } diff --git a/internal/plugin/protocol.go b/internal/plugin/protocol.go index ec4d74d..615d3a9 100644 --- a/internal/plugin/protocol.go +++ b/internal/plugin/protocol.go @@ -20,11 +20,13 @@ package plugin const APIVersion = 1 // Request is the single line godo writes to a plugin's stdin. +// +// There is no mode: a plugin is invoked to run. --preview prints the body +// without starting anything, so nothing here has to describe a dry run. A +// later --predict will add a field for it; plugins ignore fields they do not +// know, so that costs nothing today. type Request struct { API int `json:"api"` - // Mode is "run" or "preview". What preview means is the plugin's call: - // it knows what its own bodies do, and godo does not. - Mode string `json:"mode"` // Runner is the name the catalog asked for, so one plugin can provide // several. Runner string `json:"runner"` @@ -46,9 +48,6 @@ type Op struct { Argv []string `json:"argv,omitempty"` Dir string `json:"dir,omitempty"` Capture bool `json:"capture,omitempty"` - - // emit - Line string `json:"line,omitempty"` } // Result is godo's answer to an op that needs one. @@ -66,7 +65,4 @@ type Result struct { const ( // OpExec runs an argument vector and waits. Needs config proc.exec. OpExec = "exec" - // OpEmit contributes one line to --preview output. Needs nothing, and - // gets no answer. - OpEmit = "emit" ) diff --git a/internal/plugin/runner.go b/internal/plugin/runner.go index 8381fef..c002580 100644 --- a/internal/plugin/runner.go +++ b/internal/plugin/runner.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "strings" "github.com/my-rv/godo/internal/catalog" ) @@ -35,7 +36,12 @@ func (Runner) AcceptsArgs([]string) bool { return true } // RunInvocation implements catalog.InvocationRunner. func (r Runner) RunInvocation(inv catalog.Invocation) error { - out, err := r.invoke(inv, "run") + out, err := r.Plugin.Invoke(context.Background(), Request{ + Runner: string(r.Name), + Body: inv.Body, + Argv: nonNil(inv.Captures), + Args: inv.Args, + }, r.Host) if err != nil { return err } @@ -48,33 +54,14 @@ func (r Runner) RunInvocation(inv catalog.Invocation) error { return nil } -// PreviewInvocation implements catalog.InvocationRunner. +// PreviewInvocation implements catalog.InvocationRunner: the body, verbatim. // -// The plugin decides what preview means for its own bodies. A plugin that -// emits nothing previews as nothing, and one that refuses to preview says so -// by exiting non-zero. +// --preview does not start the plugin. A plugin body is a program, and the +// only faithful answer to "what will this do" without running it is the +// program itself. Anything else would be a guess dressed as a fact, and a +// guess needs its own flag rather than quietly borrowing this one. func (r Runner) PreviewInvocation(inv catalog.Invocation) ([]string, error) { - out, err := r.invoke(inv, "preview") - if err != nil { - return nil, err - } - if out.Code != 0 { - return nil, &catalog.ExitError{ - Code: out.Code, - Message: fmt.Sprintf("script %q: runner %q could not preview it", inv.Script, r.Name), - } - } - return out.Emitted, nil -} - -func (r Runner) invoke(inv catalog.Invocation, mode string) (Outcome, error) { - return r.Plugin.Invoke(context.Background(), Request{ - Mode: mode, - Runner: string(r.Name), - Body: inv.Body, - Argv: nonNil(inv.Captures), - Args: inv.Args, - }, r.Host) + return strings.Split(strings.TrimRight(inv.Body, "\n"), "\n"), nil } func nonNil(m map[string]string) map[string]string { diff --git a/internal/plugin/runtime.go b/internal/plugin/runtime.go index bac6fac..8a9205d 100644 --- a/internal/plugin/runtime.go +++ b/internal/plugin/runtime.go @@ -11,6 +11,8 @@ import ( "sync" "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/experimental" "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1" ) @@ -21,8 +23,14 @@ type Runtime struct { } // NewRuntime returns a runtime with WASI available and nothing else. +// +// Exception handling is enabled because a C interpreter compiled for WASI uses +// it for setjmp/longjmp — MicroPython's non-local return is built on it, and +// without the feature its module does not even compile ("tag section not +// supported"). It costs nothing for a plugin that does not use it. func NewRuntime(ctx context.Context) *Runtime { - rt := wazero.NewRuntime(ctx) + rt := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfig(). + WithCoreFeatures(api.CoreFeaturesV2|experimental.CoreFeaturesExceptionHandling)) wasi_snapshot_preview1.MustInstantiate(ctx, rt) return &Runtime{rt: rt} } From 4c3cd265f328d37d6b26bfe0492f5b3dac98901f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=83=9E=E3=82=A4=E3=83=8E=E3=83=AB?= <97069334+MY-RV@users.noreply.github.com> Date: Sun, 20 Sep 2026 18:47:17 -0600 Subject: [PATCH 05/14] feat: add the out and slink ops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A plugin's script printing had nowhere to go: stdout is the protocol channel, so a print would corrupt it. The out op carries a line to the host's stdout and is deliberately one-way — a plugin that waited for an answer would hang. It needs no capability, because printing is not a side effect on the machine. slink is the one filesystem operation a mounted directory does not solve. os.symlink is not portable: Unix makes a symlink and Windows wants a junction, and a script cannot paper over that itself. It needs config fs.slink. Paths resolve against the catalog's directory but are not confined to it. The first draft confined them, which broke the case the op exists for: a git worktree is created beside a repository, not inside it, and linking .env into it is the whole point. Confinement would also have been theatre — proc.exec can run 'ln -s' anywhere, so a slink narrower than exec protects nothing. The grant is the boundary; withhold it from a plugin you would not hand a shell. An absolute path is taken as given rather than reinterpreted: a script naming /tmp means /tmp. --- docs/dev/plugin-protocol.md | 20 +++- internal/plugin/invoke.go | 51 +++++++++ internal/plugin/invoke_test.go | 185 +++++++++++++++++++++++++++++---- internal/plugin/protocol.go | 12 +++ 4 files changed, 246 insertions(+), 22 deletions(-) diff --git a/docs/dev/plugin-protocol.md b/docs/dev/plugin-protocol.md index b2696ad..35b7c01 100644 --- a/docs/dev/plugin-protocol.md +++ b/docs/dev/plugin-protocol.md @@ -21,7 +21,7 @@ small: there is no memory-sharing ABI to get right, any language that targets WASI can write one, and a plugin can be tested as an ordinary program — ```bash -echo '{"api":1,"mode":"preview","body":"echo hi","argv":{},"args":[]}' | ./plugin +echo '{"api":1,"body":"echo hi","argv":{},"args":[]}' | ./plugin ``` ## The sandbox @@ -43,12 +43,21 @@ import socket the module is not there |-------|-------| | `proc.exec` | the `exec` op | | `fs.mount` | the catalog's own directory, as the guest's root | +| `fs.slink` | the `slink` op | +| `fs.slink` | the `slink` op | | `time.wall` | the real clock instead of a frozen one | `fs.mount` is a bool, not a path: what gets mounted is the directory the `godo.yaml` lives in, never somewhere the file names. A catalog that chose its own mount point could ask for `/`, and the grant would mean nothing. +`fs.slink` is not confined the same way. A relative path resolves against the +catalog's directory, but it may leave it — a git worktree is created *beside* a +repository, and linking into it is the use case. Confining it would also be +theatre: `proc.exec` can run `ln -s` anywhere, so a `slink` narrower than +`exec` protects nothing. The grant is the boundary; withhold it from a plugin +you would not hand a shell. + ## Request One line, written once, before anything else. @@ -81,9 +90,13 @@ unbuffered, so a plugin that spoke first would deadlock. | Op | Answered | Needs | |----|----------|-------| | `exec` | yes | `config.proc.exec` | +| `out` | no | — | +| `slink` | yes | `config.fs.slink` | ```json {"op":"exec","argv":["git","status"],"dir":"","capture":false} +{"op":"out","text":"updated dependencies"} +{"op":"slink","src":"bin/godo","dst":".bin/godo","force":false} ``` ### Result @@ -179,5 +192,6 @@ release archives. - **Size.** A Go plugin carries Go's runtime — the example is ~4.5 MB. TinyGo or a C-family language produces far smaller wasm. - **One instantiation per step.** Fine at godo's scale; it is not a server. -- **`exec` only.** No spawn. Filesystem access is the mount, not an op. - `config` is shaped as sections so more can be added without moving anything. +- **No spawn.** Filesystem access is limited to the mount and `slink` within + the catalog. `config` is shaped as sections so more can be added without + moving anything. diff --git a/internal/plugin/invoke.go b/internal/plugin/invoke.go index 4997960..2a32f24 100644 --- a/internal/plugin/invoke.go +++ b/internal/plugin/invoke.go @@ -9,6 +9,7 @@ import ( "io" "os" "os/exec" + "path/filepath" "strings" "github.com/tetratelabs/wazero" @@ -130,11 +131,20 @@ func (p *Plugin) serve(r io.Reader, w io.Writer, host Host) error { return fmt.Errorf("plugin %s: unreadable op %q: %w", p.Source, line, err) } switch op.Op { + case OpOut: + if _, err := fmt.Fprintln(orStd(host.Stdout, os.Stdout), op.Text); err != nil { + return fmt.Errorf("plugin %s: %w", p.Source, err) + } case OpExec: res := p.execOp(op, host) if err := enc.Encode(res); err != nil { return fmt.Errorf("plugin %s: %w", p.Source, err) } + case OpSlink: + res := p.slinkOp(op, host) + if err := enc.Encode(res); err != nil { + return fmt.Errorf("plugin %s: %w", p.Source, err) + } default: if err := enc.Encode(Result{Error: fmt.Sprintf("unknown op %q", op.Op)}); err != nil { return fmt.Errorf("plugin %s: %w", p.Source, err) @@ -178,6 +188,47 @@ func (p *Plugin) execOp(op Op, host Host) Result { return res } +func (p *Plugin) slinkOp(op Op, host Host) Result { + if !p.granted("fs", "slink") { + return Result{Error: "not granted: fs.slink — enable it under engine.plugins[].config.fs.slink"} + } + // Paths resolve against the catalog's directory but are not confined to + // it. A worktree is created beside a repository, not inside it, and the + // link into it is the point. Confinement here would also be theatre: + // proc.exec can run "ln -s" anywhere, so a slink narrower than exec + // protects nothing. The grant is the boundary — withhold fs.slink from a + // plugin you would not hand a shell. + src := resolveAgainst(host.Dir, op.Src) + dst := resolveAgainst(host.Dir, op.Dst) + + if _, err := os.Lstat(dst); err == nil { + if !op.Force { + return Result{Error: fmt.Sprintf("slink: dst exists: %s", op.Dst)} + } + if err := os.Remove(dst); err != nil { + return Result{Error: fmt.Sprintf("slink: remove dst: %v", err)} + } + } else if !errors.Is(err, os.ErrNotExist) { + return Result{Error: fmt.Sprintf("slink: dst: %v", err)} + } + + // Windows support for os.Symlink is unverified on a real Windows host. + if err := os.Symlink(src, dst); err != nil { + return Result{Error: fmt.Sprintf("slink: %v", err)} + } + return Result{Code: 0, OK: true} +} + +// resolveAgainst turns a plugin-supplied path into an absolute one. +// +// An absolute path is taken as given: a script that names /tmp means /tmp. +func resolveAgainst(dir, path string) string { + if filepath.IsAbs(path) || dir == "" { + return path + } + return filepath.Join(dir, path) +} + func runAndCode(cmd *exec.Cmd, res *Result) int { err := cmd.Run() if err == nil { diff --git a/internal/plugin/invoke_test.go b/internal/plugin/invoke_test.go index 9312aa1..5c5b00f 100644 --- a/internal/plugin/invoke_test.go +++ b/internal/plugin/invoke_test.go @@ -1,15 +1,15 @@ -package plugin_test +package plugin import ( + "bytes" "context" + "encoding/json" "os" "os/exec" "path/filepath" "strings" "sync" "testing" - - "github.com/my-rv/godo/internal/plugin" ) var ( @@ -59,15 +59,15 @@ func exampleWasm(t *testing.T) string { return wasmPath } -func load(t *testing.T, config map[string]any) (*plugin.Plugin, func()) { +func load(t *testing.T, config map[string]any) (*Plugin, func()) { t.Helper() path := exampleWasm(t) - digest, err := plugin.Digest(path) + digest, err := Digest(path) if err != nil { t.Fatal(err) } ctx := context.Background() - rt := plugin.NewRuntime(ctx) + rt := NewRuntime(ctx) p, err := rt.Load(ctx, path, digest, "", []string{"runner:lines"}, config) if err != nil { t.Fatal(err) @@ -78,7 +78,7 @@ func load(t *testing.T, config map[string]any) (*plugin.Plugin, func()) { func TestLoad_refusesAWrongDigest(t *testing.T) { path := exampleWasm(t) ctx := context.Background() - rt := plugin.NewRuntime(ctx) + rt := NewRuntime(ctx) defer rt.Close(ctx) _, err := rt.Load(ctx, path, strings.Repeat("0", 64), "", nil, nil) @@ -98,12 +98,12 @@ func TestLoad_refusesSomethingThatIsNotWasm(t *testing.T) { if err := os.WriteFile(path, []byte("hello"), 0o644); err != nil { t.Fatal(err) } - digest, err := plugin.Digest(path) + digest, err := Digest(path) if err != nil { t.Fatal(err) } ctx := context.Background() - rt := plugin.NewRuntime(ctx) + rt := NewRuntime(ctx) defer rt.Close(ctx) if _, err := rt.Load(ctx, path, digest, "", nil, nil); err == nil { t.Fatal("want a compile error") @@ -115,10 +115,10 @@ func TestInvoke_execRunsWhenGranted(t *testing.T) { p, done := load(t, map[string]any{"proc": map[string]any{"exec": true}}) defer done() - out, err := p.Invoke(context.Background(), plugin.Request{ + out, err := p.Invoke(context.Background(), Request{ Body: "touch marker\ntouch ${NAME}", Argv: map[string]string{"NAME": "second"}, - }, plugin.Host{Dir: dir}) + }, Host{Dir: dir}) if err != nil { t.Fatal(err) } @@ -138,9 +138,9 @@ func TestInvoke_execRefusedWhenNotGranted(t *testing.T) { p, done := load(t, nil) defer done() - out, err := p.Invoke(context.Background(), plugin.Request{ + out, err := p.Invoke(context.Background(), Request{ Body: "touch marker", - }, plugin.Host{Dir: dir}) + }, Host{Dir: dir}) if err != nil { t.Fatal(err) } @@ -161,7 +161,7 @@ func TestInvoke_configMustSayTrue(t *testing.T) { {"fs": map[string]any{"exec": true}}, } { p, done := load(t, cfg) - out, err := p.Invoke(context.Background(), plugin.Request{Body: "touch marker"}, plugin.Host{Dir: dir}) + out, err := p.Invoke(context.Background(), Request{Body: "touch marker"}, Host{Dir: dir}) done() if err != nil { t.Fatalf("%v: %v", cfg, err) @@ -180,10 +180,10 @@ func TestInvoke_exitCodeIsTheChilds(t *testing.T) { p, done := load(t, map[string]any{"proc": map[string]any{"exec": true}}) defer done() - out, err := p.Invoke(context.Background(), plugin.Request{ + out, err := p.Invoke(context.Background(), Request{ Body: "sh -c ${SCRIPT}\ntouch should-not-exist", Argv: map[string]string{"SCRIPT": "exit 42"}, - }, plugin.Host{Dir: t.TempDir()}) + }, Host{Dir: t.TempDir()}) if err != nil { t.Fatal(err) } @@ -198,10 +198,10 @@ func TestInvoke_optionalLineDoesNotStopTheBody(t *testing.T) { p, done := load(t, map[string]any{"proc": map[string]any{"exec": true}}) defer done() - out, err := p.Invoke(context.Background(), plugin.Request{ + out, err := p.Invoke(context.Background(), Request{ Body: "?sh -c ${SCRIPT}\ntouch after", Argv: map[string]string{"SCRIPT": "exit 3"}, - }, plugin.Host{Dir: dir}) + }, Host{Dir: dir}) if err != nil { t.Fatal(err) } @@ -224,7 +224,7 @@ func TestInvoke_noFilesystemUnlessGranted(t *testing.T) { // "ls" here runs on the host through exec, which is granted; what is not // granted is the guest seeing the directory itself. - out, err := p.Invoke(context.Background(), plugin.Request{Body: "true"}, plugin.Host{Dir: dir}) + out, err := p.Invoke(context.Background(), Request{Body: "true"}, Host{Dir: dir}) if err != nil { t.Fatal(err) } @@ -267,3 +267,150 @@ func TestInvoke_mountIsGatedAndScoped(t *testing.T) { }) } } + +func TestServe_outWritesWithoutAnswer(t *testing.T) { + var stdout, answers bytes.Buffer + p := &Plugin{} + if err := p.serve(strings.NewReader(`{"op":"out","text":"hello"}`+"\n"), &answers, Host{Stdout: &stdout}); err != nil { + t.Fatal(err) + } + if got := stdout.String(); got != "hello\n" { + t.Fatalf("stdout=%q", got) + } + if got := answers.String(); got != "" { + t.Fatalf("out unexpectedly received an answer: %q", got) + } +} + +func TestServe_slinkCreatesLinkWhenGranted(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "source"), []byte("linked"), 0o644); err != nil { + t.Fatal(err) + } + p := &Plugin{Config: map[string]any{"fs": map[string]any{"slink": true}}} + res := serveOp(t, p, Host{Dir: dir}, Op{Op: OpSlink, Src: "source", Dst: "link"}) + if !res.OK || res.Code != 0 || res.Error != "" { + t.Fatalf("result=%+v", res) + } + data, err := os.ReadFile(filepath.Join(dir, "link")) + if err != nil { + t.Fatal(err) + } + if got := string(data); got != "linked" { + t.Fatalf("linked content=%q", got) + } +} + +func TestServe_slinkRefusesWithoutGrant(t *testing.T) { + dir := t.TempDir() + p := &Plugin{} + res := serveOp(t, p, Host{Dir: dir}, Op{Op: OpSlink, Src: "source", Dst: "link"}) + const want = "not granted: fs.slink — enable it under engine.plugins[].config.fs.slink" + if res.Error != want { + t.Fatalf("error=%q, want %q", res.Error, want) + } + if _, err := os.Lstat(filepath.Join(dir, "link")); !os.IsNotExist(err) { + t.Fatalf("ungranted slink created dst: %v", err) + } +} + +// The catalog's neighbour is reachable on purpose: a git worktree is created +// beside a repository, and linking .env into it is the whole use case. +// +// Confinement here would also be theatre — proc.exec can run "ln -s" anywhere, +// so a slink narrower than exec protects nothing. The grant is the boundary. +func TestServe_slinkReachesBesideTheCatalog(t *testing.T) { + root := t.TempDir() + catalog := filepath.Join(root, "repo") + neighbour := filepath.Join(root, "repo-wt") + for _, d := range []string{catalog, neighbour} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(catalog, ".env"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + p := &Plugin{Config: map[string]any{"fs": map[string]any{"slink": true}}} + + res := serveOp(t, p, Host{Dir: catalog}, Op{Op: OpSlink, Src: ".env", Dst: "../repo-wt/.env"}) + if res.Error != "" { + t.Fatalf("error=%q", res.Error) + } + if !res.OK { + t.Fatal("slink reported failure") + } + if _, err := os.Lstat(filepath.Join(neighbour, ".env")); err != nil { + t.Fatalf("the link was not created: %v", err) + } +} + +// An absolute path is taken as given rather than reinterpreted. +func TestServe_slinkAcceptsAnAbsolutePath(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "source") + if err := os.WriteFile(target, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + p := &Plugin{Config: map[string]any{"fs": map[string]any{"slink": true}}} + + res := serveOp(t, p, Host{Dir: dir}, Op{Op: OpSlink, Src: target, Dst: "link"}) + if res.Error != "" { + t.Fatalf("error=%q", res.Error) + } + got, err := os.Readlink(filepath.Join(dir, "link")) + if err != nil { + t.Fatal(err) + } + if got != target { + t.Fatalf("link points at %q, want %q", got, target) + } +} + +func TestServe_slinkForceReplacesExistingDestination(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "source"), []byte("replacement"), 0o644); err != nil { + t.Fatal(err) + } + dst := filepath.Join(dir, "link") + if err := os.Symlink("missing", dst); err != nil { + t.Fatal(err) + } + p := &Plugin{Config: map[string]any{"fs": map[string]any{"slink": true}}} + res := serveOp(t, p, Host{Dir: dir}, Op{Op: OpSlink, Src: "source", Dst: "link"}) + if res.Error == "" { + t.Fatal("force=false replaced an existing destination") + } + if target, err := os.Readlink(dst); err != nil || target != "missing" { + t.Fatalf("force=false changed destination: target=%q err=%v", target, err) + } + + res = serveOp(t, p, Host{Dir: dir}, Op{Op: OpSlink, Src: "source", Dst: "link", Force: true}) + if !res.OK || res.Code != 0 || res.Error != "" { + t.Fatalf("result=%+v", res) + } + data, err := os.ReadFile(dst) + if err != nil { + t.Fatal(err) + } + if got := string(data); got != "replacement" { + t.Fatalf("replacement content=%q", got) + } +} + +func serveOp(t *testing.T, p *Plugin, host Host, op Op) Result { + t.Helper() + line, err := json.Marshal(op) + if err != nil { + t.Fatal(err) + } + var answers bytes.Buffer + if err := p.serve(bytes.NewReader(append(line, '\n')), &answers, host); err != nil { + t.Fatal(err) + } + var res Result + if err := json.NewDecoder(&answers).Decode(&res); err != nil { + t.Fatalf("decode result %q: %v", answers.String(), err) + } + return res +} diff --git a/internal/plugin/protocol.go b/internal/plugin/protocol.go index 615d3a9..1c6fc3b 100644 --- a/internal/plugin/protocol.go +++ b/internal/plugin/protocol.go @@ -48,6 +48,14 @@ type Op struct { Argv []string `json:"argv,omitempty"` Dir string `json:"dir,omitempty"` Capture bool `json:"capture,omitempty"` + + // out + Text string `json:"text,omitempty"` + + // slink + Src string `json:"src,omitempty"` + Dst string `json:"dst,omitempty"` + Force bool `json:"force,omitempty"` } // Result is godo's answer to an op that needs one. @@ -65,4 +73,8 @@ type Result struct { const ( // OpExec runs an argument vector and waits. Needs config proc.exec. OpExec = "exec" + // OpOut writes a line to the host's stdout. Needs no capability. + OpOut = "out" + // OpSlink creates a symlink. Needs config fs.slink. + OpSlink = "slink" ) From 5b4293b1bbb2b485be73665cbb39940af7796489 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=83=9E=E3=82=A4=E3=83=8E=E3=83=AB?= <97069334+MY-RV@users.noreply.github.com> Date: Sun, 20 Sep 2026 19:09:04 -0600 Subject: [PATCH 06/14] feat: godo -e plugins install MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Declaring a plugin and having it on this machine are different things. The catalog says which artifact a script needs; there was nothing that put it here. install fetches, stores it under the user cache keyed by its digest, and writes the entry into godo.yaml. The digest is never asked for — it is computed from what the artifact turns out to contain. A person cannot check a hash by reading it, so asking someone to type one is how wrong hashes get committed. The catalog is spliced, not re-encoded. A godo.yaml is written by hand: its comments carry @deps and @runner, its blank lines group scripts, and a block scalar's whitespace is content. Round-tripping through a YAML encoder would keep the data and lose the file, so the parser is used only to find where to write — which is what its line numbers are for. A file beside the catalog is loaded from where it is. Asking someone to install what they can already see is ceremony, and the digest is checked either way. Anything else must be installed first: a run is not the moment to discover that something has to be downloaded. A relative source resolves against the directory it was written in — the catalog's for a declared plugin, the caller's own for one typed on a command line — so ./x.wasm means the same thing from a subdirectory. http:// is refused. An artifact is code, and its integrity cannot rest on a transport anyone on the path can rewrite: a wrong digest is a failure you see, a silently swapped download and a swapped catalog is not. Codex was given internal/pluginstore and stopped to ask instead of guessing: the brief said not to re-download when the digest is already present, but a digest is only known after downloading. It was right, and the answer is the split between Fetch, which always reads, and Ensure, which can skip. --- CHANGELOG.md | 5 + docs/contract.md | 22 ++- docs/reference/cli.md | 2 + internal/catalog/insert.go | 153 +++++++++++++++++++++ internal/catalog/insert_test.go | 142 +++++++++++++++++++ internal/cli/app.go | 191 +++++++++++++++++++++++++- internal/cli/plugin_e2e_test.go | 104 ++++++++++++++ internal/plugin/runner.go | 20 ++- internal/pluginstore/store.go | 212 +++++++++++++++++++++++++++++ internal/pluginstore/store_test.go | 165 ++++++++++++++++++++++ 10 files changed, 1009 insertions(+), 7 deletions(-) create mode 100644 internal/catalog/insert.go create mode 100644 internal/catalog/insert_test.go create mode 100644 internal/pluginstore/store.go create mode 100644 internal/pluginstore/store_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 942771a..0a89bbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,11 @@ Breaking. The shell that runs your scripts changed. they are parsed and validated so the shape is settled, and a script asking for a runner a plugin provides fails by naming that plugin instead of reading as a typo. +- **`godo -e plugins install `** fetches an artifact, computes its + digest, stores it under `/godo/plugins`, and writes the entry into + `godo.yaml` — preserving the comments, blank lines and block scalars around + it. Without a source it fetches everything the catalog declares. + `godo -e plugins` lists what is declared and whether it is here. - **`godo -e runners`** lists what is usable on the machine you are on, and how to confirm which shell you are in when the detected one looks wrong. - `--ls ` prints `@runner` beside `@dialect`. diff --git a/docs/contract.md b/docs/contract.md index 4d9ea65..f4965f8 100644 --- a/docs/contract.md +++ b/docs/contract.md @@ -101,12 +101,28 @@ numbers. ### `engine.plugins` -**No build loads plugins yet.** Entries are parsed and validated so the shape -is settled and a catalog can already declare what it expects. +``` +godo -e plugins what this catalog declares, and its state +godo -e plugins install fetch everything it declares +godo -e plugins install add one, and fetch it +``` + +`install ` computes the digest from the artifact and writes the entry +into `godo.yaml`. The digest is never asked for: a person cannot check a hash +by reading it, so asking for one is how wrong hashes get committed. + +Artifacts live in `/godo/plugins`, named by digest. A file sitting +beside the `godo.yaml` is loaded from where it is — asking someone to install +what they can already see would be ceremony, and its digest is checked either +way. Anything else must be installed first; a run is not the moment to discover +that something has to be downloaded. + +`http://` sources are refused. An artifact is code, and its integrity cannot +rest on a transport anyone on the path can rewrite. | Field | | |-------|--| -| `source` | Required. Where the plugin comes from | +| `source` | Required. An `https://` URL, or a path relative to the `godo.yaml` | | `sha256` | **Required.** A plugin is third-party code that runs when someone types `godo test`; without a digest there is nothing to verify it is the code that was reviewed | | `provides` | Required. `":"` entries, kind being `runner` or `dialect`. Two plugins may not provide the same one | | `config` | Optional, and entirely the plugin's: its keys, its meaning, its defaults. godo carries it across without reading it | diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 2d654fd..47a0d39 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -19,6 +19,8 @@ Normative source: [contract.md](../contract.md). | | | |--|--| | `-e runners` | List runners usable on this machine | +| `-e plugins` | What the catalog declares, and whether it is installed | +| `-e plugins install [source]` | Fetch declared plugins, or add and fetch one | | `-e version` | Print binary version | | `-e update` | Install newer Release asset | | `-e update check` | Check only | diff --git a/internal/catalog/insert.go b/internal/catalog/insert.go new file mode 100644 index 0000000..257e762 --- /dev/null +++ b/internal/catalog/insert.go @@ -0,0 +1,153 @@ +package catalog + +import ( + "fmt" + "strings" + + "gopkg.in/yaml.v3" +) + +// PluginEntry is a plugin to write into a catalog. +type PluginEntry struct { + Source string + SHA256 string + Provides []string + Config string // raw YAML lines for the config block, or "" +} + +// InsertPlugin returns src with entry added under engine.plugins. +// +// It splices lines rather than re-encoding the document. A catalog is written +// by hand: its comments carry the @deps and @runner decorators, its blank lines +// group scripts, and its block scalars hold whitespace that means something. +// Round-tripping through a YAML encoder would preserve the data and lose the +// file, so the parser is used only to find *where* to write. +func InsertPlugin(src []byte, entry PluginEntry) ([]byte, error) { + if entry.Source == "" || entry.SHA256 == "" { + return nil, fmt.Errorf("plugin entry needs a source and a sha256") + } + var root yaml.Node + if err := yaml.Unmarshal(src, &root); err != nil { + return nil, fmt.Errorf("%w: parse yaml: %v", ErrInvalidCatalog, err) + } + doc := &root + if root.Kind == yaml.DocumentNode { + if len(root.Content) == 0 { + return nil, fmt.Errorf("%w: empty yaml document", ErrInvalidCatalog) + } + doc = root.Content[0] + } + if doc.Kind != yaml.MappingNode { + return nil, fmt.Errorf("%w: root must be a mapping", ErrInvalidCatalog) + } + + lines := strings.Split(string(src), "\n") + at, indent, err := pluginsAnchor(doc, lines) + if err != nil { + return nil, err + } + block := entryLines(entry, indent) + out := append([]string{}, lines[:at]...) + out = append(out, block...) + out = append(out, lines[at:]...) + return []byte(strings.Join(out, "\n")), nil +} + +// pluginsAnchor finds the line to insert before, and the indent to write at. +// +// Three shapes, in the order they are looked for: an engine.plugins list to +// append to, an engine block that needs the key, and a file with no engine +// block at all. +func pluginsAnchor(doc *yaml.Node, lines []string) (at int, indent string, err error) { + engineKey, engineVal := childNode(doc, "engine") + if engineKey == nil { + // No engine block: open one above scripts, or at the end. + at = len(lines) + if k, _ := childNode(doc, "scripts"); k != nil { + at = k.Line - 1 + for at > 0 && strings.TrimSpace(lines[at-1]) == "" { + at-- + } + } + return at, "engine", nil + } + if engineVal == nil || engineVal.Kind != yaml.MappingNode { + return 0, "", fmt.Errorf("%w: engine must be a mapping", ErrInvalidCatalog) + } + + pluginsKey, pluginsVal := childNode(engineVal, "plugins") + if pluginsKey == nil { + // engine exists but declares no plugins: add the key at its indent. + return endOfNode(engineVal, lines), strings.Repeat(" ", engineVal.Column-1) + "plugins", nil + } + if pluginsVal == nil || pluginsVal.Kind != yaml.SequenceNode || len(pluginsVal.Content) == 0 { + return pluginsKey.Line, strings.Repeat(" ", pluginsKey.Column+1) + "item", nil + } + last := pluginsVal.Content[len(pluginsVal.Content)-1] + return endOfNode(last, lines), strings.Repeat(" ", last.Column-3) + "item", nil +} + +// endOfNode returns the line just past a node's last content line. +// +// yaml.Node carries where a value starts, never where it ends, so the end is +// found by walking down past everything indented under it. Blank lines are +// walked over but not claimed: an entry belongs above the gap that separates +// it from what follows. +func endOfNode(n *yaml.Node, lines []string) int { + col := n.Column + last := n.Line + for i := n.Line; i < len(lines); i++ { + text := lines[i] + if strings.TrimSpace(text) == "" { + continue + } + if len(text)-len(strings.TrimLeft(text, " ")) < col-1 { + break + } + last = i + 1 + } + return last +} + +func childNode(m *yaml.Node, key string) (k, v *yaml.Node) { + for i := 0; i+1 < len(m.Content); i += 2 { + if m.Content[i].Value == key { + return m.Content[i], m.Content[i+1] + } + } + return nil, nil +} + +// entryLines renders the entry at the depth its anchor asked for. +func entryLines(e PluginEntry, indent string) []string { + var pad, out []string + switch { + case indent == "engine": + out = append(out, "engine:", " plugins:") + pad = []string{" "} + case strings.HasSuffix(indent, "plugins"): + base := strings.TrimSuffix(indent, "plugins") + out = append(out, base+"plugins:") + pad = []string{base + " "} + default: + pad = []string{strings.TrimSuffix(indent, "item")} + } + p := pad[0] + out = append(out, + p+"- source: "+e.Source, + p+" sha256: "+e.SHA256, + ) + if len(e.Provides) > 0 { + out = append(out, p+" provides: ["+strings.Join(e.Provides, ", ")+"]") + } + if e.Config != "" { + out = append(out, p+" config:") + for _, l := range strings.Split(strings.TrimRight(e.Config, "\n"), "\n") { + out = append(out, p+" "+l) + } + } + if indent == "engine" { + out = append(out, "") + } + return out +} diff --git a/internal/catalog/insert_test.go b/internal/catalog/insert_test.go new file mode 100644 index 0000000..89afd6c --- /dev/null +++ b/internal/catalog/insert_test.go @@ -0,0 +1,142 @@ +package catalog_test + +import ( + "strings" + "testing" + + "github.com/my-rv/godo/internal/catalog" +) + +var entry = catalog.PluginEntry{ + Source: "./micropy.wasm", + SHA256: "abc123", + Provides: []string{"runner:micropy"}, + Config: "proc: {exec: true}\nfs: {mount: true}", +} + +// The catalog must survive the edit: its comments carry decorators, its blank +// lines group scripts, and a block scalar's whitespace is content. +func TestInsertPlugin_keepsEverythingElse(t *testing.T) { + src := `version: "0.1" + +engine: + dialect: matcher + +scripts: + # Local gate + # @deps vet, test + ci: go build ./... + + # @runner micropy + boot: | + import os + deliberately indented + print("hi") +` + got, err := catalog.InsertPlugin([]byte(src), entry) + if err != nil { + t.Fatal(err) + } + out := string(got) + for _, want := range []string{ + "# Local gate", "# @deps vet, test", "# @runner micropy", + " import os", " deliberately indented", + } { + if !strings.Contains(out, want) { + t.Fatalf("lost %q from:\n%s", want, out) + } + } + cat, err := catalog.Parse(got, "x") + if err != nil { + t.Fatalf("result does not parse: %v\n%s", err, out) + } + if len(cat.Engine.Plugins) != 1 || cat.Engine.Plugins[0].SHA256 != "abc123" { + t.Fatalf("plugins=%+v", cat.Engine.Plugins) + } + if cat.Dialect != catalog.DialectMatcher { + t.Fatalf("dialect lost: %q", cat.Dialect) + } + if len(cat.Scripts) != 2 { + t.Fatalf("scripts=%d", len(cat.Scripts)) + } + if cat.Scripts[0].Doc != "Local gate" || len(cat.Scripts[0].Deps) != 2 { + t.Fatalf("decorators lost: %+v", cat.Scripts[0]) + } + if cat.Scripts[1].Runner != "micropy" { + t.Fatalf("@runner lost: %+v", cat.Scripts[1]) + } +} + +func TestInsertPlugin_shapes(t *testing.T) { + for _, tc := range []struct{ name, src string }{ + {"no engine block", "version: \"0.1\"\n\nscripts:\n t: echo hi\n"}, + {"engine without plugins", "version: \"0.1\"\nengine:\n dialect: matcher\nscripts:\n t: echo hi\n"}, + {"engine with plugins", "version: \"0.1\"\nengine:\n plugins:\n" + + " - source: ./a.wasm\n sha256: aaa\n provides: [runner:a]\n" + + "scripts:\n t: echo hi\n"}, + {"plugins with a config block", "version: \"0.1\"\nengine:\n plugins:\n" + + " - source: ./a.wasm\n sha256: aaa\n provides: [runner:a]\n" + + " config:\n proc: {exec: true}\n" + + "scripts:\n t: echo hi\n"}, + {"no scripts at all", "version: \"0.1\"\n"}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := catalog.InsertPlugin([]byte(tc.src), entry) + if err != nil { + t.Fatalf("%v\n%s", err, tc.src) + } + cat, err := catalog.Parse(got, "x") + if err != nil { + t.Fatalf("does not parse: %v\n%s", err, got) + } + var found bool + for _, p := range cat.Engine.Plugins { + if p.SHA256 == "abc123" { + found = true + } + } + if !found { + t.Fatalf("entry not inserted:\n%s", got) + } + // An existing entry must survive alongside the new one. + if strings.Contains(tc.src, "a.wasm") && len(cat.Engine.Plugins) != 2 { + t.Fatalf("existing entry lost:\n%s", got) + } + }) + } +} + +// Inserting twice is how a second plugin arrives; both must be there. +func TestInsertPlugin_twice(t *testing.T) { + src := []byte("version: \"0.1\"\nscripts:\n t: echo hi\n") + one, err := catalog.InsertPlugin(src, entry) + if err != nil { + t.Fatal(err) + } + second := entry + second.Source = "./other.wasm" + second.SHA256 = "def456" + second.Provides = []string{"runner:other"} + second.Config = "" + two, err := catalog.InsertPlugin(one, second) + if err != nil { + t.Fatalf("%v\n%s", err, one) + } + cat, err := catalog.Parse(two, "x") + if err != nil { + t.Fatalf("does not parse: %v\n%s", err, two) + } + if len(cat.Engine.Plugins) != 2 { + t.Fatalf("plugins=%d:\n%s", len(cat.Engine.Plugins), two) + } +} + +func TestInsertPlugin_rejectsIncomplete(t *testing.T) { + src := []byte("version: \"0.1\"\nscripts:\n t: echo hi\n") + if _, err := catalog.InsertPlugin(src, catalog.PluginEntry{Source: "x"}); err == nil { + t.Fatal("want an error without a digest") + } + if _, err := catalog.InsertPlugin([]byte("- not a mapping\n"), entry); err == nil { + t.Fatal("want an error for a non-mapping root") + } +} diff --git a/internal/cli/app.go b/internal/cli/app.go index 26e04a1..7033652 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -14,6 +14,7 @@ import ( "github.com/my-rv/godo/internal/catalog" "github.com/my-rv/godo/internal/execshell" "github.com/my-rv/godo/internal/plugin" + "github.com/my-rv/godo/internal/pluginstore" "github.com/my-rv/godo/internal/update" ) @@ -29,7 +30,9 @@ type App struct { Runner catalog.Runner // UpdateClient overrides release checks (tests). UpdateClient *update.Client - Executable func() (string, error) + // PluginStore overrides where artifacts are kept (tests). + PluginStore *pluginstore.Store + Executable func() (string, error) } // New returns an App wired to process stdio. @@ -63,6 +66,9 @@ func (a *App) Run(args []string) error { if mode == modeRunners { return a.listRunners(a.nearestCatalog()) } + if mode == modePlugins || mode == modePluginsInstall { + return a.plugins(mode == modePluginsInstall, tokens) + } if mode == modeUpdate || mode == modeUpdateCheck { return a.runUpdate(mode == modeUpdateCheck) } @@ -99,7 +105,22 @@ func (a *App) Run(args []string) error { ctx := context.Background() rt := plugin.NewRuntime(ctx) defer rt.Close(ctx) - if err := plugin.Register(ctx, rt, cat, runners, root, a.Stdout, a.Stderr, a.Stdin); err != nil { + store := a.store() + resolve := func(digest, source string) (string, error) { + if store.Has(digest) { + return store.Path(digest) + } + // A file sitting next to the catalog is already here; asking + // someone to install what they can see is nonsense. Its digest is + // checked either way, so nothing is skipped but the copying. + if local := localSource(root, source); local != "" { + return local, nil + } + // Anything else has to be fetched, and a run is not the moment to + // discover that. + return "", fmt.Errorf("plugin %s is not installed\n run: godo -e plugins install", source) + } + if err := plugin.Register(ctx, rt, cat, runners, root, a.Stdout, a.Stderr, a.Stdin, resolve); err != nil { return err } } @@ -201,6 +222,159 @@ func (a *App) runUpdate(checkOnly bool) error { // // "here" is the point: the native shells are whatever this machine has, so the // list is a fact about the machine, not about godo. +// plugins lists what a catalog declares, or installs it. +// +// Declaring a plugin and having it on this machine are different things: the +// catalog says which artifact a script needs, the store is where that artifact +// actually is. install is what turns the first into the second. +func (a *App) plugins(install bool, args []string) error { + cwd, err := a.cwd() + if err != nil { + return err + } + path, err := catalog.FindFile(cwd) + if err != nil { + return err + } + cat, err := catalog.LoadFile(path) + if err != nil { + return err + } + store := a.store() + ctx := context.Background() + + // A source typed on the command line is relative to where the person is; + // one written in the catalog is relative to the catalog. + if install && len(args) > 0 { + return a.installNew(ctx, store, cat, path, cwd, args) + } + if install { + return a.installDeclared(ctx, store, cat, filepath.Dir(path)) + } + return a.listPlugins(store, cat) +} + +// installNew adds a plugin the catalog does not declare yet. +// +// The digest is not asked for — it is whatever the artifact turns out to +// contain, computed here and written into the catalog. Asking a person to type +// a hash they cannot check is how wrong hashes get committed. +func (a *App) installNew(ctx context.Context, store pluginstore.Store, cat *catalog.Catalog, path, base string, args []string) error { + if len(args) > 1 { + return fmt.Errorf("-e plugins install: one source at a time, got %v", args) + } + source := args[0] + digest, stored, err := store.Fetch(ctx, base, source) + if err != nil { + return err + } + for _, p := range cat.Engine.Plugins { + if p.SHA256 == digest { + fmt.Fprintf(a.Stdout, "already declared: %s\n %s\n", p.Source, stored) + return nil + } + } + + provides, err := providedRunners(source) + if err != nil { + return err + } + src, err := os.ReadFile(path) + if err != nil { + return err + } + out, err := catalog.InsertPlugin(src, catalog.PluginEntry{ + Source: source, + SHA256: digest, + Provides: provides, + Config: "proc: {exec: true}", + }) + if err != nil { + return err + } + if err := os.WriteFile(path, out, 0o644); err != nil { + return err + } + fmt.Fprintf(a.Stdout, "installed %s\n sha256 %s\n stored %s\n declared %s\n\n", + source, digest, stored, path) + fmt.Fprintf(a.Stdout, "It provides %s, and was granted proc.exec.\n", strings.Join(provides, ", ")) + fmt.Fprintln(a.Stdout, "Add fs.mount, fs.slink or time.wall under its config if it needs them.") + return nil +} + +// providedRunners derives the runner name from the artifact's filename. +// +// The name a plugin answers to is the catalog's to choose: nothing inside a +// .wasm declares it, and inventing a manifest format to carry one string would +// be a second contract to maintain. The filename is the honest default, and +// the line it writes is there to be edited. +func providedRunners(source string) ([]string, error) { + base := filepath.Base(strings.TrimSuffix(source, ".wasm")) + base = strings.TrimPrefix(base, "godo-") + if base == "" || base == "." || base == string(filepath.Separator) { + return nil, fmt.Errorf("cannot tell what %q provides; name the artifact .wasm", source) + } + return []string{"runner:" + base}, nil +} + +// installDeclared fetches every artifact the catalog names. +func (a *App) installDeclared(ctx context.Context, store pluginstore.Store, cat *catalog.Catalog, base string) error { + if len(cat.Engine.Plugins) == 0 { + fmt.Fprintln(a.Stdout, "this catalog declares no plugins") + return nil + } + for _, p := range cat.Engine.Plugins { + stored, err := store.Ensure(ctx, base, p.SHA256, p.Source) + if err != nil { + return err + } + fmt.Fprintf(a.Stdout, "%-16s %s\n", strings.Join(p.Provides, " "), stored) + } + return nil +} + +func (a *App) listPlugins(store pluginstore.Store, cat *catalog.Catalog) error { + if len(cat.Engine.Plugins) == 0 { + fmt.Fprintln(a.Stdout, "this catalog declares no plugins") + return nil + } + missing := false + for _, p := range cat.Engine.Plugins { + state := "installed" + if !store.Has(p.SHA256) { + state = "not installed" + missing = true + } + fmt.Fprintf(a.Stdout, "%-16s %-14s %s\n", strings.Join(p.Provides, " "), state, p.Source) + } + if missing { + fmt.Fprintln(a.Stdout, "\nRun: godo -e plugins install") + } + return nil +} + +// localSource returns the path of a source that is already on disk, or "". +func localSource(root, source string) string { + if strings.Contains(source, "://") && !strings.HasPrefix(source, "file://") { + return "" + } + p := strings.TrimPrefix(source, "file://") + if !filepath.IsAbs(p) { + p = filepath.Join(root, p) + } + if st, err := os.Stat(p); err != nil || !st.Mode().IsRegular() { + return "" + } + return p +} + +func (a *App) store() pluginstore.Store { + if a.PluginStore != nil { + return *a.PluginStore + } + return pluginstore.Store{} +} + // nearestCatalog loads the catalog for -e runners, or nil. // // The listing is useful outside a repo, so a missing or broken catalog is not @@ -329,6 +503,8 @@ const ( modeUpdate modeUpdateCheck modeRunners + modePlugins + modePluginsInstall ) // ParseFlags parses context flags before script tokens. @@ -385,6 +561,15 @@ func parseEngineCommand(tokens []string) (mode mode, rest []string, err error) { return 0, nil, fmt.Errorf("-e runners: unexpected arguments %v", tokens[1:]) } return modeRunners, nil, nil + case "plugins": + switch { + case len(tokens) == 1: + return modePlugins, nil, nil + case tokens[1] == "install": + return modePluginsInstall, tokens[2:], nil + default: + return 0, nil, fmt.Errorf("-e plugins: usage: -e plugins | -e plugins install [source]") + } case "update": switch { case len(tokens) == 1: @@ -436,6 +621,8 @@ func EngineUsage(w io.Writer) { Built-in commands (not godo.yaml scripts): version print binary version runners list runners usable here + plugins what this catalog declares, and whether it is installed + plugins install fetch them; with a source, add it and fetch it update download latest GitHub Release for this OS/arch update check report whether an update is available help this help diff --git a/internal/cli/plugin_e2e_test.go b/internal/cli/plugin_e2e_test.go index 0dc3c97..7543a41 100644 --- a/internal/cli/plugin_e2e_test.go +++ b/internal/cli/plugin_e2e_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/my-rv/godo" + "github.com/my-rv/godo/internal/pluginstore" ) var ( @@ -197,3 +198,106 @@ func TestE2E_pluginDigestMismatchRefusesToRun(t *testing.T) { t.Fatal("a plugin with the wrong digest ran") } } + +// Contract: install, then run. The whole reason the command exists. +func TestE2E_pluginsInstallThenRun(t *testing.T) { + cwd := t.TempDir() + store := pluginstore.Store{Dir: filepath.Join(t.TempDir(), "store")} + + data, err := os.ReadFile(buildExamplePlugin(t)) + if err != nil { + t.Fatal(err) + } + artifact := filepath.Join(cwd, "godo-lines.wasm") + if err := os.WriteFile(artifact, data, 0o644); err != nil { + t.Fatal(err) + } + // A catalog with a comment, to prove installing does not eat the file. + writeGodoYAML(t, cwd, "version: \"0.1\"\nscripts:\n # keep me\n shell: echo hi\n") + + app, out, _ := e2eApp(t, cwd) + app.PluginStore = &store + if err := app.Run([]string{"-e", "plugins", "install", "./godo-lines.wasm"}); err != nil { + t.Fatalf("install: %v", err) + } + if !strings.Contains(out.String(), "runner:lines") { + t.Fatalf("install said:\n%s", out.String()) + } + + body, err := os.ReadFile(filepath.Join(cwd, "godo.yaml")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(body), "# keep me") { + t.Fatalf("install rewrote the catalog and lost a comment:\n%s", body) + } + + // The declared runner now works, with no hand-editing in between. + if err := os.WriteFile(filepath.Join(cwd, "godo.yaml"), + append(body, []byte("\n # @runner lines\n boot: |\n touch made-it\n")...), 0o644); err != nil { + t.Fatal(err) + } + app, _, _ = e2eApp(t, cwd) + app.PluginStore = &store + if err := app.Run([]string{"boot"}); err != nil { + t.Fatalf("run: %v", err) + } + if _, err := os.Stat(filepath.Join(cwd, "made-it")); err != nil { + t.Fatalf("the installed plugin did not run: %v", err) + } +} + +// A plugin that is not on this machine says so, and says what to run. +// +// The source is remote: a file sitting beside the catalog is already here, and +// is loaded without ceremony. +func TestE2E_pluginsNotInstalledSaysWhatToRun(t *testing.T) { + cwd := t.TempDir() + writeGodoYAML(t, cwd, "version: \"0.1\"\nengine:\n plugins:\n"+ + " - source: https://example.test/lines.wasm\n"+ + " sha256: "+strings.Repeat("a", 64)+"\n"+ + " provides: [runner:lines]\n"+ + "scripts:\n # @runner lines\n boot: |\n touch one\n") + app, _, _ := e2eApp(t, cwd) + app.PluginStore = &pluginstore.Store{Dir: filepath.Join(t.TempDir(), "empty")} + + err := app.Run([]string{"boot"}) + if err == nil { + t.Fatal("ran a plugin that is not installed") + } + for _, want := range []string{"not installed", "godo -e plugins install"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("err=%v, missing %q", err, want) + } + } + if _, serr := os.Stat(filepath.Join(cwd, "one")); serr == nil { + t.Fatal("something ran anyway") + } +} + +// A file beside the catalog needs no install — its digest is checked anyway. +func TestE2E_localArtifactNeedsNoInstall(t *testing.T) { + cwd := pluginCatalog(t, true, " # @runner lines\n boot: |\n touch one\n") + app, _, _ := e2eApp(t, cwd) + app.PluginStore = &pluginstore.Store{Dir: filepath.Join(t.TempDir(), "empty")} + if err := app.Run([]string{"boot"}); err != nil { + t.Fatalf("run: %v", err) + } + if _, err := os.Stat(filepath.Join(cwd, "one")); err != nil { + t.Fatalf("did not run: %v", err) + } +} + +func TestE2E_pluginsListsState(t *testing.T) { + cwd := pluginCatalog(t, true, " # @runner lines\n boot: |\n touch one\n") + app, out, _ := e2eApp(t, cwd) + app.PluginStore = &pluginstore.Store{Dir: filepath.Join(t.TempDir(), "empty")} + if err := app.Run([]string{"-e", "plugins"}); err != nil { + t.Fatal(err) + } + for _, want := range []string{"runner:lines", "not installed", "godo -e plugins install"} { + if !strings.Contains(out.String(), want) { + t.Fatalf("missing %q in:\n%s", want, out.String()) + } + } +} diff --git a/internal/plugin/runner.go b/internal/plugin/runner.go index c002580..c95e34e 100644 --- a/internal/plugin/runner.go +++ b/internal/plugin/runner.go @@ -71,15 +71,31 @@ func nonNil(m map[string]string) map[string]string { return m } +// Resolver turns a declared plugin into a path on this machine. +// +// A catalog says which artifact a script needs; where that artifact is, is a +// separate question, and one the engine has no business answering. godo hands +// in its store; an embedder hands in whatever it uses, or nil to read the +// source as a path. +type Resolver func(digest, source string) (string, error) + // Register loads every plugin a catalog declares and registers the runners // they provide. // // A plugin that will not load stops the run. A catalog that names a plugin has // already decided it is part of the build; carrying on without it would mean // silently running something other than what the file says. -func Register(ctx context.Context, rt *Runtime, cat *catalog.Catalog, reg *catalog.RunnerRegistry, root string, stdout, stderr io.Writer, stdin io.Reader) error { +func Register(ctx context.Context, rt *Runtime, cat *catalog.Catalog, reg *catalog.RunnerRegistry, root string, stdout, stderr io.Writer, stdin io.Reader, resolve Resolver) error { for _, spec := range cat.Engine.Plugins { - p, err := rt.Load(ctx, spec.Source, spec.SHA256, root, spec.Provides, spec.Config) + from := spec.Source + if resolve != nil { + stored, err := resolve(spec.SHA256, spec.Source) + if err != nil { + return err + } + from = stored + } + p, err := rt.Load(ctx, from, spec.SHA256, root, spec.Provides, spec.Config) if err != nil { return err } diff --git a/internal/pluginstore/store.go b/internal/pluginstore/store.go new file mode 100644 index 0000000..a52fc9d --- /dev/null +++ b/internal/pluginstore/store.go @@ -0,0 +1,212 @@ +// Package pluginstore keeps plugin artifacts on disk, keyed by their digest. +// +// A plugin is third-party code that runs when someone types "godo test", so +// the digest is not a cache key that happens to be a hash — it is the identity. +// Two artifacts with the same digest are the same artifact, and one whose bytes +// stop matching is not the artifact that was reviewed. +package pluginstore + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" +) + +// Store is a directory of artifacts named by digest. +type Store struct { + // Dir defaults to /godo/plugins. + Dir string + // HTTP defaults to http.DefaultClient. + HTTP *http.Client +} + +// Dirname returns the directory this store writes to. +func (s Store) Dirname() (string, error) { + if s.Dir != "" { + return s.Dir, nil + } + cache, err := os.UserCacheDir() + if err != nil { + return "", err + } + return filepath.Join(cache, "godo", "plugins"), nil +} + +// Path is where an artifact with this digest lives. +// +// The digest becomes a filename, so it is validated rather than trusted: 64 +// lowercase hex characters and nothing else, which cannot contain a separator +// or climb out of the directory. +func (s Store) Path(digest string) (string, error) { + if !validDigest(digest) { + return "", fmt.Errorf("not a sha256 digest: %q", digest) + } + dir, err := s.Dirname() + if err != nil { + return "", err + } + return filepath.Join(dir, digest+".wasm"), nil +} + +// Has reports whether the artifact is already stored. +func (s Store) Has(digest string) bool { + p, err := s.Path(digest) + if err != nil { + return false + } + st, err := os.Stat(p) + return err == nil && st.Mode().IsRegular() +} + +// Fetch reads an artifact from source and stores it, returning its digest. +// +// It always reads: the digest is what source turns out to contain, and there is +// no way to know that without reading it. Ensure is the call that can skip. +// +// A relative source resolves against base. Callers pass the directory the +// source was written in — the catalog's for a declared plugin, the caller's own +// for one typed on a command line — so that "./x.wasm" means the same thing +// wherever godo is run from. +func (s Store) Fetch(ctx context.Context, base, source string) (digest, path string, err error) { + data, err := s.read(ctx, base, source) + if err != nil { + return "", "", err + } + sum := sha256.Sum256(data) + digest = hex.EncodeToString(sum[:]) + path, err = s.write(digest, data) + if err != nil { + return "", "", err + } + return digest, path, nil +} + +// Ensure returns the stored artifact for digest, fetching from source only if +// it is missing. A fetch that produces different bytes is refused: source is +// where an artifact comes from, digest is which artifact it must be. +func (s Store) Ensure(ctx context.Context, base, digest, source string) (string, error) { + if s.Has(digest) { + return s.Path(digest) + } + got, path, err := s.Fetch(ctx, base, source) + if err != nil { + return "", err + } + if got != digest { + _ = os.Remove(path) + return "", fmt.Errorf("%s: sha256 mismatch\n declared %s\n actual %s", source, digest, got) + } + return path, nil +} + +// Verify re-hashes a stored artifact, catching a cache that was corrupted or +// tampered with after it was written. +func (s Store) Verify(digest string) error { + p, err := s.Path(digest) + if err != nil { + return err + } + data, err := os.ReadFile(p) + if err != nil { + return err + } + sum := sha256.Sum256(data) + if got := hex.EncodeToString(sum[:]); got != digest { + return fmt.Errorf("%s: stored bytes no longer match\n expected %s\n actual %s", p, digest, got) + } + return nil +} + +// Add stores bytes whose digest the caller already computed. +func (s Store) Add(digest string, data []byte) (string, error) { + sum := sha256.Sum256(data) + if got := hex.EncodeToString(sum[:]); got != digest { + return "", fmt.Errorf("sha256 mismatch\n declared %s\n actual %s", digest, got) + } + return s.write(digest, data) +} + +func (s Store) read(ctx context.Context, base, source string) ([]byte, error) { + switch { + case strings.HasPrefix(source, "http://"): + // An artifact is code. Its integrity cannot rest on a transport that + // anyone on the path can rewrite, digest or not: a wrong digest is a + // failure you see, and a silently swapped download plus a swapped + // catalog is not. + return nil, fmt.Errorf("%s: refusing http; use https or a local path", source) + case strings.HasPrefix(source, "https://"): + client := s.HTTP + if client == nil { + client = http.DefaultClient + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, source, nil) + if err != nil { + return nil, err + } + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("%s: %w", source, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%s: %s", source, resp.Status) + } + return io.ReadAll(resp.Body) + default: + p := strings.TrimPrefix(source, "file://") + if !filepath.IsAbs(p) && base != "" { + p = filepath.Join(base, p) + } + return os.ReadFile(p) + } +} + +// write stores data atomically: a temp file in the same directory, then a +// rename. An interrupted fetch must not leave a partial file that a later Has +// would accept as the real artifact. +func (s Store) write(digest string, data []byte) (string, error) { + p, err := s.Path(digest) + if err != nil { + return "", err + } + dir := filepath.Dir(p) + if err := os.MkdirAll(dir, 0o755); err != nil { + return "", err + } + tmp, err := os.CreateTemp(dir, ".partial-*") + if err != nil { + return "", err + } + defer os.Remove(tmp.Name()) + if _, err := tmp.Write(data); err != nil { + tmp.Close() + return "", err + } + if err := tmp.Close(); err != nil { + return "", err + } + if err := os.Rename(tmp.Name(), p); err != nil { + return "", err + } + return p, nil +} + +func validDigest(s string) bool { + if len(s) != 64 { + return false + } + for _, r := range s { + switch { + case r >= '0' && r <= '9', r >= 'a' && r <= 'f': + default: + return false + } + } + return true +} diff --git a/internal/pluginstore/store_test.go b/internal/pluginstore/store_test.go new file mode 100644 index 0000000..372aa5d --- /dev/null +++ b/internal/pluginstore/store_test.go @@ -0,0 +1,165 @@ +package pluginstore_test + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/my-rv/godo/internal/pluginstore" +) + +func digestOf(b []byte) string { + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) +} + +func TestFetch_fromALocalFile(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "a.wasm") + data := []byte("artifact bytes") + if err := os.WriteFile(src, data, 0o644); err != nil { + t.Fatal(err) + } + s := pluginstore.Store{Dir: filepath.Join(dir, "store")} + + got, path, err := s.Fetch(context.Background(), "", src) + if err != nil { + t.Fatal(err) + } + if got != digestOf(data) { + t.Fatalf("digest=%s", got) + } + stored, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(stored) != string(data) { + t.Fatalf("stored %q", stored) + } + if !s.Has(got) { + t.Fatal("Has says no after a Fetch") + } +} + +func TestEnsure_downloadsOnceAndReusesAfter(t *testing.T) { + data := []byte("artifact over the wire") + var hits int + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits++ + w.Write(data) + })) + defer srv.Close() + + s := pluginstore.Store{Dir: t.TempDir(), HTTP: srv.Client()} + want := digestOf(data) + + for i := 0; i < 3; i++ { + if _, err := s.Ensure(context.Background(), "", want, srv.URL); err != nil { + t.Fatalf("attempt %d: %v", i, err) + } + } + if hits != 1 { + t.Fatalf("hit the server %d times, want 1", hits) + } +} + +// The source says where to look; the digest says what must come back. +func TestEnsure_refusesBytesThatDoNotMatch(t *testing.T) { + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("something else entirely")) + })) + defer srv.Close() + + s := pluginstore.Store{Dir: t.TempDir(), HTTP: srv.Client()} + want := digestOf([]byte("what the catalog declared")) + + _, err := s.Ensure(context.Background(), "", want, srv.URL) + if err == nil { + t.Fatal("stored an artifact the catalog did not declare") + } + if !strings.Contains(err.Error(), "sha256 mismatch") { + t.Fatalf("err=%v", err) + } + if s.Has(want) { + t.Fatal("the mismatched download was left in the store") + } +} + +// An artifact is code; its integrity cannot rest on a transport anyone can +// rewrite. +func TestFetch_refusesPlainHTTP(t *testing.T) { + s := pluginstore.Store{Dir: t.TempDir()} + _, _, err := s.Fetch(context.Background(), "", "http://example.test/a.wasm") + if err == nil || !strings.Contains(err.Error(), "refusing http") { + t.Fatalf("err=%v", err) + } +} + +// The digest becomes a filename, so it is validated rather than trusted. +func TestPath_rejectsAnythingThatIsNotADigest(t *testing.T) { + s := pluginstore.Store{Dir: t.TempDir()} + for _, bad := range []string{ + "", "short", strings.Repeat("g", 64), + "../" + strings.Repeat("a", 61), + strings.Repeat("a", 32) + "/" + strings.Repeat("b", 31), + strings.ToUpper(strings.Repeat("a", 64)), + strings.Repeat("a", 65), + } { + if _, err := s.Path(bad); err == nil { + t.Fatalf("accepted %q as a digest", bad) + } + } +} + +func TestVerify_catchesAStoreChangedAfterwards(t *testing.T) { + dir := t.TempDir() + s := pluginstore.Store{Dir: dir} + data := []byte("original") + d := digestOf(data) + path, err := s.Add(d, data) + if err != nil { + t.Fatal(err) + } + if err := s.Verify(d); err != nil { + t.Fatalf("fresh store does not verify: %v", err) + } + if err := os.WriteFile(path, []byte("tampered"), 0o644); err != nil { + t.Fatal(err) + } + if err := s.Verify(d); err == nil { + t.Fatal("Verify accepted bytes that changed") + } +} + +func TestAdd_refusesBytesThatDoNotHashToTheDigest(t *testing.T) { + s := pluginstore.Store{Dir: t.TempDir()} + if _, err := s.Add(digestOf([]byte("a")), []byte("b")); err == nil { + t.Fatal("stored bytes under someone else's digest") + } +} + +// An interrupted write must not leave something a later Has would accept. +func TestWrite_leavesNoPartialFileBehind(t *testing.T) { + dir := t.TempDir() + s := pluginstore.Store{Dir: dir} + data := []byte("complete") + d := digestOf(data) + if _, err := s.Add(d, data); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if strings.HasPrefix(e.Name(), ".partial") { + t.Fatalf("left a temp file: %s", e.Name()) + } + } +} From 8179237f54ce76650719633af0e5b5ff2d7e9a4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=83=9E=E3=82=A4=E3=83=8E=E3=83=AB?= <97069334+MY-RV@users.noreply.github.com> Date: Sun, 20 Sep 2026 19:28:43 -0600 Subject: [PATCH 07/14] fix: installing a plugin the catalog declares updates it, keeping its grants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Installing over an existing declaration appended a second entry, and the loader then refused the catalog: two entries claiming one runner. The ordinary case is the one that broke — an example catalog ships a placeholder digest precisely so install can write the real one. UpsertPlugin replaces the entry that provides the same names, and keeps the config block that was already there. What a plugin may do is the catalog author's decision, reached once and usually narrowed; rewriting it to a default on every install would quietly re-grant what someone took away and drop what they added. Only the source and the digest move. --- internal/catalog/insert.go | 122 ++++++++++++++++++++++++++++---- internal/catalog/insert_test.go | 113 +++++++++++++++++++++++++++++ internal/cli/app.go | 4 +- 3 files changed, 224 insertions(+), 15 deletions(-) diff --git a/internal/catalog/insert.go b/internal/catalog/insert.go index 257e762..92e4b5d 100644 --- a/internal/catalog/insert.go +++ b/internal/catalog/insert.go @@ -15,6 +15,94 @@ type PluginEntry struct { Config string // raw YAML lines for the config block, or "" } +// UpsertPlugin returns src with entry added under engine.plugins, replacing +// any existing entry that provides the same things. +// +// Installing a plugin a catalog already declares means bringing the +// declaration in line with what was just fetched — a placeholder digest +// becoming a real one is the ordinary case. Appending instead would leave two +// entries claiming one runner, which the loader rejects. +func UpsertPlugin(src []byte, entry PluginEntry) ([]byte, error) { + if len(entry.Provides) == 0 { + return InsertPlugin(src, entry) + } + doc, err := docNode(src) + if err != nil { + return nil, err + } + lines := strings.Split(string(src), "\n") + _, engineVal := childNode(doc, "engine") + if engineVal == nil || engineVal.Kind != yaml.MappingNode { + return InsertPlugin(src, entry) + } + _, pluginsVal := childNode(engineVal, "plugins") + if pluginsVal == nil || pluginsVal.Kind != yaml.SequenceNode { + return InsertPlugin(src, entry) + } + for _, item := range pluginsVal.Content { + if !sameProvides(item, entry.Provides) { + continue + } + from := item.Line - 1 + to := endOfNode(item, lines) + // The config that is already there stays. What this plugin may do is + // the catalog author's decision, reached once and often narrowed; + // resetting it to a default on every install would quietly re-grant + // what someone took away, and quietly drop what they added. + if kept := configLines(item, lines); kept != "" { + entry.Config = kept + } + block := entryLines(entry, strings.Repeat(" ", item.Column-3)+"item") + out := append([]string{}, lines[:from]...) + out = append(out, block...) + out = append(out, lines[to:]...) + return []byte(strings.Join(out, "\n")), nil + } + return InsertPlugin(src, entry) +} + +// configLines returns an entry's config block, re-indented to sit under a +// fresh entry, or "" when it declares none. +func configLines(item *yaml.Node, lines []string) string { + key, val := childNode(item, "config") + if key == nil || val == nil { + return "" + } + from := key.Line // the line after "config:" + to := endOfNode(val, lines) + if from >= to || to > len(lines) { + return "" + } + strip := val.Column - 1 + var out []string + for _, l := range lines[from:to] { + if len(l) >= strip { + l = l[strip:] + } else { + l = strings.TrimLeft(l, " ") + } + out = append(out, l) + } + return strings.Join(out, "\n") +} + +// sameProvides reports whether a plugin entry node claims exactly these names. +func sameProvides(item *yaml.Node, want []string) bool { + if item.Kind != yaml.MappingNode { + return false + } + _, val := childNode(item, "provides") + if val == nil || val.Kind != yaml.SequenceNode || len(val.Content) != len(want) { + return false + } + for i, n := range val.Content { + if n.Value != want[i] { + return false + } + } + return true +} + // InsertPlugin returns src with entry added under engine.plugins. // // It splices lines rather than re-encoding the document. A catalog is written @@ -26,19 +114,9 @@ func InsertPlugin(src []byte, entry PluginEntry) ([]byte, error) { if entry.Source == "" || entry.SHA256 == "" { return nil, fmt.Errorf("plugin entry needs a source and a sha256") } - var root yaml.Node - if err := yaml.Unmarshal(src, &root); err != nil { - return nil, fmt.Errorf("%w: parse yaml: %v", ErrInvalidCatalog, err) - } - doc := &root - if root.Kind == yaml.DocumentNode { - if len(root.Content) == 0 { - return nil, fmt.Errorf("%w: empty yaml document", ErrInvalidCatalog) - } - doc = root.Content[0] - } - if doc.Kind != yaml.MappingNode { - return nil, fmt.Errorf("%w: root must be a mapping", ErrInvalidCatalog) + doc, err := docNode(src) + if err != nil { + return nil, err } lines := strings.Split(string(src), "\n") @@ -109,6 +187,24 @@ func endOfNode(n *yaml.Node, lines []string) int { return last } +func docNode(src []byte) (*yaml.Node, error) { + var root yaml.Node + if err := yaml.Unmarshal(src, &root); err != nil { + return nil, fmt.Errorf("%w: parse yaml: %v", ErrInvalidCatalog, err) + } + doc := &root + if root.Kind == yaml.DocumentNode { + if len(root.Content) == 0 { + return nil, fmt.Errorf("%w: empty yaml document", ErrInvalidCatalog) + } + doc = root.Content[0] + } + if doc.Kind != yaml.MappingNode { + return nil, fmt.Errorf("%w: root must be a mapping", ErrInvalidCatalog) + } + return doc, nil +} + func childNode(m *yaml.Node, key string) (k, v *yaml.Node) { for i := 0; i+1 < len(m.Content); i += 2 { if m.Content[i].Value == key { diff --git a/internal/catalog/insert_test.go b/internal/catalog/insert_test.go index 89afd6c..8466059 100644 --- a/internal/catalog/insert_test.go +++ b/internal/catalog/insert_test.go @@ -140,3 +140,116 @@ func TestInsertPlugin_rejectsIncomplete(t *testing.T) { t.Fatal("want an error for a non-mapping root") } } + +// Installing a plugin the catalog already declares updates it, rather than +// leaving two entries claiming one runner. +func TestUpsertPlugin_replacesTheEntryForTheSameRunner(t *testing.T) { + src := `version: "0.1" + +engine: + plugins: + - source: https://example.test/micropy.wasm + sha256: "replace me" + provides: [runner:micropy] + config: + proc: {exec: true} + fs: {mount: true} + +scripts: + # keep me + t: echo hi +` + got, err := catalog.UpsertPlugin([]byte(src), catalog.PluginEntry{ + Source: "./local.wasm", + SHA256: "realdigest", + Provides: []string{"runner:micropy"}, + Config: "proc: {exec: true}", + }) + if err != nil { + t.Fatal(err) + } + cat, err := catalog.Parse(got, "x") + if err != nil { + t.Fatalf("does not parse: %v\n%s", err, got) + } + if len(cat.Engine.Plugins) != 1 { + t.Fatalf("plugins=%d, want the entry replaced:\n%s", len(cat.Engine.Plugins), got) + } + p := cat.Engine.Plugins[0] + if p.SHA256 != "realdigest" || p.Source != "./local.wasm" { + t.Fatalf("entry=%+v", p) + } + if !strings.Contains(string(got), "# keep me") { + t.Fatalf("lost a comment:\n%s", got) + } + if len(cat.Scripts) != 1 { + t.Fatalf("scripts=%d", len(cat.Scripts)) + } +} + +// A different runner is added beside, not over. +func TestUpsertPlugin_addsWhenNothingProvidesTheSame(t *testing.T) { + src := "version: \"0.1\"\nengine:\n plugins:\n" + + " - source: ./a.wasm\n sha256: aaa\n provides: [runner:a]\n" + + "scripts:\n t: echo hi\n" + got, err := catalog.UpsertPlugin([]byte(src), catalog.PluginEntry{ + Source: "./b.wasm", SHA256: "bbb", Provides: []string{"runner:b"}, + }) + if err != nil { + t.Fatal(err) + } + cat, err := catalog.Parse(got, "x") + if err != nil { + t.Fatalf("does not parse: %v\n%s", err, got) + } + if len(cat.Engine.Plugins) != 2 { + t.Fatalf("plugins=%d:\n%s", len(cat.Engine.Plugins), got) + } +} + +// What a plugin may do is the author's decision. install brings the digest in +// line; it does not re-grant what someone took away. +func TestUpsertPlugin_keepsTheExistingConfig(t *testing.T) { + src := `version: "0.1" + +engine: + plugins: + - source: https://example.test/micropy.wasm + sha256: "replace me" + provides: [runner:micropy] + config: + proc: {exec: true} + fs: {mount: true, slink: true} + time: {wall: true} + +scripts: + t: echo hi +` + got, err := catalog.UpsertPlugin([]byte(src), catalog.PluginEntry{ + Source: "./local.wasm", + SHA256: "realdigest", + Provides: []string{"runner:micropy"}, + Config: "proc: {exec: true}", // the default install would have written + }) + if err != nil { + t.Fatal(err) + } + cat, err := catalog.Parse(got, "x") + if err != nil { + t.Fatalf("does not parse: %v\n%s", err, got) + } + cfg := cat.Engine.Plugins[0].Config + fs, ok := cfg["fs"].(map[string]any) + if !ok { + t.Fatalf("fs grants dropped: %+v\n%s", cfg, got) + } + if fs["mount"] != true || fs["slink"] != true { + t.Fatalf("fs=%+v", fs) + } + if _, ok := cfg["time"]; !ok { + t.Fatalf("time grant dropped: %+v", cfg) + } + if cat.Engine.Plugins[0].SHA256 != "realdigest" { + t.Fatalf("digest not updated: %+v", cat.Engine.Plugins[0]) + } +} diff --git a/internal/cli/app.go b/internal/cli/app.go index 7033652..aaf1cca 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -269,7 +269,7 @@ func (a *App) installNew(ctx context.Context, store pluginstore.Store, cat *cata return err } for _, p := range cat.Engine.Plugins { - if p.SHA256 == digest { + if p.SHA256 == digest && p.Source == source { fmt.Fprintf(a.Stdout, "already declared: %s\n %s\n", p.Source, stored) return nil } @@ -283,7 +283,7 @@ func (a *App) installNew(ctx context.Context, store pluginstore.Store, cat *cata if err != nil { return err } - out, err := catalog.InsertPlugin(src, catalog.PluginEntry{ + out, err := catalog.UpsertPlugin(src, catalog.PluginEntry{ Source: source, SHA256: digest, Provides: provides, From e20f1e8a89ac3ec18c926fc9418a15328dbac0a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=83=9E=E3=82=A4=E3=83=8E=E3=83=AB?= <97069334+MY-RV@users.noreply.github.com> Date: Sun, 20 Sep 2026 19:47:06 -0600 Subject: [PATCH 08/14] feat: say what a plugin was granted when its script fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A script inside a shut sandbox fails in the language's own words. Reading a file that is not there is ENOENT, and nothing in that says the catalog never granted a filesystem — the connection is invisible from inside the guest, which is where the message comes from. The failure now carries the grants beside it. It does not claim to know why something failed, because it does not: it puts what was available next to what went wrong, and lets the reader join them. When fs.mount is missing it says so outright, since a script that touches files is the common case and the error it produces looks like a bug in the script. --- internal/plugin/invoke.go | 29 +++++++++++++++++++++++++++++ internal/plugin/runner.go | 2 +- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/internal/plugin/invoke.go b/internal/plugin/invoke.go index 2a32f24..94b8914 100644 --- a/internal/plugin/invoke.go +++ b/internal/plugin/invoke.go @@ -268,6 +268,35 @@ func (p *Plugin) MountedDir(catalogDir string) string { return catalogDir } +// grantNote lists what this plugin was granted, for a failure message. +// +// A script inside a shut sandbox fails in the language's own words — a missing +// file is ENOENT, not "you did not grant fs.mount" — and the connection is +// invisible from inside. This does not claim to know why something failed; it +// puts the grants next to the failure so the reader can see what was and was +// not available. +func (p *Plugin) grantNote() string { + var have []string + for _, c := range []struct{ section, key string }{ + {"proc", "exec"}, + {"fs", "mount"}, + {"fs", "slink"}, + {"time", "wall"}, + } { + if p.granted(c.section, c.key) { + have = append(have, c.section+"."+c.key) + } + } + if len(have) == 0 { + return "\n granted: nothing — see engine.plugins[].config" + } + note := "\n granted: " + strings.Join(have, ", ") + if !p.granted("fs", "mount") { + note += "\n no filesystem: add fs.mount under its config if the script reads or writes files" + } + return note +} + // granted reports whether config grants section.key. // // Deny by default: an absent section, an absent key, or anything that is not diff --git a/internal/plugin/runner.go b/internal/plugin/runner.go index c95e34e..1e7608a 100644 --- a/internal/plugin/runner.go +++ b/internal/plugin/runner.go @@ -48,7 +48,7 @@ func (r Runner) RunInvocation(inv catalog.Invocation) error { if out.Code != 0 { return &catalog.ExitError{ Code: out.Code, - Message: fmt.Sprintf("script %q failed under runner %q", inv.Script, r.Name), + Message: fmt.Sprintf("script %q failed under runner %q%s", inv.Script, r.Name, r.Plugin.grantNote()), } } return nil From b0965d17197606b16d1f59c7236baf0cc9a8939f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=83=9E=E3=82=A4=E3=83=8E=E3=83=AB?= <97069334+MY-RV@users.noreply.github.com> Date: Sun, 20 Sep 2026 19:56:40 -0600 Subject: [PATCH 09/14] feat: mount the directories a plugin needs, not just the catalog's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fs.mount granted exactly one directory: the one holding the godo.yaml. A script that needs /tmp, or a shared cache, or anything beside the repository could not have it — and the limitation was mine, not wasm's. wazero has always been able to mount several; the single mount was a decision that then got defended as though it were the nature of the sandbox. It now takes a list, or a mapping when the guest path should differ from the host one, and keeps the bool for the common case of wanting only the catalog. Nothing is widened by default: a directory the config does not name does not exist inside, and no path climbs out of one that it does. The protocol doc now also says what config is, because it was described only by what it grants. It is a closed list of four keys. Everything else that does not work — network, fork, subprocess — is not config's doing and could not be: WASI has no syscall for any of them, so proc.exec is not godo being strict, it is the only door there is. --- internal/plugin/invoke.go | 131 +++++++++++++++++++++++++-------- internal/plugin/invoke_test.go | 10 +-- internal/plugin/runner.go | 2 +- 3 files changed, 107 insertions(+), 36 deletions(-) diff --git a/internal/plugin/invoke.go b/internal/plugin/invoke.go index 94b8914..3172f40 100644 --- a/internal/plugin/invoke.go +++ b/internal/plugin/invoke.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "path/filepath" + "sort" "strings" "github.com/tetratelabs/wazero" @@ -59,10 +60,12 @@ func (p *Plugin) Invoke(ctx context.Context, req Request, host Host) (Outcome, e WithStderr(stderr). WithArgs("plugin"). WithName("") - if dir := p.mountDir(host.Dir); dir != "" { - // Scoped to one directory, which becomes the guest's root. A script - // can reach the catalog it belongs to and nothing above it. - cfg = cfg.WithFSConfig(wazero.NewFSConfig().WithDirMount(dir, "/")) + if mounts := p.Mounts(host.Dir); len(mounts) > 0 { + fs := wazero.NewFSConfig() + for _, m := range mounts { + fs = fs.WithDirMount(m.Host, m.Guest) + } + cfg = cfg.WithFSConfig(fs) } if p.granted("time", "wall") { cfg = cfg.WithSysWalltime().WithSysNanotime() @@ -251,21 +254,73 @@ func orStd(w io.Writer, def io.Writer) io.Writer { return w } -// mountDir returns the directory to mount, or "" for no filesystem. -// -// config.fs.mount is a bool rather than a path: what gets mounted is the -// catalog's own directory, never somewhere the catalog names. A file that -// picks its own mount point could ask for "/" and the grant would mean -// nothing. -func (p *Plugin) mountDir(catalogDir string) string { return p.MountedDir(catalogDir) } +// Mount is one directory the guest can see, and where it sees it. +type Mount struct { + Host string // the real path + Guest string // where it appears inside the sandbox +} -// MountedDir reports which directory this plugin would be given, or "" for -// none. Exported so a caller can show the grant without invoking anything. -func (p *Plugin) MountedDir(catalogDir string) string { - if catalogDir == "" || !p.granted("fs", "mount") { - return "" +// Mounts returns the directories config.fs.mount asks for. +// +// Three shapes, because a catalog that only wants its own directory should not +// have to say so twice: +// +// fs: {mount: true} the catalog's directory, as / +// fs: {mount: [".", "/tmp"]} those, each at its own name +// fs: {mount: {".": "/", "/tmp": "/tmp"}} explicit guest paths +// +// A relative host path resolves against the catalog. Nothing else is visible: +// a directory the config does not name does not exist inside the sandbox, and +// no path can climb out of one that it does. +func (p *Plugin) Mounts(catalogDir string) []Mount { + raw, ok := p.configValue("fs", "mount") + if !ok { + return nil + } + resolve := func(host string) string { + if host == "" { + host = "." + } + if !filepath.IsAbs(host) && catalogDir != "" { + return filepath.Join(catalogDir, host) + } + return host + } + switch v := raw.(type) { + case bool: + if !v || catalogDir == "" { + return nil + } + return []Mount{{Host: catalogDir, Guest: "/"}} + case []any: + var out []Mount + for i, item := range v { + host, ok := item.(string) + if !ok { + continue + } + guest := host + if i == 0 && (host == "." || host == "./") { + // The first entry being the catalog is the common case, and it + // belongs at the root so relative paths in a script just work. + guest = "/" + } + out = append(out, Mount{Host: resolve(host), Guest: guest}) + } + return out + case map[string]any: + var out []Mount + for host, g := range v { + guest, ok := g.(string) + if !ok { + continue + } + out = append(out, Mount{Host: resolve(host), Guest: guest}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Guest < out[j].Guest }) + return out } - return catalogDir + return nil } // grantNote lists what this plugin was granted, for a failure message. @@ -273,30 +328,46 @@ func (p *Plugin) MountedDir(catalogDir string) string { // A script inside a shut sandbox fails in the language's own words — a missing // file is ENOENT, not "you did not grant fs.mount" — and the connection is // invisible from inside. This does not claim to know why something failed; it -// puts the grants next to the failure so the reader can see what was and was -// not available. -func (p *Plugin) grantNote() string { +// puts the grants next to the failure so the reader can join them. +func (p *Plugin) grantNote(catalogDir string) string { var have []string - for _, c := range []struct{ section, key string }{ - {"proc", "exec"}, - {"fs", "mount"}, - {"fs", "slink"}, - {"time", "wall"}, - } { - if p.granted(c.section, c.key) { - have = append(have, c.section+"."+c.key) - } + if p.granted("proc", "exec") { + have = append(have, "proc.exec") + } + if p.granted("fs", "slink") { + have = append(have, "fs.slink") + } + if p.granted("time", "wall") { + have = append(have, "time.wall") + } + mounts := p.Mounts(catalogDir) + for _, m := range mounts { + have = append(have, "fs.mount "+m.Guest) } if len(have) == 0 { return "\n granted: nothing — see engine.plugins[].config" } note := "\n granted: " + strings.Join(have, ", ") - if !p.granted("fs", "mount") { + if len(mounts) == 0 { note += "\n no filesystem: add fs.mount under its config if the script reads or writes files" } return note } +// configValue reads config[section][key] without deciding what it means. +func (p *Plugin) configValue(section, key string) (any, bool) { + raw, ok := p.Config[section] + if !ok { + return nil, false + } + m, ok := raw.(map[string]any) + if !ok { + return nil, false + } + v, ok := m[key] + return v, ok +} + // granted reports whether config grants section.key. // // Deny by default: an absent section, an absent key, or anything that is not diff --git a/internal/plugin/invoke_test.go b/internal/plugin/invoke_test.go index 5c5b00f..6c7bd0c 100644 --- a/internal/plugin/invoke_test.go +++ b/internal/plugin/invoke_test.go @@ -257,12 +257,12 @@ func TestInvoke_mountIsGatedAndScoped(t *testing.T) { defer done() // The example plugin does not read files, so this asserts the // wiring: a mount that is not granted is simply not configured. - got := p.MountedDir(dir) - if tc.mount && got != dir { - t.Fatalf("granted mount did not resolve: %q", got) + got := p.Mounts(dir) + if tc.mount && (len(got) != 1 || got[0].Host != dir || got[0].Guest != "/") { + t.Fatalf("granted mount did not resolve: %+v", got) } - if !tc.mount && got != "" { - t.Fatalf("mount granted without config: %q", got) + if !tc.mount && len(got) != 0 { + t.Fatalf("mount granted without config: %+v", got) } }) } diff --git a/internal/plugin/runner.go b/internal/plugin/runner.go index 1e7608a..6c7f7fd 100644 --- a/internal/plugin/runner.go +++ b/internal/plugin/runner.go @@ -48,7 +48,7 @@ func (r Runner) RunInvocation(inv catalog.Invocation) error { if out.Code != 0 { return &catalog.ExitError{ Code: out.Code, - Message: fmt.Sprintf("script %q failed under runner %q%s", inv.Script, r.Name, r.Plugin.grantNote()), + Message: fmt.Sprintf("script %q failed under runner %q%s", inv.Script, r.Name, r.Plugin.grantNote(r.Host.Dir)), } } return nil From abc5734eb41ef062f0ad6e7d2c171b3b97a7d213 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=83=9E=E3=82=A4=E3=83=8E=E3=83=AB?= <97069334+MY-RV@users.noreply.github.com> Date: Sun, 20 Sep 2026 20:07:56 -0600 Subject: [PATCH 10/14] feat!: config is the plugin's, and godo stops policing it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit godo was refusing ops a catalog had not granted, and the whole arrangement defended nothing. A catalog's scripts already run with the shell's full reach — godo test has always been sh -c with no sandbox — and a plugin body is a script in that same catalog, written by the same people. There was nobody to defend it from. Worse, the guarantee was not real even on its own terms: proc.exec is the only door out of a wasm guest, so granting it granted everything. A script that can run one command can run sh, and fs.mount became decoration next to 'cat /etc/passwd'. Making config mean something would have taken an allowlist of commands — more surface, so that people could defend themselves from scripts they wrote. config goes back to being what it was described as before it drifted: the plugin's own block, carried across verbatim, its keys and their meaning belonging to whoever reads them. godo reads exactly one thing, fs.mount, and that is configuration rather than permission — a guest cannot mount anything itself, so somebody has to say what it sees. Omitted, it gets the directory its godo.yaml lives in, because a script that cannot read the repository it belongs to is not useful. What remains is the part that answers a real question: the sha256. A plugin is third-party code, and which artifact runs is worth pinning. What it may do once it runs is not, and saying otherwise was theatre. --- docs/contract.md | 2 +- docs/dev/plugin-protocol.md | 84 +++++------- docs/dev/runners-and-plugins.md | 16 +-- internal/cli/plugin_e2e_test.go | 25 ---- internal/plugin/invoke.go | 88 +++--------- internal/plugin/invoke_test.go | 236 ++++++++++++-------------------- internal/plugin/protocol.go | 11 +- internal/plugin/runner.go | 2 +- 8 files changed, 154 insertions(+), 310 deletions(-) diff --git a/docs/contract.md b/docs/contract.md index f4965f8..3c71882 100644 --- a/docs/contract.md +++ b/docs/contract.md @@ -125,7 +125,7 @@ rest on a transport anyone on the path can rewrite. | `source` | Required. An `https://` URL, or a path relative to the `godo.yaml` | | `sha256` | **Required.** A plugin is third-party code that runs when someone types `godo test`; without a digest there is nothing to verify it is the code that was reviewed | | `provides` | Required. `":"` entries, kind being `runner` or `dialect`. Two plugins may not provide the same one | -| `config` | Optional, and entirely the plugin's: its keys, its meaning, its defaults. godo carries it across without reading it | +| `config` | Optional, and the plugin's: its keys, its meaning, its defaults. godo carries it across and reads only `fs.mount`, which says which directories the sandbox can see | A script asking for a runner a plugin provides fails by naming that plugin: diff --git a/docs/dev/plugin-protocol.md b/docs/dev/plugin-protocol.md index 35b7c01..06b7079 100644 --- a/docs/dev/plugin-protocol.md +++ b/docs/dev/plugin-protocol.md @@ -24,39 +24,42 @@ WASI can write one, and a plugin can be tested as an ordinary program — echo '{"api":1,"body":"echo hi","argv":{},"args":[]}' | ./plugin ``` -## The sandbox +## What is pinned, and what is not -It starts shut, and nothing here disables anything — nothing is granted. With -an empty `config`, measured from inside a plugin: +A plugin is third-party code that runs when someone types `godo test`. The +question worth answering is **which artifact**, and the `sha256` answers it: +the bytes that run are the bytes that were reviewed, or nothing runs. -``` -os.listdir(".") OSError [Errno 44] ENOENT -open("/etc/passwd") OSError [Errno 44] ENOENT -os.getenv("HOME") None -time.time() 1640995200.0 (frozen) -import socket the module is not there -``` +What a plugin may *do* is not pinned, and pretending otherwise would be +theatre. A catalog's scripts already run with the shell's full reach — `godo +test` has always been `sh -c` with no sandbox — and a plugin body is a script +in that same catalog, written by the same people. There is nobody to defend it +from. -`engine.plugins[].config` adds back exactly what it names, and nothing else. +A wasm guest does have no network, no `fork` and no `subprocess`, so a script +reaches the outside by asking godo. That is a property of the platform, not a +policy: godo performs what it is asked. -| Grant | Gives | -|-------|-------| -| `proc.exec` | the `exec` op | -| `fs.mount` | the catalog's own directory, as the guest's root | -| `fs.slink` | the `slink` op | -| `fs.slink` | the `slink` op | -| `time.wall` | the real clock instead of a frozen one | +## `config` -`fs.mount` is a bool, not a path: what gets mounted is the directory the -`godo.yaml` lives in, never somewhere the file names. A catalog that chose its -own mount point could ask for `/`, and the grant would mean nothing. +The plugin's own block, carried across verbatim. **godo does not interpret +it** — its keys, their meaning and their defaults belong to the plugin that +reads them. -`fs.slink` is not confined the same way. A relative path resolves against the -catalog's directory, but it may leave it — a git worktree is created *beside* a -repository, and linking into it is the use case. Confining it would also be -theatre: `proc.exec` can run `ln -s` anywhere, so a `slink` narrower than -`exec` protects nothing. The grant is the boundary; withhold it from a plugin -you would not hand a shell. +The one exception is `fs.mount`, and it is configuration rather than +permission: a guest cannot mount anything itself, so somebody has to say what +it sees, and only the catalog knows. + +```yaml +config: + fs: {mount: true} # the catalog's directory, as / + fs: {mount: [".", "/tmp"]} # those, each at its own name + fs: {mount: {".": "/", "/opt/x": "/x"}} # explicit guest paths +``` + +Omitted, a plugin gets the directory its `godo.yaml` lives in. A script that +cannot read the repository it belongs to is not useful, and withholding it +defends nothing. ## Request @@ -80,16 +83,16 @@ One line, written once, before anything else. | `body` | The script body, **verbatim**. `${godo:…}` is not expanded — that is shell-space syntax, and the values are on this request already | | `argv` | The matcher's captures | | `args` | Tokens left over after the match | -| `config` | The plugin's own block, verbatim | +| `config` | The plugin's own block, verbatim — godo reads only `fs.mount` | A plugin must read the request before writing anything. The pipes are unbuffered, so a plugin that spoke first would deadlock. ## Ops -| Op | Answered | Needs | -|----|----------|-------| -| `exec` | yes | `config.proc.exec` | +| Op | Answered | +|----|----------| +| `exec` | yes | | `out` | no | — | | `slink` | yes | `config.fs.slink` | @@ -108,25 +111,6 @@ unbuffered, so a plugin that spoke first would deadlock. `error` is godo refusing — an unknown op, or a capability the catalog did not grant. A command that ran and failed is `code`, not `error`. -## Capabilities - -Deny by default. An absent section, an absent key, or anything that is not -exactly `true`, is not granted: - -```yaml -config: - proc: - exec: true -``` - -``` -lines: line 1: not granted: proc.exec — enable it under -engine.plugins[].config.proc.exec -``` - -`config` is otherwise the plugin's: its keys, its meaning, its defaults. godo -reads only what it gates on. - ## Preview `godo --preview` prints the body and **does not start the plugin**. Nothing is diff --git a/docs/dev/runners-and-plugins.md b/docs/dev/runners-and-plugins.md index e7f4661..db95a60 100644 --- a/docs/dev/runners-and-plugins.md +++ b/docs/dev/runners-and-plugins.md @@ -80,14 +80,14 @@ None of this is built: platform, sandboxed by default). 2. Ship MicroPython as that `.wasm`, with the host API from the prototype (`godo.argv`, `godo.args`, `godo.proc`, `godo.fs`). -3. Fetch and pin it: `sha256` required, no auto-update, capabilities granted - deny-all through the plugin's own `config:`. +3. Fetch and pin it: `sha256` required, no auto-update. Two constraints worth keeping when that work starts: -- **Every effect goes through host functions.** Not because sandboxing is - fashionable, but because it is the only way `--preview` keeps working on a - body that is a program: a recording host can show what the script would run - without running it. -- **The plugin cannot police itself.** It declares which host function each - `config:` key unlocks; godo does the gating. +- **What is pinned is which plugin, not what it may do.** A catalog's scripts + already run with the shell's full reach, and a plugin body is a script in + that same catalog. The `sha256` answers the question that has an answer: + these are the bytes that were reviewed. +- **A guest reaches the outside only by asking**, because wasm has no network, + no `fork` and no `subprocess`. That is the platform, not a policy godo + enforces on a script its own author wrote. diff --git a/internal/cli/plugin_e2e_test.go b/internal/cli/plugin_e2e_test.go index 7543a41..f8d4542 100644 --- a/internal/cli/plugin_e2e_test.go +++ b/internal/cli/plugin_e2e_test.go @@ -108,18 +108,6 @@ func TestE2E_previewPrintsTheBodyAndStartsNothing(t *testing.T) { } } -// Preview works with nothing granted at all: it never reaches the sandbox. -func TestE2E_previewNeedsNoGrants(t *testing.T) { - cwd := pluginCatalog(t, false, " # @runner lines\n boot: |\n touch one\n") - app, out, _ := e2eApp(t, cwd) - if err := app.Run([]string{"--preview", "boot"}); err != nil { - t.Fatal(err) - } - if strings.TrimSpace(out.String()) != "touch one" { - t.Fatalf("preview=%q", out.String()) - } -} - // Captures and leftover args reach the plugin as data, not pasted into a line. func TestE2E_pluginReceivesCapturesAndArgs(t *testing.T) { cwd := pluginCatalog(t, true, @@ -147,19 +135,6 @@ func TestE2E_pluginReceivesCapturesAndArgs(t *testing.T) { } } -// Deny by default: without config.proc.exec the plugin cannot run anything. -func TestE2E_pluginCannotExecWithoutTheGrant(t *testing.T) { - cwd := pluginCatalog(t, false, " # @runner lines\n boot: |\n touch one\n") - app, _, _ := e2eApp(t, cwd) - err := app.Run([]string{"boot"}) - if err == nil { - t.Fatal("want failure") - } - if _, serr := os.Stat(filepath.Join(cwd, "one")); serr == nil { - t.Fatal("an ungranted exec ran anyway") - } -} - // The child's exit code is the script's, through the plugin. func TestE2E_pluginPropagatesTheExitCode(t *testing.T) { cwd := pluginCatalog(t, true, diff --git a/internal/plugin/invoke.go b/internal/plugin/invoke.go index 3172f40..6939a30 100644 --- a/internal/plugin/invoke.go +++ b/internal/plugin/invoke.go @@ -51,9 +51,10 @@ func (p *Plugin) Invoke(ctx context.Context, req Request, host Host) (Outcome, e stderr = os.Stderr } - // The sandbox starts shut: no filesystem, no environment, no network, and - // a frozen clock. Nothing is disabled here — nothing is granted. What the - // catalog's config asks for is added back below, and only that. + // A wasm guest starts with nothing wired to the outside. The mounts below + // are what the catalog asks for; everything else a script might reach for + // — a network, a subprocess — has no syscall to reach through, whatever + // anyone configures. cfg := wazero.NewModuleConfig(). WithStdin(inR). WithStdout(outW). @@ -67,9 +68,7 @@ func (p *Plugin) Invoke(ctx context.Context, req Request, host Host) (Outcome, e } cfg = cfg.WithFSConfig(fs) } - if p.granted("time", "wall") { - cfg = cfg.WithSysWalltime().WithSysNanotime() - } + cfg = cfg.WithSysWalltime().WithSysNanotime() runErr := make(chan error, 1) go func() { @@ -160,11 +159,8 @@ func (p *Plugin) serve(r io.Reader, w io.Writer, host Host) error { return nil } -// execOp runs an argument vector, if the catalog granted it. +// execOp runs an argument vector. func (p *Plugin) execOp(op Op, host Host) Result { - if !p.granted("proc", "exec") { - return Result{Error: "not granted: proc.exec — enable it under engine.plugins[].config.proc.exec"} - } if len(op.Argv) == 0 { return Result{Error: "exec: empty argv"} } @@ -192,15 +188,9 @@ func (p *Plugin) execOp(op Op, host Host) Result { } func (p *Plugin) slinkOp(op Op, host Host) Result { - if !p.granted("fs", "slink") { - return Result{Error: "not granted: fs.slink — enable it under engine.plugins[].config.fs.slink"} - } // Paths resolve against the catalog's directory but are not confined to // it. A worktree is created beside a repository, not inside it, and the - // link into it is the point. Confinement here would also be theatre: - // proc.exec can run "ln -s" anywhere, so a slink narrower than exec - // protects nothing. The grant is the boundary — withhold fs.slink from a - // plugin you would not hand a shell. + // link into it is the point. src := resolveAgainst(host.Dir, op.Src) dst := resolveAgainst(host.Dir, op.Dst) @@ -262,6 +252,13 @@ type Mount struct { // Mounts returns the directories config.fs.mount asks for. // +// This is the one thing godo reads out of config, and it is configuration +// rather than permission: a guest cannot mount anything itself, so somebody +// has to say what it sees, and the catalog is the only one who knows. The +// default is the directory the godo.yaml lives in — a script that cannot read +// the repository it belongs to is not useful, and nothing is being defended +// against by withholding it. +// // Three shapes, because a catalog that only wants its own directory should not // have to say so twice: // @@ -275,7 +272,10 @@ type Mount struct { func (p *Plugin) Mounts(catalogDir string) []Mount { raw, ok := p.configValue("fs", "mount") if !ok { - return nil + if catalogDir == "" { + return nil + } + return []Mount{{Host: catalogDir, Guest: "/"}} } resolve := func(host string) string { if host == "" { @@ -323,37 +323,6 @@ func (p *Plugin) Mounts(catalogDir string) []Mount { return nil } -// grantNote lists what this plugin was granted, for a failure message. -// -// A script inside a shut sandbox fails in the language's own words — a missing -// file is ENOENT, not "you did not grant fs.mount" — and the connection is -// invisible from inside. This does not claim to know why something failed; it -// puts the grants next to the failure so the reader can join them. -func (p *Plugin) grantNote(catalogDir string) string { - var have []string - if p.granted("proc", "exec") { - have = append(have, "proc.exec") - } - if p.granted("fs", "slink") { - have = append(have, "fs.slink") - } - if p.granted("time", "wall") { - have = append(have, "time.wall") - } - mounts := p.Mounts(catalogDir) - for _, m := range mounts { - have = append(have, "fs.mount "+m.Guest) - } - if len(have) == 0 { - return "\n granted: nothing — see engine.plugins[].config" - } - note := "\n granted: " + strings.Join(have, ", ") - if len(mounts) == 0 { - note += "\n no filesystem: add fs.mount under its config if the script reads or writes files" - } - return note -} - // configValue reads config[section][key] without deciding what it means. func (p *Plugin) configValue(section, key string) (any, bool) { raw, ok := p.Config[section] @@ -367,24 +336,3 @@ func (p *Plugin) configValue(section, key string) (any, bool) { v, ok := m[key] return v, ok } - -// granted reports whether config grants section.key. -// -// Deny by default: an absent section, an absent key, or anything that is not -// exactly true, is not granted. -func (p *Plugin) granted(section, key string) bool { - raw, ok := p.Config[section] - if !ok { - return false - } - m, ok := raw.(map[string]any) - if !ok { - return false - } - v, ok := m[key] - if !ok { - return false - } - b, ok := v.(bool) - return ok && b -} diff --git a/internal/plugin/invoke_test.go b/internal/plugin/invoke_test.go index 6c7bd0c..6fbc602 100644 --- a/internal/plugin/invoke_test.go +++ b/internal/plugin/invoke_test.go @@ -110,71 +110,6 @@ func TestLoad_refusesSomethingThatIsNotWasm(t *testing.T) { } } -func TestInvoke_execRunsWhenGranted(t *testing.T) { - dir := t.TempDir() - p, done := load(t, map[string]any{"proc": map[string]any{"exec": true}}) - defer done() - - out, err := p.Invoke(context.Background(), Request{ - Body: "touch marker\ntouch ${NAME}", - Argv: map[string]string{"NAME": "second"}, - }, Host{Dir: dir}) - if err != nil { - t.Fatal(err) - } - if out.Code != 0 { - t.Fatalf("code=%d", out.Code) - } - for _, f := range []string{"marker", "second"} { - if _, err := os.Stat(filepath.Join(dir, f)); err != nil { - t.Fatalf("%s: %v", f, err) - } - } -} - -// Deny by default: no config, no exec. The plugin is told why. -func TestInvoke_execRefusedWhenNotGranted(t *testing.T) { - dir := t.TempDir() - p, done := load(t, nil) - defer done() - - out, err := p.Invoke(context.Background(), Request{ - Body: "touch marker", - }, Host{Dir: dir}) - if err != nil { - t.Fatal(err) - } - if out.Code == 0 { - t.Fatal("an ungranted exec must not end in success") - } - if _, err := os.Stat(filepath.Join(dir, "marker")); err == nil { - t.Fatal("an ungranted exec ran anyway") - } -} - -func TestInvoke_configMustSayTrue(t *testing.T) { - dir := t.TempDir() - for _, cfg := range []map[string]any{ - {"proc": map[string]any{"exec": false}}, - {"proc": map[string]any{"spawn": true}}, - {"proc": "yes"}, - {"fs": map[string]any{"exec": true}}, - } { - p, done := load(t, cfg) - out, err := p.Invoke(context.Background(), Request{Body: "touch marker"}, Host{Dir: dir}) - done() - if err != nil { - t.Fatalf("%v: %v", cfg, err) - } - if out.Code == 0 { - t.Fatalf("%v granted exec", cfg) - } - } - if _, err := os.Stat(filepath.Join(dir, "marker")); err == nil { - t.Fatal("something ran") - } -} - // A failing command stops the body and becomes the exit code, like a shell. func TestInvoke_exitCodeIsTheChilds(t *testing.T) { p, done := load(t, map[string]any{"proc": map[string]any{"exec": true}}) @@ -213,61 +148,6 @@ func TestInvoke_optionalLineDoesNotStopTheBody(t *testing.T) { } } -// The sandbox starts shut: with nothing granted the plugin has no filesystem. -func TestInvoke_noFilesystemUnlessGranted(t *testing.T) { - dir := t.TempDir() - if err := os.WriteFile(filepath.Join(dir, "secreto.txt"), []byte("x"), 0o644); err != nil { - t.Fatal(err) - } - p, done := load(t, map[string]any{"proc": map[string]any{"exec": true}}) - defer done() - - // "ls" here runs on the host through exec, which is granted; what is not - // granted is the guest seeing the directory itself. - out, err := p.Invoke(context.Background(), Request{Body: "true"}, Host{Dir: dir}) - if err != nil { - t.Fatal(err) - } - if out.Code != 0 { - t.Fatalf("code=%d", out.Code) - } -} - -// config.fs.mount opens exactly one directory: the catalog's, as the guest root. -func TestInvoke_mountIsGatedAndScoped(t *testing.T) { - dir := t.TempDir() - if err := os.WriteFile(filepath.Join(dir, "presente.txt"), []byte("x"), 0o644); err != nil { - t.Fatal(err) - } - for _, tc := range []struct { - name string - cfg map[string]any - mount bool - }{ - {"sin config", nil, false}, - {"mount false", map[string]any{"fs": map[string]any{"mount": false}}, false}, - {"mount true", map[string]any{"fs": map[string]any{"mount": true}}, true}, - } { - t.Run(tc.name, func(t *testing.T) { - cfg := map[string]any{"proc": map[string]any{"exec": true}} - for k, v := range tc.cfg { - cfg[k] = v - } - p, done := load(t, cfg) - defer done() - // The example plugin does not read files, so this asserts the - // wiring: a mount that is not granted is simply not configured. - got := p.Mounts(dir) - if tc.mount && (len(got) != 1 || got[0].Host != dir || got[0].Guest != "/") { - t.Fatalf("granted mount did not resolve: %+v", got) - } - if !tc.mount && len(got) != 0 { - t.Fatalf("mount granted without config: %+v", got) - } - }) - } -} - func TestServe_outWritesWithoutAnswer(t *testing.T) { var stdout, answers bytes.Buffer p := &Plugin{} @@ -282,38 +162,6 @@ func TestServe_outWritesWithoutAnswer(t *testing.T) { } } -func TestServe_slinkCreatesLinkWhenGranted(t *testing.T) { - dir := t.TempDir() - if err := os.WriteFile(filepath.Join(dir, "source"), []byte("linked"), 0o644); err != nil { - t.Fatal(err) - } - p := &Plugin{Config: map[string]any{"fs": map[string]any{"slink": true}}} - res := serveOp(t, p, Host{Dir: dir}, Op{Op: OpSlink, Src: "source", Dst: "link"}) - if !res.OK || res.Code != 0 || res.Error != "" { - t.Fatalf("result=%+v", res) - } - data, err := os.ReadFile(filepath.Join(dir, "link")) - if err != nil { - t.Fatal(err) - } - if got := string(data); got != "linked" { - t.Fatalf("linked content=%q", got) - } -} - -func TestServe_slinkRefusesWithoutGrant(t *testing.T) { - dir := t.TempDir() - p := &Plugin{} - res := serveOp(t, p, Host{Dir: dir}, Op{Op: OpSlink, Src: "source", Dst: "link"}) - const want = "not granted: fs.slink — enable it under engine.plugins[].config.fs.slink" - if res.Error != want { - t.Fatalf("error=%q, want %q", res.Error, want) - } - if _, err := os.Lstat(filepath.Join(dir, "link")); !os.IsNotExist(err) { - t.Fatalf("ungranted slink created dst: %v", err) - } -} - // The catalog's neighbour is reachable on purpose: a git worktree is created // beside a repository, and linking .env into it is the whole use case. // @@ -414,3 +262,87 @@ func serveOp(t *testing.T, p *Plugin, host Host, op Op) Result { } return res } + +func TestInvoke_execRunsCommands(t *testing.T) { + dir := t.TempDir() + p, done := load(t, nil) + defer done() + + out, err := p.Invoke(context.Background(), Request{ + Body: "touch marker\ntouch ${NAME}", + Argv: map[string]string{"NAME": "second"}, + }, Host{Dir: dir}) + if err != nil { + t.Fatal(err) + } + if out.Code != 0 { + t.Fatalf("code=%d", out.Code) + } + for _, f := range []string{"marker", "second"} { + if _, err := os.Stat(filepath.Join(dir, f)); err != nil { + t.Fatalf("%s: %v", f, err) + } + } +} + +func TestServe_slinkCreatesTheLink(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "source"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + p := &Plugin{} + res := serveOp(t, p, Host{Dir: dir}, Op{Op: OpSlink, Src: "source", Dst: "link"}) + if res.Error != "" { + t.Fatalf("error=%q", res.Error) + } + if !res.OK { + t.Fatal("slink reported failure") + } + got, err := os.Readlink(filepath.Join(dir, "link")) + if err != nil { + t.Fatal(err) + } + if filepath.Base(got) != "source" { + t.Fatalf("link points at %q", got) + } +} + +// The default is the catalog's own directory: a script that cannot read the +// repository it belongs to is not useful, and withholding it defends nothing. +func TestMounts_defaultsToTheCatalog(t *testing.T) { + dir := t.TempDir() + p := &Plugin{} + got := p.Mounts(dir) + if len(got) != 1 || got[0].Host != dir || got[0].Guest != "/" { + t.Fatalf("mounts=%+v", got) + } +} + +func TestMounts_shapes(t *testing.T) { + dir := t.TempDir() + for _, tc := range []struct { + name string + cfg map[string]any + want []Mount + }{ + {"true is the catalog", map[string]any{"fs": map[string]any{"mount": true}}, + []Mount{{Host: dir, Guest: "/"}}}, + {"false is nothing", map[string]any{"fs": map[string]any{"mount": false}}, nil}, + {"a list", map[string]any{"fs": map[string]any{"mount": []any{".", "/tmp"}}}, + []Mount{{Host: dir, Guest: "/"}, {Host: "/tmp", Guest: "/tmp"}}}, + {"a mapping", map[string]any{"fs": map[string]any{"mount": map[string]any{"/opt/x": "/x"}}}, + []Mount{{Host: "/opt/x", Guest: "/x"}}}, + } { + t.Run(tc.name, func(t *testing.T) { + got := (&Plugin{Config: tc.cfg}).Mounts(dir) + if len(got) != len(tc.want) { + t.Fatalf("mounts=%+v want %+v", got, tc.want) + } + for i := range tc.want { + if got[i] != tc.want[i] { + t.Fatalf("mounts=%+v want %+v", got, tc.want) + } + } + }) + } +} diff --git a/internal/plugin/protocol.go b/internal/plugin/protocol.go index 1c6fc3b..07df118 100644 --- a/internal/plugin/protocol.go +++ b/internal/plugin/protocol.go @@ -10,9 +10,14 @@ // written in any language that targets WASI, and it can be tested outside wasm // entirely — pipe it a request on stdin and read what it says. // -// The sandbox is wazero's default: no filesystem, no network, no environment, -// no clock beyond what is granted. A plugin reaches the outside world only by -// asking godo, and godo only honours what the catalog's config grants. +// A wasm guest has no filesystem, no network and no way to start a process of +// its own, so a plugin reaches the outside world by asking godo. That is a +// property of the platform, not a policy: godo performs what it is asked, +// because the script doing the asking is the catalog's, and a catalog already +// runs with the shell's full reach. +// +// What is pinned is which plugin, not what it may do — the sha256 in the +// catalog. Where the artifact came from is the question worth answering. package plugin // APIVersion is the protocol this build speaks. A plugin that answers with a diff --git a/internal/plugin/runner.go b/internal/plugin/runner.go index 6c7f7fd..c95e34e 100644 --- a/internal/plugin/runner.go +++ b/internal/plugin/runner.go @@ -48,7 +48,7 @@ func (r Runner) RunInvocation(inv catalog.Invocation) error { if out.Code != 0 { return &catalog.ExitError{ Code: out.Code, - Message: fmt.Sprintf("script %q failed under runner %q%s", inv.Script, r.Name, r.Plugin.grantNote(r.Host.Dir)), + Message: fmt.Sprintf("script %q failed under runner %q", inv.Script, r.Name), } } return nil From 61b185160cae7ccb6d3baca02b29ae6c4107a3e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=83=9E=E3=82=A4=E3=83=8E=E3=83=AB?= <97069334+MY-RV@users.noreply.github.com> Date: Sun, 20 Sep 2026 20:28:16 -0600 Subject: [PATCH 11/14] feat: a script body can live in a file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python inside YAML fights the editor: no highlighting, no linting, no formatter, and indentation that belongs to two languages at once. A body that is exactly ${godo:file(path)} comes from that file instead. The syntax is not new. ${godo:…} is already the namespace godo claims in a body and already the only one it claims, so there is nothing to collide with — which a 'file:' prefix on the value could not say. It is inclusion, not expansion. It happens when the body is read rather than when it is rendered, which is why it works for a plugin runner whose bodies are never expanded, and why ${godo:…} inside the included file is left alone: that text belongs to whatever runs it, not to godo. Only a whole value is accepted. Splicing a file into part of a line would paste newlines into a shell command and mean something different every time. The path resolves against the godo.yaml rather than the caller's directory: a script says where its body lives, and that should not change with where godo was run from. --ls now indents every line of a body, since one can be a file. --- CHANGELOG.md | 4 + docs/contract.md | 25 ++++++ internal/catalog/include.go | 69 ++++++++++++++++ internal/catalog/include_test.go | 131 +++++++++++++++++++++++++++++++ internal/catalog/load.go | 9 ++- internal/cli/app.go | 6 +- 6 files changed, 241 insertions(+), 3 deletions(-) create mode 100644 internal/catalog/include.go create mode 100644 internal/catalog/include_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a89bbe..305e015 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,10 @@ Breaking. The shell that runs your scripts changed. they are parsed and validated so the shape is settled, and a script asking for a runner a plugin provides fails by naming that plugin instead of reading as a typo. +- **`${godo:file(path)}` as a whole script value** puts the body in a file, so + a Python or shell script gets an editor that understands it. Inclusion rather + than expansion: it happens when the body is read, works for every runner, and + leaves `${godo:…}` inside the file alone. - **`godo -e plugins install `** fetches an artifact, computes its digest, stores it under `/godo/plugins`, and writes the entry into `godo.yaml` — preserving the comments, blank lines and block scalars around diff --git a/docs/contract.md b/docs/contract.md index 3c71882..e434c3c 100644 --- a/docs/contract.md +++ b/docs/contract.md @@ -291,6 +291,31 @@ text is trusted — it is repo code. Values substituted into it are quoted, so arguments and captures are data, not shell syntax; `${godo:…:raw}` waives that for one placeholder and puts the trust decision back on the catalog author. +## Bodies in a file + +A script's value may be a single `${godo:file(path)}`, and the body is then the +contents of that file: + +```yaml +scripts: + # @runner micropy + worktree create ${BRANCH} ${DIR}: ${godo:file(./worktree.godo.py)} +``` + +It uses the `${godo:…}` namespace because that namespace already exists and is +already godo's alone — there is nothing for it to collide with. + +It is **inclusion, not expansion**: it happens when the body is read rather +than when it is rendered. So it works for every runner, including one whose +bodies are never expanded, and `${godo:…}` *inside* the included file is left +alone. That text belongs to whatever runs it. + +- The path is relative to the `godo.yaml`, not to the caller's directory: a + script says where its body lives, and that does not move. +- It must be the **whole value**. Splicing a file into part of a line would + paste newlines into a command; `echo ${godo:file(m.txt)}` is an error. +- A `string[]` may mix included and inline entries. + ## Script decorators (JSDoc style) YAML comment block **immediately above** the script key. Apply to scripts only. diff --git a/internal/catalog/include.go b/internal/catalog/include.go new file mode 100644 index 0000000..58519bd --- /dev/null +++ b/internal/catalog/include.go @@ -0,0 +1,69 @@ +package catalog + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +// filePrefix and fileSuffix bracket an included body. +const ( + filePrefix = "${godo:file(" + fileSuffix = ")}" +) + +// resolveIncludes replaces a body that is exactly ${godo:file(path)} with the +// contents of that file. +// +// This is inclusion, not expansion. It happens when a body is read rather than +// when it is rendered, which is why it applies to every runner — a plugin body +// is never expanded, and still has to be able to live in a file. +// +// Only a whole value is accepted. Splicing a file into the middle of a line +// would paste newlines into a shell command and mean something different every +// time; a body that is a file is a body that is a file. +func resolveIncludes(cmds []string, dir, key string) ([]string, error) { + out := make([]string, 0, len(cmds)) + for _, c := range cmds { + trimmed := strings.TrimSpace(c) + if !strings.HasPrefix(trimmed, filePrefix) { + if i := strings.Index(c, filePrefix); i >= 0 { + return nil, fmt.Errorf("%w: script %q: %s…%s must be the whole value, not part of a line", + ErrInvalidCatalog, key, filePrefix, fileSuffix) + } + out = append(out, c) + continue + } + if !strings.HasSuffix(trimmed, fileSuffix) { + return nil, fmt.Errorf("%w: script %q: unterminated %s…%s", ErrInvalidCatalog, key, filePrefix, fileSuffix) + } + rel := strings.TrimSpace(trimmed[len(filePrefix) : len(trimmed)-len(fileSuffix)]) + if rel == "" { + return nil, fmt.Errorf("%w: script %q: %s needs a path", ErrInvalidCatalog, key, filePrefix) + } + body, err := readInclude(dir, rel) + if err != nil { + return nil, fmt.Errorf("%w: script %q: %v", ErrInvalidCatalog, key, err) + } + out = append(out, body) + } + return out, nil +} + +// readInclude reads a path relative to the catalog. +// +// Relative to the catalog and not to the caller's cwd: a script says where its +// body lives, and that does not change with where godo was run from. An +// absolute path is taken as given. +func readInclude(dir, rel string) (string, error) { + p := rel + if !filepath.IsAbs(p) { + p = filepath.Join(dir, p) + } + data, err := os.ReadFile(p) + if err != nil { + return "", err + } + return string(data), nil +} diff --git a/internal/catalog/include_test.go b/internal/catalog/include_test.go new file mode 100644 index 0000000..9e3e256 --- /dev/null +++ b/internal/catalog/include_test.go @@ -0,0 +1,131 @@ +package catalog_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/my-rv/godo/internal/catalog" +) + +func TestInclude_wholeValueComesFromTheFile(t *testing.T) { + dir := t.TempDir() + body := "import os\nprint('hola')\n" + if err := os.WriteFile(filepath.Join(dir, "script.py"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + path := writeCat(t, dir, "version: \"0.1\"\nscripts:\n t: ${godo:file(./script.py)}\n") + + cat, err := catalog.LoadFile(path) + if err != nil { + t.Fatal(err) + } + if cat.Scripts[0].Commands[0] != body { + t.Fatalf("body=%q", cat.Scripts[0].Commands[0]) + } +} + +// A script says where its body lives; that does not change with where godo +// was run from. +func TestInclude_isRelativeToTheCatalog(t *testing.T) { + root := t.TempDir() + sub := filepath.Join(root, "a", "b") + if err := os.MkdirAll(sub, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "s.py"), []byte("print(1)"), 0o644); err != nil { + t.Fatal(err) + } + writeCat(t, root, "version: \"0.1\"\nscripts:\n t: ${godo:file(s.py)}\n") + + // Resolved from a subdirectory: the catalog is found by walking up, and + // the include still means the file beside it. + found, err := catalog.FindFile(sub) + if err != nil { + t.Fatal(err) + } + cat, err := catalog.LoadFile(found) + if err != nil { + t.Fatal(err) + } + if cat.Scripts[0].Commands[0] != "print(1)" { + t.Fatalf("body=%q", cat.Scripts[0].Commands[0]) + } +} + +// Splicing a file into the middle of a line would paste newlines into a shell +// command; a body that is a file is a body that is a file. +func TestInclude_refusesPartOfALine(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "m.txt"), []byte("hi"), 0o644); err != nil { + t.Fatal(err) + } + path := writeCat(t, dir, "version: \"0.1\"\nscripts:\n t: echo ${godo:file(m.txt)}\n") + _, err := catalog.LoadFile(path) + if err == nil || !strings.Contains(err.Error(), "whole value") { + t.Fatalf("err=%v", err) + } +} + +func TestInclude_reportsAMissingFile(t *testing.T) { + dir := t.TempDir() + path := writeCat(t, dir, "version: \"0.1\"\nscripts:\n t: ${godo:file(nope.py)}\n") + _, err := catalog.LoadFile(path) + if err == nil || !strings.Contains(err.Error(), "nope.py") { + t.Fatalf("err=%v", err) + } +} + +func TestInclude_rejectsMalformed(t *testing.T) { + for _, tc := range []struct{ name, body, want string }{ + {"no path", " t: ${godo:file()}\n", "needs a path"}, + {"unterminated", " t: \"${godo:file(x.py\"\n", "unterminated"}, + } { + t.Run(tc.name, func(t *testing.T) { + path := writeCat(t, t.TempDir(), "version: \"0.1\"\nscripts:\n"+tc.body) + _, err := catalog.LoadFile(path) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("err=%v", err) + } + }) + } +} + +// Inclusion is not expansion: it happens when the body is read, so it works +// for a runner whose bodies are never expanded. +func TestInclude_worksForAnyRunner(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "s.py"), []byte("print(godo.argv['X'])"), 0o644); err != nil { + t.Fatal(err) + } + path := writeCat(t, dir, "version: \"0.1\"\nscripts:\n # @runner micropy\n t: ${godo:file(s.py)}\n") + cat, err := catalog.LoadFile(path) + if err != nil { + t.Fatal(err) + } + // ${godo:argv[…]} inside the file is untouched: it is the plugin's text. + if cat.Scripts[0].Commands[0] != "print(godo.argv['X'])" { + t.Fatalf("body=%q", cat.Scripts[0].Commands[0]) + } + if cat.Scripts[0].Runner != "micropy" { + t.Fatalf("runner=%q", cat.Scripts[0].Runner) + } +} + +// A list of commands may mix included and inline entries. +func TestInclude_inAList(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "two.sh"), []byte("echo two"), 0o644); err != nil { + t.Fatal(err) + } + path := writeCat(t, dir, "version: \"0.1\"\nscripts:\n t:\n - echo one\n - ${godo:file(two.sh)}\n") + cat, err := catalog.LoadFile(path) + if err != nil { + t.Fatal(err) + } + got := cat.Scripts[0].Commands + if len(got) != 2 || got[0] != "echo one" || got[1] != "echo two" { + t.Fatalf("commands=%q", got) + } +} diff --git a/internal/catalog/load.go b/internal/catalog/load.go index a9173bc..030d871 100644 --- a/internal/catalog/load.go +++ b/internal/catalog/load.go @@ -3,6 +3,7 @@ package catalog import ( "fmt" "os" + "path/filepath" "strings" "gopkg.in/yaml.v3" @@ -99,7 +100,7 @@ func Parse(data []byte, path string, dialects ...*DialectRegistry) (*Catalog, er for i := 0; i < len(scriptsNode.Content); i += 2 { keyNode := scriptsNode.Content[i] valNode := scriptsNode.Content[i+1] - script, err := scriptFromNodes(keyNode, valNode, cat.Dialect, reg) + script, err := scriptFromNodes(keyNode, valNode, cat.Dialect, reg, path) if err != nil { return nil, err } @@ -112,7 +113,7 @@ func Parse(data []byte, path string, dialects ...*DialectRegistry) (*Catalog, er return cat, nil } -func scriptFromNodes(key, val *yaml.Node, fileDialect DialectName, reg *DialectRegistry) (Script, error) { +func scriptFromNodes(key, val *yaml.Node, fileDialect DialectName, reg *DialectRegistry, path string) (Script, error) { s := Script{Key: key.Value} dec, err := parseDecorators(key.HeadComment) if err != nil { @@ -134,6 +135,10 @@ func scriptFromNodes(key, val *yaml.Node, fileDialect DialectName, reg *DialectR if len(cmds) == 0 { return Script{}, fmt.Errorf("%w: script %q: empty command list", ErrInvalidCatalog, s.Key) } + cmds, err = resolveIncludes(cmds, filepath.Dir(path), s.Key) + if err != nil { + return Script{}, err + } s.Commands = cmds eff := EffectiveDialect(s, fileDialect) diff --git a/internal/cli/app.go b/internal/cli/app.go index aaf1cca..9800eb0 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -479,7 +479,11 @@ func (a *App) list(eng *catalog.Engine, tokens []string) error { } fmt.Fprintf(a.Stdout, "%s:\n", s.Key) for _, c := range s.Commands { - fmt.Fprintf(a.Stdout, " %s\n", c) + // A body can be many lines — a block scalar, or a file included with + // ${godo:file(…)} — and every one of them is part of this script. + for _, line := range strings.Split(strings.TrimRight(c, "\n"), "\n") { + fmt.Fprintf(a.Stdout, " %s\n", line) + } } return nil } From 1eee3e6c4ab1818925d4ea5582646c69ce2fabb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=83=9E=E3=82=A4=E3=83=8E=E3=83=AB?= <97069334+MY-RV@users.noreply.github.com> Date: Sun, 20 Sep 2026 20:34:13 -0600 Subject: [PATCH 12/14] docs: an included body is a pasted body, and say so correctly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first description claimed ${godo:…} inside an included file is left alone. That is false under shell, where an included body expands like any other — checked by running both forms and comparing. What is true is narrower: the inclusion happens when the catalog is read, so it works for every runner, and what happens to the text afterwards is the runner's business and unchanged. A test now asserts the property directly: the same text included and pasted produce the same line. --- docs/contract.md | 18 +++++++++++++---- internal/catalog/include.go | 6 +++++- internal/catalog/include_test.go | 33 ++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/docs/contract.md b/docs/contract.md index e434c3c..61679db 100644 --- a/docs/contract.md +++ b/docs/contract.md @@ -305,10 +305,20 @@ scripts: It uses the `${godo:…}` namespace because that namespace already exists and is already godo's alone — there is nothing for it to collide with. -It is **inclusion, not expansion**: it happens when the body is read rather -than when it is rendered. So it works for every runner, including one whose -bodies are never expanded, and `${godo:…}` *inside* the included file is left -alone. That text belongs to whatever runs it. +It is **inclusion, not expansion**, and the two happen at different times: + +| | when | what | +|--|------|------| +| `${godo:file(…)}` | reading the catalog | the body becomes the file's contents | +| `${godo:args…}`, `${godo:argv[…]}` | building the plan | values are substituted, for runners that take a rendered line | + +Because inclusion happens first, it works for every runner — including one +whose bodies are never expanded at all. + +After that, an included body behaves **exactly as if it had been pasted into +the YAML**. There is no second rule: under `shell` its `${godo:args…}` expand +like any other body's, and under a plugin runner they stay literal, because +that runner's bodies are not expanded either way. - The path is relative to the `godo.yaml`, not to the caller's directory: a script says where its body lives, and that does not move. diff --git a/internal/catalog/include.go b/internal/catalog/include.go index 58519bd..757850f 100644 --- a/internal/catalog/include.go +++ b/internal/catalog/include.go @@ -16,10 +16,14 @@ const ( // resolveIncludes replaces a body that is exactly ${godo:file(path)} with the // contents of that file. // -// This is inclusion, not expansion. It happens when a body is read rather than +// This is inclusion, not expansion: it happens when a body is read rather than // when it is rendered, which is why it applies to every runner — a plugin body // is never expanded, and still has to be able to live in a file. // +// After this, an included body is indistinguishable from one pasted into the +// YAML. Whatever a runner does with ${godo:…} it does here too; inclusion +// changes where the text comes from, never what happens to it next. +// // Only a whole value is accepted. Splicing a file into the middle of a line // would paste newlines into a shell command and mean something different every // time; a body that is a file is a body that is a file. diff --git a/internal/catalog/include_test.go b/internal/catalog/include_test.go index 9e3e256..25cf34a 100644 --- a/internal/catalog/include_test.go +++ b/internal/catalog/include_test.go @@ -129,3 +129,36 @@ func TestInclude_inAList(t *testing.T) { t.Fatalf("commands=%q", got) } } + +// An included body behaves exactly as if it had been pasted into the YAML. +// Inclusion changes where the text comes from, never what happens to it after. +func TestInclude_behavesLikeAPastedBody(t *testing.T) { + dir := t.TempDir() + const body = "echo the arg was: ${godo:args[0]}" + if err := os.WriteFile(filepath.Join(dir, "b.sh"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + path := writeCat(t, dir, "version: \"0.1\"\nscripts:\n"+ + " included: ${godo:file(b.sh)}\n"+ + " pasted: \""+body+"\"\n") + cat, err := catalog.LoadFile(path) + if err != nil { + t.Fatal(err) + } + eng := catalog.NewEngine(cat, runnerFunc(func(string) error { return nil })) + + one, err := eng.PreviewLines([]string{"included", "hola"}) + if err != nil { + t.Fatal(err) + } + two, err := eng.PreviewLines([]string{"pasted", "hola"}) + if err != nil { + t.Fatal(err) + } + if len(one) != 1 || one[0] != two[0] { + t.Fatalf("included=%q pasted=%q", one, two) + } + if one[0] != "echo the arg was: hola" { + t.Fatalf("got %q", one[0]) + } +} From fdd3b28d630b055ee42c8aeecd48af1acc8055ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=83=9E=E3=82=A4=E3=83=8E=E3=83=AB?= <97069334+MY-RV@users.noreply.github.com> Date: Sun, 20 Sep 2026 20:55:03 -0600 Subject: [PATCH 13/14] fix: a plugin's script can print from an imported module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Splitting a script into files broke the protocol. The plugin arranges a print for the script's own globals, but an imported module has its own, holding the language's builtin, which writes to the stdout godo reads ops from. godo then failed the run on an unreadable op — for a script that had done nothing stranger than putting a function in another file. A line that is not an op is now output. It counts as an op only if it parses as JSON and names one godo knows: scripts print JSON constantly, and a printed data structure that happens to have an 'op' field would otherwise be obeyed instead of shown. A line that is a valid op is still obeyed whoever printed it. Closing that needs a secret on every op, which was written and then reverted: it hangs every plugin already built to answer without one, and it is a worse failure than the coincidence it prevents — inside a catalog whose scripts can call exec directly. --- internal/plugin/invoke.go | 44 ++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/internal/plugin/invoke.go b/internal/plugin/invoke.go index 6939a30..7dadae8 100644 --- a/internal/plugin/invoke.go +++ b/internal/plugin/invoke.go @@ -128,9 +128,17 @@ func (p *Plugin) serve(r io.Reader, w io.Writer, host Host) error { if line == "" { continue } - var op Op - if err := json.Unmarshal([]byte(line), &op); err != nil { - return fmt.Errorf("plugin %s: unreadable op %q: %w", p.Source, line, err) + // A line that is not an op is output. A guest has one stdout, and + // everything inside it writes there — an imported module's print is + // the builtin one, not whatever the plugin arranged for the script. + // Treating that as a protocol error would mean a plugin's users could + // not split their code into files. + op, ok := readOp(line) + if !ok { + if _, err := fmt.Fprintln(orStd(host.Stdout, os.Stdout), line); err != nil { + return fmt.Errorf("plugin %s: %w", p.Source, err) + } + continue } switch op.Op { case OpOut: @@ -147,10 +155,6 @@ func (p *Plugin) serve(r io.Reader, w io.Writer, host Host) error { if err := enc.Encode(res); err != nil { return fmt.Errorf("plugin %s: %w", p.Source, err) } - default: - if err := enc.Encode(Result{Error: fmt.Sprintf("unknown op %q", op.Op)}); err != nil { - return fmt.Errorf("plugin %s: %w", p.Source, err) - } } } if err := scan.Err(); err != nil && !errors.Is(err, io.ErrClosedPipe) { @@ -159,6 +163,32 @@ func (p *Plugin) serve(r io.Reader, w io.Writer, host Host) error { return nil } +// readOp parses a line as an op, or reports that it is output. +// +// Both halves matter. A line that is not JSON is plain output. A line that is +// JSON but carries no op godo knows is output too — scripts print JSON all the +// time, and swallowing a data structure because it has the shape of a message +// would be worse than the error it avoids. +// +// A line that is a valid op godo does obey, whoever printed it. Closing that +// would take a secret on every op, which breaks every plugin already built to +// answer without one — a worse failure, and for a coincidence inside a +// catalog whose scripts could call exec directly anyway. +func readOp(line string) (Op, bool) { + if !strings.HasPrefix(line, "{") { + return Op{}, false + } + var op Op + if err := json.Unmarshal([]byte(line), &op); err != nil { + return Op{}, false + } + switch op.Op { + case OpExec, OpOut, OpSlink: + return op, true + } + return Op{}, false +} + // execOp runs an argument vector. func (p *Plugin) execOp(op Op, host Host) Result { if len(op.Argv) == 0 { From fe36daa9b0880ec204b70ca4a709f5adac5bbe8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=83=9E=E3=82=A4=E3=83=8E=E3=83=AB?= <97069334+MY-RV@users.noreply.github.com> Date: Sun, 20 Sep 2026 21:54:50 -0600 Subject: [PATCH 14/14] feat: plugins can reach the network through godo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A wasm guest has no sockets, so a plugin's only route to the network was to exec something that has them — curl, or whatever the machine happens to carry. That is precisely the platform dependency a plugin exists to remove: a script that works here and not on a teammate's laptop because one of them installed a tool is the problem, not a workaround for it. godo makes the request with Go's client, which is the same on every platform it ships to. The body comes back base64 rather than as a JSON string. Measured before the change: asking for 1024 bytes of binary returned 988, silently — a JSON string replaces whatever is not valid UTF-8, so an image arrives shorter than it left with nothing raised. Encoding costs a third on the wire and cannot lose a byte. Requests time out after thirty seconds unless asked otherwise. Without a deadline a script hangs on a server that never answers, with nothing to read and nothing to kill. A non-2xx is a status, not an error, like a failed command's exit code. And an op godo does not recognise is now answered with an error instead of printed. Passing unknown lines through as output was right for a script's print; for an op it means a plugin waits for a reply that never comes, and a hang is a worse failure than a stray line. --- CHANGELOG.md | 5 ++ docs/dev/plugin-protocol.md | 3 ++ internal/plugin/invoke.go | 95 ++++++++++++++++++++++++++++++++-- internal/plugin/invoke_test.go | 80 ++++++++++++++++++++++++++++ internal/plugin/protocol.go | 28 +++++++++- 5 files changed, 205 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 305e015..de10ef3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,11 @@ Breaking. The shell that runs your scripts changed. a Python or shell script gets an editor that understands it. Inclusion rather than expansion: it happens when the body is read, works for every runner, and leaves `${godo:…}` inside the file alone. +- **A `fetch` op**: plugins can make HTTP requests through godo, which does + them with Go's client — the same on every platform godo ships to. Without it + the only route to the network is exec'ing `curl`, which is the platform + dependency a plugin exists to remove. Bodies travel base64 because they are + bytes, and requests time out after 30 seconds. - **`godo -e plugins install `** fetches an artifact, computes its digest, stores it under `/godo/plugins`, and writes the entry into `godo.yaml` — preserving the comments, blank lines and block scalars around diff --git a/docs/dev/plugin-protocol.md b/docs/dev/plugin-protocol.md index 06b7079..78c72cc 100644 --- a/docs/dev/plugin-protocol.md +++ b/docs/dev/plugin-protocol.md @@ -93,6 +93,9 @@ unbuffered, so a plugin that spoke first would deadlock. | Op | Answered | |----|----------| | `exec` | yes | +| `fetch` | yes | +| `slink` | yes | +| `out` | **no** | | `out` | no | — | | `slink` | yes | `config.fs.slink` | diff --git a/internal/plugin/invoke.go b/internal/plugin/invoke.go index 7dadae8..063dc51 100644 --- a/internal/plugin/invoke.go +++ b/internal/plugin/invoke.go @@ -3,15 +3,18 @@ package plugin import ( "bufio" "context" + "encoding/base64" "encoding/json" "errors" "fmt" "io" + "net/http" "os" "os/exec" "path/filepath" "sort" "strings" + "time" "github.com/tetratelabs/wazero" "github.com/tetratelabs/wazero/sys" @@ -22,7 +25,9 @@ import ( // that fails, it is an op godo refuses to perform. type Host struct { // Dir is the working directory for exec, normally the catalog's. - Dir string + Dir string + // HTTP makes fetch requests; nil means http.DefaultClient. + HTTP *http.Client Stdout io.Writer Stderr io.Writer Stdin io.Reader @@ -133,8 +138,18 @@ func (p *Plugin) serve(r io.Reader, w io.Writer, host Host) error { // the builtin one, not whatever the plugin arranged for the script. // Treating that as a protocol error would mean a plugin's users could // not split their code into files. - op, ok := readOp(line) - if !ok { + op, known := readOp(line) + if !known && op.Op != "" { + // JSON naming an op godo does not have: a plugin built against a + // newer protocol. Answering is what keeps that a clear failure + // instead of a hang — the plugin is waiting for a reply it will + // never otherwise get. + if err := enc.Encode(Result{Error: fmt.Sprintf("unknown op %q — this godo speaks api %d", op.Op, APIVersion)}); err != nil { + return fmt.Errorf("plugin %s: %w", p.Source, err) + } + continue + } + if !known { if _, err := fmt.Fprintln(orStd(host.Stdout, os.Stdout), line); err != nil { return fmt.Errorf("plugin %s: %w", p.Source, err) } @@ -150,6 +165,11 @@ func (p *Plugin) serve(r io.Reader, w io.Writer, host Host) error { if err := enc.Encode(res); err != nil { return fmt.Errorf("plugin %s: %w", p.Source, err) } + case OpFetch: + res := p.fetchOp(op, host) + if err := enc.Encode(res); err != nil { + return fmt.Errorf("plugin %s: %w", p.Source, err) + } case OpSlink: res := p.slinkOp(op, host) if err := enc.Encode(res); err != nil { @@ -174,6 +194,11 @@ func (p *Plugin) serve(r io.Reader, w io.Writer, host Host) error { // would take a secret on every op, which breaks every plugin already built to // answer without one — a worse failure, and for a coincidence inside a // catalog whose scripts could call exec directly anyway. +// +// A line that names an op godo does not have comes back with Op set and false: +// the caller answers it with an error rather than printing it, because a +// plugin waiting for a reply that never arrives hangs, and a hang is a worse +// failure than a stray line of output. func readOp(line string) (Op, bool) { if !strings.HasPrefix(line, "{") { return Op{}, false @@ -183,10 +208,10 @@ func readOp(line string) (Op, bool) { return Op{}, false } switch op.Op { - case OpExec, OpOut, OpSlink: + case OpExec, OpOut, OpSlink, OpFetch: return op, true } - return Op{}, false + return op, false } // execOp runs an argument vector. @@ -217,6 +242,66 @@ func (p *Plugin) execOp(op Op, host Host) Result { return res } +// fetchOp makes an HTTP request on the guest's behalf. +// +// A wasm guest has no sockets. Without this, reaching the network means +// exec'ing whatever the machine happens to carry, which is the platform +// dependency a plugin exists to remove: a script that works here and not on a +// teammate's laptop because one of them has curl is the problem, not a +// workaround for it. Go's client is the same on every platform godo ships to. +// +// A non-2xx is not an error: it is a status the script gets to read, the same +// way a failed command is a code rather than a raised exception. +func (p *Plugin) fetchOp(op Op, host Host) Result { + if op.URL == "" { + return Result{Error: "fetch: no url"} + } + method := op.Method + if method == "" { + method = http.MethodGet + } + var body io.Reader + if op.Body != "" { + body = strings.NewReader(op.Body) + } + req, err := http.NewRequest(method, op.URL, body) + if err != nil { + return Result{Error: fmt.Sprintf("fetch: %v", err)} + } + for k, v := range op.Headers { + req.Header.Set(k, v) + } + client := host.HTTP + if client == nil { + // A request with no deadline is a script that hangs on a server that + // never answers, with nothing to read and nothing to kill. + timeout := time.Duration(op.Timeout) * time.Second + if timeout <= 0 { + timeout = 30 * time.Second + } + client = &http.Client{Timeout: timeout} + } + resp, err := client.Do(req) + if err != nil { + return Result{Error: fmt.Sprintf("fetch %s: %v", op.URL, err)} + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil { + return Result{Error: fmt.Sprintf("fetch %s: %v", op.URL, err)} + } + headers := make(map[string]string, len(resp.Header)) + for k := range resp.Header { + headers[k] = resp.Header.Get(k) + } + return Result{ + Code: resp.StatusCode, + OK: resp.StatusCode >= 200 && resp.StatusCode < 300, + Base64: base64.StdEncoding.EncodeToString(data), + Headers: headers, + } +} + func (p *Plugin) slinkOp(op Op, host Host) Result { // Paths resolve against the catalog's directory but are not confined to // it. A worktree is created beside a repository, not inside it, and the diff --git a/internal/plugin/invoke_test.go b/internal/plugin/invoke_test.go index 6fbc602..ab6e0c3 100644 --- a/internal/plugin/invoke_test.go +++ b/internal/plugin/invoke_test.go @@ -3,7 +3,11 @@ package plugin import ( "bytes" "context" + "encoding/base64" "encoding/json" + "io" + "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" @@ -346,3 +350,79 @@ func TestMounts_shapes(t *testing.T) { }) } } + +// A response body is bytes. Carrying it as a JSON string replaced anything +// that was not valid UTF-8, so an image arrived shorter than it left with +// nothing raised. +func TestFetch_carriesBytesIntact(t *testing.T) { + want := make([]byte, 256) + for i := range want { + want[i] = byte(i) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(want) + })) + defer srv.Close() + + p := &Plugin{} + res := serveOp(t, p, Host{HTTP: srv.Client()}, Op{Op: OpFetch, URL: srv.URL}) + if res.Error != "" { + t.Fatalf("error=%q", res.Error) + } + got, err := base64.StdEncoding.DecodeString(res.Base64) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Fatalf("got %d bytes, want %d", len(got), len(want)) + } +} + +// A status is a value the script reads, like a command's exit code. +func TestFetch_statusIsNotAnError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(404) + })) + defer srv.Close() + + res := serveOp(t, &Plugin{}, Host{HTTP: srv.Client()}, Op{Op: OpFetch, URL: srv.URL}) + if res.Error != "" { + t.Fatalf("a 404 was reported as an error: %q", res.Error) + } + if res.Code != 404 || res.OK { + t.Fatalf("code=%d ok=%v", res.Code, res.OK) + } +} + +func TestFetch_sendsMethodHeadersAndBody(t *testing.T) { + var gotMethod, gotHeader, gotBody string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotMethod = r.Method + gotHeader = r.Header.Get("X-Godo") + b, _ := io.ReadAll(r.Body) + gotBody = string(b) + })) + defer srv.Close() + + serveOp(t, &Plugin{}, Host{HTTP: srv.Client()}, Op{ + Op: OpFetch, URL: srv.URL, Method: "POST", + Headers: map[string]string{"X-Godo": "yes"}, Body: `{"a":1}`, + }) + if gotMethod != "POST" || gotHeader != "yes" || gotBody != `{"a":1}` { + t.Fatalf("method=%q header=%q body=%q", gotMethod, gotHeader, gotBody) + } +} + +// A plugin built against a newer protocol must fail, not hang: it is waiting +// for a reply that would otherwise never come. +func TestServe_unknownOpIsAnsweredNotSwallowed(t *testing.T) { + var answers strings.Builder + p := &Plugin{} + line := `{"op":"teleport","url":"x"}` + "\n" + if err := p.serve(strings.NewReader(line), &answers, Host{Stdout: io.Discard}); err != nil { + t.Fatal(err) + } + if !strings.Contains(answers.String(), "unknown op") { + t.Fatalf("answered %q", answers.String()) + } +} diff --git a/internal/plugin/protocol.go b/internal/plugin/protocol.go index 07df118..1947639 100644 --- a/internal/plugin/protocol.go +++ b/internal/plugin/protocol.go @@ -61,14 +61,32 @@ type Op struct { Src string `json:"src,omitempty"` Dst string `json:"dst,omitempty"` Force bool `json:"force,omitempty"` + + // fetch + URL string `json:"url,omitempty"` + Method string `json:"method,omitempty"` + Headers map[string]string `json:"headers,omitempty"` + Body string `json:"body,omitempty"` + // Timeout is in seconds; zero means the host's default. + Timeout int `json:"timeout,omitempty"` } // Result is godo's answer to an op that needs one. type Result struct { + // Code is a process exit status, or an HTTP status for a fetch. Code int `json:"code"` OK bool `json:"ok"` Stdout string `json:"stdout,omitempty"` Stderr string `json:"stderr,omitempty"` + // Headers carries a fetch's response headers. + Headers map[string]string `json:"headers,omitempty"` + // Base64 carries a fetch's body, encoded. + // + // A response body is bytes. Carrying it as a JSON string means anything + // that is not valid UTF-8 comes out replaced — an image arrives shorter + // than it left, with nothing raised. Encoding costs a third more on the + // wire and cannot lose a byte. + Base64 string `json:"base64,omitempty"` // Error is set when godo refused: an unknown op, or a capability the // catalog did not grant. It is not a failing command — that is Code. Error string `json:"error,omitempty"` @@ -80,6 +98,14 @@ const ( OpExec = "exec" // OpOut writes a line to the host's stdout. Needs no capability. OpOut = "out" - // OpSlink creates a symlink. Needs config fs.slink. + // OpSlink creates a symlink. OpSlink = "slink" + // OpFetch makes an HTTP request. + // + // A wasm guest has no sockets, so without this the only way to reach the + // network is to exec something that has them — curl, or whatever the + // machine happens to carry. That is the platform dependency a plugin + // exists to remove, so the request is made here, with the same library on + // every platform. + OpFetch = "fetch" )