diff --git a/.goreleaser.yaml b/.goreleaser.yaml index dc9a45b..3f1f2c7 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -28,6 +28,13 @@ builds: archives: - id: archive + # Default files are LICENSE*/README*/CHANGELOG*. The binaries statically + # link Apache-2.0 code (wazero), whose attribution has to ship with them. + files: + - LICENSE + - THIRD_PARTY_LICENSES.md + - README.md + - CHANGELOG.md formats: [tar.gz] format_overrides: - goos: windows diff --git a/CHANGELOG.md b/CHANGELOG.md index 942771a..de10ef3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,20 @@ Breaking. The shell that runs your scripts changed. they are parsed and validated so the shape is settled, and a script asking for a runner a plugin provides fails by naming that plugin instead of reading as a typo. +- **`${godo:file(path)}` as a whole script value** puts the body in a file, so + a Python or shell script gets an editor that understands it. Inclusion rather + than expansion: it happens when the body is read, works for every runner, and + leaves `${godo:…}` inside the file alone. +- **A `fetch` op**: plugins can make HTTP requests through godo, which does + them with Go's client — the same on every platform godo ships to. Without it + the only route to the network is exec'ing `curl`, which is the platform + dependency a plugin exists to remove. Bodies travel base64 because they are + bytes, and requests time out after 30 seconds. +- **`godo -e plugins install `** fetches an artifact, computes its + digest, stores it under `/godo/plugins`, and writes the entry into + `godo.yaml` — preserving the comments, blank lines and block scalars around + it. Without a source it fetches everything the catalog declares. + `godo -e plugins` lists what is declared and whether it is here. - **`godo -e runners`** lists what is usable on the machine you are on, and how to confirm which shell you are in when the detected one looks wrong. - `--ls ` prints `@runner` beside `@dialect`. diff --git a/THIRD_PARTY_LICENSES.md b/THIRD_PARTY_LICENSES.md new file mode 100644 index 0000000..233b77a --- /dev/null +++ b/THIRD_PARTY_LICENSES.md @@ -0,0 +1,251 @@ +# Third-party licenses + +GoDo is MIT (see [LICENSE](./LICENSE)). The binaries shipped on GitHub Releases +statically link the libraries below, so their notices travel with them. + +Source dependencies are listed in [go.mod](./go.mod). + +Every licence here is permissive. None of them asks GoDo to change its own, and +none of them reaches into a plugin: a plugin is a separate artifact from a +separate repository, and GoDo does not ship one. What a plugin contains is +that repository's inventory to publish, next to its own artifact. + +--- + +## github.com/tetratelabs/wazero + +WebAssembly runtime. Used to run plugins. + +Licensed under the Apache License, Version 2.0. + +### NOTICE + +``` +wazero +Copyright 2020-2023 wazero authors +``` + +### License + +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020-2023 wazero authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` + +--- + +## golang.org/x/sys + +Low-level OS primitives. Pulled in by wazero. + +Licensed under the BSD 3-Clause License, `Copyright (c) 2009 The Go Authors`. +Same terms as the Go standard library, whose runtime is already part of every +Go binary. + +--- + +## gopkg.in/yaml.v3 + +YAML parser. Used to read `godo.yaml`. + +Licensed under the MIT License, with portions under the Apache License, +Version 2.0. See the module's `LICENSE` and `NOTICE` files. diff --git a/docs/contract.md b/docs/contract.md index 4d9ea65..61679db 100644 --- a/docs/contract.md +++ b/docs/contract.md @@ -101,15 +101,31 @@ numbers. ### `engine.plugins` -**No build loads plugins yet.** Entries are parsed and validated so the shape -is settled and a catalog can already declare what it expects. +``` +godo -e plugins what this catalog declares, and its state +godo -e plugins install fetch everything it declares +godo -e plugins install add one, and fetch it +``` + +`install ` computes the digest from the artifact and writes the entry +into `godo.yaml`. The digest is never asked for: a person cannot check a hash +by reading it, so asking for one is how wrong hashes get committed. + +Artifacts live in `/godo/plugins`, named by digest. A file sitting +beside the `godo.yaml` is loaded from where it is — asking someone to install +what they can already see would be ceremony, and its digest is checked either +way. Anything else must be installed first; a run is not the moment to discover +that something has to be downloaded. + +`http://` sources are refused. An artifact is code, and its integrity cannot +rest on a transport anyone on the path can rewrite. | Field | | |-------|--| -| `source` | Required. Where the plugin comes from | +| `source` | Required. An `https://` URL, or a path relative to the `godo.yaml` | | `sha256` | **Required.** A plugin is third-party code that runs when someone types `godo test`; without a digest there is nothing to verify it is the code that was reviewed | | `provides` | Required. `":"` entries, kind being `runner` or `dialect`. Two plugins may not provide the same one | -| `config` | Optional, and entirely the plugin's: its keys, its meaning, its defaults. godo carries it across without reading it | +| `config` | Optional, and the plugin's: its keys, its meaning, its defaults. godo carries it across and reads only `fs.mount`, which says which directories the sandbox can see | A script asking for a runner a plugin provides fails by naming that plugin: @@ -275,6 +291,41 @@ text is trusted — it is repo code. Values substituted into it are quoted, so arguments and captures are data, not shell syntax; `${godo:…:raw}` waives that for one placeholder and puts the trust decision back on the catalog author. +## Bodies in a file + +A script's value may be a single `${godo:file(path)}`, and the body is then the +contents of that file: + +```yaml +scripts: + # @runner micropy + worktree create ${BRANCH} ${DIR}: ${godo:file(./worktree.godo.py)} +``` + +It uses the `${godo:…}` namespace because that namespace already exists and is +already godo's alone — there is nothing for it to collide with. + +It is **inclusion, not expansion**, and the two happen at different times: + +| | when | what | +|--|------|------| +| `${godo:file(…)}` | reading the catalog | the body becomes the file's contents | +| `${godo:args…}`, `${godo:argv[…]}` | building the plan | values are substituted, for runners that take a rendered line | + +Because inclusion happens first, it works for every runner — including one +whose bodies are never expanded at all. + +After that, an included body behaves **exactly as if it had been pasted into +the YAML**. There is no second rule: under `shell` its `${godo:args…}` expand +like any other body's, and under a plugin runner they stay literal, because +that runner's bodies are not expanded either way. + +- The path is relative to the `godo.yaml`, not to the caller's directory: a + script says where its body lives, and that does not move. +- It must be the **whole value**. Splicing a file into part of a line would + paste newlines into a command; `echo ${godo:file(m.txt)}` is an error. +- A `string[]` may mix included and inline entries. + ## Script decorators (JSDoc style) YAML comment block **immediately above** the script key. Apply to scripts only. diff --git a/docs/dev/README.md b/docs/dev/README.md index f30e276..08d5d75 100644 --- a/docs/dev/README.md +++ b/docs/dev/README.md @@ -8,6 +8,7 @@ Internals and contributor conventions. Product docs live in [`docs/`](../README. | [Standards](./standards.md) | Coding bar | | [Versioning](./versioning.md) | SemVer + binary vs file `version:` | | [Runners and plugins](./runners-and-plugins.md) | Premise: godo is a proxy; cross-OS is a plugin's job | +| [Plugin protocol](./plugin-protocol.md) | The wire: WASI command, JSON ops, capabilities | | [Archive](./archive/README.md) | Brainstorm notes (historical) | Start with root [`CONTRIBUTING.md`](../../CONTRIBUTING.md). diff --git a/docs/dev/plugin-protocol.md b/docs/dev/plugin-protocol.md new file mode 100644 index 0000000..78c72cc --- /dev/null +++ b/docs/dev/plugin-protocol.md @@ -0,0 +1,184 @@ +# Plugin protocol + +How godo talks to a plugin. Not the contract — [`contract.md`](../contract.md) +covers the `godo.yaml` side; this is the wire. + +## What a plugin is + +A **WASI command**: a normal program with a `main()`, compiled to wasm. + +``` +godo → plugin stdin one JSON request +godo ← plugin stdout JSON ops, one per line +godo → plugin stdin one JSON result per op that needs one +plugin exits its exit code is the script's +``` + +`--preview` never starts a plugin. It prints the body. + +Being a command rather than a set of exported functions keeps the contract +small: there is no memory-sharing ABI to get right, any language that targets +WASI can write one, and a plugin can be tested as an ordinary program — + +```bash +echo '{"api":1,"body":"echo hi","argv":{},"args":[]}' | ./plugin +``` + +## What is pinned, and what is not + +A plugin is third-party code that runs when someone types `godo test`. The +question worth answering is **which artifact**, and the `sha256` answers it: +the bytes that run are the bytes that were reviewed, or nothing runs. + +What a plugin may *do* is not pinned, and pretending otherwise would be +theatre. A catalog's scripts already run with the shell's full reach — `godo +test` has always been `sh -c` with no sandbox — and a plugin body is a script +in that same catalog, written by the same people. There is nobody to defend it +from. + +A wasm guest does have no network, no `fork` and no `subprocess`, so a script +reaches the outside by asking godo. That is a property of the platform, not a +policy: godo performs what it is asked. + +## `config` + +The plugin's own block, carried across verbatim. **godo does not interpret +it** — its keys, their meaning and their defaults belong to the plugin that +reads them. + +The one exception is `fs.mount`, and it is configuration rather than +permission: a guest cannot mount anything itself, so somebody has to say what +it sees, and only the catalog knows. + +```yaml +config: + fs: {mount: true} # the catalog's directory, as / + fs: {mount: [".", "/tmp"]} # those, each at its own name + fs: {mount: {".": "/", "/opt/x": "/x"}} # explicit guest paths +``` + +Omitted, a plugin gets the directory its `godo.yaml` lives in. A script that +cannot read the repository it belongs to is not useful, and withholding it +defends nothing. + +## Request + +One line, written once, before anything else. + +```json +{ + "api": 1, + "runner": "lines", + "body": "git status\n?pnpm install", + "argv": {"BRANCH": "wt/example"}, + "args": ["--no-install"], + "config": {"proc": {"exec": true}} +} +``` + +| Field | | +|-------|--| +| `api` | Protocol version. A plugin that speaks another major should exit non-zero rather than guess | +| `runner` | The name the catalog asked for, so one plugin can provide several | +| `body` | The script body, **verbatim**. `${godo:…}` is not expanded — that is shell-space syntax, and the values are on this request already | +| `argv` | The matcher's captures | +| `args` | Tokens left over after the match | +| `config` | The plugin's own block, verbatim — godo reads only `fs.mount` | + +A plugin must read the request before writing anything. The pipes are +unbuffered, so a plugin that spoke first would deadlock. + +## Ops + +| Op | Answered | +|----|----------| +| `exec` | yes | +| `fetch` | yes | +| `slink` | yes | +| `out` | **no** | +| `out` | no | — | +| `slink` | yes | `config.fs.slink` | + +```json +{"op":"exec","argv":["git","status"],"dir":"","capture":false} +{"op":"out","text":"updated dependencies"} +{"op":"slink","src":"bin/godo","dst":".bin/godo","force":false} +``` + +### Result + +```json +{"code":0,"ok":true,"stdout":"","stderr":"","error":""} +``` + +`error` is godo refusing — an unknown op, or a capability the catalog did not +grant. A command that ran and failed is `code`, not `error`. + +## Preview + +`godo --preview` prints the body and **does not start the plugin**. Nothing is +compiled, nothing is instantiated, no grant is needed. + +A plugin body is a program. The only faithful answer to "what will this do" +without running it is the program itself; anything else is a guess. A guess is +useful, but it deserves its own flag rather than quietly borrowing this one — +that is a later `--predict`, which will add back a mode field and an output op. +Plugins ignore fields they do not know, so nothing written today breaks then. + +## Exit codes + +The plugin's exit code is the script's. A plugin propagating a child's code +gets godo exiting with that code, like a shell would. + +## Worked example + +[`examples/plugins/lines`](../../examples/plugins/lines) — a runner whose body +is one command per line. Under 200 lines of ordinary Go, no wasm-specific code. + +```bash +GOOS=wasip1 GOARCH=wasm go build -o lines.wasm ./examples/plugins/lines +shasum -a 256 lines.wasm +``` + +```yaml +engine: + plugins: + - source: ./lines.wasm + sha256: "…" + provides: [runner:lines] + config: + proc: {exec: true} + +scripts: + # @runner lines + boot: | + git status + ?pnpm install +``` + +## Licensing + +A plugin is a separate artifact from a separate repository, and godo does not +ship one. So godo's own licensing is unaffected by what a plugin contains — +but the repository that *does* ship one carries whatever is inside it. + +A repository distributing a `.wasm` is distributing everything compiled into +it. Whatever the interpreter or runtime inside it is licensed under, its notice +travels with the artifact. Check the build's own license inventory rather than +assuming, and ship the notices next to the `.wasm`, not only in the source +tree — the artifact is what people download. + +godo's own third-party notices are in +[THIRD_PARTY_LICENSES.md](../../THIRD_PARTY_LICENSES.md) and ship inside the +release archives. + +## Known limits + +- **Local paths only.** `source` is a file on this machine. Fetching belongs + with a lockfile and is not built. +- **Size.** A Go plugin carries Go's runtime — the example is ~4.5 MB. TinyGo + or a C-family language produces far smaller wasm. +- **One instantiation per step.** Fine at godo's scale; it is not a server. +- **No spawn.** Filesystem access is limited to the mount and `slink` within + the catalog. `config` is shaped as sections so more can be added without + moving anything. diff --git a/docs/dev/runners-and-plugins.md b/docs/dev/runners-and-plugins.md index e7f4661..db95a60 100644 --- a/docs/dev/runners-and-plugins.md +++ b/docs/dev/runners-and-plugins.md @@ -80,14 +80,14 @@ None of this is built: platform, sandboxed by default). 2. Ship MicroPython as that `.wasm`, with the host API from the prototype (`godo.argv`, `godo.args`, `godo.proc`, `godo.fs`). -3. Fetch and pin it: `sha256` required, no auto-update, capabilities granted - deny-all through the plugin's own `config:`. +3. Fetch and pin it: `sha256` required, no auto-update. Two constraints worth keeping when that work starts: -- **Every effect goes through host functions.** Not because sandboxing is - fashionable, but because it is the only way `--preview` keeps working on a - body that is a program: a recording host can show what the script would run - without running it. -- **The plugin cannot police itself.** It declares which host function each - `config:` key unlocks; godo does the gating. +- **What is pinned is which plugin, not what it may do.** A catalog's scripts + already run with the shell's full reach, and a plugin body is a script in + that same catalog. The `sha256` answers the question that has an answer: + these are the bytes that were reviewed. +- **A guest reaches the outside only by asking**, because wasm has no network, + no `fork` and no `subprocess`. That is the platform, not a policy godo + enforces on a script its own author wrote. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 2d654fd..47a0d39 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -19,6 +19,8 @@ Normative source: [contract.md](../contract.md). | | | |--|--| | `-e runners` | List runners usable on this machine | +| `-e plugins` | What the catalog declares, and whether it is installed | +| `-e plugins install [source]` | Fetch declared plugins, or add and fetch one | | `-e version` | Print binary version | | `-e update` | Install newer Release asset | | `-e update check` | Check only | diff --git a/examples/plugins/lines/main.go b/examples/plugins/lines/main.go new file mode 100644 index 0000000..039b682 --- /dev/null +++ b/examples/plugins/lines/main.go @@ -0,0 +1,196 @@ +// Command lines is a godo plugin: a worked example of the protocol. +// +// It provides the runner "lines", whose body is one command per line: +// +// # @runner lines +// boot: | +// git status +// ?pnpm install +// echo done +// +// Lines run in order and stop at the first failure, like a script body. A line +// starting with "?" may fail without stopping the rest. ${NAME} is replaced +// with a capture; $1, $2, … with a leftover argument. +// +// There is no preview branch: godo prints the body itself and never starts a +// plugin for --preview. +// +// Build: +// +// GOOS=wasip1 GOARCH=wasm go build -o lines.wasm ./examples/plugins/lines +// +// There is nothing wasm-specific in here. It reads stdin, writes stdout, and +// exits with a code — which is why it can be tested as an ordinary program. +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "strconv" + "strings" +) + +// request is godo's opening line. Only the fields this plugin uses. +type request struct { + API int `json:"api"` + Runner string `json:"runner"` + Body string `json:"body"` + Argv map[string]string `json:"argv"` + Args []string `json:"args"` +} + +type op struct { + Op string `json:"op"` + Argv []string `json:"argv,omitempty"` + Capture bool `json:"capture,omitempty"` +} + +type result struct { + Code int `json:"code"` + OK bool `json:"ok"` + Error string `json:"error,omitempty"` +} + +const apiVersion = 1 + +func main() { + in := bufio.NewScanner(os.Stdin) + in.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + out := json.NewEncoder(os.Stdout) + + req, err := readRequest(in) + if err != nil { + fail(err) + } + if req.API != apiVersion { + fail(fmt.Errorf("plugin speaks api %d, godo speaks %d", apiVersion, req.API)) + } + + for n, raw := range strings.Split(req.Body, "\n") { + line := strings.TrimSpace(raw) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + mayFail := strings.HasPrefix(line, "?") + line = strings.TrimSpace(strings.TrimPrefix(line, "?")) + + argv, err := substitute(line, req) + if err != nil { + fail(fmt.Errorf("line %d: %w", n+1, err)) + } + if len(argv) == 0 { + continue + } + + res, err := exec(in, out, argv) + if err != nil { + fail(err) + } + if res.Error != "" { + fail(fmt.Errorf("line %d: %s", n+1, res.Error)) + } + if !res.OK && !mayFail { + // The command's code becomes the script's, like a shell would. + os.Exit(res.Code) + } + } + os.Exit(0) +} + +func readRequest(in *bufio.Scanner) (request, error) { + if !in.Scan() { + if err := in.Err(); err != nil { + return request{}, err + } + return request{}, fmt.Errorf("no request on stdin") + } + var req request + if err := json.Unmarshal(in.Bytes(), &req); err != nil { + return request{}, fmt.Errorf("unreadable request: %w", err) + } + return req, nil +} + +// substitute splits a line into argv, replacing ${NAME} with a capture and +// $1, $2, … with a leftover argument. A value with a space stays one argument: +// substitution happens after splitting, never before. +func substitute(line string, req request) ([]string, error) { + var argv []string + for _, word := range strings.Fields(line) { + v, err := expandWord(word, req) + if err != nil { + return nil, err + } + argv = append(argv, v) + } + return argv, nil +} + +func expandWord(word string, req request) (string, error) { + var b strings.Builder + rest := word + for { + i := strings.Index(rest, "$") + if i < 0 || i == len(rest)-1 { + break + } + b.WriteString(rest[:i]) + rest = rest[i+1:] + + if strings.HasPrefix(rest, "{") { + end := strings.Index(rest, "}") + if end < 0 { + return "", fmt.Errorf("unterminated ${ in %q", word) + } + name := rest[1:end] + v, ok := req.Argv[name] + if !ok { + return "", fmt.Errorf("unknown capture %q", name) + } + b.WriteString(v) + rest = rest[end+1:] + continue + } + + digits := 0 + for digits < len(rest) && rest[digits] >= '0' && rest[digits] <= '9' { + digits++ + } + if digits == 0 { + b.WriteString("$") + continue + } + n, _ := strconv.Atoi(rest[:digits]) + if n < 1 || n > len(req.Args) { + return "", fmt.Errorf("$%d out of range (%d args)", n, len(req.Args)) + } + b.WriteString(req.Args[n-1]) + rest = rest[digits:] + } + b.WriteString(rest) + return b.String(), nil +} + +func exec(in *bufio.Scanner, out *json.Encoder, argv []string) (result, error) { + if err := out.Encode(op{Op: "exec", Argv: argv}); err != nil { + return result{}, err + } + if !in.Scan() { + if err := in.Err(); err != nil { + return result{}, err + } + return result{}, fmt.Errorf("godo closed the connection") + } + var res result + if err := json.Unmarshal(in.Bytes(), &res); err != nil { + return result{}, fmt.Errorf("unreadable result: %w", err) + } + return res, nil +} + +func fail(err error) { + fmt.Fprintln(os.Stderr, "lines:", err) + os.Exit(1) +} diff --git a/go.mod b/go.mod index e6b35d1..1354bf4 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,10 @@ module github.com/my-rv/godo -go 1.22 +go 1.26.0 require gopkg.in/yaml.v3 v3.0.1 + +require ( + github.com/tetratelabs/wazero v1.12.0 // indirect + golang.org/x/sys v0.44.0 // indirect +) diff --git a/go.sum b/go.sum index a62c313..d385a7e 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,9 @@ +github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZBf/I= +github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM= +github.com/tetratelabs/wazero v1.12.0 h1:DuWcpNu/FzgEXgGBDp8J1Spc+CWOvvtvVyjKlaZopYU= +github.com/tetratelabs/wazero v1.12.0/go.mod h1:LvKtzl2RqO4gyF27BiXU+nKAjcV8f38U+kP/q2vgxh0= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/catalog/engine.go b/internal/catalog/engine.go index c1d4094..0f85f0f 100644 --- a/internal/catalog/engine.go +++ b/internal/catalog/engine.go @@ -13,6 +13,35 @@ type Runner interface { Run(command string) error } +// Invocation is what a runner needs that a rendered command line cannot carry. +// +// A shell runner only ever wanted the finished line. A runner whose bodies are +// not shell text wants the body as written and the values as data — a plugin +// script reads a capture by name, it does not read a line godo already pasted +// it into. +type Invocation struct { + Script string // the catalog key, for messages + Runner RunnerName + Body string // the body as written + Captures map[string]string + Args []string +} + +// InvocationRunner receives the bound invocation instead of a rendered line, +// and answers for its own --preview. +// +// Preview is the runner's to answer because only it knows what its bodies do. +// godo can render a shell line because it wrote it; it cannot render a program +// it does not interpret, so it asks. +// +// A body reaching one of these is not expanded: ${godo:…} is shell-space +// syntax, and the values it would paste in are on the Invocation already. +type InvocationRunner interface { + Runner + RunInvocation(inv Invocation) error + PreviewInvocation(inv Invocation) ([]string, error) +} + // ArgsAwareRunner is a Runner that decides for itself whether a body takes the // tokens left over after the match. // @@ -135,7 +164,7 @@ func (e *Engine) stepRunner(s Script) (RunnerName, error) { // A runner the catalog declared a plugin for is a different failure // from a typo, and saying so saves the reader the hunt. if p, ok := e.Catalog.Engine.ProviderOf("runner", string(name)); ok { - return "", fmt.Errorf("%w: script %q asks for runner %q, provided by plugin %s — this build cannot load plugins", + return "", fmt.Errorf("%w: script %q asks for runner %q, which plugin %s should provide but did not", ErrUnknownRunner, s.Key, name, p.Source) } // Otherwise the registry's resolver knows why (unknown name, shell not @@ -155,8 +184,57 @@ type Plan struct { type PlanStep struct { Kind string // "dep" | "body" Source string - Command string + Command string // the line, for runners that take one Runner RunnerName // effective runner for this step + // Inv is set when the step's runner takes the bound invocation instead of + // a rendered line. + Inv *Invocation +} + +// steps turns one resolved match into plan steps. +// +// A runner that takes the bound invocation gets the body as written: ${godo:…} +// is shell-space syntax, and the values it would paste in travel on the +// Invocation instead. +func (e *Engine) steps(kind, source string, m *Match, runner RunnerName) ([]PlanStep, error) { + if _, ok := e.invocationRunner(runner); ok { + out := make([]PlanStep, 0, len(m.Script.Commands)) + for _, body := range m.Script.Commands { + out = append(out, PlanStep{ + Kind: kind, Source: source, Command: body, Runner: runner, + Inv: &Invocation{ + Script: m.Script.Key, + Runner: runner, + Body: body, + Captures: m.Captures, + Args: m.Args, + }, + }) + } + return out, nil + } + cmds, err := expand.ExpandAll(m.Script.Commands, m.Captures, m.Args) + if err != nil { + return nil, err + } + out := make([]PlanStep, 0, len(cmds)) + for _, c := range cmds { + out = append(out, PlanStep{Kind: kind, Source: source, Command: c, Runner: runner}) + } + return out, nil +} + +// invocationRunner reports whether the runner takes the bound invocation. +func (e *Engine) invocationRunner(name RunnerName) (InvocationRunner, bool) { + if r, err := e.Runners.Lookup(name); err == nil { + ir, ok := r.(InvocationRunner) + return ir, ok + } + if name == "" { + ir, ok := e.Runner.(InvocationRunner) + return ir, ok + } + return nil, false } // BuildPlan expands deps + body without executing. @@ -183,13 +261,11 @@ func (e *Engine) BuildPlan(tokens []string) (*Plan, error) { if err := e.appendDeps(plan, m, stack, done); err != nil { return nil, err } - cmds, err := expand.ExpandAll(m.Script.Commands, m.Captures, m.Args) + steps, err := e.steps("body", m.Script.Key, m, runner) if err != nil { return nil, err } - for _, c := range cmds { - plan.Steps = append(plan.Steps, PlanStep{Kind: "body", Source: m.Script.Key, Command: c, Runner: runner}) - } + plan.Steps = append(plan.Steps, steps...) return plan, nil } @@ -267,13 +343,11 @@ func (e *Engine) appendDeps(plan *Plan, m *Match, stack, done map[string]bool) e if err := e.appendDeps(plan, depMatch, stack, done); err != nil { return err } - cmds, err := expand.ExpandAll(depMatch.Script.Commands, depMatch.Captures, depMatch.Args) + steps, err := e.steps("dep", invKey, depMatch, depRunner) if err != nil { return err } - for _, c := range cmds { - plan.Steps = append(plan.Steps, PlanStep{Kind: "dep", Source: invKey, Command: c, Runner: depRunner}) - } + plan.Steps = append(plan.Steps, steps...) delete(stack, invKey) done[invKey] = true } @@ -291,6 +365,12 @@ func (e *Engine) Run(tokens []string) error { if err != nil { return err } + if ir, ok := runner.(InvocationRunner); ok && step.Inv != nil { + if err := ir.RunInvocation(*step.Inv); err != nil { + return err + } + continue + } if err := runner.Run(step.Command); err != nil { return err } @@ -298,15 +378,31 @@ func (e *Engine) Run(tokens []string) error { return nil } -// PreviewLines returns expanded commands in order (deps + body). +// PreviewLines returns the lines that would run, in order (deps + body). +// +// A runner that takes the bound invocation renders its own: godo can show a +// shell line because it wrote it, and has to ask for anything else. func (e *Engine) PreviewLines(tokens []string) ([]string, error) { plan, err := e.BuildPlan(tokens) if err != nil { return nil, err } - out := make([]string, len(plan.Steps)) - for i, s := range plan.Steps { - out[i] = s.Command + var out []string + for _, s := range plan.Steps { + if s.Inv == nil { + out = append(out, s.Command) + continue + } + ir, ok := e.invocationRunner(s.Runner) + if !ok { + out = append(out, s.Command) + continue + } + lines, err := ir.PreviewInvocation(*s.Inv) + if err != nil { + return nil, err + } + out = append(out, lines...) } return out, nil } diff --git a/internal/catalog/engine_block_test.go b/internal/catalog/engine_block_test.go index 7195245..3d4158b 100644 --- a/internal/catalog/engine_block_test.go +++ b/internal/catalog/engine_block_test.go @@ -107,6 +107,10 @@ func TestParse_engineRejects(t *testing.T) { } // A runner a plugin declares fails by naming the plugin, not as a typo. +// +// The engine does not load plugins — internal/plugin does, and the CLI wires +// it up. An embedder that registers nothing sees this message rather than +// "unknown runner", which would point at the catalog instead of the wiring. func TestBuildPlan_pluginRunnerNamesItsPlugin(t *testing.T) { cat, err := catalog.Parse([]byte(pluginBlock), "x") if err != nil { @@ -117,7 +121,7 @@ func TestBuildPlan_pluginRunnerNamesItsPlugin(t *testing.T) { if !errors.Is(err, catalog.ErrUnknownRunner) { t.Fatalf("err=%v", err) } - for _, want := range []string{"micropy", "godo-micropy", "cannot load plugins"} { + for _, want := range []string{"micropy", "godo-micropy", "should provide but did not"} { if !strings.Contains(err.Error(), want) { t.Fatalf("err=%v, missing %q", err, want) } diff --git a/internal/catalog/include.go b/internal/catalog/include.go new file mode 100644 index 0000000..757850f --- /dev/null +++ b/internal/catalog/include.go @@ -0,0 +1,73 @@ +package catalog + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +// filePrefix and fileSuffix bracket an included body. +const ( + filePrefix = "${godo:file(" + fileSuffix = ")}" +) + +// resolveIncludes replaces a body that is exactly ${godo:file(path)} with the +// contents of that file. +// +// This is inclusion, not expansion: it happens when a body is read rather than +// when it is rendered, which is why it applies to every runner — a plugin body +// is never expanded, and still has to be able to live in a file. +// +// After this, an included body is indistinguishable from one pasted into the +// YAML. Whatever a runner does with ${godo:…} it does here too; inclusion +// changes where the text comes from, never what happens to it next. +// +// Only a whole value is accepted. Splicing a file into the middle of a line +// would paste newlines into a shell command and mean something different every +// time; a body that is a file is a body that is a file. +func resolveIncludes(cmds []string, dir, key string) ([]string, error) { + out := make([]string, 0, len(cmds)) + for _, c := range cmds { + trimmed := strings.TrimSpace(c) + if !strings.HasPrefix(trimmed, filePrefix) { + if i := strings.Index(c, filePrefix); i >= 0 { + return nil, fmt.Errorf("%w: script %q: %s…%s must be the whole value, not part of a line", + ErrInvalidCatalog, key, filePrefix, fileSuffix) + } + out = append(out, c) + continue + } + if !strings.HasSuffix(trimmed, fileSuffix) { + return nil, fmt.Errorf("%w: script %q: unterminated %s…%s", ErrInvalidCatalog, key, filePrefix, fileSuffix) + } + rel := strings.TrimSpace(trimmed[len(filePrefix) : len(trimmed)-len(fileSuffix)]) + if rel == "" { + return nil, fmt.Errorf("%w: script %q: %s needs a path", ErrInvalidCatalog, key, filePrefix) + } + body, err := readInclude(dir, rel) + if err != nil { + return nil, fmt.Errorf("%w: script %q: %v", ErrInvalidCatalog, key, err) + } + out = append(out, body) + } + return out, nil +} + +// readInclude reads a path relative to the catalog. +// +// Relative to the catalog and not to the caller's cwd: a script says where its +// body lives, and that does not change with where godo was run from. An +// absolute path is taken as given. +func readInclude(dir, rel string) (string, error) { + p := rel + if !filepath.IsAbs(p) { + p = filepath.Join(dir, p) + } + data, err := os.ReadFile(p) + if err != nil { + return "", err + } + return string(data), nil +} diff --git a/internal/catalog/include_test.go b/internal/catalog/include_test.go new file mode 100644 index 0000000..25cf34a --- /dev/null +++ b/internal/catalog/include_test.go @@ -0,0 +1,164 @@ +package catalog_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/my-rv/godo/internal/catalog" +) + +func TestInclude_wholeValueComesFromTheFile(t *testing.T) { + dir := t.TempDir() + body := "import os\nprint('hola')\n" + if err := os.WriteFile(filepath.Join(dir, "script.py"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + path := writeCat(t, dir, "version: \"0.1\"\nscripts:\n t: ${godo:file(./script.py)}\n") + + cat, err := catalog.LoadFile(path) + if err != nil { + t.Fatal(err) + } + if cat.Scripts[0].Commands[0] != body { + t.Fatalf("body=%q", cat.Scripts[0].Commands[0]) + } +} + +// A script says where its body lives; that does not change with where godo +// was run from. +func TestInclude_isRelativeToTheCatalog(t *testing.T) { + root := t.TempDir() + sub := filepath.Join(root, "a", "b") + if err := os.MkdirAll(sub, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "s.py"), []byte("print(1)"), 0o644); err != nil { + t.Fatal(err) + } + writeCat(t, root, "version: \"0.1\"\nscripts:\n t: ${godo:file(s.py)}\n") + + // Resolved from a subdirectory: the catalog is found by walking up, and + // the include still means the file beside it. + found, err := catalog.FindFile(sub) + if err != nil { + t.Fatal(err) + } + cat, err := catalog.LoadFile(found) + if err != nil { + t.Fatal(err) + } + if cat.Scripts[0].Commands[0] != "print(1)" { + t.Fatalf("body=%q", cat.Scripts[0].Commands[0]) + } +} + +// Splicing a file into the middle of a line would paste newlines into a shell +// command; a body that is a file is a body that is a file. +func TestInclude_refusesPartOfALine(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "m.txt"), []byte("hi"), 0o644); err != nil { + t.Fatal(err) + } + path := writeCat(t, dir, "version: \"0.1\"\nscripts:\n t: echo ${godo:file(m.txt)}\n") + _, err := catalog.LoadFile(path) + if err == nil || !strings.Contains(err.Error(), "whole value") { + t.Fatalf("err=%v", err) + } +} + +func TestInclude_reportsAMissingFile(t *testing.T) { + dir := t.TempDir() + path := writeCat(t, dir, "version: \"0.1\"\nscripts:\n t: ${godo:file(nope.py)}\n") + _, err := catalog.LoadFile(path) + if err == nil || !strings.Contains(err.Error(), "nope.py") { + t.Fatalf("err=%v", err) + } +} + +func TestInclude_rejectsMalformed(t *testing.T) { + for _, tc := range []struct{ name, body, want string }{ + {"no path", " t: ${godo:file()}\n", "needs a path"}, + {"unterminated", " t: \"${godo:file(x.py\"\n", "unterminated"}, + } { + t.Run(tc.name, func(t *testing.T) { + path := writeCat(t, t.TempDir(), "version: \"0.1\"\nscripts:\n"+tc.body) + _, err := catalog.LoadFile(path) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("err=%v", err) + } + }) + } +} + +// Inclusion is not expansion: it happens when the body is read, so it works +// for a runner whose bodies are never expanded. +func TestInclude_worksForAnyRunner(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "s.py"), []byte("print(godo.argv['X'])"), 0o644); err != nil { + t.Fatal(err) + } + path := writeCat(t, dir, "version: \"0.1\"\nscripts:\n # @runner micropy\n t: ${godo:file(s.py)}\n") + cat, err := catalog.LoadFile(path) + if err != nil { + t.Fatal(err) + } + // ${godo:argv[…]} inside the file is untouched: it is the plugin's text. + if cat.Scripts[0].Commands[0] != "print(godo.argv['X'])" { + t.Fatalf("body=%q", cat.Scripts[0].Commands[0]) + } + if cat.Scripts[0].Runner != "micropy" { + t.Fatalf("runner=%q", cat.Scripts[0].Runner) + } +} + +// A list of commands may mix included and inline entries. +func TestInclude_inAList(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "two.sh"), []byte("echo two"), 0o644); err != nil { + t.Fatal(err) + } + path := writeCat(t, dir, "version: \"0.1\"\nscripts:\n t:\n - echo one\n - ${godo:file(two.sh)}\n") + cat, err := catalog.LoadFile(path) + if err != nil { + t.Fatal(err) + } + got := cat.Scripts[0].Commands + if len(got) != 2 || got[0] != "echo one" || got[1] != "echo two" { + t.Fatalf("commands=%q", got) + } +} + +// An included body behaves exactly as if it had been pasted into the YAML. +// Inclusion changes where the text comes from, never what happens to it after. +func TestInclude_behavesLikeAPastedBody(t *testing.T) { + dir := t.TempDir() + const body = "echo the arg was: ${godo:args[0]}" + if err := os.WriteFile(filepath.Join(dir, "b.sh"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + path := writeCat(t, dir, "version: \"0.1\"\nscripts:\n"+ + " included: ${godo:file(b.sh)}\n"+ + " pasted: \""+body+"\"\n") + cat, err := catalog.LoadFile(path) + if err != nil { + t.Fatal(err) + } + eng := catalog.NewEngine(cat, runnerFunc(func(string) error { return nil })) + + one, err := eng.PreviewLines([]string{"included", "hola"}) + if err != nil { + t.Fatal(err) + } + two, err := eng.PreviewLines([]string{"pasted", "hola"}) + if err != nil { + t.Fatal(err) + } + if len(one) != 1 || one[0] != two[0] { + t.Fatalf("included=%q pasted=%q", one, two) + } + if one[0] != "echo the arg was: hola" { + t.Fatalf("got %q", one[0]) + } +} diff --git a/internal/catalog/insert.go b/internal/catalog/insert.go new file mode 100644 index 0000000..92e4b5d --- /dev/null +++ b/internal/catalog/insert.go @@ -0,0 +1,249 @@ +package catalog + +import ( + "fmt" + "strings" + + "gopkg.in/yaml.v3" +) + +// PluginEntry is a plugin to write into a catalog. +type PluginEntry struct { + Source string + SHA256 string + Provides []string + Config string // raw YAML lines for the config block, or "" +} + +// UpsertPlugin returns src with entry added under engine.plugins, replacing +// any existing entry that provides the same things. +// +// Installing a plugin a catalog already declares means bringing the +// declaration in line with what was just fetched — a placeholder digest +// becoming a real one is the ordinary case. Appending instead would leave two +// entries claiming one runner, which the loader rejects. +func UpsertPlugin(src []byte, entry PluginEntry) ([]byte, error) { + if len(entry.Provides) == 0 { + return InsertPlugin(src, entry) + } + doc, err := docNode(src) + if err != nil { + return nil, err + } + lines := strings.Split(string(src), "\n") + _, engineVal := childNode(doc, "engine") + if engineVal == nil || engineVal.Kind != yaml.MappingNode { + return InsertPlugin(src, entry) + } + _, pluginsVal := childNode(engineVal, "plugins") + if pluginsVal == nil || pluginsVal.Kind != yaml.SequenceNode { + return InsertPlugin(src, entry) + } + for _, item := range pluginsVal.Content { + if !sameProvides(item, entry.Provides) { + continue + } + from := item.Line - 1 + to := endOfNode(item, lines) + // The config that is already there stays. What this plugin may do is + // the catalog author's decision, reached once and often narrowed; + // resetting it to a default on every install would quietly re-grant + // what someone took away, and quietly drop what they added. + if kept := configLines(item, lines); kept != "" { + entry.Config = kept + } + block := entryLines(entry, strings.Repeat(" ", item.Column-3)+"item") + out := append([]string{}, lines[:from]...) + out = append(out, block...) + out = append(out, lines[to:]...) + return []byte(strings.Join(out, "\n")), nil + } + return InsertPlugin(src, entry) +} + +// configLines returns an entry's config block, re-indented to sit under a +// fresh entry, or "" when it declares none. +func configLines(item *yaml.Node, lines []string) string { + key, val := childNode(item, "config") + if key == nil || val == nil { + return "" + } + from := key.Line // the line after "config:" + to := endOfNode(val, lines) + if from >= to || to > len(lines) { + return "" + } + strip := val.Column - 1 + var out []string + for _, l := range lines[from:to] { + if len(l) >= strip { + l = l[strip:] + } else { + l = strings.TrimLeft(l, " ") + } + out = append(out, l) + } + return strings.Join(out, "\n") +} + +// sameProvides reports whether a plugin entry node claims exactly these names. +func sameProvides(item *yaml.Node, want []string) bool { + if item.Kind != yaml.MappingNode { + return false + } + _, val := childNode(item, "provides") + if val == nil || val.Kind != yaml.SequenceNode || len(val.Content) != len(want) { + return false + } + for i, n := range val.Content { + if n.Value != want[i] { + return false + } + } + return true +} + +// InsertPlugin returns src with entry added under engine.plugins. +// +// It splices lines rather than re-encoding the document. A catalog is written +// by hand: its comments carry the @deps and @runner decorators, its blank lines +// group scripts, and its block scalars hold whitespace that means something. +// Round-tripping through a YAML encoder would preserve the data and lose the +// file, so the parser is used only to find *where* to write. +func InsertPlugin(src []byte, entry PluginEntry) ([]byte, error) { + if entry.Source == "" || entry.SHA256 == "" { + return nil, fmt.Errorf("plugin entry needs a source and a sha256") + } + doc, err := docNode(src) + if err != nil { + return nil, err + } + + lines := strings.Split(string(src), "\n") + at, indent, err := pluginsAnchor(doc, lines) + if err != nil { + return nil, err + } + block := entryLines(entry, indent) + out := append([]string{}, lines[:at]...) + out = append(out, block...) + out = append(out, lines[at:]...) + return []byte(strings.Join(out, "\n")), nil +} + +// pluginsAnchor finds the line to insert before, and the indent to write at. +// +// Three shapes, in the order they are looked for: an engine.plugins list to +// append to, an engine block that needs the key, and a file with no engine +// block at all. +func pluginsAnchor(doc *yaml.Node, lines []string) (at int, indent string, err error) { + engineKey, engineVal := childNode(doc, "engine") + if engineKey == nil { + // No engine block: open one above scripts, or at the end. + at = len(lines) + if k, _ := childNode(doc, "scripts"); k != nil { + at = k.Line - 1 + for at > 0 && strings.TrimSpace(lines[at-1]) == "" { + at-- + } + } + return at, "engine", nil + } + if engineVal == nil || engineVal.Kind != yaml.MappingNode { + return 0, "", fmt.Errorf("%w: engine must be a mapping", ErrInvalidCatalog) + } + + pluginsKey, pluginsVal := childNode(engineVal, "plugins") + if pluginsKey == nil { + // engine exists but declares no plugins: add the key at its indent. + return endOfNode(engineVal, lines), strings.Repeat(" ", engineVal.Column-1) + "plugins", nil + } + if pluginsVal == nil || pluginsVal.Kind != yaml.SequenceNode || len(pluginsVal.Content) == 0 { + return pluginsKey.Line, strings.Repeat(" ", pluginsKey.Column+1) + "item", nil + } + last := pluginsVal.Content[len(pluginsVal.Content)-1] + return endOfNode(last, lines), strings.Repeat(" ", last.Column-3) + "item", nil +} + +// endOfNode returns the line just past a node's last content line. +// +// yaml.Node carries where a value starts, never where it ends, so the end is +// found by walking down past everything indented under it. Blank lines are +// walked over but not claimed: an entry belongs above the gap that separates +// it from what follows. +func endOfNode(n *yaml.Node, lines []string) int { + col := n.Column + last := n.Line + for i := n.Line; i < len(lines); i++ { + text := lines[i] + if strings.TrimSpace(text) == "" { + continue + } + if len(text)-len(strings.TrimLeft(text, " ")) < col-1 { + break + } + last = i + 1 + } + return last +} + +func docNode(src []byte) (*yaml.Node, error) { + var root yaml.Node + if err := yaml.Unmarshal(src, &root); err != nil { + return nil, fmt.Errorf("%w: parse yaml: %v", ErrInvalidCatalog, err) + } + doc := &root + if root.Kind == yaml.DocumentNode { + if len(root.Content) == 0 { + return nil, fmt.Errorf("%w: empty yaml document", ErrInvalidCatalog) + } + doc = root.Content[0] + } + if doc.Kind != yaml.MappingNode { + return nil, fmt.Errorf("%w: root must be a mapping", ErrInvalidCatalog) + } + return doc, nil +} + +func childNode(m *yaml.Node, key string) (k, v *yaml.Node) { + for i := 0; i+1 < len(m.Content); i += 2 { + if m.Content[i].Value == key { + return m.Content[i], m.Content[i+1] + } + } + return nil, nil +} + +// entryLines renders the entry at the depth its anchor asked for. +func entryLines(e PluginEntry, indent string) []string { + var pad, out []string + switch { + case indent == "engine": + out = append(out, "engine:", " plugins:") + pad = []string{" "} + case strings.HasSuffix(indent, "plugins"): + base := strings.TrimSuffix(indent, "plugins") + out = append(out, base+"plugins:") + pad = []string{base + " "} + default: + pad = []string{strings.TrimSuffix(indent, "item")} + } + p := pad[0] + out = append(out, + p+"- source: "+e.Source, + p+" sha256: "+e.SHA256, + ) + if len(e.Provides) > 0 { + out = append(out, p+" provides: ["+strings.Join(e.Provides, ", ")+"]") + } + if e.Config != "" { + out = append(out, p+" config:") + for _, l := range strings.Split(strings.TrimRight(e.Config, "\n"), "\n") { + out = append(out, p+" "+l) + } + } + if indent == "engine" { + out = append(out, "") + } + return out +} diff --git a/internal/catalog/insert_test.go b/internal/catalog/insert_test.go new file mode 100644 index 0000000..8466059 --- /dev/null +++ b/internal/catalog/insert_test.go @@ -0,0 +1,255 @@ +package catalog_test + +import ( + "strings" + "testing" + + "github.com/my-rv/godo/internal/catalog" +) + +var entry = catalog.PluginEntry{ + Source: "./micropy.wasm", + SHA256: "abc123", + Provides: []string{"runner:micropy"}, + Config: "proc: {exec: true}\nfs: {mount: true}", +} + +// The catalog must survive the edit: its comments carry decorators, its blank +// lines group scripts, and a block scalar's whitespace is content. +func TestInsertPlugin_keepsEverythingElse(t *testing.T) { + src := `version: "0.1" + +engine: + dialect: matcher + +scripts: + # Local gate + # @deps vet, test + ci: go build ./... + + # @runner micropy + boot: | + import os + deliberately indented + print("hi") +` + got, err := catalog.InsertPlugin([]byte(src), entry) + if err != nil { + t.Fatal(err) + } + out := string(got) + for _, want := range []string{ + "# Local gate", "# @deps vet, test", "# @runner micropy", + " import os", " deliberately indented", + } { + if !strings.Contains(out, want) { + t.Fatalf("lost %q from:\n%s", want, out) + } + } + cat, err := catalog.Parse(got, "x") + if err != nil { + t.Fatalf("result does not parse: %v\n%s", err, out) + } + if len(cat.Engine.Plugins) != 1 || cat.Engine.Plugins[0].SHA256 != "abc123" { + t.Fatalf("plugins=%+v", cat.Engine.Plugins) + } + if cat.Dialect != catalog.DialectMatcher { + t.Fatalf("dialect lost: %q", cat.Dialect) + } + if len(cat.Scripts) != 2 { + t.Fatalf("scripts=%d", len(cat.Scripts)) + } + if cat.Scripts[0].Doc != "Local gate" || len(cat.Scripts[0].Deps) != 2 { + t.Fatalf("decorators lost: %+v", cat.Scripts[0]) + } + if cat.Scripts[1].Runner != "micropy" { + t.Fatalf("@runner lost: %+v", cat.Scripts[1]) + } +} + +func TestInsertPlugin_shapes(t *testing.T) { + for _, tc := range []struct{ name, src string }{ + {"no engine block", "version: \"0.1\"\n\nscripts:\n t: echo hi\n"}, + {"engine without plugins", "version: \"0.1\"\nengine:\n dialect: matcher\nscripts:\n t: echo hi\n"}, + {"engine with plugins", "version: \"0.1\"\nengine:\n plugins:\n" + + " - source: ./a.wasm\n sha256: aaa\n provides: [runner:a]\n" + + "scripts:\n t: echo hi\n"}, + {"plugins with a config block", "version: \"0.1\"\nengine:\n plugins:\n" + + " - source: ./a.wasm\n sha256: aaa\n provides: [runner:a]\n" + + " config:\n proc: {exec: true}\n" + + "scripts:\n t: echo hi\n"}, + {"no scripts at all", "version: \"0.1\"\n"}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := catalog.InsertPlugin([]byte(tc.src), entry) + if err != nil { + t.Fatalf("%v\n%s", err, tc.src) + } + cat, err := catalog.Parse(got, "x") + if err != nil { + t.Fatalf("does not parse: %v\n%s", err, got) + } + var found bool + for _, p := range cat.Engine.Plugins { + if p.SHA256 == "abc123" { + found = true + } + } + if !found { + t.Fatalf("entry not inserted:\n%s", got) + } + // An existing entry must survive alongside the new one. + if strings.Contains(tc.src, "a.wasm") && len(cat.Engine.Plugins) != 2 { + t.Fatalf("existing entry lost:\n%s", got) + } + }) + } +} + +// Inserting twice is how a second plugin arrives; both must be there. +func TestInsertPlugin_twice(t *testing.T) { + src := []byte("version: \"0.1\"\nscripts:\n t: echo hi\n") + one, err := catalog.InsertPlugin(src, entry) + if err != nil { + t.Fatal(err) + } + second := entry + second.Source = "./other.wasm" + second.SHA256 = "def456" + second.Provides = []string{"runner:other"} + second.Config = "" + two, err := catalog.InsertPlugin(one, second) + if err != nil { + t.Fatalf("%v\n%s", err, one) + } + cat, err := catalog.Parse(two, "x") + if err != nil { + t.Fatalf("does not parse: %v\n%s", err, two) + } + if len(cat.Engine.Plugins) != 2 { + t.Fatalf("plugins=%d:\n%s", len(cat.Engine.Plugins), two) + } +} + +func TestInsertPlugin_rejectsIncomplete(t *testing.T) { + src := []byte("version: \"0.1\"\nscripts:\n t: echo hi\n") + if _, err := catalog.InsertPlugin(src, catalog.PluginEntry{Source: "x"}); err == nil { + t.Fatal("want an error without a digest") + } + if _, err := catalog.InsertPlugin([]byte("- not a mapping\n"), entry); err == nil { + t.Fatal("want an error for a non-mapping root") + } +} + +// Installing a plugin the catalog already declares updates it, rather than +// leaving two entries claiming one runner. +func TestUpsertPlugin_replacesTheEntryForTheSameRunner(t *testing.T) { + src := `version: "0.1" + +engine: + plugins: + - source: https://example.test/micropy.wasm + sha256: "replace me" + provides: [runner:micropy] + config: + proc: {exec: true} + fs: {mount: true} + +scripts: + # keep me + t: echo hi +` + got, err := catalog.UpsertPlugin([]byte(src), catalog.PluginEntry{ + Source: "./local.wasm", + SHA256: "realdigest", + Provides: []string{"runner:micropy"}, + Config: "proc: {exec: true}", + }) + if err != nil { + t.Fatal(err) + } + cat, err := catalog.Parse(got, "x") + if err != nil { + t.Fatalf("does not parse: %v\n%s", err, got) + } + if len(cat.Engine.Plugins) != 1 { + t.Fatalf("plugins=%d, want the entry replaced:\n%s", len(cat.Engine.Plugins), got) + } + p := cat.Engine.Plugins[0] + if p.SHA256 != "realdigest" || p.Source != "./local.wasm" { + t.Fatalf("entry=%+v", p) + } + if !strings.Contains(string(got), "# keep me") { + t.Fatalf("lost a comment:\n%s", got) + } + if len(cat.Scripts) != 1 { + t.Fatalf("scripts=%d", len(cat.Scripts)) + } +} + +// A different runner is added beside, not over. +func TestUpsertPlugin_addsWhenNothingProvidesTheSame(t *testing.T) { + src := "version: \"0.1\"\nengine:\n plugins:\n" + + " - source: ./a.wasm\n sha256: aaa\n provides: [runner:a]\n" + + "scripts:\n t: echo hi\n" + got, err := catalog.UpsertPlugin([]byte(src), catalog.PluginEntry{ + Source: "./b.wasm", SHA256: "bbb", Provides: []string{"runner:b"}, + }) + if err != nil { + t.Fatal(err) + } + cat, err := catalog.Parse(got, "x") + if err != nil { + t.Fatalf("does not parse: %v\n%s", err, got) + } + if len(cat.Engine.Plugins) != 2 { + t.Fatalf("plugins=%d:\n%s", len(cat.Engine.Plugins), got) + } +} + +// What a plugin may do is the author's decision. install brings the digest in +// line; it does not re-grant what someone took away. +func TestUpsertPlugin_keepsTheExistingConfig(t *testing.T) { + src := `version: "0.1" + +engine: + plugins: + - source: https://example.test/micropy.wasm + sha256: "replace me" + provides: [runner:micropy] + config: + proc: {exec: true} + fs: {mount: true, slink: true} + time: {wall: true} + +scripts: + t: echo hi +` + got, err := catalog.UpsertPlugin([]byte(src), catalog.PluginEntry{ + Source: "./local.wasm", + SHA256: "realdigest", + Provides: []string{"runner:micropy"}, + Config: "proc: {exec: true}", // the default install would have written + }) + if err != nil { + t.Fatal(err) + } + cat, err := catalog.Parse(got, "x") + if err != nil { + t.Fatalf("does not parse: %v\n%s", err, got) + } + cfg := cat.Engine.Plugins[0].Config + fs, ok := cfg["fs"].(map[string]any) + if !ok { + t.Fatalf("fs grants dropped: %+v\n%s", cfg, got) + } + if fs["mount"] != true || fs["slink"] != true { + t.Fatalf("fs=%+v", fs) + } + if _, ok := cfg["time"]; !ok { + t.Fatalf("time grant dropped: %+v", cfg) + } + if cat.Engine.Plugins[0].SHA256 != "realdigest" { + t.Fatalf("digest not updated: %+v", cat.Engine.Plugins[0]) + } +} diff --git a/internal/catalog/load.go b/internal/catalog/load.go index a9173bc..030d871 100644 --- a/internal/catalog/load.go +++ b/internal/catalog/load.go @@ -3,6 +3,7 @@ package catalog import ( "fmt" "os" + "path/filepath" "strings" "gopkg.in/yaml.v3" @@ -99,7 +100,7 @@ func Parse(data []byte, path string, dialects ...*DialectRegistry) (*Catalog, er for i := 0; i < len(scriptsNode.Content); i += 2 { keyNode := scriptsNode.Content[i] valNode := scriptsNode.Content[i+1] - script, err := scriptFromNodes(keyNode, valNode, cat.Dialect, reg) + script, err := scriptFromNodes(keyNode, valNode, cat.Dialect, reg, path) if err != nil { return nil, err } @@ -112,7 +113,7 @@ func Parse(data []byte, path string, dialects ...*DialectRegistry) (*Catalog, er return cat, nil } -func scriptFromNodes(key, val *yaml.Node, fileDialect DialectName, reg *DialectRegistry) (Script, error) { +func scriptFromNodes(key, val *yaml.Node, fileDialect DialectName, reg *DialectRegistry, path string) (Script, error) { s := Script{Key: key.Value} dec, err := parseDecorators(key.HeadComment) if err != nil { @@ -134,6 +135,10 @@ func scriptFromNodes(key, val *yaml.Node, fileDialect DialectName, reg *DialectR if len(cmds) == 0 { return Script{}, fmt.Errorf("%w: script %q: empty command list", ErrInvalidCatalog, s.Key) } + cmds, err = resolveIncludes(cmds, filepath.Dir(path), s.Key) + if err != nil { + return Script{}, err + } s.Commands = cmds eff := EffectiveDialect(s, fileDialect) diff --git a/internal/cli/app.go b/internal/cli/app.go index cc8ac18..9800eb0 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -1,6 +1,7 @@ package cli import ( + "context" "fmt" "io" "os" @@ -12,6 +13,8 @@ import ( "github.com/my-rv/godo" "github.com/my-rv/godo/internal/catalog" "github.com/my-rv/godo/internal/execshell" + "github.com/my-rv/godo/internal/plugin" + "github.com/my-rv/godo/internal/pluginstore" "github.com/my-rv/godo/internal/update" ) @@ -27,7 +30,9 @@ type App struct { Runner catalog.Runner // UpdateClient overrides release checks (tests). UpdateClient *update.Client - Executable func() (string, error) + // PluginStore overrides where artifacts are kept (tests). + PluginStore *pluginstore.Store + Executable func() (string, error) } // New returns an App wired to process stdio. @@ -61,6 +66,9 @@ func (a *App) Run(args []string) error { if mode == modeRunners { return a.listRunners(a.nearestCatalog()) } + if mode == modePlugins || mode == modePluginsInstall { + return a.plugins(mode == modePluginsInstall, tokens) + } if mode == modeUpdate || mode == modeUpdateCheck { return a.runUpdate(mode == modeUpdateCheck) } @@ -93,6 +101,29 @@ func (a *App) Run(args []string) error { if err := runners.Register(catalog.RunnerInherit, execshell.InheritRunner{Dir: root}); err != nil { return err } + if len(cat.Engine.Plugins) > 0 { + ctx := context.Background() + rt := plugin.NewRuntime(ctx) + defer rt.Close(ctx) + store := a.store() + resolve := func(digest, source string) (string, error) { + if store.Has(digest) { + return store.Path(digest) + } + // A file sitting next to the catalog is already here; asking + // someone to install what they can see is nonsense. Its digest is + // checked either way, so nothing is skipped but the copying. + if local := localSource(root, source); local != "" { + return local, nil + } + // Anything else has to be fetched, and a run is not the moment to + // discover that. + return "", fmt.Errorf("plugin %s is not installed\n run: godo -e plugins install", source) + } + if err := plugin.Register(ctx, rt, cat, runners, root, a.Stdout, a.Stderr, a.Stdin, resolve); err != nil { + return err + } + } // Any other name is a shell the catalog asked for by name. godo does not // manage those — it proxies to whatever is on PATH. runners.Resolve = func(name catalog.RunnerName) (catalog.Runner, error) { @@ -191,6 +222,159 @@ func (a *App) runUpdate(checkOnly bool) error { // // "here" is the point: the native shells are whatever this machine has, so the // list is a fact about the machine, not about godo. +// plugins lists what a catalog declares, or installs it. +// +// Declaring a plugin and having it on this machine are different things: the +// catalog says which artifact a script needs, the store is where that artifact +// actually is. install is what turns the first into the second. +func (a *App) plugins(install bool, args []string) error { + cwd, err := a.cwd() + if err != nil { + return err + } + path, err := catalog.FindFile(cwd) + if err != nil { + return err + } + cat, err := catalog.LoadFile(path) + if err != nil { + return err + } + store := a.store() + ctx := context.Background() + + // A source typed on the command line is relative to where the person is; + // one written in the catalog is relative to the catalog. + if install && len(args) > 0 { + return a.installNew(ctx, store, cat, path, cwd, args) + } + if install { + return a.installDeclared(ctx, store, cat, filepath.Dir(path)) + } + return a.listPlugins(store, cat) +} + +// installNew adds a plugin the catalog does not declare yet. +// +// The digest is not asked for — it is whatever the artifact turns out to +// contain, computed here and written into the catalog. Asking a person to type +// a hash they cannot check is how wrong hashes get committed. +func (a *App) installNew(ctx context.Context, store pluginstore.Store, cat *catalog.Catalog, path, base string, args []string) error { + if len(args) > 1 { + return fmt.Errorf("-e plugins install: one source at a time, got %v", args) + } + source := args[0] + digest, stored, err := store.Fetch(ctx, base, source) + if err != nil { + return err + } + for _, p := range cat.Engine.Plugins { + if p.SHA256 == digest && p.Source == source { + fmt.Fprintf(a.Stdout, "already declared: %s\n %s\n", p.Source, stored) + return nil + } + } + + provides, err := providedRunners(source) + if err != nil { + return err + } + src, err := os.ReadFile(path) + if err != nil { + return err + } + out, err := catalog.UpsertPlugin(src, catalog.PluginEntry{ + Source: source, + SHA256: digest, + Provides: provides, + Config: "proc: {exec: true}", + }) + if err != nil { + return err + } + if err := os.WriteFile(path, out, 0o644); err != nil { + return err + } + fmt.Fprintf(a.Stdout, "installed %s\n sha256 %s\n stored %s\n declared %s\n\n", + source, digest, stored, path) + fmt.Fprintf(a.Stdout, "It provides %s, and was granted proc.exec.\n", strings.Join(provides, ", ")) + fmt.Fprintln(a.Stdout, "Add fs.mount, fs.slink or time.wall under its config if it needs them.") + return nil +} + +// providedRunners derives the runner name from the artifact's filename. +// +// The name a plugin answers to is the catalog's to choose: nothing inside a +// .wasm declares it, and inventing a manifest format to carry one string would +// be a second contract to maintain. The filename is the honest default, and +// the line it writes is there to be edited. +func providedRunners(source string) ([]string, error) { + base := filepath.Base(strings.TrimSuffix(source, ".wasm")) + base = strings.TrimPrefix(base, "godo-") + if base == "" || base == "." || base == string(filepath.Separator) { + return nil, fmt.Errorf("cannot tell what %q provides; name the artifact .wasm", source) + } + return []string{"runner:" + base}, nil +} + +// installDeclared fetches every artifact the catalog names. +func (a *App) installDeclared(ctx context.Context, store pluginstore.Store, cat *catalog.Catalog, base string) error { + if len(cat.Engine.Plugins) == 0 { + fmt.Fprintln(a.Stdout, "this catalog declares no plugins") + return nil + } + for _, p := range cat.Engine.Plugins { + stored, err := store.Ensure(ctx, base, p.SHA256, p.Source) + if err != nil { + return err + } + fmt.Fprintf(a.Stdout, "%-16s %s\n", strings.Join(p.Provides, " "), stored) + } + return nil +} + +func (a *App) listPlugins(store pluginstore.Store, cat *catalog.Catalog) error { + if len(cat.Engine.Plugins) == 0 { + fmt.Fprintln(a.Stdout, "this catalog declares no plugins") + return nil + } + missing := false + for _, p := range cat.Engine.Plugins { + state := "installed" + if !store.Has(p.SHA256) { + state = "not installed" + missing = true + } + fmt.Fprintf(a.Stdout, "%-16s %-14s %s\n", strings.Join(p.Provides, " "), state, p.Source) + } + if missing { + fmt.Fprintln(a.Stdout, "\nRun: godo -e plugins install") + } + return nil +} + +// localSource returns the path of a source that is already on disk, or "". +func localSource(root, source string) string { + if strings.Contains(source, "://") && !strings.HasPrefix(source, "file://") { + return "" + } + p := strings.TrimPrefix(source, "file://") + if !filepath.IsAbs(p) { + p = filepath.Join(root, p) + } + if st, err := os.Stat(p); err != nil || !st.Mode().IsRegular() { + return "" + } + return p +} + +func (a *App) store() pluginstore.Store { + if a.PluginStore != nil { + return *a.PluginStore + } + return pluginstore.Store{} +} + // nearestCatalog loads the catalog for -e runners, or nil. // // The listing is useful outside a repo, so a missing or broken catalog is not @@ -240,9 +424,9 @@ func (a *App) printDeclaredPlugins(cat *catalog.Catalog) { if len(cat.Engine.Plugins) == 0 { return } - fmt.Fprintln(a.Stdout, "\ndeclared by this catalog (no plugin loader in this build):") + fmt.Fprintln(a.Stdout, "\nfrom plugins declared by this catalog:") for _, p := range cat.Engine.Plugins { - fmt.Fprintf(a.Stdout, " %-10s %s\n", strings.Join(p.Provides, " "), p.Source) + fmt.Fprintf(a.Stdout, " %-16s %s\n", strings.Join(p.Provides, " "), p.Source) } } @@ -295,7 +479,11 @@ func (a *App) list(eng *catalog.Engine, tokens []string) error { } fmt.Fprintf(a.Stdout, "%s:\n", s.Key) for _, c := range s.Commands { - fmt.Fprintf(a.Stdout, " %s\n", c) + // A body can be many lines — a block scalar, or a file included with + // ${godo:file(…)} — and every one of them is part of this script. + for _, line := range strings.Split(strings.TrimRight(c, "\n"), "\n") { + fmt.Fprintf(a.Stdout, " %s\n", line) + } } return nil } @@ -319,6 +507,8 @@ const ( modeUpdate modeUpdateCheck modeRunners + modePlugins + modePluginsInstall ) // ParseFlags parses context flags before script tokens. @@ -375,6 +565,15 @@ func parseEngineCommand(tokens []string) (mode mode, rest []string, err error) { return 0, nil, fmt.Errorf("-e runners: unexpected arguments %v", tokens[1:]) } return modeRunners, nil, nil + case "plugins": + switch { + case len(tokens) == 1: + return modePlugins, nil, nil + case tokens[1] == "install": + return modePluginsInstall, tokens[2:], nil + default: + return 0, nil, fmt.Errorf("-e plugins: usage: -e plugins | -e plugins install [source]") + } case "update": switch { case len(tokens) == 1: @@ -426,6 +625,8 @@ func EngineUsage(w io.Writer) { Built-in commands (not godo.yaml scripts): version print binary version runners list runners usable here + plugins what this catalog declares, and whether it is installed + plugins install fetch them; with a source, add it and fetch it update download latest GitHub Release for this OS/arch update check report whether an update is available help this help diff --git a/internal/cli/plugin_e2e_test.go b/internal/cli/plugin_e2e_test.go new file mode 100644 index 0000000..f8d4542 --- /dev/null +++ b/internal/cli/plugin_e2e_test.go @@ -0,0 +1,278 @@ +package cli_test + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/my-rv/godo" + "github.com/my-rv/godo/internal/pluginstore" +) + +var ( + wasmOnce sync.Once + wasmPath string + wasmErr error +) + +// buildExamplePlugin compiles examples/plugins/lines for wasip1. +// +// Built rather than committed: a checked-in binary is something nobody can +// review, and what this asserts is that the artifact matches the source next +// to it. +func buildExamplePlugin(t *testing.T) string { + t.Helper() + wasmOnce.Do(func() { + f, err := os.CreateTemp("", "lines-*.wasm") + if err != nil { + wasmErr = err + return + } + f.Close() + cmd := exec.Command("go", "build", "-o", f.Name(), "../../examples/plugins/lines") + cmd.Env = append(os.Environ(), "GOOS=wasip1", "GOARCH=wasm") + if b, err := cmd.CombinedOutput(); err != nil { + wasmErr = err + t.Logf("build: %s", b) + return + } + wasmPath = f.Name() + }) + if wasmErr != nil { + t.Fatalf("building the example plugin: %v", wasmErr) + } + return wasmPath +} + +// pluginCatalog writes a catalog next to a copy of the built plugin. +func pluginCatalog(t *testing.T, grantExec bool, scripts string) string { + t.Helper() + cwd := t.TempDir() + data, err := os.ReadFile(buildExamplePlugin(t)) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(cwd, "lines.wasm"), data, 0o644); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(data) + config := "" + if grantExec { + config = " config:\n proc:\n exec: true\n" + } + writeGodoYAML(t, cwd, "version: \"0.1\"\nengine:\n plugins:\n"+ + " - source: ./lines.wasm\n"+ + " sha256: "+hex.EncodeToString(sum[:])+"\n"+ + " provides: [runner:lines]\n"+ + config+ + "scripts:\n"+scripts) + return cwd +} + +// Contract: a plugin provides a runner, and a script names it like any other. +func TestE2E_pluginRunnerRunsTheScript(t *testing.T) { + cwd := pluginCatalog(t, true, " # @runner lines\n boot: |\n touch one\n touch two\n") + app, _, _ := e2eApp(t, cwd) + if err := app.Run([]string{"boot"}); err != nil { + t.Fatalf("run: %v", err) + } + for _, f := range []string{"one", "two"} { + if _, err := os.Stat(filepath.Join(cwd, f)); err != nil { + t.Fatalf("%s: %v", f, err) + } + } +} + +// --preview prints the body and never starts the plugin. +// +// A plugin body is a program; the only faithful answer to "what will this do" +// without running it is the program itself. Guessing the flow is a separate +// flag's job, not this one's. +func TestE2E_previewPrintsTheBodyAndStartsNothing(t *testing.T) { + cwd := pluginCatalog(t, true, " # @runner lines\n boot: |\n touch one\n ?false\n") + app, out, _ := e2eApp(t, cwd) + if err := app.Run([]string{"--preview", "boot"}); err != nil { + t.Fatal(err) + } + got := strings.TrimSpace(out.String()) + if got != "touch one\n?false" { + t.Fatalf("preview=%q", got) + } + if _, err := os.Stat(filepath.Join(cwd, "one")); err == nil { + t.Fatal("preview ran the body") + } +} + +// Captures and leftover args reach the plugin as data, not pasted into a line. +func TestE2E_pluginReceivesCapturesAndArgs(t *testing.T) { + cwd := pluginCatalog(t, true, + " # @runner lines\n make ${NAME}: |\n touch ${NAME}\n touch $1\n") + // engine.dialect must be matcher for a capture in the key. + body, err := os.ReadFile(filepath.Join(cwd, "godo.yaml")) + if err != nil { + t.Fatal(err) + } + fixed := strings.Replace(string(body), "engine:\n", "engine:\n dialect: matcher\n", 1) + if err := os.WriteFile(filepath.Join(cwd, "godo.yaml"), []byte(fixed), 0o644); err != nil { + t.Fatal(err) + } + + app, _, _ := e2eApp(t, cwd) + if err := app.Run([]string{"make", "from capture", "from arg"}); err != nil { + t.Fatalf("run: %v", err) + } + // A value with a space stays one argument: the plugin substitutes after + // splitting, so nothing re-splits it. + for _, f := range []string{"from capture", "from arg"} { + if _, err := os.Stat(filepath.Join(cwd, f)); err != nil { + t.Fatalf("%q: %v", f, err) + } + } +} + +// The child's exit code is the script's, through the plugin. +func TestE2E_pluginPropagatesTheExitCode(t *testing.T) { + cwd := pluginCatalog(t, true, + " # @runner lines\n boom ${CODE}: |\n sh -c ${CODE}\n touch never\n") + body, _ := os.ReadFile(filepath.Join(cwd, "godo.yaml")) + fixed := strings.Replace(string(body), "engine:\n", "engine:\n dialect: matcher\n", 1) + if err := os.WriteFile(filepath.Join(cwd, "godo.yaml"), []byte(fixed), 0o644); err != nil { + t.Fatal(err) + } + + app, _, _ := e2eApp(t, cwd) + err := app.Run([]string{"boom", "exit 42"}) + if got := godo.ExitCode(err); got != 42 { + t.Fatalf("exit=%d (%v)", got, err) + } + if _, serr := os.Stat(filepath.Join(cwd, "never")); serr == nil { + t.Fatal("the body carried on after a failure") + } +} + +// A digest that does not match the bytes stops the run before anything loads. +func TestE2E_pluginDigestMismatchRefusesToRun(t *testing.T) { + cwd := pluginCatalog(t, true, " # @runner lines\n boot: |\n touch one\n") + body, _ := os.ReadFile(filepath.Join(cwd, "godo.yaml")) + broken := strings.Replace(string(body), "sha256: ", "sha256: 00", 1) + if err := os.WriteFile(filepath.Join(cwd, "godo.yaml"), []byte(broken), 0o644); err != nil { + t.Fatal(err) + } + + app, _, _ := e2eApp(t, cwd) + err := app.Run([]string{"boot"}) + if err == nil || !strings.Contains(err.Error(), "sha256 mismatch") { + t.Fatalf("err=%v", err) + } + if _, serr := os.Stat(filepath.Join(cwd, "one")); serr == nil { + t.Fatal("a plugin with the wrong digest ran") + } +} + +// Contract: install, then run. The whole reason the command exists. +func TestE2E_pluginsInstallThenRun(t *testing.T) { + cwd := t.TempDir() + store := pluginstore.Store{Dir: filepath.Join(t.TempDir(), "store")} + + data, err := os.ReadFile(buildExamplePlugin(t)) + if err != nil { + t.Fatal(err) + } + artifact := filepath.Join(cwd, "godo-lines.wasm") + if err := os.WriteFile(artifact, data, 0o644); err != nil { + t.Fatal(err) + } + // A catalog with a comment, to prove installing does not eat the file. + writeGodoYAML(t, cwd, "version: \"0.1\"\nscripts:\n # keep me\n shell: echo hi\n") + + app, out, _ := e2eApp(t, cwd) + app.PluginStore = &store + if err := app.Run([]string{"-e", "plugins", "install", "./godo-lines.wasm"}); err != nil { + t.Fatalf("install: %v", err) + } + if !strings.Contains(out.String(), "runner:lines") { + t.Fatalf("install said:\n%s", out.String()) + } + + body, err := os.ReadFile(filepath.Join(cwd, "godo.yaml")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(body), "# keep me") { + t.Fatalf("install rewrote the catalog and lost a comment:\n%s", body) + } + + // The declared runner now works, with no hand-editing in between. + if err := os.WriteFile(filepath.Join(cwd, "godo.yaml"), + append(body, []byte("\n # @runner lines\n boot: |\n touch made-it\n")...), 0o644); err != nil { + t.Fatal(err) + } + app, _, _ = e2eApp(t, cwd) + app.PluginStore = &store + if err := app.Run([]string{"boot"}); err != nil { + t.Fatalf("run: %v", err) + } + if _, err := os.Stat(filepath.Join(cwd, "made-it")); err != nil { + t.Fatalf("the installed plugin did not run: %v", err) + } +} + +// A plugin that is not on this machine says so, and says what to run. +// +// The source is remote: a file sitting beside the catalog is already here, and +// is loaded without ceremony. +func TestE2E_pluginsNotInstalledSaysWhatToRun(t *testing.T) { + cwd := t.TempDir() + writeGodoYAML(t, cwd, "version: \"0.1\"\nengine:\n plugins:\n"+ + " - source: https://example.test/lines.wasm\n"+ + " sha256: "+strings.Repeat("a", 64)+"\n"+ + " provides: [runner:lines]\n"+ + "scripts:\n # @runner lines\n boot: |\n touch one\n") + app, _, _ := e2eApp(t, cwd) + app.PluginStore = &pluginstore.Store{Dir: filepath.Join(t.TempDir(), "empty")} + + err := app.Run([]string{"boot"}) + if err == nil { + t.Fatal("ran a plugin that is not installed") + } + for _, want := range []string{"not installed", "godo -e plugins install"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("err=%v, missing %q", err, want) + } + } + if _, serr := os.Stat(filepath.Join(cwd, "one")); serr == nil { + t.Fatal("something ran anyway") + } +} + +// A file beside the catalog needs no install — its digest is checked anyway. +func TestE2E_localArtifactNeedsNoInstall(t *testing.T) { + cwd := pluginCatalog(t, true, " # @runner lines\n boot: |\n touch one\n") + app, _, _ := e2eApp(t, cwd) + app.PluginStore = &pluginstore.Store{Dir: filepath.Join(t.TempDir(), "empty")} + if err := app.Run([]string{"boot"}); err != nil { + t.Fatalf("run: %v", err) + } + if _, err := os.Stat(filepath.Join(cwd, "one")); err != nil { + t.Fatalf("did not run: %v", err) + } +} + +func TestE2E_pluginsListsState(t *testing.T) { + cwd := pluginCatalog(t, true, " # @runner lines\n boot: |\n touch one\n") + app, out, _ := e2eApp(t, cwd) + app.PluginStore = &pluginstore.Store{Dir: filepath.Join(t.TempDir(), "empty")} + if err := app.Run([]string{"-e", "plugins"}); err != nil { + t.Fatal(err) + } + for _, want := range []string{"runner:lines", "not installed", "godo -e plugins install"} { + if !strings.Contains(out.String(), want) { + t.Fatalf("missing %q in:\n%s", want, out.String()) + } + } +} diff --git a/internal/plugin/invoke.go b/internal/plugin/invoke.go new file mode 100644 index 0000000..063dc51 --- /dev/null +++ b/internal/plugin/invoke.go @@ -0,0 +1,453 @@ +package plugin + +import ( + "bufio" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/sys" +) + +// Host is what a plugin can reach through godo. Everything here is gated by +// the catalog's config: a capability the catalog did not grant is not a stub +// that fails, it is an op godo refuses to perform. +type Host struct { + // Dir is the working directory for exec, normally the catalog's. + Dir string + // HTTP makes fetch requests; nil means http.DefaultClient. + HTTP *http.Client + Stdout io.Writer + Stderr io.Writer + Stdin io.Reader +} + +// Outcome is what an invocation produced. +type Outcome struct { + // Code is the plugin's exit code, and so the script's. + Code int +} + +// Invoke runs the plugin once. +// +// The plugin is a WASI command, so this instantiates the module, writes the +// request to its stdin, and answers ops until it exits. Its exit code is the +// result; its stderr goes to godo's, so a plugin can explain itself. +func (p *Plugin) Invoke(ctx context.Context, req Request, host Host) (Outcome, error) { + req.API = APIVersion + req.Config = p.Config + + inR, inW := io.Pipe() + outR, outW := io.Pipe() + + stderr := host.Stderr + if stderr == nil { + stderr = os.Stderr + } + + // A wasm guest starts with nothing wired to the outside. The mounts below + // are what the catalog asks for; everything else a script might reach for + // — a network, a subprocess — has no syscall to reach through, whatever + // anyone configures. + cfg := wazero.NewModuleConfig(). + WithStdin(inR). + WithStdout(outW). + WithStderr(stderr). + WithArgs("plugin"). + WithName("") + if mounts := p.Mounts(host.Dir); len(mounts) > 0 { + fs := wazero.NewFSConfig() + for _, m := range mounts { + fs = fs.WithDirMount(m.Host, m.Guest) + } + cfg = cfg.WithFSConfig(fs) + } + cfg = cfg.WithSysWalltime().WithSysNanotime() + + runErr := make(chan error, 1) + go func() { + _, err := p.rt.InstantiateModule(ctx, p.compiled, cfg) + _ = outW.Close() + runErr <- err + }() + + // The request goes out on its own goroutine: an io.Pipe has no buffer, so + // writing it inline would deadlock against a plugin that spoke first. + writeErr := make(chan error, 1) + go func() { + line, err := json.Marshal(req) + if err != nil { + writeErr <- err + return + } + _, err = inW.Write(append(line, '\n')) + writeErr <- err + }() + + out := Outcome{} + loopErr := p.serve(outR, inW, host) + + _ = inW.Close() + _ = inR.Close() + + err := <-runErr + _ = outR.Close() + if werr := <-writeErr; werr != nil && loopErr == nil { + loopErr = werr + } + + var exit *sys.ExitError + switch { + case err == nil: + out.Code = 0 + case errors.As(err, &exit): + out.Code = int(exit.ExitCode()) + default: + return out, fmt.Errorf("plugin %s: %w", p.Source, err) + } + if loopErr != nil { + return out, loopErr + } + return out, nil +} + +// serve answers ops until the plugin's stdout closes. +func (p *Plugin) serve(r io.Reader, w io.Writer, host Host) error { + scan := bufio.NewScanner(r) + scan.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + enc := json.NewEncoder(w) + + for scan.Scan() { + line := strings.TrimSpace(scan.Text()) + if line == "" { + continue + } + // A line that is not an op is output. A guest has one stdout, and + // everything inside it writes there — an imported module's print is + // the builtin one, not whatever the plugin arranged for the script. + // Treating that as a protocol error would mean a plugin's users could + // not split their code into files. + op, known := readOp(line) + if !known && op.Op != "" { + // JSON naming an op godo does not have: a plugin built against a + // newer protocol. Answering is what keeps that a clear failure + // instead of a hang — the plugin is waiting for a reply it will + // never otherwise get. + if err := enc.Encode(Result{Error: fmt.Sprintf("unknown op %q — this godo speaks api %d", op.Op, APIVersion)}); err != nil { + return fmt.Errorf("plugin %s: %w", p.Source, err) + } + continue + } + if !known { + if _, err := fmt.Fprintln(orStd(host.Stdout, os.Stdout), line); err != nil { + return fmt.Errorf("plugin %s: %w", p.Source, err) + } + continue + } + switch op.Op { + case OpOut: + if _, err := fmt.Fprintln(orStd(host.Stdout, os.Stdout), op.Text); err != nil { + return fmt.Errorf("plugin %s: %w", p.Source, err) + } + case OpExec: + res := p.execOp(op, host) + if err := enc.Encode(res); err != nil { + return fmt.Errorf("plugin %s: %w", p.Source, err) + } + case OpFetch: + res := p.fetchOp(op, host) + if err := enc.Encode(res); err != nil { + return fmt.Errorf("plugin %s: %w", p.Source, err) + } + case OpSlink: + res := p.slinkOp(op, host) + if err := enc.Encode(res); err != nil { + return fmt.Errorf("plugin %s: %w", p.Source, err) + } + } + } + if err := scan.Err(); err != nil && !errors.Is(err, io.ErrClosedPipe) { + return fmt.Errorf("plugin %s: %w", p.Source, err) + } + return nil +} + +// readOp parses a line as an op, or reports that it is output. +// +// Both halves matter. A line that is not JSON is plain output. A line that is +// JSON but carries no op godo knows is output too — scripts print JSON all the +// time, and swallowing a data structure because it has the shape of a message +// would be worse than the error it avoids. +// +// A line that is a valid op godo does obey, whoever printed it. Closing that +// would take a secret on every op, which breaks every plugin already built to +// answer without one — a worse failure, and for a coincidence inside a +// catalog whose scripts could call exec directly anyway. +// +// A line that names an op godo does not have comes back with Op set and false: +// the caller answers it with an error rather than printing it, because a +// plugin waiting for a reply that never arrives hangs, and a hang is a worse +// failure than a stray line of output. +func readOp(line string) (Op, bool) { + if !strings.HasPrefix(line, "{") { + return Op{}, false + } + var op Op + if err := json.Unmarshal([]byte(line), &op); err != nil { + return Op{}, false + } + switch op.Op { + case OpExec, OpOut, OpSlink, OpFetch: + return op, true + } + return op, false +} + +// execOp runs an argument vector. +func (p *Plugin) execOp(op Op, host Host) Result { + if len(op.Argv) == 0 { + return Result{Error: "exec: empty argv"} + } + cmd := exec.Command(op.Argv[0], op.Argv[1:]...) + cmd.Dir = host.Dir + if op.Dir != "" { + cmd.Dir = op.Dir + } + var res Result + if op.Capture { + var stdout, stderr strings.Builder + cmd.Stdout = &stdout + cmd.Stderr = &stderr + res.Code = runAndCode(cmd, &res) + res.Stdout = stdout.String() + res.Stderr = stderr.String() + } else { + cmd.Stdout = orStd(host.Stdout, os.Stdout) + cmd.Stderr = orStd(host.Stderr, os.Stderr) + cmd.Stdin = host.Stdin + res.Code = runAndCode(cmd, &res) + } + res.OK = res.Code == 0 + return res +} + +// fetchOp makes an HTTP request on the guest's behalf. +// +// A wasm guest has no sockets. Without this, reaching the network means +// exec'ing whatever the machine happens to carry, which is the platform +// dependency a plugin exists to remove: a script that works here and not on a +// teammate's laptop because one of them has curl is the problem, not a +// workaround for it. Go's client is the same on every platform godo ships to. +// +// A non-2xx is not an error: it is a status the script gets to read, the same +// way a failed command is a code rather than a raised exception. +func (p *Plugin) fetchOp(op Op, host Host) Result { + if op.URL == "" { + return Result{Error: "fetch: no url"} + } + method := op.Method + if method == "" { + method = http.MethodGet + } + var body io.Reader + if op.Body != "" { + body = strings.NewReader(op.Body) + } + req, err := http.NewRequest(method, op.URL, body) + if err != nil { + return Result{Error: fmt.Sprintf("fetch: %v", err)} + } + for k, v := range op.Headers { + req.Header.Set(k, v) + } + client := host.HTTP + if client == nil { + // A request with no deadline is a script that hangs on a server that + // never answers, with nothing to read and nothing to kill. + timeout := time.Duration(op.Timeout) * time.Second + if timeout <= 0 { + timeout = 30 * time.Second + } + client = &http.Client{Timeout: timeout} + } + resp, err := client.Do(req) + if err != nil { + return Result{Error: fmt.Sprintf("fetch %s: %v", op.URL, err)} + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil { + return Result{Error: fmt.Sprintf("fetch %s: %v", op.URL, err)} + } + headers := make(map[string]string, len(resp.Header)) + for k := range resp.Header { + headers[k] = resp.Header.Get(k) + } + return Result{ + Code: resp.StatusCode, + OK: resp.StatusCode >= 200 && resp.StatusCode < 300, + Base64: base64.StdEncoding.EncodeToString(data), + Headers: headers, + } +} + +func (p *Plugin) slinkOp(op Op, host Host) Result { + // Paths resolve against the catalog's directory but are not confined to + // it. A worktree is created beside a repository, not inside it, and the + // link into it is the point. + src := resolveAgainst(host.Dir, op.Src) + dst := resolveAgainst(host.Dir, op.Dst) + + if _, err := os.Lstat(dst); err == nil { + if !op.Force { + return Result{Error: fmt.Sprintf("slink: dst exists: %s", op.Dst)} + } + if err := os.Remove(dst); err != nil { + return Result{Error: fmt.Sprintf("slink: remove dst: %v", err)} + } + } else if !errors.Is(err, os.ErrNotExist) { + return Result{Error: fmt.Sprintf("slink: dst: %v", err)} + } + + // Windows support for os.Symlink is unverified on a real Windows host. + if err := os.Symlink(src, dst); err != nil { + return Result{Error: fmt.Sprintf("slink: %v", err)} + } + return Result{Code: 0, OK: true} +} + +// resolveAgainst turns a plugin-supplied path into an absolute one. +// +// An absolute path is taken as given: a script that names /tmp means /tmp. +func resolveAgainst(dir, path string) string { + if filepath.IsAbs(path) || dir == "" { + return path + } + return filepath.Join(dir, path) +} + +func runAndCode(cmd *exec.Cmd, res *Result) int { + err := cmd.Run() + if err == nil { + return 0 + } + var ee *exec.ExitError + if errors.As(err, &ee) { + return ee.ExitCode() + } + // Could not start at all: a missing binary is the plugin's problem to + // report, not godo's to crash on. + res.Error = err.Error() + return 127 +} + +func orStd(w io.Writer, def io.Writer) io.Writer { + if w == nil { + return def + } + return w +} + +// Mount is one directory the guest can see, and where it sees it. +type Mount struct { + Host string // the real path + Guest string // where it appears inside the sandbox +} + +// Mounts returns the directories config.fs.mount asks for. +// +// This is the one thing godo reads out of config, and it is configuration +// rather than permission: a guest cannot mount anything itself, so somebody +// has to say what it sees, and the catalog is the only one who knows. The +// default is the directory the godo.yaml lives in — a script that cannot read +// the repository it belongs to is not useful, and nothing is being defended +// against by withholding it. +// +// Three shapes, because a catalog that only wants its own directory should not +// have to say so twice: +// +// fs: {mount: true} the catalog's directory, as / +// fs: {mount: [".", "/tmp"]} those, each at its own name +// fs: {mount: {".": "/", "/tmp": "/tmp"}} explicit guest paths +// +// A relative host path resolves against the catalog. Nothing else is visible: +// a directory the config does not name does not exist inside the sandbox, and +// no path can climb out of one that it does. +func (p *Plugin) Mounts(catalogDir string) []Mount { + raw, ok := p.configValue("fs", "mount") + if !ok { + if catalogDir == "" { + return nil + } + return []Mount{{Host: catalogDir, Guest: "/"}} + } + resolve := func(host string) string { + if host == "" { + host = "." + } + if !filepath.IsAbs(host) && catalogDir != "" { + return filepath.Join(catalogDir, host) + } + return host + } + switch v := raw.(type) { + case bool: + if !v || catalogDir == "" { + return nil + } + return []Mount{{Host: catalogDir, Guest: "/"}} + case []any: + var out []Mount + for i, item := range v { + host, ok := item.(string) + if !ok { + continue + } + guest := host + if i == 0 && (host == "." || host == "./") { + // The first entry being the catalog is the common case, and it + // belongs at the root so relative paths in a script just work. + guest = "/" + } + out = append(out, Mount{Host: resolve(host), Guest: guest}) + } + return out + case map[string]any: + var out []Mount + for host, g := range v { + guest, ok := g.(string) + if !ok { + continue + } + out = append(out, Mount{Host: resolve(host), Guest: guest}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Guest < out[j].Guest }) + return out + } + return nil +} + +// configValue reads config[section][key] without deciding what it means. +func (p *Plugin) configValue(section, key string) (any, bool) { + raw, ok := p.Config[section] + if !ok { + return nil, false + } + m, ok := raw.(map[string]any) + if !ok { + return nil, false + } + v, ok := m[key] + return v, ok +} diff --git a/internal/plugin/invoke_test.go b/internal/plugin/invoke_test.go new file mode 100644 index 0000000..ab6e0c3 --- /dev/null +++ b/internal/plugin/invoke_test.go @@ -0,0 +1,428 @@ +package plugin + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" +) + +var ( + buildOnce sync.Once + wasmPath string + buildErr error +) + +// exampleWasm builds examples/plugins/lines for wasip1 once per run. +// +// Built rather than committed: a checked-in binary is a thing nobody can +// review, and the point of this test is that the artifact matches the source +// beside it. +func exampleWasm(t *testing.T) string { + t.Helper() + buildOnce.Do(func() { + dir := t.TempDir() + out := filepath.Join(dir, "lines.wasm") + cmd := exec.Command("go", "build", "-o", out, "../../examples/plugins/lines") + cmd.Env = append(os.Environ(), "GOOS=wasip1", "GOARCH=wasm") + if b, err := cmd.CombinedOutput(); err != nil { + buildErr = err + t.Logf("build: %s", b) + return + } + // Keep it outside t.TempDir's cleanup by copying to the package temp. + data, err := os.ReadFile(out) + if err != nil { + buildErr = err + return + } + keep, err := os.CreateTemp("", "lines-*.wasm") + if err != nil { + buildErr = err + return + } + defer keep.Close() + if _, err := keep.Write(data); err != nil { + buildErr = err + return + } + wasmPath = keep.Name() + }) + if buildErr != nil { + t.Fatalf("building the example plugin: %v", buildErr) + } + return wasmPath +} + +func load(t *testing.T, config map[string]any) (*Plugin, func()) { + t.Helper() + path := exampleWasm(t) + digest, err := Digest(path) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + rt := NewRuntime(ctx) + p, err := rt.Load(ctx, path, digest, "", []string{"runner:lines"}, config) + if err != nil { + t.Fatal(err) + } + return p, func() { _ = rt.Close(ctx) } +} + +func TestLoad_refusesAWrongDigest(t *testing.T) { + path := exampleWasm(t) + ctx := context.Background() + rt := NewRuntime(ctx) + defer rt.Close(ctx) + + _, err := rt.Load(ctx, path, strings.Repeat("0", 64), "", nil, nil) + if err == nil { + t.Fatal("a plugin whose bytes do not match its digest must not load") + } + for _, want := range []string{"sha256 mismatch", "declared", "actual"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("err=%v, missing %q", err, want) + } + } +} + +func TestLoad_refusesSomethingThatIsNotWasm(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "not.wasm") + if err := os.WriteFile(path, []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + digest, err := Digest(path) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + rt := NewRuntime(ctx) + defer rt.Close(ctx) + if _, err := rt.Load(ctx, path, digest, "", nil, nil); err == nil { + t.Fatal("want a compile error") + } +} + +// A failing command stops the body and becomes the exit code, like a shell. +func TestInvoke_exitCodeIsTheChilds(t *testing.T) { + p, done := load(t, map[string]any{"proc": map[string]any{"exec": true}}) + defer done() + + out, err := p.Invoke(context.Background(), Request{ + Body: "sh -c ${SCRIPT}\ntouch should-not-exist", + Argv: map[string]string{"SCRIPT": "exit 42"}, + }, Host{Dir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + if out.Code != 42 { + t.Fatalf("code=%d, want 42", out.Code) + } +} + +// "?" marks a line that may fail without stopping the rest. +func TestInvoke_optionalLineDoesNotStopTheBody(t *testing.T) { + dir := t.TempDir() + p, done := load(t, map[string]any{"proc": map[string]any{"exec": true}}) + defer done() + + out, err := p.Invoke(context.Background(), Request{ + Body: "?sh -c ${SCRIPT}\ntouch after", + Argv: map[string]string{"SCRIPT": "exit 3"}, + }, Host{Dir: dir}) + if err != nil { + t.Fatal(err) + } + if out.Code != 0 { + t.Fatalf("code=%d", out.Code) + } + if _, err := os.Stat(filepath.Join(dir, "after")); err != nil { + t.Fatalf("the body stopped at an optional failure: %v", err) + } +} + +func TestServe_outWritesWithoutAnswer(t *testing.T) { + var stdout, answers bytes.Buffer + p := &Plugin{} + if err := p.serve(strings.NewReader(`{"op":"out","text":"hello"}`+"\n"), &answers, Host{Stdout: &stdout}); err != nil { + t.Fatal(err) + } + if got := stdout.String(); got != "hello\n" { + t.Fatalf("stdout=%q", got) + } + if got := answers.String(); got != "" { + t.Fatalf("out unexpectedly received an answer: %q", got) + } +} + +// The catalog's neighbour is reachable on purpose: a git worktree is created +// beside a repository, and linking .env into it is the whole use case. +// +// Confinement here would also be theatre — proc.exec can run "ln -s" anywhere, +// so a slink narrower than exec protects nothing. The grant is the boundary. +func TestServe_slinkReachesBesideTheCatalog(t *testing.T) { + root := t.TempDir() + catalog := filepath.Join(root, "repo") + neighbour := filepath.Join(root, "repo-wt") + for _, d := range []string{catalog, neighbour} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(catalog, ".env"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + p := &Plugin{Config: map[string]any{"fs": map[string]any{"slink": true}}} + + res := serveOp(t, p, Host{Dir: catalog}, Op{Op: OpSlink, Src: ".env", Dst: "../repo-wt/.env"}) + if res.Error != "" { + t.Fatalf("error=%q", res.Error) + } + if !res.OK { + t.Fatal("slink reported failure") + } + if _, err := os.Lstat(filepath.Join(neighbour, ".env")); err != nil { + t.Fatalf("the link was not created: %v", err) + } +} + +// An absolute path is taken as given rather than reinterpreted. +func TestServe_slinkAcceptsAnAbsolutePath(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "source") + if err := os.WriteFile(target, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + p := &Plugin{Config: map[string]any{"fs": map[string]any{"slink": true}}} + + res := serveOp(t, p, Host{Dir: dir}, Op{Op: OpSlink, Src: target, Dst: "link"}) + if res.Error != "" { + t.Fatalf("error=%q", res.Error) + } + got, err := os.Readlink(filepath.Join(dir, "link")) + if err != nil { + t.Fatal(err) + } + if got != target { + t.Fatalf("link points at %q, want %q", got, target) + } +} + +func TestServe_slinkForceReplacesExistingDestination(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "source"), []byte("replacement"), 0o644); err != nil { + t.Fatal(err) + } + dst := filepath.Join(dir, "link") + if err := os.Symlink("missing", dst); err != nil { + t.Fatal(err) + } + p := &Plugin{Config: map[string]any{"fs": map[string]any{"slink": true}}} + res := serveOp(t, p, Host{Dir: dir}, Op{Op: OpSlink, Src: "source", Dst: "link"}) + if res.Error == "" { + t.Fatal("force=false replaced an existing destination") + } + if target, err := os.Readlink(dst); err != nil || target != "missing" { + t.Fatalf("force=false changed destination: target=%q err=%v", target, err) + } + + res = serveOp(t, p, Host{Dir: dir}, Op{Op: OpSlink, Src: "source", Dst: "link", Force: true}) + if !res.OK || res.Code != 0 || res.Error != "" { + t.Fatalf("result=%+v", res) + } + data, err := os.ReadFile(dst) + if err != nil { + t.Fatal(err) + } + if got := string(data); got != "replacement" { + t.Fatalf("replacement content=%q", got) + } +} + +func serveOp(t *testing.T, p *Plugin, host Host, op Op) Result { + t.Helper() + line, err := json.Marshal(op) + if err != nil { + t.Fatal(err) + } + var answers bytes.Buffer + if err := p.serve(bytes.NewReader(append(line, '\n')), &answers, host); err != nil { + t.Fatal(err) + } + var res Result + if err := json.NewDecoder(&answers).Decode(&res); err != nil { + t.Fatalf("decode result %q: %v", answers.String(), err) + } + return res +} + +func TestInvoke_execRunsCommands(t *testing.T) { + dir := t.TempDir() + p, done := load(t, nil) + defer done() + + out, err := p.Invoke(context.Background(), Request{ + Body: "touch marker\ntouch ${NAME}", + Argv: map[string]string{"NAME": "second"}, + }, Host{Dir: dir}) + if err != nil { + t.Fatal(err) + } + if out.Code != 0 { + t.Fatalf("code=%d", out.Code) + } + for _, f := range []string{"marker", "second"} { + if _, err := os.Stat(filepath.Join(dir, f)); err != nil { + t.Fatalf("%s: %v", f, err) + } + } +} + +func TestServe_slinkCreatesTheLink(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "source"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + p := &Plugin{} + res := serveOp(t, p, Host{Dir: dir}, Op{Op: OpSlink, Src: "source", Dst: "link"}) + if res.Error != "" { + t.Fatalf("error=%q", res.Error) + } + if !res.OK { + t.Fatal("slink reported failure") + } + got, err := os.Readlink(filepath.Join(dir, "link")) + if err != nil { + t.Fatal(err) + } + if filepath.Base(got) != "source" { + t.Fatalf("link points at %q", got) + } +} + +// The default is the catalog's own directory: a script that cannot read the +// repository it belongs to is not useful, and withholding it defends nothing. +func TestMounts_defaultsToTheCatalog(t *testing.T) { + dir := t.TempDir() + p := &Plugin{} + got := p.Mounts(dir) + if len(got) != 1 || got[0].Host != dir || got[0].Guest != "/" { + t.Fatalf("mounts=%+v", got) + } +} + +func TestMounts_shapes(t *testing.T) { + dir := t.TempDir() + for _, tc := range []struct { + name string + cfg map[string]any + want []Mount + }{ + {"true is the catalog", map[string]any{"fs": map[string]any{"mount": true}}, + []Mount{{Host: dir, Guest: "/"}}}, + {"false is nothing", map[string]any{"fs": map[string]any{"mount": false}}, nil}, + {"a list", map[string]any{"fs": map[string]any{"mount": []any{".", "/tmp"}}}, + []Mount{{Host: dir, Guest: "/"}, {Host: "/tmp", Guest: "/tmp"}}}, + {"a mapping", map[string]any{"fs": map[string]any{"mount": map[string]any{"/opt/x": "/x"}}}, + []Mount{{Host: "/opt/x", Guest: "/x"}}}, + } { + t.Run(tc.name, func(t *testing.T) { + got := (&Plugin{Config: tc.cfg}).Mounts(dir) + if len(got) != len(tc.want) { + t.Fatalf("mounts=%+v want %+v", got, tc.want) + } + for i := range tc.want { + if got[i] != tc.want[i] { + t.Fatalf("mounts=%+v want %+v", got, tc.want) + } + } + }) + } +} + +// A response body is bytes. Carrying it as a JSON string replaced anything +// that was not valid UTF-8, so an image arrived shorter than it left with +// nothing raised. +func TestFetch_carriesBytesIntact(t *testing.T) { + want := make([]byte, 256) + for i := range want { + want[i] = byte(i) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(want) + })) + defer srv.Close() + + p := &Plugin{} + res := serveOp(t, p, Host{HTTP: srv.Client()}, Op{Op: OpFetch, URL: srv.URL}) + if res.Error != "" { + t.Fatalf("error=%q", res.Error) + } + got, err := base64.StdEncoding.DecodeString(res.Base64) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Fatalf("got %d bytes, want %d", len(got), len(want)) + } +} + +// A status is a value the script reads, like a command's exit code. +func TestFetch_statusIsNotAnError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(404) + })) + defer srv.Close() + + res := serveOp(t, &Plugin{}, Host{HTTP: srv.Client()}, Op{Op: OpFetch, URL: srv.URL}) + if res.Error != "" { + t.Fatalf("a 404 was reported as an error: %q", res.Error) + } + if res.Code != 404 || res.OK { + t.Fatalf("code=%d ok=%v", res.Code, res.OK) + } +} + +func TestFetch_sendsMethodHeadersAndBody(t *testing.T) { + var gotMethod, gotHeader, gotBody string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotMethod = r.Method + gotHeader = r.Header.Get("X-Godo") + b, _ := io.ReadAll(r.Body) + gotBody = string(b) + })) + defer srv.Close() + + serveOp(t, &Plugin{}, Host{HTTP: srv.Client()}, Op{ + Op: OpFetch, URL: srv.URL, Method: "POST", + Headers: map[string]string{"X-Godo": "yes"}, Body: `{"a":1}`, + }) + if gotMethod != "POST" || gotHeader != "yes" || gotBody != `{"a":1}` { + t.Fatalf("method=%q header=%q body=%q", gotMethod, gotHeader, gotBody) + } +} + +// A plugin built against a newer protocol must fail, not hang: it is waiting +// for a reply that would otherwise never come. +func TestServe_unknownOpIsAnsweredNotSwallowed(t *testing.T) { + var answers strings.Builder + p := &Plugin{} + line := `{"op":"teleport","url":"x"}` + "\n" + if err := p.serve(strings.NewReader(line), &answers, Host{Stdout: io.Discard}); err != nil { + t.Fatal(err) + } + if !strings.Contains(answers.String(), "unknown op") { + t.Fatalf("answered %q", answers.String()) + } +} diff --git a/internal/plugin/protocol.go b/internal/plugin/protocol.go new file mode 100644 index 0000000..1947639 --- /dev/null +++ b/internal/plugin/protocol.go @@ -0,0 +1,111 @@ +// Package plugin runs godo plugins as WebAssembly. +// +// A plugin is a WASI command: a normal program with a main(), compiled to +// wasm. godo writes one JSON request to its stdin, the plugin writes JSON ops +// to its stdout, and godo answers the ones that need answering. The plugin's +// exit code is the script's exit code. +// +// Being a command rather than a set of exported functions is what keeps the +// contract small. There is no memory-sharing ABI to get right, a plugin can be +// written in any language that targets WASI, and it can be tested outside wasm +// entirely — pipe it a request on stdin and read what it says. +// +// A wasm guest has no filesystem, no network and no way to start a process of +// its own, so a plugin reaches the outside world by asking godo. That is a +// property of the platform, not a policy: godo performs what it is asked, +// because the script doing the asking is the catalog's, and a catalog already +// runs with the shell's full reach. +// +// What is pinned is which plugin, not what it may do — the sha256 in the +// catalog. Where the artifact came from is the question worth answering. +package plugin + +// APIVersion is the protocol this build speaks. A plugin that answers with a +// different major refuses to run rather than guessing. +const APIVersion = 1 + +// Request is the single line godo writes to a plugin's stdin. +// +// There is no mode: a plugin is invoked to run. --preview prints the body +// without starting anything, so nothing here has to describe a dry run. A +// later --predict will add a field for it; plugins ignore fields they do not +// know, so that costs nothing today. +type Request struct { + API int `json:"api"` + // Runner is the name the catalog asked for, so one plugin can provide + // several. + Runner string `json:"runner"` + // Body is the script body, verbatim. godo does not expand ${godo:…} for a + // plugin: a plugin body is not shell text, and the values are right here. + Body string `json:"body"` + // Argv holds the matcher's captures; Args the tokens left over. + Argv map[string]string `json:"argv"` + Args []string `json:"args"` + // Config is the plugin's own block from godo.yaml, verbatim. + Config map[string]any `json:"config,omitempty"` +} + +// Op is one line a plugin writes to its stdout. +type Op struct { + Op string `json:"op"` + + // exec + Argv []string `json:"argv,omitempty"` + Dir string `json:"dir,omitempty"` + Capture bool `json:"capture,omitempty"` + + // out + Text string `json:"text,omitempty"` + + // slink + Src string `json:"src,omitempty"` + Dst string `json:"dst,omitempty"` + Force bool `json:"force,omitempty"` + + // fetch + URL string `json:"url,omitempty"` + Method string `json:"method,omitempty"` + Headers map[string]string `json:"headers,omitempty"` + Body string `json:"body,omitempty"` + // Timeout is in seconds; zero means the host's default. + Timeout int `json:"timeout,omitempty"` +} + +// Result is godo's answer to an op that needs one. +type Result struct { + // Code is a process exit status, or an HTTP status for a fetch. + Code int `json:"code"` + OK bool `json:"ok"` + Stdout string `json:"stdout,omitempty"` + Stderr string `json:"stderr,omitempty"` + // Headers carries a fetch's response headers. + Headers map[string]string `json:"headers,omitempty"` + // Base64 carries a fetch's body, encoded. + // + // A response body is bytes. Carrying it as a JSON string means anything + // that is not valid UTF-8 comes out replaced — an image arrives shorter + // than it left, with nothing raised. Encoding costs a third more on the + // wire and cannot lose a byte. + Base64 string `json:"base64,omitempty"` + // Error is set when godo refused: an unknown op, or a capability the + // catalog did not grant. It is not a failing command — that is Code. + Error string `json:"error,omitempty"` +} + +// Op names. +const ( + // OpExec runs an argument vector and waits. Needs config proc.exec. + OpExec = "exec" + // OpOut writes a line to the host's stdout. Needs no capability. + OpOut = "out" + // OpSlink creates a symlink. + OpSlink = "slink" + // OpFetch makes an HTTP request. + // + // A wasm guest has no sockets, so without this the only way to reach the + // network is to exec something that has them — curl, or whatever the + // machine happens to carry. That is the platform dependency a plugin + // exists to remove, so the request is made here, with the same library on + // every platform. + OpFetch = "fetch" +) diff --git a/internal/plugin/runner.go b/internal/plugin/runner.go new file mode 100644 index 0000000..c95e34e --- /dev/null +++ b/internal/plugin/runner.go @@ -0,0 +1,129 @@ +package plugin + +import ( + "context" + "fmt" + "io" + "strings" + + "github.com/my-rv/godo/internal/catalog" +) + +// Runner adapts a loaded plugin to one runner name. +// +// One plugin may provide several runners; the name travels in the request so +// the plugin can tell which one it was asked for. +type Runner struct { + Plugin *Plugin + Name catalog.RunnerName + Host Host +} + +// Run implements catalog.Runner. +// +// It always fails: a rendered line has lost the values a plugin reads by name. +// The engine hands a plugin the bound invocation instead. +func (r Runner) Run(string) error { + return fmt.Errorf("runner %q is a plugin and takes the bound invocation, not a rendered line", r.Name) +} + +// AcceptsArgs implements catalog.ArgsAwareRunner. +// +// A plugin body is a program, not a shell template, so there is no +// ${godo:args…} to look for. Whether the leftover tokens mean anything is the +// plugin's business; they are handed over either way. +func (Runner) AcceptsArgs([]string) bool { return true } + +// RunInvocation implements catalog.InvocationRunner. +func (r Runner) RunInvocation(inv catalog.Invocation) error { + out, err := r.Plugin.Invoke(context.Background(), Request{ + Runner: string(r.Name), + Body: inv.Body, + Argv: nonNil(inv.Captures), + Args: inv.Args, + }, r.Host) + if err != nil { + return err + } + if out.Code != 0 { + return &catalog.ExitError{ + Code: out.Code, + Message: fmt.Sprintf("script %q failed under runner %q", inv.Script, r.Name), + } + } + return nil +} + +// PreviewInvocation implements catalog.InvocationRunner: the body, verbatim. +// +// --preview does not start the plugin. A plugin body is a program, and the +// only faithful answer to "what will this do" without running it is the +// program itself. Anything else would be a guess dressed as a fact, and a +// guess needs its own flag rather than quietly borrowing this one. +func (r Runner) PreviewInvocation(inv catalog.Invocation) ([]string, error) { + return strings.Split(strings.TrimRight(inv.Body, "\n"), "\n"), nil +} + +func nonNil(m map[string]string) map[string]string { + if m == nil { + return map[string]string{} + } + return m +} + +// Resolver turns a declared plugin into a path on this machine. +// +// A catalog says which artifact a script needs; where that artifact is, is a +// separate question, and one the engine has no business answering. godo hands +// in its store; an embedder hands in whatever it uses, or nil to read the +// source as a path. +type Resolver func(digest, source string) (string, error) + +// Register loads every plugin a catalog declares and registers the runners +// they provide. +// +// A plugin that will not load stops the run. A catalog that names a plugin has +// already decided it is part of the build; carrying on without it would mean +// silently running something other than what the file says. +func Register(ctx context.Context, rt *Runtime, cat *catalog.Catalog, reg *catalog.RunnerRegistry, root string, stdout, stderr io.Writer, stdin io.Reader, resolve Resolver) error { + for _, spec := range cat.Engine.Plugins { + from := spec.Source + if resolve != nil { + stored, err := resolve(spec.SHA256, spec.Source) + if err != nil { + return err + } + from = stored + } + p, err := rt.Load(ctx, from, spec.SHA256, root, spec.Provides, spec.Config) + if err != nil { + return err + } + for _, entry := range spec.Provides { + kind, name, ok := cutKind(entry) + if !ok || kind != "runner" { + // dialect plugins are declared the same way and are not + // implemented; the catalog already validated the shape. + continue + } + runner := Runner{ + Plugin: p, + Name: catalog.RunnerName(name), + Host: Host{Dir: root, Stdout: stdout, Stderr: stderr, Stdin: stdin}, + } + if err := reg.Register(catalog.RunnerName(name), runner); err != nil { + return fmt.Errorf("plugin %s: %w", spec.Source, err) + } + } + } + return nil +} + +func cutKind(entry string) (kind, name string, ok bool) { + for i := 0; i < len(entry); i++ { + if entry[i] == ':' { + return entry[:i], entry[i+1:], true + } + } + return "", "", false +} diff --git a/internal/plugin/runtime.go b/internal/plugin/runtime.go new file mode 100644 index 0000000..8a9205d --- /dev/null +++ b/internal/plugin/runtime.go @@ -0,0 +1,111 @@ +package plugin + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/experimental" + "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1" +) + +// Runtime compiles and holds plugins for one godo invocation. +type Runtime struct { + rt wazero.Runtime + once sync.Once +} + +// NewRuntime returns a runtime with WASI available and nothing else. +// +// Exception handling is enabled because a C interpreter compiled for WASI uses +// it for setjmp/longjmp — MicroPython's non-local return is built on it, and +// without the feature its module does not even compile ("tag section not +// supported"). It costs nothing for a plugin that does not use it. +func NewRuntime(ctx context.Context) *Runtime { + rt := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfig(). + WithCoreFeatures(api.CoreFeaturesV2|experimental.CoreFeaturesExceptionHandling)) + wasi_snapshot_preview1.MustInstantiate(ctx, rt) + return &Runtime{rt: rt} +} + +// Close releases every compiled module. +func (r *Runtime) Close(ctx context.Context) error { + var err error + r.once.Do(func() { err = r.rt.Close(ctx) }) + return err +} + +// Plugin is one verified, compiled artifact. +type Plugin struct { + Source string + Provides []string + Config map[string]any + + rt wazero.Runtime + compiled wazero.CompiledModule +} + +// Load reads an artifact, checks it against its digest, and compiles it. +// +// The digest is checked before the bytes reach a compiler, not after: a +// plugin is third-party code, and "this is the artifact that was reviewed" is +// the one thing a catalog can actually assert about it. +// +// source is resolved relative to root when it is not absolute. Only local +// paths are understood today; fetching is a separate concern and belongs with +// a lockfile, not here. +func (r *Runtime) Load(ctx context.Context, source, digest, root string, provides []string, config map[string]any) (*Plugin, error) { + path, err := localPath(source, root) + if err != nil { + return nil, err + } + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("plugin %s: %w", source, err) + } + sum := sha256.Sum256(data) + got := hex.EncodeToString(sum[:]) + if !strings.EqualFold(got, strings.TrimSpace(digest)) { + return nil, fmt.Errorf("plugin %s: sha256 mismatch\n declared %s\n actual %s", source, digest, got) + } + compiled, err := r.rt.CompileModule(ctx, data) + if err != nil { + return nil, fmt.Errorf("plugin %s: not a usable wasm module: %w", source, err) + } + return &Plugin{ + Source: source, + Provides: provides, + Config: config, + rt: r.rt, + compiled: compiled, + }, nil +} + +// localPath resolves a source that names a file on this machine. +func localPath(source, root string) (string, error) { + p := strings.TrimPrefix(source, "file://") + if p == source && strings.Contains(source, "://") { + return "", fmt.Errorf("plugin %s: only local paths are supported in this build", source) + } + if filepath.IsAbs(p) { + return p, nil + } + return filepath.Join(root, p), nil +} + +// Digest returns the sha256 of a file, for writing into a catalog. +func Digest(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} diff --git a/internal/pluginstore/store.go b/internal/pluginstore/store.go new file mode 100644 index 0000000..a52fc9d --- /dev/null +++ b/internal/pluginstore/store.go @@ -0,0 +1,212 @@ +// Package pluginstore keeps plugin artifacts on disk, keyed by their digest. +// +// A plugin is third-party code that runs when someone types "godo test", so +// the digest is not a cache key that happens to be a hash — it is the identity. +// Two artifacts with the same digest are the same artifact, and one whose bytes +// stop matching is not the artifact that was reviewed. +package pluginstore + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" +) + +// Store is a directory of artifacts named by digest. +type Store struct { + // Dir defaults to /godo/plugins. + Dir string + // HTTP defaults to http.DefaultClient. + HTTP *http.Client +} + +// Dirname returns the directory this store writes to. +func (s Store) Dirname() (string, error) { + if s.Dir != "" { + return s.Dir, nil + } + cache, err := os.UserCacheDir() + if err != nil { + return "", err + } + return filepath.Join(cache, "godo", "plugins"), nil +} + +// Path is where an artifact with this digest lives. +// +// The digest becomes a filename, so it is validated rather than trusted: 64 +// lowercase hex characters and nothing else, which cannot contain a separator +// or climb out of the directory. +func (s Store) Path(digest string) (string, error) { + if !validDigest(digest) { + return "", fmt.Errorf("not a sha256 digest: %q", digest) + } + dir, err := s.Dirname() + if err != nil { + return "", err + } + return filepath.Join(dir, digest+".wasm"), nil +} + +// Has reports whether the artifact is already stored. +func (s Store) Has(digest string) bool { + p, err := s.Path(digest) + if err != nil { + return false + } + st, err := os.Stat(p) + return err == nil && st.Mode().IsRegular() +} + +// Fetch reads an artifact from source and stores it, returning its digest. +// +// It always reads: the digest is what source turns out to contain, and there is +// no way to know that without reading it. Ensure is the call that can skip. +// +// A relative source resolves against base. Callers pass the directory the +// source was written in — the catalog's for a declared plugin, the caller's own +// for one typed on a command line — so that "./x.wasm" means the same thing +// wherever godo is run from. +func (s Store) Fetch(ctx context.Context, base, source string) (digest, path string, err error) { + data, err := s.read(ctx, base, source) + if err != nil { + return "", "", err + } + sum := sha256.Sum256(data) + digest = hex.EncodeToString(sum[:]) + path, err = s.write(digest, data) + if err != nil { + return "", "", err + } + return digest, path, nil +} + +// Ensure returns the stored artifact for digest, fetching from source only if +// it is missing. A fetch that produces different bytes is refused: source is +// where an artifact comes from, digest is which artifact it must be. +func (s Store) Ensure(ctx context.Context, base, digest, source string) (string, error) { + if s.Has(digest) { + return s.Path(digest) + } + got, path, err := s.Fetch(ctx, base, source) + if err != nil { + return "", err + } + if got != digest { + _ = os.Remove(path) + return "", fmt.Errorf("%s: sha256 mismatch\n declared %s\n actual %s", source, digest, got) + } + return path, nil +} + +// Verify re-hashes a stored artifact, catching a cache that was corrupted or +// tampered with after it was written. +func (s Store) Verify(digest string) error { + p, err := s.Path(digest) + if err != nil { + return err + } + data, err := os.ReadFile(p) + if err != nil { + return err + } + sum := sha256.Sum256(data) + if got := hex.EncodeToString(sum[:]); got != digest { + return fmt.Errorf("%s: stored bytes no longer match\n expected %s\n actual %s", p, digest, got) + } + return nil +} + +// Add stores bytes whose digest the caller already computed. +func (s Store) Add(digest string, data []byte) (string, error) { + sum := sha256.Sum256(data) + if got := hex.EncodeToString(sum[:]); got != digest { + return "", fmt.Errorf("sha256 mismatch\n declared %s\n actual %s", digest, got) + } + return s.write(digest, data) +} + +func (s Store) read(ctx context.Context, base, source string) ([]byte, error) { + switch { + case strings.HasPrefix(source, "http://"): + // An artifact is code. Its integrity cannot rest on a transport that + // anyone on the path can rewrite, digest or not: a wrong digest is a + // failure you see, and a silently swapped download plus a swapped + // catalog is not. + return nil, fmt.Errorf("%s: refusing http; use https or a local path", source) + case strings.HasPrefix(source, "https://"): + client := s.HTTP + if client == nil { + client = http.DefaultClient + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, source, nil) + if err != nil { + return nil, err + } + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("%s: %w", source, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%s: %s", source, resp.Status) + } + return io.ReadAll(resp.Body) + default: + p := strings.TrimPrefix(source, "file://") + if !filepath.IsAbs(p) && base != "" { + p = filepath.Join(base, p) + } + return os.ReadFile(p) + } +} + +// write stores data atomically: a temp file in the same directory, then a +// rename. An interrupted fetch must not leave a partial file that a later Has +// would accept as the real artifact. +func (s Store) write(digest string, data []byte) (string, error) { + p, err := s.Path(digest) + if err != nil { + return "", err + } + dir := filepath.Dir(p) + if err := os.MkdirAll(dir, 0o755); err != nil { + return "", err + } + tmp, err := os.CreateTemp(dir, ".partial-*") + if err != nil { + return "", err + } + defer os.Remove(tmp.Name()) + if _, err := tmp.Write(data); err != nil { + tmp.Close() + return "", err + } + if err := tmp.Close(); err != nil { + return "", err + } + if err := os.Rename(tmp.Name(), p); err != nil { + return "", err + } + return p, nil +} + +func validDigest(s string) bool { + if len(s) != 64 { + return false + } + for _, r := range s { + switch { + case r >= '0' && r <= '9', r >= 'a' && r <= 'f': + default: + return false + } + } + return true +} diff --git a/internal/pluginstore/store_test.go b/internal/pluginstore/store_test.go new file mode 100644 index 0000000..372aa5d --- /dev/null +++ b/internal/pluginstore/store_test.go @@ -0,0 +1,165 @@ +package pluginstore_test + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/my-rv/godo/internal/pluginstore" +) + +func digestOf(b []byte) string { + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) +} + +func TestFetch_fromALocalFile(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "a.wasm") + data := []byte("artifact bytes") + if err := os.WriteFile(src, data, 0o644); err != nil { + t.Fatal(err) + } + s := pluginstore.Store{Dir: filepath.Join(dir, "store")} + + got, path, err := s.Fetch(context.Background(), "", src) + if err != nil { + t.Fatal(err) + } + if got != digestOf(data) { + t.Fatalf("digest=%s", got) + } + stored, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(stored) != string(data) { + t.Fatalf("stored %q", stored) + } + if !s.Has(got) { + t.Fatal("Has says no after a Fetch") + } +} + +func TestEnsure_downloadsOnceAndReusesAfter(t *testing.T) { + data := []byte("artifact over the wire") + var hits int + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits++ + w.Write(data) + })) + defer srv.Close() + + s := pluginstore.Store{Dir: t.TempDir(), HTTP: srv.Client()} + want := digestOf(data) + + for i := 0; i < 3; i++ { + if _, err := s.Ensure(context.Background(), "", want, srv.URL); err != nil { + t.Fatalf("attempt %d: %v", i, err) + } + } + if hits != 1 { + t.Fatalf("hit the server %d times, want 1", hits) + } +} + +// The source says where to look; the digest says what must come back. +func TestEnsure_refusesBytesThatDoNotMatch(t *testing.T) { + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("something else entirely")) + })) + defer srv.Close() + + s := pluginstore.Store{Dir: t.TempDir(), HTTP: srv.Client()} + want := digestOf([]byte("what the catalog declared")) + + _, err := s.Ensure(context.Background(), "", want, srv.URL) + if err == nil { + t.Fatal("stored an artifact the catalog did not declare") + } + if !strings.Contains(err.Error(), "sha256 mismatch") { + t.Fatalf("err=%v", err) + } + if s.Has(want) { + t.Fatal("the mismatched download was left in the store") + } +} + +// An artifact is code; its integrity cannot rest on a transport anyone can +// rewrite. +func TestFetch_refusesPlainHTTP(t *testing.T) { + s := pluginstore.Store{Dir: t.TempDir()} + _, _, err := s.Fetch(context.Background(), "", "http://example.test/a.wasm") + if err == nil || !strings.Contains(err.Error(), "refusing http") { + t.Fatalf("err=%v", err) + } +} + +// The digest becomes a filename, so it is validated rather than trusted. +func TestPath_rejectsAnythingThatIsNotADigest(t *testing.T) { + s := pluginstore.Store{Dir: t.TempDir()} + for _, bad := range []string{ + "", "short", strings.Repeat("g", 64), + "../" + strings.Repeat("a", 61), + strings.Repeat("a", 32) + "/" + strings.Repeat("b", 31), + strings.ToUpper(strings.Repeat("a", 64)), + strings.Repeat("a", 65), + } { + if _, err := s.Path(bad); err == nil { + t.Fatalf("accepted %q as a digest", bad) + } + } +} + +func TestVerify_catchesAStoreChangedAfterwards(t *testing.T) { + dir := t.TempDir() + s := pluginstore.Store{Dir: dir} + data := []byte("original") + d := digestOf(data) + path, err := s.Add(d, data) + if err != nil { + t.Fatal(err) + } + if err := s.Verify(d); err != nil { + t.Fatalf("fresh store does not verify: %v", err) + } + if err := os.WriteFile(path, []byte("tampered"), 0o644); err != nil { + t.Fatal(err) + } + if err := s.Verify(d); err == nil { + t.Fatal("Verify accepted bytes that changed") + } +} + +func TestAdd_refusesBytesThatDoNotHashToTheDigest(t *testing.T) { + s := pluginstore.Store{Dir: t.TempDir()} + if _, err := s.Add(digestOf([]byte("a")), []byte("b")); err == nil { + t.Fatal("stored bytes under someone else's digest") + } +} + +// An interrupted write must not leave something a later Has would accept. +func TestWrite_leavesNoPartialFileBehind(t *testing.T) { + dir := t.TempDir() + s := pluginstore.Store{Dir: dir} + data := []byte("complete") + d := digestOf(data) + if _, err := s.Add(d, data); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if strings.HasPrefix(e.Name(), ".partial") { + t.Fatalf("left a temp file: %s", e.Name()) + } + } +}