From 6c1272468da69f7152a74a5e890a705c75f8b6b9 Mon Sep 17 00:00:00 2001 From: barckcode Date: Mon, 26 Oct 2026 19:59:47 +0100 Subject: [PATCH 1/4] feat(context): nan context ls|show|create|push|pull|edit|rm (agent contexts C1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CLI side of agent contexts (helmcode/nan-devops AGENT-CONTEXT.md §6, C1), against the /v1/contexts API merged in cloud-api (#391, #395). - internal/contexts/path.go is a verbatim copy of cloud-api's validator, with its parity table (path_test.go) and a test of the shared pins. - push: walks DIR, NFC-normalises names (macOS gives NFD), skips dotfiles with a notice, prints every refused file with its reason and exits 65 before uploading unless --skip-invalid (CLAUDE.md gets a rename hint), checks the version limits, hashes, asks blobs/check, uploads only missing blobs (4 at a time; single PUT up to 16 MiB, multipart above, resumed within a push via the upload status and across pushes via a remembered upload id), publishes with base_version and an Idempotency-Key. Handles blob_missing (re-check, re-upload), version_conflict (exit 1, or --force shows the diff and rebases), context_secret_detected (--allow-secrets), per-file refusals and Retry-After (up to 2 minutes). --dry-run prints the add/change/remove list and the bytes to upload. - pull: refuses a non-empty DIR (--force overrides), validates every server path with the shared validator before writing anything, writes through os.OpenRoot, pins downloads to the version read and verifies sha256/size. - golang.org/x/text becomes a direct dependency at v0.39.0. CI gains a job (scripts/check-cloud-api-pins.sh, secret CLOUD_API_READ_TOKEN) comparing x/text, the Go toolchain's Unicode tables and path.go with cloud-api. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 26 + README.md | 25 + cmd/context.go | 1498 +++++++++++++++++ cmd/context_fake_test.go | 495 ++++++ cmd/context_test.go | 673 ++++++++ cmd/json_output_test.go | 17 + cmd/testdata/contexts/ls.golden | 3 + cmd/testdata/contexts/pull.golden | 1 + cmd/testdata/contexts/push.golden | 1 + cmd/testdata/contexts/push_dry_run.golden | 6 + .../contexts/push_dry_run.json.golden | 20 + .../contexts/push_refused.stderr.golden | 10 + cmd/testdata/contexts/show.golden | 12 + go.mod | 6 +- go.sum | 1 + internal/api/contexts.go | 455 +++++ internal/api/contexts_test.go | 161 ++ internal/contexts/path.go | 290 ++++ internal/contexts/path_test.go | 257 +++ internal/contexts/pins_test.go | 47 + scripts/check-cloud-api-pins.sh | 120 ++ 21 files changed, 4121 insertions(+), 3 deletions(-) create mode 100644 cmd/context.go create mode 100644 cmd/context_fake_test.go create mode 100644 cmd/context_test.go create mode 100644 cmd/testdata/contexts/ls.golden create mode 100644 cmd/testdata/contexts/pull.golden create mode 100644 cmd/testdata/contexts/push.golden create mode 100644 cmd/testdata/contexts/push_dry_run.golden create mode 100644 cmd/testdata/contexts/push_dry_run.json.golden create mode 100644 cmd/testdata/contexts/push_refused.stderr.golden create mode 100644 cmd/testdata/contexts/show.golden create mode 100644 internal/api/contexts.go create mode 100644 internal/api/contexts_test.go create mode 100644 internal/contexts/path.go create mode 100644 internal/contexts/path_test.go create mode 100644 internal/contexts/pins_test.go create mode 100755 scripts/check-cloud-api-pins.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46f40c4..f8cab40 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -193,6 +193,32 @@ jobs: } Write-Host "installer parses, web requests basic-parsed, no unguarded exit, arch survives without .NET, no preference leaks" + # `nan context` refuses or accepts a file name with a copy of cloud-api's + # rules, and those rules read Unicode tables from the Go toolchain and from + # golang.org/x/text. A dependency bump on either side would make pull refuse + # a name the server stored, quietly, for some names only. This compares the + # x/text version, the Go toolchain's tables and the copied path.go with + # cloud-api's (AGENT-CONTEXT.md §1.5). cloud-api is private: a pull request + # from a fork gets no secret, and is left to the unit test that checks the + # recorded pins (internal/contexts/pins_test.go). + cloud-api-pins: + name: Context path rules match cloud-api + runs-on: ubuntu-latest + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Compare golang.org/x/text, Go and path.go with cloud-api + env: + CLOUD_API_TOKEN: ${{ secrets.CLOUD_API_READ_TOKEN }} + run: scripts/check-cloud-api-pins.sh + # Four vulnerabilities in the standard library this repo compiled against # went unnoticed until somebody ran this by hand. Nothing here was looking, # so nothing was going to say so - not on the next release either. diff --git a/README.md b/README.md index 579f787..1810531 100644 --- a/README.md +++ b/README.md @@ -264,6 +264,31 @@ A platform token needs `notifications:read` to read and notifications on your account, these commands exit 64 and point at the portal. Docs: . +### Agent contexts + +Keep instructions (`AGENTS.md`), skills (`skills//SKILL.md`) and +reference files (`files/...`) on NaN as a versioned context for your runs and +automations: + +```bash +nan context ls [--json] +nan context show [--json] +nan context create NAME [--description TEXT] [--from DIR] [--json] +nan context push DIR [--create] [--dry-run] [--force] [--allow-secrets] [--skip-invalid] [--json] +nan context pull DIR [--force] +nan context edit [--name NEW] [--description TEXT] [--json] +nan context rm [--yes] +``` + +`push` checks every file name before uploading anything (dotfiles are +skipped; other refused names are listed with their reason and the push exits +65 unless `--skip-invalid`), uploads only what NaN does not already hold +(large files in resumable 16 MiB parts) and publishes a new version. Text +that looks like a credential is refused unless `--allow-secrets`. `pull` +writes into an empty folder only, after checking every path with the same +rules. Without contexts on your account, these commands exit 64 and point at +the portal. Docs: . + ### Exit codes `nan run`, `nan runs logs -f` and `nan automations run` exit with the run's outcome, so a script can diff --git a/cmd/context.go b/cmd/context.go new file mode 100644 index 0000000..d1b0309 --- /dev/null +++ b/cmd/context.go @@ -0,0 +1,1498 @@ +package cmd + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "regexp" + "runtime" + "sort" + "strings" + "sync" + "text/tabwriter" + "time" + + "github.com/nxssie/nan-cli/internal/api" + "github.com/nxssie/nan-cli/internal/contexts" + "github.com/nxssie/nan-cli/internal/runs" + "github.com/spf13/cobra" + "golang.org/x/text/unicode/norm" +) + +// Agent contexts (helmcode/nan-devops AGENT-CONTEXT.md §6): a versioned file +// tree a member keeps on NaN and hands to their runs and automations: +// +// AGENTS.md instructions +// skills//... skills (SKILL.md and its files) +// files/... reference files +// +// push hashes a local folder, uploads only the blobs the platform lacks +// (one PUT up to 16 MiB, a resumable multipart upload above) and publishes +// the tree as a new version based on the one it read. pull writes a version +// into an empty folder, through os.OpenRoot and only after every path passed +// the same validator the platform uses (internal/contexts, a verbatim copy). + +const ( + contextsDocs = "https://nan.builders/docs/contexts" + + scopeContextsRead = "contexts:read" + scopeContextsWrite = "contexts:write" + + // exitDataErr is sysexits' EX_DATAERR: the content was refused (a file + // name, a type, a limit, secret-like content). The same number as + // exitAuth, which is EX_NOPERM's neighbour in this CLI's table. + exitDataErr = 65 + + // Limits of one version (AGENT-CONTEXT.md §1.4), checked before any + // upload so a push that cannot be published uploads nothing. + ctxMaxFiles = 1000 + ctxMaxSkills = 30 + ctxMaxTotalBytes = 250 << 20 + ctxMaxAgentsMD = 32 << 10 + ctxMaxSkillMD = 64 << 10 + + // ctxMaxWait is the longest Retry-After a push waits out by itself. + ctxMaxWait = 2 * time.Minute + // ctxRetries is how many times one request is retried after a + // Retry-After, a part after a failure, or a publish after blob_missing + // or (with --force) version_conflict. + ctxRetries = 3 +) + +// Test seams. +var ( + // ctxSleep waits out a Retry-After. + ctxSleep = func(ctx context.Context, d time.Duration) error { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + return nil + } + } + // ctxUploadStateDir is where an unfinished multipart upload is + // remembered, so the next push resumes it instead of starting over. + ctxUploadStateDir = func() (string, error) { + d, err := os.UserCacheDir() + if err != nil { + return "", err + } + return filepath.Join(d, "nan", "context-uploads"), nil + } + // ctxConcurrency is how many blobs a push uploads at once. + ctxConcurrency = 4 + // ctxSingleMax is the largest blob sent as one PUT. + ctxSingleMax int64 = api.ContextPartSize +) + +var ( + ctxJSON bool + + ctxCreateDescription string + ctxCreateFrom string + + ctxPushCreate bool + ctxPushForce bool + ctxPushAllowSecrets bool + ctxPushDryRun bool + ctxPushSkipInvalid bool + + ctxPullForce bool + + ctxEditName string + ctxEditDescription string + + ctxRmYes bool +) + +var ctxVerbHints = map[string]string{ + "list": "ls", "get": "show", "new": "create", "add": "create", "upload": "push", "publish": "push", + "download": "pull", "clone": "pull", "update": "edit", "rename": "edit NAME --name NEW", "delete": "rm", "remove": "rm", +} + +const ctxCredentialHelp = `Authentication, first match wins: --token-file, your session from +nan auth login, NAN_TOKEN, a platform token saved with nan auth login +--api-token. A platform token needs contexts:read to list, show and pull, +and contexts:write to create, push, edit and delete. Workspace keys never +read or write contexts.` + +var contextCmd = &cobra.Command{ + Use: "context", + Aliases: []string{"contexts"}, + Short: "Keep instructions, skills and files for your agent runs", + Long: `Manage your agent contexts: versioned folders of instructions, skills and +reference files that your runs and automations hand to the agent. + +A context has this layout: + AGENTS.md instructions for the agent (at most 32 KiB) + skills// a skill: SKILL.md (name and description in its + frontmatter) plus any files it uses + files/... reference files: docs, specs, CSV, PDF, images, + Office documents + +File names may use letters of any language, digits, spaces and +. _ - ( ) + , @. Dotfiles (.env, .git) are skipped; names with emoji, +symbols, control characters or Windows reserved names (CON, NUL, ...) +are refused, each with its reason. + +Every push publishes a new version; runs pin the version they start +with, automations follow the latest. Do not put credentials in a context: +keep them in nan secrets. A push that looks like it carries one is +refused unless you add --allow-secrets. + +Exit codes: 0 done, 1 failed (the context changed while pushing), 64 +usage, 65 content refused (file names, types, limits, secret-like +content, storage quota) or credential refused, 69 nan.builders +unavailable. + +` + ctxCredentialHelp + ` + +Docs: ` + contextsDocs, + Annotations: sysexits, + Args: cobra.ArbitraryArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) > 0 { + shown := runs.Truncate(runs.SanitizeLine(args[0]), 64) + if hint, ok := ctxVerbHints[args[0]]; ok { + return usageErrorf("there is no nan context %s; try: nan context %s", shown, hint) + } + return usageErrorf("unknown command %q for \"nan context\"; try: nan context --help", shown) + } + return cmd.Help() + }, +} + +var ctxLsCmd = &cobra.Command{ + Use: "ls", + Short: "List your contexts and the storage they use", + Args: cobra.NoArgs, + Annotations: sysexits, + RunE: func(cmd *cobra.Command, args []string) error { + return withWriteEnv(func(env *runsEnv) error { return doContextLs(cmd.Context(), env) }) + }, +} + +var ctxShowCmd = &cobra.Command{ + Use: "show ", + Short: "Show a context and the files of its current version", + Args: cobra.ExactArgs(1), + Annotations: sysexits, + RunE: func(cmd *cobra.Command, args []string) error { + return withWriteEnv(func(env *runsEnv) error { return doContextShow(cmd.Context(), env, args[0]) }) + }, +} + +var ctxCreateCmd = &cobra.Command{ + Use: "create NAME [--description TEXT] [--from DIR]", + Short: "Create a context, empty or from a folder", + Long: `Create a context. NAME is lowercase letters, digits and dashes (at most +63). With --from DIR, the folder is pushed as its first content (see nan +context push --help for what a push checks). + +` + ctxCredentialHelp + ` + +Docs: ` + contextsDocs, + Example: ` nan context create acme-api --description "Acme API conventions" + nan context create acme-api --from ./acme-context`, + Args: cobra.ExactArgs(1), + Annotations: sysexits, + RunE: func(cmd *cobra.Command, args []string) error { + return withWriteEnv(func(env *runsEnv) error { return doContextCreate(cmd.Context(), env, args[0]) }) + }, +} + +var ctxPushCmd = &cobra.Command{ + Use: "push DIR", + Short: "Publish a folder as the context's next version", + Long: `Publish DIR as the next version of a context: the folder becomes the +context's whole content (files no longer in DIR are removed from the new +version; older versions are kept while a run uses them). + +Before anything is uploaded, push checks the layout and every file name: +dotfiles and dot-folders (.git, .env) are skipped with a notice; every +other refused file is printed with its reason (bad_top_level, +bad_character, case_collision, reserved_name, too_long, too_deep, +too_large, skill_invalid, symlink) and push exits 65 without uploading, +unless --skip-invalid leaves those files out. A CLAUDE.md is refused with +a hint to rename it to AGENTS.md. Names are normalised to NFC (macOS +hands out decomposed names). + +Then push hashes every file, asks the platform which ones it does not +already hold, uploads only those (16 MiB in one request, larger files in +16 MiB parts; an interrupted upload resumes on the next push) and +publishes the new version. --dry-run prints what would be added, changed +and removed, and how much would be uploaded, and changes nothing. + +If the context changed since push read it (another push, the portal), +push stops with exit 1. --force shows what changed and publishes on top +of it instead. Text that looks like a credential (API keys, tokens, +private keys) is refused with the file and line; keep credentials in nan +secrets, or publish anyway with --allow-secrets (the agent can read and +copy anything in a context). + +With --create, a context that does not exist yet is created first. + +` + ctxCredentialHelp + ` + +Docs: ` + contextsDocs, + Example: ` nan context push acme-api ./acme-context --dry-run + nan context push acme-api ./acme-context + nan context push acme-api ./acme-context --create --skip-invalid`, + Args: cobra.ExactArgs(2), + Annotations: sysexits, + RunE: func(cmd *cobra.Command, args []string) error { + return withWriteEnv(func(env *runsEnv) error { + return doContextPush(cmd.Context(), env, args[0], args[1], pushOptionsFromFlags()) + }) + }, +} + +var ctxPullCmd = &cobra.Command{ + Use: "pull DIR", + Short: "Download a context's current version into an empty folder", + Long: `Write the current version of a context into DIR, which must be empty or +not exist yet (--force writes into a folder that has files, replacing +the ones the context has). + +Every path the platform sends is checked with the same rules a push +uses, and files are written through a handle rooted at DIR: a path +that is absolute, climbs out with .., differs from another only by case, +or is a Windows reserved name is refused before anything is written. +Every file is checked against its sha256 and size. + +` + ctxCredentialHelp + ` + +Docs: ` + contextsDocs, + Example: ` nan context pull acme-api ./acme-context`, + Args: cobra.ExactArgs(2), + Annotations: sysexits, + RunE: func(cmd *cobra.Command, args []string) error { + return withWriteEnv(func(env *runsEnv) error { return doContextPull(cmd.Context(), env, args[0], args[1]) }) + }, +} + +var ctxEditCmd = &cobra.Command{ + Use: "edit [--name NEW] [--description TEXT]", + Short: "Rename a context or change its description (no new version)", + Args: cobra.ExactArgs(1), + Annotations: sysexits, + RunE: func(cmd *cobra.Command, args []string) error { + if !cmd.Flags().Changed("name") && !cmd.Flags().Changed("description") { + return usageErrorf("nothing to change: give --name or --description") + } + var patch api.ContextPatch + if cmd.Flags().Changed("name") { + if !contexts.ValidContextName(ctxEditName) { + return usageErrorf("%s", contextNameRule(ctxEditName)) + } + patch.Name = &ctxEditName + } + if cmd.Flags().Changed("description") { + patch.Description = &ctxEditDescription + } + return withWriteEnv(func(env *runsEnv) error { return doContextEdit(cmd.Context(), env, args[0], patch) }) + }, +} + +var ctxRmCmd = &cobra.Command{ + Use: "rm [--yes]", + Short: "Delete a context and all its versions", + Args: cobra.ExactArgs(1), + Annotations: sysexits, + RunE: func(cmd *cobra.Command, args []string) error { + return withWriteEnv(func(env *runsEnv) error { return doContextRm(cmd.Context(), env, args[0]) }) + }, +} + +func init() { + rootCmd.AddCommand(contextCmd) + contextCmd.AddCommand(ctxLsCmd, ctxShowCmd, ctxCreateCmd, ctxPushCmd, ctxPullCmd, ctxEditCmd, ctxRmCmd) + contextCmd.PersistentFlags().StringVar(&tokenFileFlag, "token-file", "", tokenFileUsage) + + for _, c := range []*cobra.Command{ctxLsCmd, ctxShowCmd, ctxCreateCmd, ctxPushCmd, ctxEditCmd} { + c.Flags().BoolVar(&ctxJSON, "json", false, "Print the answer as JSON") + } + ctxCreateCmd.Flags().StringVar(&ctxCreateDescription, "description", "", "What the context is for") + ctxCreateCmd.Flags().StringVar(&ctxCreateFrom, "from", "", "Push this folder as the first content (`DIR`)") + for _, c := range []*cobra.Command{ctxCreateCmd, ctxPushCmd} { + c.Flags().BoolVar(&ctxPushAllowSecrets, "allow-secrets", false, "Publish even if a text file looks like it holds a credential") + c.Flags().BoolVar(&ctxPushSkipInvalid, "skip-invalid", false, "Leave refused files out instead of stopping") + } + f := ctxPushCmd.Flags() + f.BoolVar(&ctxPushCreate, "create", false, "Create the context if it does not exist") + f.BoolVar(&ctxPushForce, "force", false, "Publish on top of a version someone else published meanwhile") + f.BoolVar(&ctxPushDryRun, "dry-run", false, "Print what would change and be uploaded; change nothing") + ctxPullCmd.Flags().BoolVar(&ctxPullForce, "force", false, "Write into a folder that is not empty") + ctxEditCmd.Flags().StringVar(&ctxEditName, "name", "", "The new name") + ctxEditCmd.Flags().StringVar(&ctxEditDescription, "description", "", "The new description") + ctxRmCmd.Flags().BoolVar(&ctxRmYes, "yes", false, "Do not ask for confirmation") +} + +type pushOptions struct { + create, force, allowSecrets, dryRun, skipInvalid, json bool +} + +func pushOptionsFromFlags() pushOptions { + return pushOptions{create: ctxPushCreate, force: ctxPushForce, allowSecrets: ctxPushAllowSecrets, dryRun: ctxPushDryRun, + skipInvalid: ctxPushSkipInvalid, json: ctxJSON} +} + +func contextNameRule(name string) string { + return fmt.Sprintf("%q is not a valid context name: use lowercase letters, digits and dashes, starting with a letter or digit, at most 63 characters (not a Windows reserved name like con or nul)", + runs.Truncate(runs.SanitizeLine(name), 80)) +} + +// ─── errors ──────────────────────────────────────────────────────────────── + +// ctxExit maps a refused contexts request onto an exit code and a message. +func ctxExit(err error, env *runsEnv, scope string) *ExitError { + var exit *ExitError + if errors.As(err, &exit) { + return exit + } + ce, ok := api.AsContextError(err) + if !ok { + return writeExit(err, env.cred, "contexts", scope) + } + msg := runs.SanitizeLine(ce.Message) + switch ce.Code { + case "not_found", "": + if ce.Status == 404 { + return apiOff("using agent contexts") + } + case "context_not_found": + return usageErrorf("no such context (see: nan context ls)") + case "context_name_taken": + return usageErrorf("you already have a context with that name") + case "too_many_contexts": + return exitf(exitDataErr, "%s", msg) + case "context_in_use": + return usageErrorf("automations use this context; remove it from them first (nan automations edit):\n%s", + joinNames(refLabels(ce.Automations))) + case "context_quota_exceeded": + return exitf(exitDataErr, "%s (%s of %s used). Delete contexts or versions you no longer need, or upgrade: %s/membership", + msg, humanBytes(ce.Used), humanBytes(ce.Quota), runs.PortalURL) + case "context_too_large", "unsupported_file_type", "invalid_path", "skill_invalid", "context_secret_detected": + return exitf(exitDataErr, "%s", msg) + case "storage_unavailable", "upload_in_progress", "blob_busy", "part_conflict", "upload_expired": + return exitf(exitUnavailable, "%s", msg) + case "rate_limited": + if ce.RetryAfter > 0 { + return exitf(exitUnavailable, "%s (try again in %s)", msg, ce.RetryAfter.Round(time.Second)) + } + return exitf(exitUnavailable, "%s", msg) + case "blob_hash_mismatch": + return exitf(exitUnavailable, "a file changed while it was being uploaded; push again (%s)", msg) + case "entitlement_required", "tier_restricted": + return exitf(exitAuth, "your membership does not include agent runs, which contexts are for: see %s/membership", runs.PortalURL) + } + return writeExit(ce.APIError, env.cred, "contexts", scope) +} + +// ctxRetry runs fn again after a Retry-After the platform asked for (rate +// limits, an upload of the same blob in progress, storage briefly away), at +// most ctxRetries times and never longer than ctxMaxWait each. +func ctxRetry(ctx context.Context, env *runsEnv, what string, fn func() error) error { + for attempt := 0; ; attempt++ { + err := fn() + ce, ok := api.AsContextError(err) + if !ok || attempt >= ctxRetries || !retryable(ce) { + return err + } + wait := ce.RetryAfter + if wait <= 0 { + wait = 5 * time.Second + } + if wait > ctxMaxWait { + return err + } + fmt.Fprintf(env.stderr, "%s: %s; retrying in %s\n", what, runs.SanitizeLine(ce.Message), wait.Round(time.Second)) + if serr := ctxSleep(ctx, wait); serr != nil { + return err + } + } +} + +func retryable(ce *api.ContextError) bool { + switch ce.Code { + case "rate_limited", "upload_in_progress", "blob_busy", "storage_unavailable": + return true + } + return ce.Status == 429 +} + +func ctxCode(err error) string { + if ce, ok := api.AsContextError(err); ok { + return ce.Code + } + return "" +} + +// ─── ls, show ────────────────────────────────────────────────────────────── + +func doContextLs(ctx context.Context, env *runsEnv) error { + list, err := env.client.ListContexts(ctx) + if err != nil { + return ctxExit(err, env, scopeContextsRead) + } + if ctxJSON { + return printJSON(env.stdout, list, true) + } + if len(list.Data) == 0 { + fmt.Fprintf(env.stderr, "no contexts yet; create one with: nan context create NAME --from DIR\n") + } else { + tw := tabwriter.NewWriter(env.stdout, 0, 4, 2, ' ', 0) + fmt.Fprintln(tw, "NAME\tVERSION\tFILES\tSIZE\tUPDATED\tDESCRIPTION") + for _, c := range list.Data { + fmt.Fprintf(tw, "%s\t%d\t%d\t%s\t%s\t%s\n", runs.SanitizeLine(c.Name), c.Version, c.FileCount, humanBytes(c.TotalBytes), + localTime(c.UpdatedAt), runs.Truncate(runs.SanitizeLine(c.Description), 48)) + } + _ = tw.Flush() + } + fmt.Fprintf(env.stdout, "storage: %s of %s used\n", humanBytes(list.Usage.StoredBytes), humanBytes(list.Usage.QuotaBytes)) + return nil +} + +func doContextShow(ctx context.Context, env *runsEnv, ref string) error { + d, err := env.client.GetContext(ctx, ref) + if err != nil { + return ctxExit(err, env, scopeContextsRead) + } + if ctxJSON { + return printJSON(env.stdout, d, true) + } + tw := tabwriter.NewWriter(env.stdout, 0, 4, 2, ' ', 0) + row := func(k, v string) { fmt.Fprintf(tw, "%s\t%s\n", k, v) } + row("name", runs.SanitizeLine(d.Name)) + row("id", runs.SanitizeLine(d.ID)) + if d.Description != "" { + row("description", runs.SanitizeLine(d.Description)) + } + row("version", fmt.Sprintf("%d (%s)", d.Version, localTime(d.UpdatedAt))) + row("files", fmt.Sprintf("%d, %s", d.FileCount, humanBytes(d.TotalBytes))) + row("instructions", yesNo(d.HasInstructions)) + skills := make([]string, len(d.Skills)) + for i, s := range d.Skills { + skills[i] = runs.SanitizeLine(s) + } + if len(skills) == 0 { + row("skills", "none") + } else { + row("skills", strings.Join(skills, ", ")) + } + _ = tw.Flush() + if len(d.Files) > 0 { + fmt.Fprintln(env.stdout) + tw = tabwriter.NewWriter(env.stdout, 0, 4, 2, ' ', 0) + fmt.Fprintln(tw, "PATH\tSIZE\tSHA256") + files := append([]api.ContextFile(nil), d.Files...) + sort.Slice(files, func(i, j int) bool { return files[i].Path < files[j].Path }) + for _, f := range files { + p := runs.SanitizeLine(f.Path) + if f.Executable { + p += " (executable)" + } + fmt.Fprintf(tw, "%s\t%s\t%s\n", p, humanBytes(f.Size), runs.Truncate(runs.SanitizeLine(f.SHA256), 12)) + } + _ = tw.Flush() + } + return nil +} + +func yesNo(b bool) string { + if b { + return "yes" + } + return "no" +} + +// humanBytes is a size as people read it (1024-based, one decimal). +func humanBytes(n int64) string { + const unit = 1024 + if n < unit { + return fmt.Sprintf("%d B", n) + } + div, exp := int64(unit), 0 + for m := n / unit; m >= unit && exp < 3; m /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGT"[exp]) +} + +// ─── create, edit, rm ────────────────────────────────────────────────────── + +func doContextCreate(ctx context.Context, env *runsEnv, name string) error { + if !contexts.ValidContextName(name) { + return usageErrorf("%s", contextNameRule(name)) + } + opts := pushOptions{allowSecrets: ctxPushAllowSecrets, skipInvalid: ctxPushSkipInvalid, json: ctxJSON} + var local *localTree + if ctxCreateFrom != "" { + // The folder is checked before the context exists, so a folder that + // cannot be pushed does not leave an empty context behind. + var err error + if local, err = scanLocal(env, ctxCreateFrom, opts.skipInvalid); err != nil { + return err + } + } + idem := newIdempotencyKey() + var c *api.Context + err := ctxRetry(ctx, env, "create", func() error { + var err error + c, err = env.client.CreateContext(ctx, name, ctxCreateDescription, idem) + return err + }) + if err != nil { + return ctxExit(err, env, scopeContextsWrite) + } + if local == nil { + if ctxJSON { + return printJSON(env.stdout, c, true) + } + fmt.Fprintf(env.stdout, "context %s created (empty, version %d)\n", runs.SanitizeLine(c.Name), c.Version) + fmt.Fprintf(env.stderr, "add content with: nan context push %s DIR\n", runs.SanitizeLine(c.Name)) + return nil + } + fmt.Fprintf(env.stderr, "context %s created\n", runs.SanitizeLine(c.Name)) + return pushTree(ctx, env, &api.ContextDetail{Context: *c, Files: []api.ContextFile{}}, local, opts) +} + +func doContextEdit(ctx context.Context, env *runsEnv, ref string, patch api.ContextPatch) error { + d, err := env.client.GetContext(ctx, ref) + if err != nil { + return ctxExit(err, env, scopeContextsRead) + } + c, err := env.client.UpdateContext(ctx, d.ID, patch) + if err != nil { + return ctxExit(err, env, scopeContextsWrite) + } + if ctxJSON { + return printJSON(env.stdout, c, true) + } + fmt.Fprintf(env.stdout, "context %s updated\n", runs.SanitizeLine(c.Name)) + return nil +} + +func doContextRm(ctx context.Context, env *runsEnv, ref string) error { + d, err := env.client.GetContext(ctx, ref) + if err != nil { + return ctxExit(err, env, scopeContextsRead) + } + name := runs.SanitizeLine(d.Name) + if err := confirm(env, ctxRmYes, "delete context "+name+" and all its versions?"); err != nil { + return err + } + if err := env.client.DeleteContext(ctx, d.ID); err != nil { + return ctxExit(err, env, scopeContextsWrite) + } + fmt.Fprintf(env.stdout, "context %s deleted\n", name) + return nil +} + +// ─── the local tree ──────────────────────────────────────────────────────── + +// localFile is one file of the folder being pushed. +type localFile struct { + path string // NFC, '/'-separated, relative to the folder + abs string + size int64 + exec bool + sha string +} + +// refusedFile is one file push will not publish, with its reason. +type refusedFile struct { + path, reason, detail string +} + +type localTree struct { + files []localFile +} + +func (t *localTree) totalBytes() int64 { + var n int64 + for _, f := range t.files { + n += f.size + } + return n +} + +// scanLocal walks dir, checks every name and the layout, applies the +// per-file limits and hashes what is left. Refused files stop the push +// (exit 65) unless skipInvalid; dotfiles are always skipped with a notice. +func scanLocal(env *runsEnv, dir string, skipInvalid bool) (*localTree, error) { + shown := runs.SanitizeLine(dir) + st, err := os.Stat(dir) + if err != nil { + return nil, usageErrorf("cannot read %s: %v", shown, unwrapPath(err)) + } + if !st.IsDir() { + return nil, usageErrorf("%s is not a folder", shown) + } + var ( + files []localFile + refused []refusedFile + dotted []string + ) + err = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if p == dir { + return nil + } + rel, rerr := filepath.Rel(dir, p) + if rerr != nil { + return rerr + } + slash := filepath.ToSlash(rel) + name := norm.NFC.String(slash) + if strings.HasPrefix(d.Name(), ".") { + dotted = append(dotted, name) + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + switch { + case d.Type()&fs.ModeSymlink != 0: + refused = append(refused, refusedFile{name, "symlink", "links are not kept; copy the file instead"}) + return nil + case d.IsDir(): + return nil + case !d.Type().IsRegular(): + refused = append(refused, refusedFile{name, "not_a_file", ""}) + return nil + } + info, ierr := d.Info() + if ierr != nil { + return ierr + } + files = append(files, localFile{path: name, abs: p, size: info.Size(), + exec: runtime.GOOS != "windows" && info.Mode().Perm()&0o111 != 0}) + return nil + }) + if err != nil { + return nil, usageErrorf("cannot read %s: %v", shown, unwrapPath(err)) + } + for _, p := range dotted { + fmt.Fprintf(env.stderr, "skipped %s (dotfiles are not kept in a context)\n", runs.SanitizeLine(p)) + } + + refused = append(refused, checkTree(files)...) + bad := map[string]bool{} + for _, r := range refused { + bad[r.path] = true + } + // A skill folder that lost its SKILL.md loses the whole skill. + for _, r := range refused { + if r.reason == "skill_invalid" && strings.HasSuffix(r.path, "/"+contexts.SkillFileName) { + prefix := strings.TrimSuffix(r.path, contexts.SkillFileName) + for _, f := range files { + if strings.HasPrefix(f.path, prefix) { + bad[f.path] = true + } + } + } + } + if len(refused) > 0 { + sort.SliceStable(refused, func(i, j int) bool { return refused[i].path < refused[j].path }) + verb := "refused" + if skipInvalid { + verb = "skipped" + } + for _, r := range refused { + line := fmt.Sprintf("%s %s: %s", verb, runs.SanitizeLine(r.path), r.reason) + if r.detail != "" { + line += " (" + r.detail + ")" + } + fmt.Fprintln(env.stderr, line) + } + if !skipInvalid { + return nil, exitf(exitDataErr, "%d %s cannot be published; nothing was uploaded. Rename or remove %s, or leave %s out with --skip-invalid", + len(refused), pluralWord(len(refused), "file", "files"), pluralWord(len(refused), "it", "them"), pluralWord(len(refused), "it", "them")) + } + } + kept := files[:0] + for _, f := range files { + if !bad[f.path] { + kept = append(kept, f) + } + } + tree := &localTree{files: kept} + if err := checkTreeLimits(tree); err != nil { + return nil, err + } + for i := range tree.files { + f := &tree.files[i] + sum, size, err := hashFile(f.abs) + if err != nil { + return nil, exitf(exitFailed, "cannot read %s: %v", runs.SanitizeLine(f.path), unwrapPath(err)) + } + f.sha, f.size = sum, size + } + sort.Slice(tree.files, func(i, j int) bool { return tree.files[i].path < tree.files[j].path }) + return tree, nil +} + +func pluralWord(n int, one, many string) string { + if n == 1 { + return one + } + return many +} + +// checkTree is the platform's path validator over the whole tree (names, +// layout, case collisions), plus the per-file limits a publish enforces. +func checkTree(files []localFile) []refusedFile { + paths := make([]string, len(files)) + for i, f := range files { + paths[i] = f.path + } + var out []refusedFile + for _, pe := range contexts.ValidateTree(paths) { + r := refusedFile{path: pe.Path, reason: string(pe.Reason)} + top := strings.SplitN(pe.Path, "/", 2)[0] + switch { + case pe.Path == "CLAUDE.md": + r.detail = "rename it to AGENTS.md: that is the file the agents read" + case pe.Reason == contexts.ReasonBadTopLevel && !strings.Contains(pe.Path, "/"): + r.detail = "only AGENTS.md lives at the top; put reference files under files/" + case pe.Reason == contexts.ReasonBadTopLevel && top != "skills" && top != "files": + r.detail = "put reference files under files/ and skills under skills//" + case pe.Reason == contexts.ReasonBadCharacter && top == "skills" && strings.Count(pe.Path, "/") >= 2: + if !contexts.ValidSkillName(strings.SplitN(pe.Path, "/", 3)[1]) { + r.detail = "a skill folder name is lowercase letters, digits and dashes" + } + } + out = append(out, r) + } + skillDirs, skillFiles := map[string]bool{}, map[string]bool{} + for _, f := range files { + switch { + case f.size > api.ContextMaxFileBytes: + out = append(out, refusedFile{f.path, "too_large", "a file is at most 100 MiB"}) + case f.path == contexts.InstructionsPath && f.size > ctxMaxAgentsMD: + out = append(out, refusedFile{f.path, "too_large", "AGENTS.md is at most 32 KiB"}) + } + parts := strings.SplitN(f.path, "/", 3) + if len(parts) == 3 && parts[0] == "skills" && contexts.ValidSkillName(parts[1]) { + skillDirs[parts[1]] = true + if parts[2] == contexts.SkillFileName { + skillFiles[parts[1]] = true + if f.size > ctxMaxSkillMD { + out = append(out, refusedFile{f.path, "skill_invalid", "SKILL.md is at most 64 KiB"}) + } + } + } + } + for dir := range skillDirs { + if !skillFiles[dir] { + out = append(out, refusedFile{"skills/" + dir + "/" + contexts.SkillFileName, "skill_invalid", "every skill folder needs a SKILL.md"}) + } + } + return out +} + +func checkTreeLimits(t *localTree) error { + if len(t.files) > ctxMaxFiles { + return exitf(exitDataErr, "%d files: a context version holds at most %d", len(t.files), ctxMaxFiles) + } + if total := t.totalBytes(); total > ctxMaxTotalBytes { + return exitf(exitDataErr, "%s in total: a context version holds at most 250 MiB", humanBytes(total)) + } + skills := map[string]bool{} + for _, f := range t.files { + if parts := strings.SplitN(f.path, "/", 3); len(parts) == 3 && parts[0] == "skills" { + skills[parts[1]] = true + } + } + if len(skills) > ctxMaxSkills { + return exitf(exitDataErr, "%d skills: a context holds at most %d", len(skills), ctxMaxSkills) + } + return nil +} + +func hashFile(p string) (string, int64, error) { + f, err := os.Open(p) + if err != nil { + return "", 0, err + } + defer f.Close() + h := sha256.New() + n, err := io.Copy(h, f) + if err != nil { + return "", 0, err + } + return hex.EncodeToString(h.Sum(nil)), n, nil +} + +// ─── push ────────────────────────────────────────────────────────────────── + +func doContextPush(ctx context.Context, env *runsEnv, ref, dir string, opts pushOptions) error { + local, err := scanLocal(env, dir, opts.skipInvalid) + if err != nil { + return err + } + d, err := env.client.GetContext(ctx, ref) + if ctxCode(err) == "context_not_found" && opts.create { + if !contexts.ValidContextName(ref) { + return usageErrorf("%s", contextNameRule(ref)) + } + if opts.dryRun { + d = &api.ContextDetail{Context: api.Context{Name: ref, Version: 0}, Files: []api.ContextFile{}} + err = nil + } else { + var c *api.Context + idem := newIdempotencyKey() + err = ctxRetry(ctx, env, "create", func() error { + var cerr error + c, cerr = env.client.CreateContext(ctx, ref, "", idem) + return cerr + }) + if err == nil { + fmt.Fprintf(env.stderr, "context %s created\n", runs.SanitizeLine(c.Name)) + d = &api.ContextDetail{Context: *c, Files: []api.ContextFile{}} + } + } + } + if err != nil { + if ctxCode(err) == "context_not_found" { + return usageErrorf("no context %s: create it with nan context create %s, or add --create", + runs.SanitizeLine(ref), runs.SanitizeLine(ref)) + } + return ctxExit(err, env, scopeContextsRead) + } + return pushTree(ctx, env, d, local, opts) +} + +// treeDiff is what a push changes against the version it is based on. +type treeDiff struct { + Added []string `json:"added"` + Changed []string `json:"changed"` + Removed []string `json:"removed"` +} + +func (d treeDiff) empty() bool { return len(d.Added)+len(d.Changed)+len(d.Removed) == 0 } + +func diffTree(remote []api.ContextFile, local *localTree) treeDiff { + d := treeDiff{Added: []string{}, Changed: []string{}, Removed: []string{}} + have := make(map[string]api.ContextFile, len(remote)) + for _, f := range remote { + have[f.Path] = f + } + seen := map[string]bool{} + for _, f := range local.files { + seen[f.path] = true + r, ok := have[f.path] + switch { + case !ok: + d.Added = append(d.Added, f.path) + case r.SHA256 != f.sha || r.Executable != f.exec: + d.Changed = append(d.Changed, f.path) + } + } + for _, f := range remote { + if !seen[f.Path] { + d.Removed = append(d.Removed, f.Path) + } + } + sort.Strings(d.Added) + sort.Strings(d.Changed) + sort.Strings(d.Removed) + return d +} + +func printDiff(w io.Writer, d treeDiff, sizes map[string]int64) { + for _, p := range d.Added { + fmt.Fprintf(w, " + %s (%s)\n", runs.SanitizeLine(p), humanBytes(sizes[p])) + } + for _, p := range d.Changed { + fmt.Fprintf(w, " ~ %s (%s)\n", runs.SanitizeLine(p), humanBytes(sizes[p])) + } + for _, p := range d.Removed { + fmt.Fprintf(w, " - %s\n", runs.SanitizeLine(p)) + } +} + +// uploadPlan is the blobs a push sends: one local file per missing sha. +type uploadPlan struct { + files []localFile + bytes int64 +} + +func planUploads(local *localTree, missing []string) uploadPlan { + want := map[string]bool{} + for _, s := range missing { + want[s] = true + } + var p uploadPlan + for _, f := range local.files { + if want[f.sha] { + p.files = append(p.files, f) + p.bytes += f.size + delete(want, f.sha) + } + } + return p +} + +func distinctShas(local *localTree) []string { + seen := map[string]bool{} + var out []string + for _, f := range local.files { + if !seen[f.sha] { + seen[f.sha] = true + out = append(out, f.sha) + } + } + sort.Strings(out) + return out +} + +type dryRunReport struct { + Context string `json:"context"` + BaseVersion int `json:"base_version"` + Diff treeDiff `json:"changes"` + Upload struct { + Files int `json:"files"` + Bytes int64 `json:"bytes"` + } `json:"upload"` +} + +func pushTree(ctx context.Context, env *runsEnv, d *api.ContextDetail, local *localTree, opts pushOptions) error { + name := runs.SanitizeLine(d.Name) + sizes := map[string]int64{} + for _, f := range local.files { + sizes[f.path] = f.size + } + diff := diffTree(d.Files, local) + if diff.empty() && d.Version > 0 { + if opts.json { + return printJSON(env.stdout, d.Context, true) + } + fmt.Fprintf(env.stdout, "context %s is up to date (version %d)\n", name, d.Version) + return nil + } + var missing []string + if len(local.files) > 0 { + err := ctxRetry(ctx, env, "check", func() error { + var err error + missing, err = env.client.CheckBlobs(ctx, distinctShas(local)) + return err + }) + if err != nil { + return ctxExit(err, env, scopeContextsWrite) + } + } + plan := planUploads(local, missing) + if opts.dryRun { + if opts.json { + r := dryRunReport{Context: d.Name, BaseVersion: d.Version, Diff: diff} + r.Upload.Files, r.Upload.Bytes = len(plan.files), plan.bytes + return printJSON(env.stdout, r, true) + } + fmt.Fprintf(env.stdout, "context %s: version %d -> %d (dry run, nothing changed)\n", name, d.Version, d.Version+1) + printDiff(env.stdout, diff, sizes) + fmt.Fprintf(env.stdout, "%d added, %d changed, %d removed; %d %s to upload (%s)\n", len(diff.Added), len(diff.Changed), + len(diff.Removed), len(plan.files), pluralWord(len(plan.files), "file", "files"), humanBytes(plan.bytes)) + return nil + } + if err := uploadBlobs(ctx, env, plan); err != nil { + return err + } + + req := api.PublishRequest{BaseVersion: d.Version, AllowSecretLike: opts.allowSecrets} + for _, f := range local.files { + req.Files = append(req.Files, api.PublishFile{Path: f.path, SHA256: f.sha, Executable: f.exec}) + } + idem := newIdempotencyKey() + missingRounds, rebases := 0, 0 + for { + var c *api.Context + err := ctxRetry(ctx, env, "publish", func() error { + var err error + c, err = env.client.PublishContext(ctx, d.ID, req, idem) + return err + }) + if err == nil { + return reportPushed(env, c, diff, plan, opts) + } + ce, ok := api.AsContextError(err) + if !ok { + return ctxExit(err, env, scopeContextsWrite) + } + switch ce.Code { + case "blob_missing": + // The platform's cleanup removed a blob between the check and + // the publish: check again and upload what it lacks. + if missingRounds >= ctxRetries { + return ctxExit(err, env, scopeContextsWrite) + } + missingRounds++ + fmt.Fprintf(env.stderr, "%d %s no longer stored on nan.builders; uploading again\n", len(ce.MissingBlobs), + pluralWord(len(ce.MissingBlobs), "file is", "files are")) + again, cerr := env.client.CheckBlobs(ctx, ce.MissingBlobs) + if cerr != nil { + return ctxExit(cerr, env, scopeContextsWrite) + } + if err := uploadBlobs(ctx, env, planUploads(local, again)); err != nil { + return err + } + continue + case "version_conflict": + current := "a newer version" + if ce.CurrentVersion != nil { + current = fmt.Sprintf("version %d", *ce.CurrentVersion) + } + if !opts.force { + return exitf(exitFailed, "context %s changed while pushing (it is now at %s; this push was based on version %d). "+ + "Nothing was published. Push again to compare with it, or add --force to publish on top of it", name, current, req.BaseVersion) + } + if rebases >= ctxRetries { + return exitf(exitFailed, "context %s keeps changing; nothing was published, try again", name) + } + rebases++ + latest, gerr := env.client.GetContext(ctx, d.ID) + if gerr != nil { + return ctxExit(gerr, env, scopeContextsRead) + } + fmt.Fprintf(env.stderr, "context %s changed to version %d meanwhile; publishing on top of it (--force). Against it, this push:\n", + name, latest.Version) + diff = diffTree(latest.Files, local) + printDiff(env.stderr, diff, sizes) + req.BaseVersion = latest.Version + idem = newIdempotencyKey() + continue + case "context_secret_detected": + for _, h := range ce.Findings { + fmt.Fprintf(env.stderr, " %s:%d looks like %s\n", runs.SanitizeLine(h.Path), h.Line, runs.SanitizeLine(h.Pattern)) + } + return exitf(exitDataErr, "some files look like they hold credentials; nothing was published. Keep credentials in nan secrets "+ + "(the agent can read and copy anything in a context), or publish anyway with --allow-secrets") + case "invalid_path", "unsupported_file_type", "skill_invalid", "context_too_large": + for _, r := range ce.Files { + line := fmt.Sprintf(" %s: %s", runs.SanitizeLine(r.Path), runs.SanitizeLine(r.Reason)) + if r.Detail != "" { + line += " (" + runs.SanitizeLine(r.Detail) + ")" + } + fmt.Fprintln(env.stderr, line) + } + return exitf(exitDataErr, "%s Nothing was published", runs.SanitizeLine(ce.Message)) + } + return ctxExit(err, env, scopeContextsWrite) + } +} + +func reportPushed(env *runsEnv, c *api.Context, diff treeDiff, plan uploadPlan, opts pushOptions) error { + if opts.json { + return printJSON(env.stdout, c, true) + } + fmt.Fprintf(env.stdout, "context %s: version %d published (%d %s, %s; %d added, %d changed, %d removed; uploaded %d %s, %s)\n", + runs.SanitizeLine(c.Name), c.Version, c.FileCount, pluralWord(c.FileCount, "file", "files"), humanBytes(c.TotalBytes), + len(diff.Added), len(diff.Changed), len(diff.Removed), len(plan.files), pluralWord(len(plan.files), "file", "files"), humanBytes(plan.bytes)) + return nil +} + +// uploadBlobs sends every blob of plan, ctxConcurrency at a time. The first +// failure stops the push after the uploads in flight finish; a multipart +// upload that was cut is resumed by the next push. +func uploadBlobs(ctx context.Context, env *runsEnv, plan uploadPlan) error { + if len(plan.files) == 0 { + return nil + } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + jobs := make(chan localFile) + var ( + mu sync.Mutex + firstErr error + wg sync.WaitGroup + ) + workers := max(1, min(ctxConcurrency, len(plan.files))) + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for f := range jobs { + if err := uploadOne(ctx, env, f); err != nil { + mu.Lock() + if firstErr == nil { + firstErr = err + cancel() + } + mu.Unlock() + } + } + }() + } + for _, f := range plan.files { + select { + case jobs <- f: + case <-ctx.Done(): + } + } + close(jobs) + wg.Wait() + return firstErr +} + +func uploadOne(ctx context.Context, env *runsEnv, f localFile) error { + if ctx.Err() != nil { + return ctx.Err() + } + var err error + if f.size <= ctxSingleMax { + err = ctxRetry(ctx, env, "upload "+runs.SanitizeLine(f.path), func() error { return putSingle(ctx, env, f) }) + } else { + err = putMultipart(ctx, env, f) + } + if err != nil { + var exit *ExitError + if errors.As(err, &exit) { + return err + } + e := ctxExit(err, env, scopeContextsWrite) + return &ExitError{Code: e.Code, Err: fmt.Errorf("upload of %s failed: %w", runs.SanitizeLine(f.path), e.Err)} + } + return nil +} + +// openChecked opens a file being uploaded and makes sure it is still the +// size it was hashed at (the platform verifies the sha256 too). +func openChecked(f localFile) (*os.File, error) { + fh, err := os.Open(f.abs) + if err != nil { + return nil, exitf(exitFailed, "cannot read %s: %v", runs.SanitizeLine(f.path), unwrapPath(err)) + } + if st, err := fh.Stat(); err != nil || st.Size() != f.size { + fh.Close() + return nil, exitf(exitFailed, "%s changed while pushing; push again", runs.SanitizeLine(f.path)) + } + return fh, nil +} + +func putSingle(ctx context.Context, env *runsEnv, f localFile) error { + fh, err := openChecked(f) + if err != nil { + return err + } + defer fh.Close() + ctx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + _, err = env.client.PutBlob(ctx, f.sha, io.NewSectionReader(fh, 0, f.size), f.size, path.Base(f.path)) + return err +} + +// uploadState is what a push remembers about a multipart upload it did not +// finish: enough to ask the platform where it stands. The id alone grants +// nothing without the member's credential. +type uploadState struct { + UploadID string `json:"upload_id"` + Size int64 `json:"size"` +} + +var shaPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) + +func uploadStatePath(sha string) (string, error) { + if !shaPattern.MatchString(sha) { + return "", fmt.Errorf("bad sha") + } + dir, err := ctxUploadStateDir() + if err != nil { + return "", err + } + return filepath.Join(dir, sha+".json"), nil +} + +func loadUploadState(sha string, size int64) *uploadState { + p, err := uploadStatePath(sha) + if err != nil { + return nil + } + b, err := os.ReadFile(p) + if err != nil { + return nil + } + var s uploadState + if json.Unmarshal(b, &s) != nil || s.Size != size || s.UploadID == "" { + return nil + } + return &s +} + +func saveUploadState(sha string, s uploadState) { + p, err := uploadStatePath(sha) + if err != nil { + return + } + if os.MkdirAll(filepath.Dir(p), 0o700) != nil { + return + } + b, _ := json.Marshal(s) + _ = os.WriteFile(p, b, 0o600) +} + +func clearUploadState(sha string) { + if p, err := uploadStatePath(sha); err == nil { + _ = os.Remove(p) + } +} + +// validUpload checks the platform's description of an upload against the +// file: a part size this client can send and the part count it implies. +func validUpload(u *api.ContextUpload, size int64) bool { + if u == nil || u.PartSize <= 0 || u.PartSize > api.ContextPartSize || u.NextPart < 1 { + return false + } + parts := (size + int64(u.PartSize) - 1) / int64(u.PartSize) + return int64(u.Parts) == parts && u.NextPart <= u.Parts+1 +} + +func putMultipart(ctx context.Context, env *runsEnv, f localFile) error { + fh, err := openChecked(f) + if err != nil { + return err + } + defer fh.Close() + shown := runs.SanitizeLine(f.path) + + var up *api.ContextUpload + if s := loadUploadState(f.sha, f.size); s != nil { + st, err := env.client.UploadStatus(ctx, f.sha, s.UploadID) + switch { + case err == nil && validUpload(st, f.size): + up = st + up.UploadID = s.UploadID + fmt.Fprintf(env.stderr, "resuming the upload of %s at part %d of %d\n", shown, up.NextPart, up.Parts) + case err == nil: + // An upload this client cannot continue: drop it, or its marker + // would refuse the new one (upload_in_progress). + _ = env.client.AbortUpload(ctx, f.sha, s.UploadID) + clearUploadState(f.sha) + case ctxCode(err) == "upload_not_found", ctxCode(err) == "upload_expired": + clearUploadState(f.sha) + default: + return err + } + } + if up == nil { + var blob *api.ContextBlob + err := ctxRetry(ctx, env, "upload "+shown, func() error { + var err error + up, blob, err = env.client.StartUpload(ctx, f.sha, f.size, path.Base(f.path)) + return err + }) + if err != nil { + return err + } + if blob != nil { + return nil // the platform already holds it + } + if !validUpload(up, f.size) { + return exitf(exitUnavailable, "nan.builders answered an upload of %s this CLI cannot send; update nan (nan --version)", shown) + } + saveUploadState(f.sha, uploadState{UploadID: up.UploadID, Size: f.size}) + } + + failures := 0 + for n := up.NextPart; n <= up.Parts; { + off := int64(n-1) * int64(up.PartSize) + size := min(int64(up.PartSize), f.size-off) + pctx, cancel := context.WithTimeout(ctx, 10*time.Minute) + next, err := env.client.PutPart(pctx, f.sha, up.UploadID, n, io.NewSectionReader(fh, off, size), size) + cancel() + if err == nil && next > n { + n = next + continue + } + if ctx.Err() != nil { + return ctx.Err() + } + if code := ctxCode(err); code == "upload_not_found" || code == "upload_expired" { + clearUploadState(f.sha) + return err + } + failures++ + if failures > ctxRetries { + if err == nil { + err = fmt.Errorf("the platform did not take part %d", n) + } + return err + } + if ce, ok := api.AsContextError(err); ok && retryable(ce) && ce.RetryAfter > 0 && ce.RetryAfter <= ctxMaxWait { + if serr := ctxSleep(ctx, ce.RetryAfter); serr != nil { + return err + } + } + // Ask where the upload stands: the part may have been stored + // before the answer was lost, or another attempt holds it. + st, serr := env.client.UploadStatus(ctx, f.sha, up.UploadID) + if serr != nil { + return err + } + if !validUpload(st, f.size) { + return exitf(exitUnavailable, "nan.builders answered an upload of %s this CLI cannot send; update nan (nan --version)", shown) + } + fmt.Fprintf(env.stderr, "part %d of %s did not go through; resuming at part %d\n", n, shown, st.NextPart) + n = st.NextPart + } + err = ctxRetry(ctx, env, "upload "+shown, func() error { + cctx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + _, err := env.client.CompleteUpload(cctx, f.sha, up.UploadID) + return err + }) + if err != nil { + if code := ctxCode(err); code == "upload_not_found" || code == "upload_expired" || code == "blob_hash_mismatch" { + clearUploadState(f.sha) + } + return err + } + clearUploadState(f.sha) + return nil +} + +// ─── pull ────────────────────────────────────────────────────────────────── + +func doContextPull(ctx context.Context, env *runsEnv, ref, dir string) error { + shown := runs.SanitizeLine(dir) + if st, err := os.Stat(dir); err == nil { + if !st.IsDir() { + return usageErrorf("%s is not a folder", shown) + } + entries, err := os.ReadDir(dir) + if err != nil { + return usageErrorf("cannot read %s: %v", shown, unwrapPath(err)) + } + if len(entries) > 0 && !ctxPullForce { + return usageErrorf("%s is not empty: pull into an empty or new folder, or add --force to write into it", shown) + } + } else if !errors.Is(err, fs.ErrNotExist) { + return usageErrorf("cannot read %s: %v", shown, unwrapPath(err)) + } + + d, err := env.client.GetContext(ctx, ref) + if err != nil { + return ctxExit(err, env, scopeContextsRead) + } + if err := checkServerTree(env, d.Files); err != nil { + return err + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return usageErrorf("cannot create %s: %v", shown, unwrapPath(err)) + } + root, err := os.OpenRoot(dir) + if err != nil { + return usageErrorf("cannot open %s: %v", shown, unwrapPath(err)) + } + defer root.Close() + + files := append([]api.ContextFile(nil), d.Files...) + sort.Slice(files, func(i, j int) bool { return files[i].Path < files[j].Path }) + var total int64 + for _, f := range files { + if err := pullOne(ctx, env, root, d, f); err != nil { + return err + } + total += f.Size + } + fmt.Fprintf(env.stdout, "context %s version %d written to %s (%d %s, %s)\n", runs.SanitizeLine(d.Name), d.Version, shown, + len(files), pluralWord(len(files), "file", "files"), humanBytes(total)) + return nil +} + +// checkServerTree refuses a file list a well-behaved platform never sends: +// paths the shared validator refuses (absolute, .., case collisions, +// reserved names, ...), bad hashes or sizes. Nothing is written then. +func checkServerTree(env *runsEnv, files []api.ContextFile) error { + if len(files) > ctxMaxFiles { + return exitf(exitDataErr, "nan.builders sent %d files, more than a context holds; nothing was written", len(files)) + } + paths := make([]string, len(files)) + var bad []string + for i, f := range files { + paths[i] = f.Path + if !shaPattern.MatchString(f.SHA256) || f.Size < 0 || f.Size > api.ContextMaxFileBytes { + bad = append(bad, fmt.Sprintf("%s: bad hash or size", runs.SanitizeLine(f.Path))) + } + } + for _, pe := range contexts.ValidateTree(paths) { + bad = append(bad, fmt.Sprintf("%s: %s", runs.SanitizeLine(pe.Path), pe.Reason)) + } + if len(bad) == 0 { + return nil + } + sort.Strings(bad) + for _, b := range bad { + fmt.Fprintf(env.stderr, "refused %s\n", b) + } + return exitf(exitDataErr, "nan.builders sent %d %s this CLI will not write; nothing was written", len(bad), pluralWord(len(bad), "path", "paths")) +} + +func pullOne(ctx context.Context, env *runsEnv, root *os.Root, d *api.ContextDetail, f api.ContextFile) error { + shown := runs.SanitizeLine(f.Path) + local := filepath.FromSlash(f.Path) + if parent := filepath.Dir(local); parent != "." { + if err := root.MkdirAll(parent, 0o755); err != nil { + return exitf(exitFailed, "cannot create the folder of %s: %v", shown, unwrapPath(err)) + } + } + flags := os.O_WRONLY | os.O_CREATE | os.O_EXCL + if ctxPullForce { + flags = os.O_WRONLY | os.O_CREATE | os.O_TRUNC + } + mode := os.FileMode(0o644) + if f.Executable { + mode = 0o755 + } + out, err := root.OpenFile(local, flags, mode) + if err != nil { + if errors.Is(err, fs.ErrExist) { + return exitf(exitFailed, "%s already exists; nothing more was written (use --force to replace files)", shown) + } + return exitf(exitFailed, "cannot write %s: %v", shown, unwrapPath(err)) + } + fail := func(e *ExitError) error { + out.Close() + _ = root.Remove(local) + return e + } + var body io.ReadCloser + err = ctxRetry(ctx, env, "download "+shown, func() error { + var err error + body, err = env.client.OpenContextFile(ctx, d.ID, f.Path, d.Version) + return err + }) + if err != nil { + return fail(ctxExit(err, env, scopeContextsRead)) + } + defer body.Close() + h := sha256.New() + n, err := io.Copy(io.MultiWriter(out, h), io.LimitReader(body, f.Size+1)) + if err != nil { + return fail(exitf(exitUnavailable, "the download of %s was cut: %v", shown, err)) + } + if n != f.Size || hex.EncodeToString(h.Sum(nil)) != f.SHA256 { + return fail(exitf(exitUnavailable, "the download of %s does not match its sha256 and size; it was not kept", shown)) + } + if err := out.Close(); err != nil { + return exitf(exitFailed, "cannot write %s: %v", shown, err) + } + if f.Executable && runtime.GOOS != "windows" { + // The umask may have taken the bits off the mode OpenFile asked for. + _ = root.Chmod(local, 0o755) + } + return nil +} diff --git a/cmd/context_fake_test.go b/cmd/context_fake_test.go new file mode 100644 index 0000000..592e629 --- /dev/null +++ b/cmd/context_fake_test.go @@ -0,0 +1,495 @@ +package cmd + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "sort" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/nxssie/nan-cli/internal/api" +) + +// fakeCtx is an in-memory /v1/contexts with the shapes, codes and status +// codes of cloud-api's handlers (internal/handlers/contexts.go, +// contexts_crud.go and internal/contexts/errors.go): the OpenAI error +// envelope with the extra fields at the top level, 201 for a new blob or +// version, 200 for a blob the member already has, Retry-After on 429/409. + +const ( + fakeCtxID = "c7000000-0000-4000-8000-000000000001" + fakeCtxStamp = "2026-10-21T10:00:00Z" + fakeSecretPat = "ghp_" +) + +type fakeContext struct { + ID, Name, Description string + Version int + Files []api.ContextFile +} + +type fakeUpload struct { + sha string + size int64 + partSize int + next int + data []byte +} + +type ctxReq struct { + Method, Path, Query, Body, Idem, Filename string + Size int64 +} + +type fakeResp struct { + status int + body map[string]any + header map[string]string +} + +type fakeCtx struct { + mu sync.Mutex + off bool + contexts []*fakeContext + blobs map[string][]byte + uploads map[string]*fakeUpload + partSize int + reqs []ctxReq + // queued answers "METHOD /path" before the normal handling. + queued map[string][]fakeResp + // onPublish runs before a publish is handled (f.mu held). + onPublish func(f *fakeCtx, n int) + publishes int + // corrupt serves wrong bytes for these paths on download. + corrupt map[string]bool + // failPart answers 500 to the first PUT of these part numbers. + failPart map[int]bool +} + +func newFakeCtx() *fakeCtx { + return &fakeCtx{ + blobs: map[string][]byte{}, + uploads: map[string]*fakeUpload{}, + partSize: api.ContextPartSize, + queued: map[string][]fakeResp{}, + corrupt: map[string]bool{}, + failPart: map[int]bool{}, + } +} + +func shaOf(b []byte) string { + s := sha256.Sum256(b) + return hex.EncodeToString(s[:]) +} + +// seed adds a context whose files hold the given contents. +func (f *fakeCtx) seed(name string, version int, files map[string]string) *fakeContext { + c := &fakeContext{ID: fakeCtxID, Name: name, Description: "Acme API conventions", Version: version} + if len(f.contexts) > 0 { + c.ID = fmt.Sprintf("c7000000-0000-4000-8000-%012d", len(f.contexts)+1) + } + for p, body := range files { + sha := shaOf([]byte(body)) + f.blobs[sha] = []byte(body) + c.Files = append(c.Files, api.ContextFile{Path: p, SHA256: sha, Size: int64(len(body)), Executable: strings.HasSuffix(p, ".sh")}) + } + sort.Slice(c.Files, func(i, j int) bool { return c.Files[i].Path < c.Files[j].Path }) + f.contexts = append(f.contexts, c) + return c +} + +func (f *fakeCtx) queue(route string, r fakeResp) { + f.mu.Lock() + defer f.mu.Unlock() + f.queued[route] = append(f.queued[route], r) +} + +func (f *fakeCtx) requests() []ctxReq { + f.mu.Lock() + defer f.mu.Unlock() + return append([]ctxReq(nil), f.reqs...) +} + +func (f *fakeCtx) sent(method, prefix string) []ctxReq { + var out []ctxReq + for _, r := range f.requests() { + if r.Method == method && strings.HasPrefix(r.Path, prefix) { + out = append(out, r) + } + } + return out +} + +func (c *fakeContext) view() map[string]any { + var total int64 + agents := false + skills := []string{} + seen := map[string]bool{} + for _, f := range c.Files { + total += f.Size + agents = agents || f.Path == "AGENTS.md" + if parts := strings.SplitN(f.Path, "/", 3); len(parts) == 3 && parts[0] == "skills" && !seen[parts[1]] { + seen[parts[1]] = true + skills = append(skills, parts[1]) + } + } + sort.Strings(skills) + return map[string]any{"id": c.ID, "object": "context", "name": c.Name, "description": c.Description, "version": c.Version, + "tree_sha256": strings.Repeat("ab", 32), "file_count": len(c.Files), "total_bytes": total, "has_instructions": agents, + "skills": skills, "created_at": fakeCtxStamp, "updated_at": fakeCtxStamp} +} + +func ctxFail(w http.ResponseWriter, status int, code, msg string, extra map[string]any, retryAfter int) { + if retryAfter > 0 { + w.Header().Set("Retry-After", strconv.Itoa(retryAfter)) + } + typ := "invalid_request_error" + if status == 429 { + typ = "rate_limit_exceeded" + } else if status >= 500 { + typ = "server_error" + } + body := map[string]any{"error": map[string]any{"message": msg, "type": typ, "param": nil, "code": code}} + for k, v := range extra { + body[k] = v + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(body) +} + +func ctxJSONResp(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} + +func (f *fakeCtx) find(ref string) *fakeContext { + for _, c := range f.contexts { + if c.ID == ref || c.Name == ref { + return c + } + } + return nil +} + +func (f *fakeCtx) blobView(sha string) map[string]any { + b := f.blobs[sha] + findings := []any{} + if bytes.Contains(b, []byte(fakeSecretPat)) { + findings = append(findings, map[string]any{"line": 1, "pattern": "github_token"}) + } + return map[string]any{"sha256": sha, "size": len(b), "format": "text", "secret_findings": findings} +} + +func (f *fakeCtx) ServeHTTP(w http.ResponseWriter, r *http.Request) { + isRaw := r.Header.Get("Content-Type") == "application/octet-stream" + body, _ := io.ReadAll(r.Body) + f.mu.Lock() + defer f.mu.Unlock() + rec := ctxReq{Method: r.Method, Path: r.URL.Path, Query: r.URL.RawQuery, Idem: r.Header.Get("Idempotency-Key"), + Filename: r.Header.Get("X-Nan-Filename"), Size: int64(len(body))} + if !isRaw { + rec.Body = string(body) + } + f.reqs = append(f.reqs, rec) + if f.off { + ctxFail(w, 404, "not_found", "Not found.", nil, 0) + return + } + route := r.Method + " " + r.URL.Path + if q := f.queued[route]; len(q) > 0 { + f.queued[route] = q[1:] + for k, v := range q[0].header { + w.Header().Set(k, v) + } + if q[0].body == nil { + w.WriteHeader(q[0].status) + return + } + ctxJSONResp(w, q[0].status, q[0].body) + return + } + rest := strings.Trim(strings.TrimPrefix(r.URL.Path, "/v1/contexts"), "/") + parts := strings.Split(rest, "/") + switch { + case rest == "" && r.Method == http.MethodGet: + data := []any{} + for _, c := range f.contexts { + data = append(data, c.view()) + } + ctxJSONResp(w, 200, map[string]any{"data": data, "usage": map[string]any{"stored_bytes": 1536, "quota_bytes": 500 << 20}}) + case rest == "" && r.Method == http.MethodPost: + var in struct{ Name, Description string } + _ = json.Unmarshal(body, &in) + if f.find(in.Name) != nil { + ctxFail(w, 409, "context_name_taken", "You already have a context with this name.", nil, 0) + return + } + c := &fakeContext{ID: fmt.Sprintf("c7000000-0000-4000-8000-%012d", len(f.contexts)+1), Name: in.Name, Description: in.Description, Version: 1} + f.contexts = append(f.contexts, c) + ctxJSONResp(w, 201, c.view()) + case parts[0] == "blobs": + f.serveBlobs(w, r, parts[1:], body) + default: + c := f.find(parts[0]) + if c == nil { + ctxFail(w, 404, "context_not_found", "context not found", nil, 0) + return + } + sub := strings.Join(parts[1:], "/") + switch r.Method + " " + sub { + case "GET ": + v := c.view() + files := []any{} + for _, fl := range c.Files { + files = append(files, map[string]any{"path": fl.Path, "size": fl.Size, "sha256": fl.SHA256, "executable": fl.Executable}) + } + v["files"] = files + ctxJSONResp(w, 200, v) + case "PATCH ": + var in map[string]*string + _ = json.Unmarshal(body, &in) + if in["name"] != nil { + c.Name = *in["name"] + } + if in["description"] != nil { + c.Description = *in["description"] + } + ctxJSONResp(w, 200, c.view()) + case "DELETE ": + for i, x := range f.contexts { + if x == c { + f.contexts = append(f.contexts[:i], f.contexts[i+1:]...) + break + } + } + w.WriteHeader(204) + case "GET file": + p := r.URL.Query().Get("path") + if v := r.URL.Query().Get("version"); v != strconv.Itoa(c.Version) { + ctxFail(w, 404, "version_not_found", "version not found", nil, 0) + return + } + for _, fl := range c.Files { + if fl.Path == p { + b := f.blobs[fl.SHA256] + if f.corrupt[p] { + b = append([]byte("X"), b[1:]...) + } + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write(b) + return + } + } + ctxFail(w, 404, "file_not_found", "file not found", nil, 0) + case "POST versions": + f.publishes++ + if f.onPublish != nil { + f.onPublish(f, f.publishes) + } + var in api.PublishRequest + _ = json.Unmarshal(body, &in) + if in.BaseVersion != c.Version { + ctxFail(w, 409, "version_conflict", "The context changed since you loaded it; reload and publish again.", + map[string]any{"current_version": c.Version}, 0) + return + } + var missing []string + for _, fl := range in.Files { + if _, ok := f.blobs[fl.SHA256]; !ok { + missing = append(missing, fl.SHA256) + } + } + if len(missing) > 0 { + ctxFail(w, 409, "blob_missing", "Some files are no longer stored; upload them again and publish.", map[string]any{"sha256": missing}, 0) + return + } + if !in.AllowSecretLike { + var hits []any + for _, fl := range in.Files { + if bytes.Contains(f.blobs[fl.SHA256], []byte(fakeSecretPat)) { + hits = append(hits, map[string]any{"path": fl.Path, "line": 1, "pattern": "github_token"}) + } + } + if len(hits) > 0 { + ctxFail(w, 422, "context_secret_detected", "Some files look like they contain secrets. Keep secrets in Secrets, or publish anyway.", + map[string]any{"findings": hits}, 0) + return + } + } + c.Version++ + c.Files = nil + for _, fl := range in.Files { + c.Files = append(c.Files, api.ContextFile{Path: fl.Path, SHA256: fl.SHA256, Size: int64(len(f.blobs[fl.SHA256])), Executable: fl.Executable}) + } + ctxJSONResp(w, 201, c.view()) + default: + ctxFail(w, 404, "not_found", "no route "+route, nil, 0) + } + } +} + +func (f *fakeCtx) serveBlobs(w http.ResponseWriter, r *http.Request, parts []string, body []byte) { + if len(parts) == 1 && parts[0] == "check" { + var in struct { + SHA256 []string `json:"sha256"` + } + _ = json.Unmarshal(body, &in) + missing := []string{} + for _, s := range in.SHA256 { + if _, ok := f.blobs[s]; !ok { + missing = append(missing, s) + } + } + ctxJSONResp(w, 200, map[string]any{"missing": missing}) + return + } + sha := parts[0] + switch { + case len(parts) == 1 && r.Method == http.MethodPut: + if int64(len(body)) > api.ContextPartSize { + ctxFail(w, 413, "invalid_request", "a single upload is at most 16 MiB", nil, 0) + return + } + if _, ok := f.blobs[sha]; ok { + ctxJSONResp(w, 200, f.blobView(sha)) + return + } + if shaOf(body) != sha { + ctxFail(w, 422, "blob_hash_mismatch", "The uploaded bytes do not match the declared sha256 and size; nothing was stored.", nil, 0) + return + } + f.blobs[sha] = body + ctxJSONResp(w, 201, f.blobView(sha)) + case len(parts) == 2 && r.Method == http.MethodPost: + if _, ok := f.blobs[sha]; ok { + ctxJSONResp(w, 200, f.blobView(sha)) + return + } + var in struct { + Size int64 `json:"size"` + Filename string `json:"filename"` + } + _ = json.Unmarshal(body, &in) + id := fmt.Sprintf("u0000000-0000-4000-8000-%012d", len(f.uploads)+1) + u := &fakeUpload{sha: sha, size: in.Size, partSize: f.partSize, next: 1} + f.uploads[id] = u + ctxJSONResp(w, 201, f.uploadView(id, u)) + case len(parts) >= 3: + id := parts[2] + u := f.uploads[id] + if u == nil || u.sha != sha { + ctxFail(w, 404, "upload_not_found", "upload not found", nil, 0) + return + } + sub := strings.Join(parts[3:], "/") + switch { + case r.Method == http.MethodGet && sub == "": + ctxJSONResp(w, 200, f.uploadView(id, u)) + case r.Method == http.MethodPut && strings.HasPrefix(sub, "parts/"): + n, _ := strconv.Atoi(strings.TrimPrefix(sub, "parts/")) + if f.failPart[n] { + delete(f.failPart, n) + ctxFail(w, 500, "internal_error", "Something went wrong. Try again.", nil, 0) + return + } + if n < u.next { + ctxJSONResp(w, 200, map[string]int{"next_part": u.next}) + return + } + if n != u.next { + ctxFail(w, 409, "part_conflict", "out of order", nil, 0) + return + } + u.data = append(u.data, body...) + u.next++ + ctxJSONResp(w, 200, map[string]int{"next_part": u.next}) + case r.Method == http.MethodPost && sub == "complete": + if int64(len(u.data)) != u.size || shaOf(u.data) != sha { + ctxFail(w, 422, "blob_hash_mismatch", "mismatch", nil, 0) + return + } + f.blobs[sha] = u.data + delete(f.uploads, id) + ctxJSONResp(w, 201, f.blobView(sha)) + case r.Method == http.MethodDelete && sub == "": + delete(f.uploads, id) + w.WriteHeader(204) + default: + ctxFail(w, 404, "not_found", "no route", nil, 0) + } + default: + ctxFail(w, 404, "not_found", "no route", nil, 0) + } +} + +func (f *fakeCtx) uploadView(id string, u *fakeUpload) map[string]any { + parts := int((u.size + int64(u.partSize) - 1) / int64(u.partSize)) + return map[string]any{"upload_id": id, "part_size": u.partSize, "parts": parts, "next_part": u.next} +} + +// ctxRun is one nan invocation against a fakeCtx. +type ctxRun struct { + code int + msg string + stdout, stderr string + sleeps []time.Duration + stateDir string +} + +type ctxSetup struct { + stdin string + tty bool + stateDir string // "" = a fresh temp dir +} + +func runCtx(t *testing.T, fake *fakeCtx, setup ctxSetup, args ...string) ctxRun { + t.Helper() + srv := httptest.NewServer(fake) + t.Cleanup(srv.Close) + fakeHome(t, cliSession) + t.Setenv(tokenEnvVar, "") + swap(t, &runsBaseURL, srv.URL) + swap(t, &writeStdin, io.Reader(strings.NewReader(setup.stdin))) + tty := setup.tty + swap(t, &writeStdinTTY, func() bool { return tty }) + var out, errOut bytes.Buffer + swap(t, &writeStdout, io.Writer(&out)) + swap(t, &writeStderr, io.Writer(&errOut)) + state := setup.stateDir + if state == "" { + state = t.TempDir() + } + swap(t, &ctxUploadStateDir, func() (string, error) { return state, nil }) + var mu sync.Mutex + var sleeps []time.Duration + swap(t, &ctxSleep, func(_ context.Context, d time.Duration) error { + mu.Lock() + defer mu.Unlock() + sleeps = append(sleeps, d) + return nil + }) + code, msg := execute(t, args...) + mu.Lock() + defer mu.Unlock() + return ctxRun{code: code, msg: msg, stdout: out.String(), stderr: errOut.String(), sleeps: sleeps, stateDir: state} +} + +func mustCtxExit(t *testing.T, r ctxRun, code int, wants ...string) { + t.Helper() + if r.code != code { + t.Fatalf("exit %d (want %d): %s\nstdout: %s\nstderr: %s", r.code, code, r.msg, r.stdout, r.stderr) + } + wantAll(t, "message", r.msg, wants...) +} diff --git a/cmd/context_test.go b/cmd/context_test.go new file mode 100644 index 0000000..1290ca1 --- /dev/null +++ b/cmd/context_test.go @@ -0,0 +1,673 @@ +package cmd + +import ( + "encoding/json" + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/nxssie/nan-cli/internal/api" +) + +// nan context (helmcode/nan-devops AGENT-CONTEXT.md §6, C1 test list), +// against fakeCtx: the shapes and codes of cloud-api's /v1/contexts. + +// writeTree creates files (path -> content) under a new temp folder. A path +// ending in ".sh" is made executable. +func writeTree(t *testing.T, files map[string]string) string { + t.Helper() + dir := t.TempDir() + for p, body := range files { + full := filepath.Join(dir, filepath.FromSlash(p)) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + mode := os.FileMode(0o644) + if strings.HasSuffix(p, ".sh") { + mode = 0o755 + } + if err := os.WriteFile(full, []byte(body), mode); err != nil { + t.Fatal(err) + } + } + return dir +} + +const skillMD = "---\nname: pr-review\ndescription: Review a pull request.\n---\nSteps.\n" + +func goodTree() map[string]string { + return map[string]string{ + "AGENTS.md": "Use tabs.\n", + "skills/pr-review/SKILL.md": skillMD, + "skills/pr-review/scripts/run.sh": "#!/bin/sh\necho ok\n", + "files/especificación técnica.md": "Spec.\n", + } +} + +func publishBody(t *testing.T, r ctxReq) api.PublishRequest { + t.Helper() + var in api.PublishRequest + if err := json.Unmarshal([]byte(r.Body), &in); err != nil { + t.Fatalf("publish body %q: %v", r.Body, err) + } + return in +} + +func publishPaths(in api.PublishRequest) []string { + var out []string + for _, f := range in.Files { + out = append(out, f.Path) + } + return out +} + +// ─── ls, show, create, edit, rm ───────────────────────────────────────────── + +func TestContextLsAndShowGoldens(t *testing.T) { + fake := newFakeCtx() + fake.seed("acme-api", 3, map[string]string{"AGENTS.md": "Use tabs.\n", "skills/pr-review/SKILL.md": skillMD, "files/spec.md": "Spec.\n"}) + + r := runCtx(t, fake, ctxSetup{}, "context", "ls") + mustCtxExit(t, r, 0) + checkGolden(t, "contexts/ls.golden", r.stdout) + + r = runCtx(t, fake, ctxSetup{}, "context", "show", "acme-api") + mustCtxExit(t, r, 0) + checkGolden(t, "contexts/show.golden", r.stdout) + + r = runCtx(t, fake, ctxSetup{}, "context", "show", "acme-api", "--json") + mustCtxExit(t, r, 0) + var d api.ContextDetail + if err := json.Unmarshal([]byte(r.stdout), &d); err != nil || d.Version != 3 || len(d.Files) != 3 { + t.Errorf("%v %s", err, r.stdout) + } + + r = runCtx(t, fake, ctxSetup{}, "context", "show", "nope") + mustCtxExit(t, r, exitUsage, "no such context") +} + +func TestContextSurfaceOff(t *testing.T) { + fake := newFakeCtx() + fake.off = true + r := runCtx(t, fake, ctxSetup{}, "context", "ls") + mustCtxExit(t, r, exitUsage, "not available for your account yet", "portal") +} + +func TestContextCreateValidatesTheName(t *testing.T) { + for _, name := range []string{"Acme", "con", "blobs", "a_b", "123e4567-e89b-12d3-a456-426614174000"} { + fake := newFakeCtx() + r := runCtx(t, fake, ctxSetup{}, "context", "create", name) + mustCtxExit(t, r, exitUsage, "not a valid context name") + if len(fake.requests()) != 0 { + t.Errorf("%s: sent %v", name, fake.requests()) + } + } +} + +func TestContextCreateFromAFolder(t *testing.T) { + fake := newFakeCtx() + dir := writeTree(t, goodTree()) + r := runCtx(t, fake, ctxSetup{}, "context", "create", "acme-api", "--description", "Acme", "--from", dir) + mustCtxExit(t, r, 0) + wantAll(t, "stdout", r.stdout, "version 2 published", "4 files") + creates := fake.sent("POST", "/v1/contexts") + if len(creates) < 1 || !idemKeyPattern.MatchString(creates[0].Idem) || !strings.Contains(creates[0].Body, `"description":"Acme"`) { + t.Errorf("create %+v", creates) + } + pubs := fake.sent("POST", "/v1/contexts/"+fake.contexts[0].ID+"/versions") + if len(pubs) != 1 || publishBody(t, pubs[0]).BaseVersion != 1 { + t.Fatalf("publishes %+v", pubs) + } + + // A folder that cannot be pushed leaves no empty context behind. + fake = newFakeCtx() + bad := writeTree(t, map[string]string{"CLAUDE.md": "x"}) + r = runCtx(t, fake, ctxSetup{}, "context", "create", "acme-api", "--from", bad) + mustCtxExit(t, r, exitDataErr) + if len(fake.requests()) != 0 { + t.Errorf("sent %v", fake.requests()) + } +} + +func TestContextEditAndRm(t *testing.T) { + fake := newFakeCtx() + fake.seed("acme-api", 2, nil) + r := runCtx(t, fake, ctxSetup{}, "context", "edit", "acme-api", "--name", "acme-web", "--description", "") + mustCtxExit(t, r, 0) + wantAll(t, "stdout", r.stdout, "context acme-web updated") + p := fake.sent("PATCH", "/v1/contexts/"+fakeCtxID) + if len(p) != 1 || p[0].Body != `{"name":"acme-web","description":""}` { + t.Errorf("patch %+v", p) + } + mustCtxExit(t, runCtx(t, fake, ctxSetup{}, "context", "edit", "acme-web"), exitUsage, "nothing to change") + mustCtxExit(t, runCtx(t, fake, ctxSetup{}, "context", "edit", "acme-web", "--name", "Bad Name"), exitUsage, "not a valid context name") + + // rm asks, and without a terminal needs --yes. + mustCtxExit(t, runCtx(t, fake, ctxSetup{}, "context", "rm", "acme-web"), exitUsage, "--yes") + fake.queue("DELETE /v1/contexts/"+fakeCtxID, fakeResp{status: 409, body: map[string]any{ + "error": map[string]any{"message": "Automations use this context; remove it from them first.", "type": "invalid_request_error", "param": nil, "code": "context_in_use"}, + "automations": []any{map[string]any{"id": testAutoID, "name": "PR review api"}}}}) + r = runCtx(t, fake, ctxSetup{}, "context", "rm", "acme-web", "--yes") + mustCtxExit(t, r, exitUsage, "automations use this context", "PR review api") + r = runCtx(t, fake, ctxSetup{}, "context", "rm", "acme-web", "--yes") + mustCtxExit(t, r, 0) + wantAll(t, "stdout", r.stdout, "context acme-web deleted") + if len(fake.contexts) != 0 { + t.Error("not deleted") + } +} + +// ─── push: local checks ───────────────────────────────────────────────────── + +func TestContextPushLayoutChecks(t *testing.T) { + fake := newFakeCtx() + fake.seed("acme-api", 1, nil) + dir := writeTree(t, map[string]string{ + "CLAUDE.md": "Use tabs.\n", + "README.md": "top level\n", + "notes/a.md": "a\n", + "skills/no-skill-md/x.md": "x\n", + "skills/Bad Skill/SKILL.md": skillMD, + "files/rocket\U0001F680.md": "emoji\n", + "files/con.txt": "reserved\n", + "files/ok.md": "ok\n", + "files/.env": "SECRET=1\n", + ".git/config": "[core]\n", + "files/sub/.hidden/inner.md": "hidden\n", + }) + r := runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, exitDataErr, "cannot be published", "nothing was uploaded", "--skip-invalid") + checkGolden(t, "contexts/push_refused.stderr.golden", r.stderr) + if n := len(fake.requests()); n != 0 { + t.Errorf("a refused push sent %d requests: %v", n, fake.requests()) + } +} + +func TestContextPushSkipInvalid(t *testing.T) { + fake := newFakeCtx() + fake.seed("acme-api", 1, nil) + dir := writeTree(t, map[string]string{"CLAUDE.md": "x\n", "files/ok.md": "ok\n", "files/a:b.md": "colon\n", + "skills/no-skill-md/x.md": "x\n", "files/.env": "S=1\n"}) + r := runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir, "--skip-invalid") + mustCtxExit(t, r, 0) + wantAll(t, "stderr", r.stderr, "skipped CLAUDE.md: bad_top_level (rename it to AGENTS.md", "skipped files/a:b.md: bad_character", + "skipped skills/no-skill-md/SKILL.md: skill_invalid", "skipped files/.env (dotfiles") + pubs := fake.sent("POST", "/v1/contexts/"+fakeCtxID+"/versions") + if len(pubs) != 1 { + t.Fatalf("publishes %v", fake.requests()) + } + if got := publishPaths(publishBody(t, pubs[0])); strings.Join(got, ",") != "files/ok.md" { + t.Errorf("published %v", got) + } +} + +// macOS hands out decomposed (NFD) names; the platform refuses non-NFC +// paths, so push normalises before hashing and sending. +func TestContextPushNormalisesNFD(t *testing.T) { + fake := newFakeCtx() + fake.seed("acme-api", 1, nil) + nfd := "files/especificación.md" + dir := writeTree(t, map[string]string{nfd: "spec\n"}) + r := runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, 0) + pubs := fake.sent("POST", "/v1/contexts/"+fakeCtxID+"/versions") + if len(pubs) != 1 { + t.Fatalf("%v", fake.requests()) + } + if got := publishPaths(publishBody(t, pubs[0])); len(got) != 1 || got[0] != "files/especificación.md" { + t.Errorf("published %q, want the NFC form", got) + } + + // Two names that are the same once normalised collide (a file system + // that is not normalisation-insensitive can hold both). + if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { + return + } + dir = writeTree(t, map[string]string{nfd: "a\n", "files/especificación.md": "b\n"}) + r = runCtx(t, newFakeCtx(), ctxSetup{}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, exitDataErr) + wantAll(t, "stderr", r.stderr, "case_collision") +} + +func TestContextPushRefusesSymlinks(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlinks need privileges on Windows") + } + fake := newFakeCtx() + fake.seed("acme-api", 1, nil) + dir := writeTree(t, map[string]string{"files/a.md": "a\n"}) + if err := os.Symlink("/etc/passwd", filepath.Join(dir, "files", "passwd")); err != nil { + t.Fatal(err) + } + r := runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, exitDataErr) + wantAll(t, "stderr", r.stderr, "refused files/passwd: symlink") +} + +func TestContextPushLimits(t *testing.T) { + fake := newFakeCtx() + fake.seed("acme-api", 1, nil) + dir := writeTree(t, map[string]string{"AGENTS.md": strings.Repeat("a", 32<<10+1)}) + r := runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, exitDataErr) + wantAll(t, "stderr", r.stderr, "refused AGENTS.md: too_large (AGENTS.md is at most 32 KiB)") +} + +// ─── push: uploads and publish ────────────────────────────────────────────── + +func TestContextPushDryRun(t *testing.T) { + fake := newFakeCtx() + fake.seed("acme-api", 3, map[string]string{"AGENTS.md": "Old rules.\n", "files/old.md": "old\n", "files/same.md": "same\n"}) + dir := writeTree(t, map[string]string{"AGENTS.md": "Use tabs.\n", "files/same.md": "same\n", "files/new.md": "new file\n", + "skills/pr-review/SKILL.md": skillMD}) + r := runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir, "--dry-run") + mustCtxExit(t, r, 0) + checkGolden(t, "contexts/push_dry_run.golden", r.stdout) + + r = runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir, "--dry-run", "--json") + mustCtxExit(t, r, 0) + checkGolden(t, "contexts/push_dry_run.json.golden", r.stdout) + + for _, req := range fake.requests() { + if req.Method != http.MethodGet && !(req.Method == http.MethodPost && req.Path == "/v1/contexts/blobs/check") { + t.Errorf("a dry run sent %s %s", req.Method, req.Path) + } + } +} + +func TestContextPushUploadsOnlyMissingBlobs(t *testing.T) { + fake := newFakeCtx() + c := fake.seed("acme-api", 2, map[string]string{"files/kept.md": "kept\n"}) + tree := goodTree() + tree["files/kept.md"] = "kept\n" + tree["files/copy-of-agents.md"] = tree["AGENTS.md"] // same blob, two paths + dir := writeTree(t, tree) + r := runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, 0) + checkGolden(t, "contexts/push.golden", r.stdout) + + puts := fake.sent("PUT", "/v1/contexts/blobs/") + if len(puts) != 4 { // AGENTS.md (once for two paths), SKILL.md, run.sh, the spec + t.Errorf("uploaded %d blobs, want 4: %+v", len(puts), puts) + } + for _, p := range puts { + if p.Path == "/v1/contexts/blobs/"+shaOf([]byte("kept\n")) { + t.Error("uploaded a blob the platform already had") + } + if p.Filename == "" || strings.Contains(p.Filename, "/") { + t.Errorf("X-Nan-Filename %q: the base name is the type hint", p.Filename) + } + } + pubs := fake.sent("POST", "/v1/contexts/"+c.ID+"/versions") + if len(pubs) != 1 { + t.Fatalf("publishes %d", len(pubs)) + } + in := publishBody(t, pubs[0]) + if in.BaseVersion != 2 || in.AllowSecretLike || len(in.Files) != 6 || !idemKeyPattern.MatchString(pubs[0].Idem) { + t.Errorf("publish %+v idem %q", in, pubs[0].Idem) + } + for _, f := range in.Files { + if f.Executable != (f.Path == "skills/pr-review/scripts/run.sh") && runtime.GOOS != "windows" { + t.Errorf("%s executable=%v", f.Path, f.Executable) + } + } + + // Pushing the same folder again changes nothing. + n := len(fake.requests()) + r = runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, 0) + wantAll(t, "stdout", r.stdout, "is up to date (version 3)") + if extra := fake.requests()[n:]; len(extra) != 1 || extra[0].Method != http.MethodGet { + t.Errorf("an up-to-date push sent %v", extra) + } +} + +func TestContextPushCreate(t *testing.T) { + fake := newFakeCtx() + dir := writeTree(t, map[string]string{"files/a.md": "a\n"}) + r := runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, exitUsage, "nan context create acme-api", "--create") + r = runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir, "--create") + mustCtxExit(t, r, 0) + wantAll(t, "stderr", r.stderr, "context acme-api created") + if len(fake.contexts) != 1 || fake.contexts[0].Version != 2 { + t.Errorf("%+v", fake.contexts) + } +} + +// A blob over the single-PUT limit goes up in parts; a part that fails is +// resumed from where the platform says the upload stands. +func TestContextPushMultipartResumesAfterAFailedPart(t *testing.T) { + swap(t, &ctxSingleMax, int64(8)) + fake := newFakeCtx() + fake.partSize = 4 + fake.failPart[2] = true + fake.seed("acme-api", 1, nil) + big := "0123456789" // 3 parts of 4, 4, 2 bytes + dir := writeTree(t, map[string]string{"files/big.csv": big, "files/small.md": "small\n"}) + r := runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, 0) + wantAll(t, "stderr", r.stderr, "part 2 of files/big.csv did not go through; resuming at part 2") + sha := shaOf([]byte(big)) + var parts []string + for _, q := range fake.sent("PUT", "/v1/contexts/blobs/"+sha+"/uploads/") { + parts = append(parts, q.Path[strings.LastIndex(q.Path, "/")+1:]) + } + if strings.Join(parts, ",") != "1,2,2,3" { + t.Errorf("parts sent %v", parts) + } + if string(fake.blobs[sha]) != big { + t.Errorf("stored %q", fake.blobs[sha]) + } + if starts := fake.sent("POST", "/v1/contexts/blobs/"+sha+"/uploads"); len(starts) != 2 { // start + complete + t.Errorf("start/complete %v", starts) + } + if entries, _ := os.ReadDir(r.stateDir); len(entries) != 0 { + t.Errorf("upload state left behind: %v", entries) + } +} + +// An upload a previous push left unfinished is resumed: no new upload, the +// stored parts are not sent again. +func TestContextPushMultipartResumesAcrossRuns(t *testing.T) { + swap(t, &ctxSingleMax, int64(8)) + fake := newFakeCtx() + fake.partSize = 4 + fake.seed("acme-api", 1, nil) + big := "0123456789" + sha := shaOf([]byte(big)) + const id = "u0000000-0000-4000-8000-000000000042" + fake.uploads[id] = &fakeUpload{sha: sha, size: int64(len(big)), partSize: 4, next: 2, data: []byte("0123")} + state := t.TempDir() + if err := os.WriteFile(filepath.Join(state, sha+".json"), []byte(`{"upload_id":"`+id+`","size":10}`), 0o600); err != nil { + t.Fatal(err) + } + dir := writeTree(t, map[string]string{"files/big.csv": big}) + r := runCtx(t, fake, ctxSetup{stateDir: state}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, 0) + wantAll(t, "stderr", r.stderr, "resuming the upload of files/big.csv at part 2 of 3") + if starts := fake.sent("POST", "/v1/contexts/blobs/"+sha+"/uploads"); len(starts) != 1 || !strings.HasSuffix(starts[0].Path, "/complete") { + t.Errorf("a new upload was started: %v", starts) + } + var parts []string + for _, q := range fake.sent("PUT", "/v1/contexts/blobs/"+sha+"/uploads/"+id+"/parts/") { + parts = append(parts, q.Path[strings.LastIndex(q.Path, "/")+1:]) + } + if strings.Join(parts, ",") != "2,3" { + t.Errorf("parts sent %v", parts) + } + if string(fake.blobs[sha]) != big { + t.Errorf("stored %q", fake.blobs[sha]) + } + + // A remembered upload the platform no longer has starts over. + fake2 := newFakeCtx() + fake2.partSize = 4 + fake2.seed("acme-api", 1, nil) + if err := os.WriteFile(filepath.Join(state, sha+".json"), []byte(`{"upload_id":"`+id+`","size":10}`), 0o600); err != nil { + t.Fatal(err) + } + r = runCtx(t, fake2, ctxSetup{stateDir: state}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, 0) + if string(fake2.blobs[sha]) != big { + t.Errorf("stored %q", fake2.blobs[sha]) + } +} + +func TestContextPushVersionConflict(t *testing.T) { + newFake := func() *fakeCtx { + fake := newFakeCtx() + fake.seed("acme-api", 2, map[string]string{"files/a.md": "a\n"}) + // Someone publishes version 3 between this push's read and its + // publish. + fake.onPublish = func(f *fakeCtx, n int) { + if n == 1 { + c := f.contexts[0] + c.Version = 3 + c.Files = append(c.Files, api.ContextFile{Path: "files/theirs.md", SHA256: shaOf([]byte("a\n")), Size: 2}) + } + } + return fake + } + dir := writeTree(t, map[string]string{"files/a.md": "a\n", "files/mine.md": "mine\n"}) + + fake := newFake() + r := runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, exitFailed, "changed while pushing", "now at version 3", "based on version 2", "--force") + if fake.contexts[0].Version != 3 { + t.Error("published anyway") + } + + fake = newFake() + r = runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir, "--force") + mustCtxExit(t, r, 0) + wantAll(t, "stderr", r.stderr, "changed to version 3 meanwhile", "- files/theirs.md") + pubs := fake.sent("POST", "/v1/contexts/"+fakeCtxID+"/versions") + if len(pubs) != 2 || publishBody(t, pubs[1]).BaseVersion != 3 || pubs[0].Idem == pubs[1].Idem { + t.Errorf("publishes %+v", pubs) + } + if fake.contexts[0].Version != 4 { + t.Errorf("version %d", fake.contexts[0].Version) + } +} + +func TestContextPushSecretLikeContent(t *testing.T) { + fake := newFakeCtx() + fake.seed("acme-api", 1, nil) + dir := writeTree(t, map[string]string{"files/setup.md": "token: " + fakeSecretPat + "abcdef\n"}) + r := runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, exitDataErr, "look like they hold credentials", "nan secrets", "--allow-secrets") + wantAll(t, "stderr", r.stderr, "files/setup.md:1 looks like github_token") + if strings.Contains(r.stderr+r.stdout+r.msg, "abcdef") { + t.Error("the secret was printed") + } + + r = runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir, "--allow-secrets") + mustCtxExit(t, r, 0) + pubs := fake.sent("POST", "/v1/contexts/"+fakeCtxID+"/versions") + if !publishBody(t, pubs[len(pubs)-1]).AllowSecretLike { + t.Error("allow_secret_like not sent") + } +} + +// The platform's cleanup can remove an unreferenced blob between the check +// and the publish: push checks again, uploads it again and publishes. +func TestContextPushBlobMissingReuploads(t *testing.T) { + fake := newFakeCtx() + fake.seed("acme-api", 1, nil) + dir := writeTree(t, map[string]string{"files/a.md": "a\n", "files/b.md": "b\n"}) + gone := shaOf([]byte("b\n")) + fake.onPublish = func(f *fakeCtx, n int) { + if n == 1 { + delete(f.blobs, gone) + } + } + r := runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, 0) + wantAll(t, "stderr", r.stderr, "1 file is no longer stored on nan.builders; uploading again") + if puts := fake.sent("PUT", "/v1/contexts/blobs/"+gone); len(puts) != 2 { + t.Errorf("uploads of the missing blob: %d", len(puts)) + } + if pubs := fake.sent("POST", "/v1/contexts/"+fakeCtxID+"/versions"); len(pubs) != 2 { + t.Errorf("publishes %d", len(pubs)) + } +} + +func TestContextPushWaitsOutRetryAfter(t *testing.T) { + fake := newFakeCtx() + fake.seed("acme-api", 1, nil) + fake.queue("POST /v1/contexts/"+fakeCtxID+"/versions", fakeResp{status: 429, header: map[string]string{"Retry-After": "7"}, + body: map[string]any{"error": map[string]any{"message": "Publish limit reached (120 attempts per hour); try again later.", + "type": "rate_limit_exceeded", "param": nil, "code": "rate_limited"}}}) + dir := writeTree(t, map[string]string{"files/a.md": "a\n"}) + r := runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, 0) + if len(r.sleeps) != 1 || r.sleeps[0] != 7*time.Second { + t.Errorf("sleeps %v", r.sleeps) + } + pubs := fake.sent("POST", "/v1/contexts/"+fakeCtxID+"/versions") + if len(pubs) != 2 || pubs[0].Idem != pubs[1].Idem { + t.Errorf("a retried publish keeps its Idempotency-Key: %+v", pubs) + } + + // A wait longer than a push waits by itself is reported, not slept. + fake = newFakeCtx() + fake.seed("acme-api", 1, nil) + fake.queue("POST /v1/contexts/"+fakeCtxID+"/versions", fakeResp{status: 429, header: map[string]string{"Retry-After": "3600"}, + body: map[string]any{"error": map[string]any{"message": "Publish limit reached.", "type": "rate_limit_exceeded", "param": nil, "code": "rate_limited"}}}) + r = runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, exitUnavailable, "try again in 1h0m0s") + if len(r.sleeps) != 0 { + t.Errorf("slept %v", r.sleeps) + } +} + +func TestContextPushServerRefusalsArePerFile(t *testing.T) { + fake := newFakeCtx() + fake.seed("acme-api", 1, nil) + fake.queue("POST /v1/contexts/"+fakeCtxID+"/versions", fakeResp{status: 415, body: map[string]any{ + "error": map[string]any{"message": "Some files are not allowed at their path.", "type": "invalid_request_error", "param": nil, "code": "unsupported_file_type"}, + "files": []any{map[string]any{"path": "files/a.md", "reason": "unsupported_file_type"}}}}) + dir := writeTree(t, map[string]string{"files/a.md": "a\n"}) + r := runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, exitDataErr, "not allowed at their path") + wantAll(t, "stderr", r.stderr, "files/a.md: unsupported_file_type") + + fake = newFakeCtx() + fake.seed("acme-api", 1, nil) + fake.queue("POST /v1/contexts/blobs/check", fakeResp{status: 413, body: map[string]any{ + "error": map[string]any{"message": "This upload does not fit in your plan's context storage.", "type": "invalid_request_error", "param": nil, "code": "context_quota_exceeded"}, + "used": 500 << 20, "quota": 500 << 20}}) + r = runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, exitDataErr, "500.0 MiB of 500.0 MiB used") +} + +// ─── pull ─────────────────────────────────────────────────────────────────── + +func TestContextPullWritesVerifiedFiles(t *testing.T) { + fake := newFakeCtx() + files := goodTree() + fake.seed("acme-api", 3, files) + dir := filepath.Join(t.TempDir(), "out") + r := runCtx(t, fake, ctxSetup{}, "context", "pull", "acme-api", dir) + mustCtxExit(t, r, 0) + checkGolden(t, "contexts/pull.golden", strings.ReplaceAll(r.stdout, dir, "DIR")) + for p, want := range files { + got, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(p))) + if err != nil || string(got) != want { + t.Errorf("%s: %q %v", p, got, err) + } + } + if runtime.GOOS != "windows" { + st, _ := os.Stat(filepath.Join(dir, "skills", "pr-review", "scripts", "run.sh")) + if st == nil || st.Mode().Perm()&0o100 == 0 { + t.Errorf("run.sh is not executable: %v", st) + } + } + for _, q := range fake.sent("GET", "/v1/contexts/"+fakeCtxID+"/file") { + if !strings.Contains(q.Query, "version=3") { + t.Errorf("a download is not pinned to the version read: %s", q.Query) + } + } + + // Round trip: what was pulled pushes as no change. + r = runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, 0) + wantAll(t, "stdout", r.stdout, "is up to date (version 3)") +} + +func TestContextPullRefusesANonEmptyFolder(t *testing.T) { + fake := newFakeCtx() + fake.seed("acme-api", 1, map[string]string{"files/a.md": "new\n"}) + dir := writeTree(t, map[string]string{"files/a.md": "mine\n"}) + r := runCtx(t, fake, ctxSetup{}, "context", "pull", "acme-api", dir) + mustCtxExit(t, r, exitUsage, "is not empty", "--force") + if len(fake.requests()) != 0 { + t.Errorf("sent %v", fake.requests()) + } + if b, _ := os.ReadFile(filepath.Join(dir, "files", "a.md")); string(b) != "mine\n" { + t.Error("overwrote a file") + } + r = runCtx(t, fake, ctxSetup{}, "context", "pull", "acme-api", dir, "--force") + mustCtxExit(t, r, 0) + if b, _ := os.ReadFile(filepath.Join(dir, "files", "a.md")); string(b) != "new\n" { + t.Errorf("--force did not write: %q", b) + } +} + +// A platform (or anything answering in its place) that sends a path the +// validator refuses gets nothing written, inside or outside DIR. +func TestContextPullRefusesHostilePaths(t *testing.T) { + for name, paths := range map[string][]string{ + "dotdot": {"files/../../evil.md"}, + "absolute": {"/tmp/evil.md"}, + "top dotdot": {"../evil.md"}, + "backslash": {"files\\..\\..\\evil.md"}, + "case collision": {"files/Readme.md", "files/README.md"}, + "file vs folder": {"files/docs", "files/Docs/a.md"}, + "reserved": {"files/CON"}, + "reserved ext": {"files/nul.txt"}, + "dotfile": {"files/.bashrc"}, + "not nfc": {"files/especificación.md"}, + "bad top": {"evil.md"}, + } { + t.Run(name, func(t *testing.T) { + fake := newFakeCtx() + c := fake.seed("acme-api", 1, nil) + for _, p := range paths { + fake.blobs[shaOf([]byte("x"))] = []byte("x") + c.Files = append(c.Files, api.ContextFile{Path: p, SHA256: shaOf([]byte("x")), Size: 1}) + } + parent := t.TempDir() + dir := filepath.Join(parent, "out") + r := runCtx(t, fake, ctxSetup{}, "context", "pull", "acme-api", dir) + mustCtxExit(t, r, exitDataErr, "nothing was written") + if len(fake.sent("GET", "/v1/contexts/"+fakeCtxID+"/file")) != 0 { + t.Error("downloaded a file") + } + entries, _ := os.ReadDir(parent) + if len(entries) != 0 { + t.Errorf("wrote %v", entries) + } + }) + } +} + +func TestContextPullRefusesBadBytes(t *testing.T) { + fake := newFakeCtx() + fake.seed("acme-api", 1, map[string]string{"files/a.md": "good\n", "files/b.md": "fine\n"}) + fake.corrupt["files/b.md"] = true + dir := filepath.Join(t.TempDir(), "out") + r := runCtx(t, fake, ctxSetup{}, "context", "pull", "acme-api", dir) + mustCtxExit(t, r, exitUnavailable, "does not match its sha256") + if _, err := os.Stat(filepath.Join(dir, "files", "b.md")); !os.IsNotExist(err) { + t.Errorf("kept a bad file: %v", err) + } +} + +func TestContextHelp(t *testing.T) { + var all strings.Builder + all.WriteString(contextCmd.Long) + for _, c := range contextCmd.Commands() { + all.WriteString(c.Long + c.Short + c.Example) + if c.InheritedFlags().Lookup("token-file") == nil { + t.Errorf("nan context %s has no --token-file", c.Name()) + } + } + for _, want := range []string{"contexts:read", "contexts:write", contextsDocs, "AGENTS.md", "--allow-secrets", "--skip-invalid", "65"} { + if !strings.Contains(all.String(), want) { + t.Errorf("nan context help lacks %q", want) + } + } + if strings.Contains(all.String(), "—") { + t.Error("an em-dash in the context help") + } + for verb, want := range map[string]string{"list": "ls", "upload": "push", "delete": "rm", "frobnicate": "unknown command"} { + r := runCtx(t, newFakeCtx(), ctxSetup{}, "context", verb) + mustCtxExit(t, r, exitUsage, want) + } +} diff --git a/cmd/json_output_test.go b/cmd/json_output_test.go index 720da25..5a85820 100644 --- a/cmd/json_output_test.go +++ b/cmd/json_output_test.go @@ -84,6 +84,9 @@ func jsonCases(t *testing.T) map[string][]jsonCase { if err := os.WriteFile(hook, []byte(testHookURL+"\n"), 0o600); err != nil { t.Fatal(err) } + // A folder to push: skipped and refused files print their notes on + // stderr only. + ctxDir := writeTree(t, map[string]string{"AGENTS.md": "Use tabs.\n", "files/.env": "S=1\n", "CLAUDE.md": "x\n"}) return map[string][]jsonCase{ "nan approvals ls": {{args: []string{"approvals", "ls", "--json"}}}, "nan approvals show": {{args: []string{"approvals", "show", testRunID, "--json"}}}, @@ -118,6 +121,14 @@ func jsonCases(t *testing.T) map[string][]jsonCase { // goes to stderr only. {args: []string{"notifications", "add", "ci", "--kind", "https", "--url-stdin", "--event", "run.failed", "--json"}, stdin: testHookHTTPS}, }, + "nan context create": {{args: []string{"context", "create", "new-ctx", "--json"}}}, + "nan context edit": {{args: []string{"context", "edit", "acme-api", "--description", "new", "--json"}}}, + "nan context ls": {{args: []string{"context", "ls", "--json"}}}, + "nan context push": { + {args: []string{"context", "push", "acme-api", ctxDir, "--skip-invalid", "--json"}}, + {args: []string{"context", "push", "acme-api", ctxDir, "--skip-invalid", "--dry-run", "--json"}}, + }, + "nan context show": {{args: []string{"context", "show", "acme-api", "--json"}}}, "nan notifications deliveries": {{args: []string{"notifications", "deliveries", "ci receiver", "--limit", "3", "--json"}}}, "nan notifications edit": { {args: []string{"notifications", "edit", "team alerts", "--disable", "--json"}}, @@ -288,8 +299,14 @@ func runJSONCase(t *testing.T, c jsonCase) (code int, msg, stdout, stderr string phase2.runGet = awaitingRunJSON() } writeHandler := write.handler(t) + ctxFake := newFakeCtx() + ctxFake.seed("acme-api", 2, map[string]string{"files/a.md": "a\n"}) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { p := r.URL.Path + if strings.HasPrefix(p, "/v1/contexts") { + ctxFake.ServeHTTP(w, r) + return + } toWrite := p == "/api/auth/me" || p == "/api/account/secrets" || strings.HasPrefix(p, "/v1/secrets") || strings.HasPrefix(p, "/v1/notification-channels") || strings.HasSuffix(p, "/triggers") || strings.Contains(p, "/triggers/") || diff --git a/cmd/testdata/contexts/ls.golden b/cmd/testdata/contexts/ls.golden new file mode 100644 index 0000000..b462f08 --- /dev/null +++ b/cmd/testdata/contexts/ls.golden @@ -0,0 +1,3 @@ +NAME VERSION FILES SIZE UPDATED DESCRIPTION +acme-api 3 3 83 B 2026-10-21 10:00 Acme API conventions +storage: 1.5 KiB of 500.0 MiB used diff --git a/cmd/testdata/contexts/pull.golden b/cmd/testdata/contexts/pull.golden new file mode 100644 index 0000000..955ac81 --- /dev/null +++ b/cmd/testdata/contexts/pull.golden @@ -0,0 +1 @@ +context acme-api version 3 written to DIR (4 files, 101 B) diff --git a/cmd/testdata/contexts/push.golden b/cmd/testdata/contexts/push.golden new file mode 100644 index 0000000..716ad67 --- /dev/null +++ b/cmd/testdata/contexts/push.golden @@ -0,0 +1 @@ +context acme-api: version 3 published (6 files, 116 B; 5 added, 0 changed, 0 removed; uploaded 4 files, 101 B) diff --git a/cmd/testdata/contexts/push_dry_run.golden b/cmd/testdata/contexts/push_dry_run.golden new file mode 100644 index 0000000..d1c84e7 --- /dev/null +++ b/cmd/testdata/contexts/push_dry_run.golden @@ -0,0 +1,6 @@ +context acme-api: version 3 -> 4 (dry run, nothing changed) + + files/new.md (9 B) + + skills/pr-review/SKILL.md (67 B) + ~ AGENTS.md (10 B) + - files/old.md +2 added, 1 changed, 1 removed; 3 files to upload (86 B) diff --git a/cmd/testdata/contexts/push_dry_run.json.golden b/cmd/testdata/contexts/push_dry_run.json.golden new file mode 100644 index 0000000..8031c09 --- /dev/null +++ b/cmd/testdata/contexts/push_dry_run.json.golden @@ -0,0 +1,20 @@ +{ + "context": "acme-api", + "base_version": 3, + "changes": { + "added": [ + "files/new.md", + "skills/pr-review/SKILL.md" + ], + "changed": [ + "AGENTS.md" + ], + "removed": [ + "files/old.md" + ] + }, + "upload": { + "files": 3, + "bytes": 86 + } +} diff --git a/cmd/testdata/contexts/push_refused.stderr.golden b/cmd/testdata/contexts/push_refused.stderr.golden new file mode 100644 index 0000000..0d2dec0 --- /dev/null +++ b/cmd/testdata/contexts/push_refused.stderr.golden @@ -0,0 +1,10 @@ +skipped .git (dotfiles are not kept in a context) +skipped files/.env (dotfiles are not kept in a context) +skipped files/sub/.hidden (dotfiles are not kept in a context) +refused CLAUDE.md: bad_top_level (rename it to AGENTS.md: that is the file the agents read) +refused README.md: bad_top_level (only AGENTS.md lives at the top; put reference files under files/) +refused files/con.txt: reserved_name +refused files/rocket🚀.md: bad_character +refused notes/a.md: bad_top_level (put reference files under files/ and skills under skills//) +refused skills/Bad Skill/SKILL.md: bad_character (a skill folder name is lowercase letters, digits and dashes) +refused skills/no-skill-md/SKILL.md: skill_invalid (every skill folder needs a SKILL.md) diff --git a/cmd/testdata/contexts/show.golden b/cmd/testdata/contexts/show.golden new file mode 100644 index 0000000..b053ea2 --- /dev/null +++ b/cmd/testdata/contexts/show.golden @@ -0,0 +1,12 @@ +name acme-api +id c7000000-0000-4000-8000-000000000001 +description Acme API conventions +version 3 (2026-10-21 10:00) +files 3, 83 B +instructions yes +skills pr-review + +PATH SIZE SHA256 +AGENTS.md 10 B 2d812d6fdf8… +files/spec.md 6 B 6b04df15eab… +skills/pr-review/SKILL.md 67 B da1195fa204… diff --git a/go.mod b/go.mod index 37ecc08..7ef4447 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,9 @@ require ( github.com/charmbracelet/lipgloss v1.1.0 github.com/charmbracelet/x/term v0.2.2 github.com/spf13/cobra v1.10.2 + github.com/spf13/pflag v1.0.9 + go.yaml.in/yaml/v3 v3.0.4 + golang.org/x/text v0.39.0 ) require ( @@ -29,9 +32,6 @@ require ( github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect - github.com/spf13/pflag v1.0.9 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/sys v0.44.0 // indirect - golang.org/x/text v0.39.0 // indirect ) diff --git a/go.sum b/go.sum index a2f04d3..092b071 100644 --- a/go.sum +++ b/go.sum @@ -60,4 +60,5 @@ golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/api/contexts.go b/internal/api/contexts.go new file mode 100644 index 0000000..e106541 --- /dev/null +++ b/internal/api/contexts.go @@ -0,0 +1,455 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "time" +) + +// Agent contexts (helmcode/nan-devops AGENT-CONTEXT.md §2): versioned file +// trees a member keeps for their runs and automations, under /v1/contexts. +// Reads take a CLI session, the member's sk- key or a platform token with +// contexts:read; writes take a CLI session or a token with contexts:write. +// +// Content goes up as content-addressed blobs (sha256): POST blobs/check says +// which ones the platform lacks, a blob of at most PartSize bytes is one PUT, +// a larger one a multipart upload of PartSize parts that can be resumed. A +// version is then published with the full file list and the version it was +// based on. This client only speaks the protocol; the push and pull logic +// lives with the command. + +// ContextPartSize is the platform's single-upload limit and multipart part +// size (16 MiB). +const ContextPartSize = 16 << 20 + +// ContextMaxFileBytes is the largest file a context holds (100 MiB). +const ContextMaxFileBytes = 100 << 20 + +// maxCheckShas is the platform's cap on one blobs/check body. +const maxCheckShas = 1000 + +// Context is a context's metadata at its current version (or, for a +// publish replay, at the version that publish created). +type Context struct { + ID string `json:"id"` + Object string `json:"object"` + Name string `json:"name"` + Description string `json:"description"` + Version int `json:"version"` + TreeSHA256 string `json:"tree_sha256"` + FileCount int `json:"file_count"` + TotalBytes int64 `json:"total_bytes"` + HasInstructions bool `json:"has_instructions"` + Skills []string `json:"skills"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +// ContextFile is one file of a version: what a pull writes. +type ContextFile struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` + Executable bool `json:"executable"` +} + +// ContextDetail is GET /v1/contexts/{id_or_name}: the context and the files +// of its current version. +type ContextDetail struct { + Context + Files []ContextFile `json:"files"` +} + +type ContextUsage struct { + StoredBytes int64 `json:"stored_bytes"` + QuotaBytes int64 `json:"quota_bytes"` +} + +type ContextList struct { + Data []Context `json:"data"` + Usage ContextUsage `json:"usage"` +} + +// ContextVersionInfo is one retained version. +type ContextVersionInfo struct { + Version int `json:"version"` + TreeSHA256 string `json:"tree_sha256"` + FileCount int `json:"file_count"` + TotalBytes int64 `json:"total_bytes"` + CreatedAt string `json:"created_at"` +} + +// SecretFinding is where a text blob looks like it holds a credential. The +// platform never returns the match itself. +type SecretFinding struct { + Line int `json:"line"` + Pattern string `json:"pattern"` +} + +// ContextBlob is a stored blob. +type ContextBlob struct { + SHA256 string `json:"sha256"` + Size int64 `json:"size"` + Format string `json:"format"` + SecretFindings []SecretFinding `json:"secret_findings"` +} + +// ContextUpload is a multipart upload in progress. +type ContextUpload struct { + UploadID string `json:"upload_id"` + PartSize int `json:"part_size"` + Parts int `json:"parts"` + NextPart int `json:"next_part"` +} + +// PublishFile is one entry of a publish. +type PublishFile struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` + Executable bool `json:"executable"` +} + +// PublishRequest is the body of POST /v1/contexts/{id}/versions. +type PublishRequest struct { + BaseVersion int `json:"base_version"` + Files []PublishFile `json:"files"` + AllowSecretLike bool `json:"allow_secret_like"` +} + +// ContextPatch is the body of PATCH /v1/contexts/{id}: nil fields stay. +type ContextPatch struct { + Name *string `json:"name,omitempty"` + Description *string `json:"description,omitempty"` +} + +// FileRefusal is one refused file of a publish: invalid_path, +// unsupported_file_type, skill_invalid (reason per file, detail optional). +type FileRefusal struct { + Path string `json:"path"` + Reason string `json:"reason"` + Detail string `json:"detail,omitempty"` +} + +// SecretHit is one finding of 422 context_secret_detected. +type SecretHit struct { + Path string `json:"path"` + Line int `json:"line"` + Pattern string `json:"pattern"` +} + +// ContextError is a refusal of the contexts API with the fields the +// platform adds next to the error envelope. It unwraps to the *APIError, so +// the generic exit mapping still sees the code. +type ContextError struct { + *APIError + // RetryAfter is the Retry-After header (0 when absent). + RetryAfter time.Duration + // CurrentVersion is set on 409 version_conflict. + CurrentVersion *int + // MissingBlobs is 409 blob_missing's sha256 list. + MissingBlobs []string + // Files is the per-file refusal list of invalid_path, + // unsupported_file_type and skill_invalid. + Files []FileRefusal + // Findings is 422 context_secret_detected's list. + Findings []SecretHit + // Used and Quota are 413 context_quota_exceeded's figures. + Used, Quota int64 +} + +func (e *ContextError) Error() string { return e.APIError.Error() } +func (e *ContextError) Unwrap() error { return e.APIError } + +// AsContextError finds a *ContextError in err. +func AsContextError(err error) (*ContextError, bool) { + var ce *ContextError + if errors.As(err, &ce) { + return ce, true + } + return nil, false +} + +func (c *RunsClient) contextError(status int, header http.Header, body []byte) error { + err := c.statusError(status, body) + apiErr, ok := err.(*APIError) + if !ok { + return err + } + ce := &ContextError{APIError: apiErr} + if s := header.Get("Retry-After"); s != "" { + if n, perr := strconv.Atoi(s); perr == nil && n >= 0 { + ce.RetryAfter = time.Duration(n) * time.Second + } + } + var extra struct { + CurrentVersion *int `json:"current_version"` + SHA256 []string `json:"sha256"` + Files []FileRefusal `json:"files"` + Findings []SecretHit `json:"findings"` + Used int64 `json:"used"` + Quota int64 `json:"quota"` + Automations []Ref `json:"automations"` + } + if json.Unmarshal(body, &extra) == nil { + ce.CurrentVersion = extra.CurrentVersion + ce.MissingBlobs = extra.SHA256 + ce.Files = extra.Files + ce.Findings = extra.Findings + ce.Used, ce.Quota = extra.Used, extra.Quota + if len(apiErr.Automations) == 0 { + apiErr.Automations = extra.Automations + } + } + return ce +} + +// ctxDo sends req with client and decodes a 2xx JSON answer into out. +func (c *RunsClient) ctxDo(client *http.Client, req *http.Request, out any) (int, error) { + resp, err := client.Do(req) + if err != nil { + return 0, fmt.Errorf("could not reach nan.builders: %w", err) + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, maxResponse)) + if err != nil { + return resp.StatusCode, fmt.Errorf("could not read the answer from nan.builders: %w", err) + } + if resp.StatusCode >= 400 { + return resp.StatusCode, c.contextError(resp.StatusCode, resp.Header, body) + } + if out != nil && len(bytes.TrimSpace(body)) > 0 { + if err := json.Unmarshal(body, out); err != nil { + return resp.StatusCode, fmt.Errorf("unexpected answer from nan.builders: %w", err) + } + } + return resp.StatusCode, nil +} + +func (c *RunsClient) ctxJSON(ctx context.Context, method, path string, body, out any, idemKey string) (int, error) { + req, err := c.newRequest(ctx, method, path, body) + if err != nil { + return 0, err + } + if idemKey != "" { + req.Header.Set("Idempotency-Key", idemKey) + } + return c.ctxDo(c.http, req, out) +} + +// rawRequest is a request whose body is streamed bytes of a known length. +func (c *RunsClient) rawRequest(ctx context.Context, method, path string, body io.Reader, size int64) (*http.Request, error) { + if body == nil || size == 0 { + body = http.NoBody + } + req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, body) + if err != nil { + return nil, err + } + req.ContentLength = size + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "nan-cli") + c.authorize(req) + return req, nil +} + +func contextPath(id string) string { return "/v1/contexts/" + url.PathEscape(id) } + +func blobPath(sha string) string { return "/v1/contexts/blobs/" + url.PathEscape(sha) } + +func uploadPath(sha, uploadID string) string { + return blobPath(sha) + "/uploads/" + url.PathEscape(uploadID) +} + +// ListContexts is GET /v1/contexts. +func (c *RunsClient) ListContexts(ctx context.Context) (*ContextList, error) { + out := &ContextList{} + _, err := c.ctxJSON(ctx, http.MethodGet, "/v1/contexts", nil, out, "") + return out, err +} + +// GetContext is GET /v1/contexts/{id_or_name}. +func (c *RunsClient) GetContext(ctx context.Context, idOrName string) (*ContextDetail, error) { + out := &ContextDetail{} + _, err := c.ctxJSON(ctx, http.MethodGet, contextPath(idOrName), nil, out, "") + return out, err +} + +// CreateContext is POST /v1/contexts: the context with an empty version 1. +func (c *RunsClient) CreateContext(ctx context.Context, name, description, idemKey string) (*Context, error) { + body := map[string]string{"name": name} + if description != "" { + body["description"] = description + } + out := &Context{} + _, err := c.ctxJSON(ctx, http.MethodPost, "/v1/contexts", body, out, idemKey) + return out, err +} + +// UpdateContext is PATCH /v1/contexts/{id} (name and description only; no +// new version). +func (c *RunsClient) UpdateContext(ctx context.Context, id string, p ContextPatch) (*Context, error) { + out := &Context{} + _, err := c.ctxJSON(ctx, http.MethodPatch, contextPath(id), p, out, "") + return out, err +} + +// DeleteContext is DELETE /v1/contexts/{id}; 409 context_in_use names the +// automations that use it. +func (c *RunsClient) DeleteContext(ctx context.Context, id string) error { + _, err := c.ctxJSON(ctx, http.MethodDelete, contextPath(id), nil, nil, "") + return err +} + +// ContextVersions is GET /v1/contexts/{id}/versions, newest first. +func (c *RunsClient) ContextVersions(ctx context.Context, id string) ([]ContextVersionInfo, error) { + var out []ContextVersionInfo + _, err := c.ctxJSON(ctx, http.MethodGet, contextPath(id)+"/versions", nil, &out, "") + return out, err +} + +// CheckBlobs is POST /v1/contexts/blobs/check: which of shas the platform +// does not hold for this member (asked in batches of the platform's cap). +func (c *RunsClient) CheckBlobs(ctx context.Context, shas []string) ([]string, error) { + missing := []string{} + for start := 0; start < len(shas); start += maxCheckShas { + end := min(start+maxCheckShas, len(shas)) + var out struct { + Missing []string `json:"missing"` + } + if _, err := c.ctxJSON(ctx, http.MethodPost, "/v1/contexts/blobs/check", map[string][]string{"sha256": shas[start:end]}, &out, ""); err != nil { + return nil, err + } + missing = append(missing, out.Missing...) + } + return missing, nil +} + +// PutBlob is PUT /v1/contexts/blobs/{sha256}: one upload of at most +// ContextPartSize bytes. filename is a type hint for the platform only +// (X-Nan-Filename); it is never stored. The request has no client timeout: +// ctx bounds it. +func (c *RunsClient) PutBlob(ctx context.Context, sha string, body io.Reader, size int64, filename string) (*ContextBlob, error) { + req, err := c.rawRequest(ctx, http.MethodPut, blobPath(sha), body, size) + if err != nil { + return nil, err + } + if filename != "" { + req.Header.Set("X-Nan-Filename", filename) + } + out := &ContextBlob{} + _, err = c.ctxDo(c.stream, req, out) + return out, err +} + +// StartUpload is POST /v1/contexts/blobs/{sha256}/uploads for a blob larger +// than ContextPartSize. When the platform already holds the blob it answers +// it instead of an upload (blob non-nil, upload nil). +func (c *RunsClient) StartUpload(ctx context.Context, sha string, size int64, filename string) (*ContextUpload, *ContextBlob, error) { + req, err := c.newRequest(ctx, http.MethodPost, blobPath(sha)+"/uploads", map[string]any{"size": size, "filename": filename}) + if err != nil { + return nil, nil, err + } + var raw json.RawMessage + status, err := c.ctxDo(c.http, req, &raw) + if err != nil { + return nil, nil, err + } + if status == http.StatusOK { + b := &ContextBlob{} + return nil, b, json.Unmarshal(raw, b) + } + u := &ContextUpload{} + return u, nil, json.Unmarshal(raw, u) +} + +// UploadStatus is GET …/uploads/{upload_id}: where a multipart upload +// stands, to resume it. +func (c *RunsClient) UploadStatus(ctx context.Context, sha, uploadID string) (*ContextUpload, error) { + out := &ContextUpload{} + _, err := c.ctxJSON(ctx, http.MethodGet, uploadPath(sha, uploadID), nil, out, "") + return out, err +} + +// PutPart is PUT …/uploads/{upload_id}/parts/{n}; it answers the next part +// the platform expects. +func (c *RunsClient) PutPart(ctx context.Context, sha, uploadID string, n int, body io.Reader, size int64) (int, error) { + req, err := c.rawRequest(ctx, http.MethodPut, uploadPath(sha, uploadID)+"/parts/"+strconv.Itoa(n), body, size) + if err != nil { + return 0, err + } + var out struct { + NextPart int `json:"next_part"` + } + _, err = c.ctxDo(c.stream, req, &out) + return out.NextPart, err +} + +// CompleteUpload is POST …/uploads/{upload_id}/complete. +func (c *RunsClient) CompleteUpload(ctx context.Context, sha, uploadID string) (*ContextBlob, error) { + req, err := c.newRequest(ctx, http.MethodPost, uploadPath(sha, uploadID)+"/complete", nil) + if err != nil { + return nil, err + } + out := &ContextBlob{} + _, err = c.ctxDo(c.stream, req, out) + return out, err +} + +// AbortUpload is DELETE …/uploads/{upload_id}. +func (c *RunsClient) AbortUpload(ctx context.Context, sha, uploadID string) error { + _, err := c.ctxJSON(ctx, http.MethodDelete, uploadPath(sha, uploadID), nil, nil, "") + return err +} + +// PublishContext is POST /v1/contexts/{id}/versions. The answer is the +// context at the new version. It can take a while (the platform rescans and +// reads every SKILL.md), so ctx bounds it instead of the client timeout. +func (c *RunsClient) PublishContext(ctx context.Context, id string, in PublishRequest, idemKey string) (*Context, error) { + if in.Files == nil { + in.Files = []PublishFile{} + } + req, err := c.newRequest(ctx, http.MethodPost, contextPath(id)+"/versions", in) + if err != nil { + return nil, err + } + if idemKey != "" { + req.Header.Set("Idempotency-Key", idemKey) + } + out := &Context{} + _, err = c.ctxDo(c.stream, req, out) + return out, err +} + +// OpenContextFile is GET /v1/contexts/{id}/file?path=…&version=N: the +// file's bytes as a stream the caller closes. The caller verifies them +// against the manifest; nothing here trusts the length or the content. +func (c *RunsClient) OpenContextFile(ctx context.Context, id, path string, version int) (io.ReadCloser, error) { + q := url.Values{} + q.Set("path", path) + if version > 0 { + q.Set("version", strconv.Itoa(version)) + } + req, err := c.newRequest(ctx, http.MethodGet, contextPath(id)+"/file?"+q.Encode(), nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/octet-stream") + resp, err := c.stream.Do(req) + if err != nil { + return nil, fmt.Errorf("could not reach nan.builders: %w", err) + } + if resp.StatusCode >= 400 { + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + return nil, c.contextError(resp.StatusCode, resp.Header, body) + } + return resp.Body, nil +} diff --git a/internal/api/contexts_test.go b/internal/api/contexts_test.go new file mode 100644 index 0000000..e74a425 --- /dev/null +++ b/internal/api/contexts_test.go @@ -0,0 +1,161 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func ctxServer(t *testing.T, h http.HandlerFunc) *RunsClient { + t.Helper() + srv := httptest.NewServer(h) + t.Cleanup(srv.Close) + return NewRunsClient("sess", "").WithBaseURL(srv.URL) +} + +// The contexts API puts its extra fields next to the error envelope (cloud-api +// ContextsHandler.fail); every one of them reaches the caller. +func TestContextErrorCarriesTheExtraFields(t *testing.T) { + for _, tc := range []struct { + body string + check func(*ContextError) bool + }{ + {`{"error":{"message":"changed","type":"invalid_request_error","param":"base_version","code":"version_conflict"},"current_version":7}`, + func(e *ContextError) bool { + return e.CurrentVersion != nil && *e.CurrentVersion == 7 && e.Param == "base_version" + }}, + {`{"error":{"message":"gone","type":"invalid_request_error","param":null,"code":"blob_missing"},"sha256":["aa","bb"]}`, + func(e *ContextError) bool { return strings.Join(e.MissingBlobs, ",") == "aa,bb" }}, + {`{"error":{"message":"names","type":"invalid_request_error","param":null,"code":"invalid_path"},"files":[{"path":"files/.env","reason":"dotfile"}]}`, + func(e *ContextError) bool { return len(e.Files) == 1 && e.Files[0].Reason == "dotfile" }}, + {`{"error":{"message":"secrets","type":"invalid_request_error","param":null,"code":"context_secret_detected"},"findings":[{"path":"AGENTS.md","line":3,"pattern":"github_token"}]}`, + func(e *ContextError) bool { return len(e.Findings) == 1 && e.Findings[0].Line == 3 }}, + {`{"error":{"message":"quota","type":"invalid_request_error","param":null,"code":"context_quota_exceeded"},"used":10,"quota":5,"growing":true}`, + func(e *ContextError) bool { return e.Used == 10 && e.Quota == 5 }}, + {`{"error":{"message":"in use","type":"invalid_request_error","param":null,"code":"context_in_use"},"automations":[{"id":"a1","name":"PR review"}]}`, + func(e *ContextError) bool { return len(e.Automations) == 1 && e.Automations[0].Name == "PR review" }}, + } { + c := ctxServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "5") + w.WriteHeader(409) + _, _ = io.WriteString(w, tc.body) + }) + _, err := c.GetContext(context.Background(), "acme") + ce, ok := AsContextError(err) + if !ok || !tc.check(ce) || ce.RetryAfter != 5*time.Second { + t.Errorf("%s: %+v", tc.body, ce) + continue + } + var apiErr *APIError + if !errors.As(err, &apiErr) || apiErr.Code == "" { + t.Errorf("does not unwrap to the APIError: %v", err) + } + } +} + +func TestContextSessionExpired(t *testing.T) { + c := ctxServer(t, func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(401) }) + if _, err := c.ListContexts(context.Background()); !errors.Is(err, ErrSessionExpired) { + t.Errorf("%v", err) + } +} + +func TestCheckBlobsBatchesAtTheCap(t *testing.T) { + var sizes []int + c := ctxServer(t, func(w http.ResponseWriter, r *http.Request) { + var in struct { + SHA256 []string `json:"sha256"` + } + _ = json.NewDecoder(r.Body).Decode(&in) + sizes = append(sizes, len(in.SHA256)) + _ = json.NewEncoder(w).Encode(map[string]any{"missing": in.SHA256[:1]}) + }) + shas := make([]string, 2001) + for i := range shas { + shas[i] = fmt.Sprintf("%064x", i) + } + missing, err := c.CheckBlobs(context.Background(), shas) + if err != nil || len(missing) != 3 || fmt.Sprint(sizes) != "[1000 1000 1]" { + t.Errorf("%v %v %v", err, missing, sizes) + } +} + +func TestPutBlobStreamsWithItsLengthAndHint(t *testing.T) { + c := ctxServer(t, func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + if r.Method != http.MethodPut || r.URL.Path != "/v1/contexts/blobs/abc" || r.ContentLength != 5 || string(b) != "hello" || + r.Header.Get("X-Nan-Filename") != "a.md" || r.Header.Get("Content-Type") != "application/octet-stream" || + r.Header.Get("Cookie") != "nan_session=sess" { + t.Errorf("%s %s len=%d %q %v", r.Method, r.URL.Path, r.ContentLength, b, r.Header) + } + w.WriteHeader(201) + _, _ = io.WriteString(w, `{"sha256":"abc","size":5,"format":"text","secret_findings":[{"line":1,"pattern":"github_token"}]}`) + }) + b, err := c.PutBlob(context.Background(), "abc", strings.NewReader("hello"), 5, "a.md") + if err != nil || b.Size != 5 || len(b.SecretFindings) != 1 { + t.Errorf("%+v %v", b, err) + } +} + +// A blob the member already has answers 200 with it instead of an upload. +func TestStartUploadTellsAStoredBlobFromAnUpload(t *testing.T) { + status := 200 + c := ctxServer(t, func(w http.ResponseWriter, r *http.Request) { + var in map[string]any + _ = json.NewDecoder(r.Body).Decode(&in) + if in["size"] != float64(20<<20) || in["filename"] != "big.pdf" { + t.Errorf("body %v", in) + } + w.WriteHeader(status) + if status == 200 { + _, _ = io.WriteString(w, `{"sha256":"abc","size":20971520,"format":"pdf","secret_findings":[]}`) + return + } + _, _ = io.WriteString(w, `{"upload_id":"u1","part_size":16777216,"parts":2,"next_part":1}`) + }) + up, blob, err := c.StartUpload(context.Background(), "abc", 20<<20, "big.pdf") + if err != nil || up != nil || blob == nil || blob.Format != "pdf" { + t.Errorf("%+v %+v %v", up, blob, err) + } + status = 201 + up, blob, err = c.StartUpload(context.Background(), "abc", 20<<20, "big.pdf") + if err != nil || blob != nil || up == nil || up.UploadID != "u1" || up.Parts != 2 { + t.Errorf("%+v %+v %v", up, blob, err) + } +} + +func TestOpenContextFileRefusalIsAContextError(t *testing.T) { + c := ctxServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("path") != "files/a b.md" || r.URL.Query().Get("version") != "3" { + t.Errorf("query %q", r.URL.RawQuery) + } + w.WriteHeader(404) + _, _ = io.WriteString(w, `{"error":{"message":"file not found","type":"invalid_request_error","param":"path","code":"file_not_found"}}`) + }) + _, err := c.OpenContextFile(context.Background(), "id", "files/a b.md", 3) + if ce, ok := AsContextError(err); !ok || ce.Code != "file_not_found" { + t.Errorf("%v", err) + } +} + +func TestPublishSendsAnEmptyListNotNull(t *testing.T) { + c := ctxServer(t, func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + if string(b) != `{"base_version":1,"files":[],"allow_secret_like":false}` || r.Header.Get("Idempotency-Key") != "k" { + t.Errorf("%s %v", b, r.Header) + } + w.WriteHeader(201) + _, _ = io.WriteString(w, `{"id":"id","name":"acme","version":2}`) + }) + out, err := c.PublishContext(context.Background(), "id", PublishRequest{BaseVersion: 1}, "k") + if err != nil || out.Version != 2 { + t.Errorf("%+v %v", out, err) + } +} diff --git a/internal/contexts/path.go b/internal/contexts/path.go new file mode 100644 index 0000000..8362424 --- /dev/null +++ b/internal/contexts/path.go @@ -0,0 +1,290 @@ +// Package contexts holds the agent-context domain of devops AGENT-CONTEXT.md: +// path validators, content typing, the Office and secret checks, the strict +// SKILL.md decoder and the quota rules. Everything in this file set is pure +// (no database, no R2): the API, the manifest builder, nan-run and the CLI +// share it. +package contexts + +import ( + "errors" + "fmt" + "regexp" + "sort" + "strings" + "unicode" + "unicode/utf8" + + "golang.org/x/text/cases" + "golang.org/x/text/unicode/norm" +) + +// Reason is the per-file refusal code the UI and CLI show next to a file +// (AGENT-CONTEXT.md §1.5). The wire values are part of the API contract. +type Reason string + +const ( + ReasonDotfile Reason = "dotfile" + ReasonBadCharacter Reason = "bad_character" + ReasonNotNFC Reason = "not_nfc" + ReasonTooLong Reason = "too_long" + ReasonTooDeep Reason = "too_deep" + ReasonReservedName Reason = "reserved_name" + ReasonCaseCollision Reason = "case_collision" + ReasonBadTopLevel Reason = "bad_top_level" + ReasonNotSupportedYet Reason = "not_supported_yet" // mcp/ (reserved for v2) +) + +// Stored-path limits (§1.5). +const ( + MaxStoredPathBytes = 512 + MaxStoredPathDepth = 8 + MaxComponentBytes = 128 + InstructionsPath = "AGENTS.md" + SkillFileName = "SKILL.md" + topSkills = "skills" + topFiles = "files" + topMCP = "mcp" + contextNameMaxBytes = 63 +) + +// PathError is one refused path with its reason. +type PathError struct { + Path string + Reason Reason +} + +func (e *PathError) Error() string { return fmt.Sprintf("contexts: invalid path (%s)", e.Reason) } + +// ErrInvalidPath matches every *PathError with errors.Is. +var ErrInvalidPath = errors.New("contexts: invalid path") + +// Is makes errors.Is(err, ErrInvalidPath) hold for a *PathError. +func (e *PathError) Is(target error) bool { return target == ErrInvalidPath } + +func pathErr(p string, r Reason) *PathError { return &PathError{Path: p, Reason: r} } + +var ( + reContextName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`) + reSkillName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,63}$`) +) + +// ValidContextName reports whether name is an acceptable context name: the +// contexts_name_shape CHECK of migration 126, and not a Windows reserved +// basename (`con`, `nul`, `com1`, ...). The second rule is not in the SQL +// CHECK: a context's name becomes a directory of the run's manifest +// (files//...), and the manifest validator refuses reserved names, so +// a context named `con` could never deliver its files. +// +// Two more names are refused because the API could never reach them by +// name: a UUID-shaped name (GET /v1/contexts/{x} always reads it as an id) +// and "blobs" (the /v1/contexts/blobs/... upload routes). +func ValidContextName(name string) bool { + return reContextName.MatchString(name) && !isWindowsReserved(name) && name != reservedBlobs && !uuidShaped(name) +} + +const reservedBlobs = "blobs" + +// uuidShaped reports the 8-4-4-4-12 lowercase hex form. +func uuidShaped(s string) bool { + if len(s) != 36 { + return false + } + for i := 0; i < len(s); i++ { + c := s[i] + switch i { + case 8, 13, 18, 23: + if c != '-' { + return false + } + default: + if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'f') { + return false + } + } + } + return true +} + +// ValidSkillName reports whether name is an acceptable skill directory name. +func ValidSkillName(name string) bool { return reSkillName.MatchString(name) } + +// ValidateStoredPath checks one path a member publishes (§1.5 "Stored +// paths"). It returns nil or a *PathError carrying the reason. Uniqueness +// (case_collision) is a property of the whole tree: see ValidateTree. +func ValidateStoredPath(p string) error { + comps, err := checkPathShape(p, MaxStoredPathBytes, MaxStoredPathDepth) + if err != nil { + return err + } + switch top := comps[0]; { + case len(comps) == 1 && top == InstructionsPath: + return nil + case top == topSkills: + // skills// + if len(comps) < 3 { + return pathErr(p, ReasonBadTopLevel) + } + if !ValidSkillName(comps[1]) { + return pathErr(p, ReasonBadCharacter) + } + return nil + case top == topFiles: + if len(comps) < 2 { + return pathErr(p, ReasonBadTopLevel) + } + return nil + case top == topMCP: + return pathErr(p, ReasonNotSupportedYet) + default: + return pathErr(p, ReasonBadTopLevel) + } +} + +// checkPathShape applies the rules shared by stored and manifest paths: +// UTF-8, NFC, byte and depth limits, and the component rules. +func checkPathShape(p string, maxBytes, maxDepth int) ([]string, error) { + if !utf8.ValidString(p) { + return nil, pathErr(p, ReasonBadCharacter) + } + if !norm.NFC.IsNormalString(p) { + return nil, pathErr(p, ReasonNotNFC) + } + if len(p) == 0 { + return nil, pathErr(p, ReasonBadCharacter) + } + if len(p) > maxBytes { + return nil, pathErr(p, ReasonTooLong) + } + comps := strings.Split(p, "/") + if len(comps) > maxDepth { + return nil, pathErr(p, ReasonTooDeep) + } + for _, c := range comps { + if r := componentReason(c); r != "" { + return nil, pathErr(p, r) + } + } + return comps, nil +} + +// componentReason returns the reason one path component is refused, or "". +func componentReason(c string) Reason { + if c == "" { + return ReasonBadCharacter // empty component: leading, trailing or doubled '/' + } + if len(c) > MaxComponentBytes { + return ReasonTooLong + } + if c[0] == '.' { + return ReasonDotfile // .git, .env.example, ., .. + } + if c[0] == ' ' || c[len(c)-1] == ' ' || c[len(c)-1] == '.' { + return ReasonBadCharacter + } + for _, r := range c { + if !allowedRune(r) { + return ReasonBadCharacter + } + } + if isWindowsReserved(c) { + return ReasonReservedName + } + return "" +} + +// allowedRune: Unicode letters (L*), combining marks (Mn, Mc), decimal +// digits (Nd), the ASCII space and `. _ - ( ) + , @`, minus every +// Default_Ignorable_Code_Point. Everything else (Cc, Cf, other separators, +// symbols, emoji, private use, unassigned, shell/Windows-hostile ASCII) is +// refused. +func allowedRune(r rune) bool { + switch r { + case ' ', '.', '_', '-', '(', ')', '+', ',', '@': + return true + } + if !(unicode.IsLetter(r) || unicode.In(r, unicode.Mn, unicode.Mc, unicode.Nd)) { + return false + } + return !defaultIgnorable(r) +} + +// defaultIgnorable covers Default_Ignorable_Code_Point among the categories +// allowedRune admits: the derived property is Other_Default_Ignorable_Code_Point +// + Cf + Variation_Selector (minus a few code points that are Cf, refused +// anyway). So Hangul fillers (Lo), U+034F and the Khmer inherent vowels (Mn) +// and every variation selector (Mn) are refused. +func defaultIgnorable(r rune) bool { + return unicode.In(r, unicode.Other_Default_Ignorable_Code_Point, unicode.Variation_Selector, unicode.Cf) +} + +var windowsReserved = map[string]bool{"CON": true, "PRN": true, "AUX": true, "NUL": true} + +func init() { + for i := 1; i <= 9; i++ { + windowsReserved[fmt.Sprintf("COM%d", i)] = true + windowsReserved[fmt.Sprintf("LPT%d", i)] = true + } +} + +// isWindowsReserved: the basename before the first '.' (any extension, any +// case) is CON, PRN, AUX, NUL, COM1..COM9 or LPT1..LPT9. +func isWindowsReserved(c string) bool { + base := c + if i := strings.IndexByte(c, '.'); i >= 0 { + base = c[:i] + } + return windowsReserved[strings.ToUpper(base)] +} + +var folder = cases.Fold() + +// FoldKey is the case-insensitive identity of a path: Unicode full case +// folding of the NFC form, re-normalised (folding İ yields a combining dot). +// Two paths with the same key collide on a case-insensitive file system. +func FoldKey(p string) string { + return norm.NFC.String(folder.String(norm.NFC.String(p))) +} + +// ValidateTree validates every path of one version and their uniqueness: no +// two paths with the same FoldKey, and no path whose folded form is also a +// folder of another path (a file and a folder cannot share a name on any +// file system). It returns every refusal (sorted by path), not only the +// first, so the UI and CLI can mark each file. +func ValidateTree(paths []string) []PathError { + var out []PathError + seen := make(map[string]string, len(paths)) // fold key -> first path + valid := make([]string, 0, len(paths)) + for _, p := range paths { + if err := ValidateStoredPath(p); err != nil { + var pe *PathError + errors.As(err, &pe) + out = append(out, *pe) + continue + } + k := FoldKey(p) + if _, dup := seen[k]; dup { + out = append(out, PathError{Path: p, Reason: ReasonCaseCollision}) + continue + } + seen[k] = p + valid = append(valid, p) + } + // File/folder clash: a valid path whose folded form is a folder prefix + // of another valid path. + dirs := make(map[string]bool) + for _, p := range valid { + k := FoldKey(p) + for i := 0; i < len(k); i++ { + if k[i] == '/' { + dirs[k[:i]] = true + } + } + } + for _, p := range valid { + if dirs[FoldKey(p)] { + out = append(out, PathError{Path: p, Reason: ReasonCaseCollision}) + } + } + sort.SliceStable(out, func(i, j int) bool { return out[i].Path < out[j].Path }) + return out +} diff --git a/internal/contexts/path_test.go b/internal/contexts/path_test.go new file mode 100644 index 0000000..67d2215 --- /dev/null +++ b/internal/contexts/path_test.go @@ -0,0 +1,257 @@ +package contexts + +import ( + "errors" + "reflect" + "strings" + "testing" +) + +// Parity table: copied from cloud-api internal/contexts/path_test.go (the +// middleware name-shape test is left out: nan-cli has no route table). The +// copy must stay in step with the server; see scripts/check-cloud-api-pins.sh. +// +// Stored paths (devops AGENT-CONTEXT.md section 1.5): every refusal carries its +// per-file reason; the table covers the Unicode rules and the exact maxima. + +func reasonOf(err error) Reason { + var pe *PathError + if errors.As(err, &pe) { + return pe.Reason + } + return "" +} + +// rep builds a component of n bytes from an ASCII letter. +func rep(n int) string { return strings.Repeat("a", n) } + +// pathOf builds a files/ path of exactly total bytes with depth components +// (files + depth-1 components), every component at most 128 bytes. +func pathOf(t *testing.T, total, depth int) string { + t.Helper() + comps := []string{"files"} + left := total - len("files") + n := depth - 1 + for i := 0; i < n; i++ { + remainingComps := n - i + // bytes for this component: its share of what is left after the separators + avail := left - remainingComps // one '/' per remaining component + share := avail / remainingComps + if i == n-1 { + share = avail + } + if share > MaxComponentBytes { + share = MaxComponentBytes + } + comps = append(comps, rep(share)) + left -= share + 1 + } + p := strings.Join(comps, "/") + if len(p) != total || len(comps) != depth { + t.Fatalf("pathOf(%d,%d) built %d bytes, %d comps", total, depth, len(p), len(comps)) + } + return p +} + +func TestStoredPathTable(t *testing.T) { + for _, tc := range []struct { + name string + path string + want Reason // "" = accepted + }{ + // Layout. + {"instructions", "AGENTS.md", ""}, + {"skill file", "skills/pr-review/SKILL.md", ""}, + {"skill aux", "skills/pr-review/scripts/run.sh", ""}, + {"file", "files/spec.pdf", ""}, + {"nested file", "files/docs/api/v1.md", ""}, + {"agents not top", "files/AGENTS.md", ""}, + {"agents lowercase", "agents.md", ReasonBadTopLevel}, + {"agents as dir", "AGENTS.md/x", ReasonBadTopLevel}, + {"unknown top", "notes/a.md", ReasonBadTopLevel}, + {"bare files", "files", ReasonBadTopLevel}, + {"skill without file", "skills/pr-review", ReasonBadTopLevel}, + {"file directly in skills", "skills/readme.md", ReasonBadTopLevel}, + {"skill uppercase", "skills/PR/SKILL.md", ReasonBadCharacter}, + {"skill accent", "skills/revisión/SKILL.md", ReasonBadCharacter}, + {"skill 64 bytes", "skills/" + rep(64) + "/SKILL.md", ""}, + {"skill 65 bytes", "skills/" + rep(65) + "/SKILL.md", ReasonBadCharacter}, + {"mcp reserved", "mcp/server.json", ReasonNotSupportedYet}, + // Spaces and Unicode letters are accepted (Lead decision 2026-10-21). + {"spaces", "files/Meeting notes 2026.md", ""}, + {"accents", "files/especificación técnica.md", ""}, + {"cjk", "files/仕様書.md", ""}, + {"cyrillic", "files/документ.txt", ""}, + {"arabic digits", "files/ملف١٢.txt", ""}, + {"devanagari marks", "files/ह\u093Fन\u094Dद\u0940.txt", ""}, + {"punctuation set", "files/a_b-c(d)+e,f@g.txt", ""}, + {"dotless i", "files/ılık.md", ""}, + {"eszett", "files/Straße.md", ""}, + // NFC is required, never rewritten. + {"decomposed e acute", "files/especificacio\u0301n.md", ReasonNotNFC}, + {"composed e acute", "files/especificación.md", ""}, + // Format and ignorable characters. + {"rtl override", "files/invoice\u202Etxt.exe", ReasonBadCharacter}, + {"zero width joiner", "files/a\u200Db.md", ReasonBadCharacter}, + {"zero width space", "files/a\u200Bb.md", ReasonBadCharacter}, + {"bom", "files/\uFEFFa.md", ReasonBadCharacter}, + {"soft hyphen", "files/a\u00ADb.md", ReasonBadCharacter}, + {"nbsp", "files/a\u00A0b.md", ReasonBadCharacter}, + {"ideographic space", "files/a\u3000b.md", ReasonBadCharacter}, + {"tab", "files/a\tb.md", ReasonBadCharacter}, + {"newline", "files/a\nb.md", ReasonBadCharacter}, + {"nul", "files/a\x00b.md", ReasonBadCharacter}, + {"variation selector 16", "files/a\uFE0Fb.md", ReasonBadCharacter}, + {"variation selector supplement", "files/a\U000E0100b.md", ReasonBadCharacter}, + {"mongolian fvs", "files/a\u180Bb.md", ReasonBadCharacter}, + {"combining grapheme joiner", "files/a\u034Fb.md", ReasonBadCharacter}, + {"hangul filler", "files/a\u3164b.md", ReasonBadCharacter}, + {"hangul choseong filler", "files/\u115Fa.md", ReasonBadCharacter}, + {"hangul jungseong filler", "files/\u1160a.md", ReasonBadCharacter}, + {"halfwidth hangul filler", "files/a\uFFA0b.md", ReasonBadCharacter}, + {"khmer inherent vowel", "files/a\u17B4b.md", ReasonBadCharacter}, + {"emoji", "files/rocket\U0001F680.md", ReasonBadCharacter}, + {"symbol", "files/a\u00A9b.md", ReasonBadCharacter}, + {"private use", "files/a\uE000b.md", ReasonBadCharacter}, + {"unassigned", "files/a\U000E0080b.md", ReasonBadCharacter}, + {"invalid utf8", "files/a\xffb.md", ReasonBadCharacter}, + // Shell and Windows hostile ASCII. + {"backslash", "files/a\\b.md", ReasonBadCharacter}, + {"colon", "files/a:b.md", ReasonBadCharacter}, + {"star", "files/a*.md", ReasonBadCharacter}, + {"question", "files/a?.md", ReasonBadCharacter}, + {"quote", "files/a\"b.md", ReasonBadCharacter}, + {"lt", "files/ab.md", ReasonBadCharacter}, + {"pipe", "files/a|b.md", ReasonBadCharacter}, + {"apostrophe", "files/don't.md", ReasonBadCharacter}, + {"backtick", "files/a`b.md", ReasonBadCharacter}, + {"dollar", "files/$HOME.md", ReasonBadCharacter}, + // Dotfiles, leading and trailing space or dot. + {"dotfile", "files/.env", ReasonDotfile}, + {"dot env example", "files/.env.example", ReasonDotfile}, + {"git dir", "files/.git/config", ReasonDotfile}, + {"pi dir", "skills/x/.pi/a.md", ReasonDotfile}, + {"dot", "files/./a.md", ReasonDotfile}, + {"dotdot", "files/../AGENTS.md", ReasonDotfile}, + {"leading space", "files/ a.md", ReasonBadCharacter}, + {"trailing space", "files/a.md ", ReasonBadCharacter}, + {"trailing dot", "files/a.", ReasonBadCharacter}, + {"trailing dot dir", "files/dir./a.md", ReasonBadCharacter}, + {"inner dots ok", "files/v1.2.3.tar.md", ""}, + // Empty components and absolute paths. + {"empty", "", ReasonBadCharacter}, + {"absolute", "/files/a.md", ReasonBadCharacter}, + {"double slash", "files//a.md", ReasonBadCharacter}, + {"trailing slash", "files/a/", ReasonBadCharacter}, + // Windows reserved basenames, any extension, any case. + {"con", "files/CON", ReasonReservedName}, + {"con lower ext", "files/con.txt", ReasonReservedName}, + {"nul double ext", "files/Nul.tar.md", ReasonReservedName}, + {"com1", "files/com1.md", ReasonReservedName}, + {"lpt9 dir", "files/LPT9/a.md", ReasonReservedName}, + {"aux", "files/aux", ReasonReservedName}, + {"prn", "files/prn.md", ReasonReservedName}, + {"com0 ok", "files/com0.md", ""}, + {"console ok", "files/console.md", ""}, + {"con- ok", "files/con-1.md", ""}, + // Exact maxima (512 bytes, 8 components, 128 bytes per component). + {"512 bytes", pathOf(t, 512, 6), ""}, + {"513 bytes", pathOf(t, 513, 6), ReasonTooLong}, + {"8 components", "files/a/b/c/d/e/f/g.md", ""}, + {"9 components", "files/a/b/c/d/e/f/g/h.md", ReasonTooDeep}, + {"component 128", "files/" + rep(128), ""}, + {"component 129", "files/" + rep(129), ReasonTooLong}, + {"component 128 multibyte", "files/" + strings.Repeat("é", 64), ""}, + {"component 129 multibyte", "files/" + strings.Repeat("é", 64) + "a", ReasonTooLong}, + } { + err := ValidateStoredPath(tc.path) + if got := reasonOf(err); got != tc.want { + t.Errorf("%s: %q: reason %q, want %q (err %v)", tc.name, tc.path, got, tc.want, err) + } + if err != nil && !errors.Is(err, ErrInvalidPath) { + t.Errorf("%s: error does not match ErrInvalidPath", tc.name) + } + } +} + +func TestValidContextName(t *testing.T) { + for name, want := range map[string]bool{ + "acme": true, "a": true, "0": true, "pr-review": true, rep(63): true, + rep(64): false, "": false, "-a": false, "Acme": false, "a_b": false, "a.b": false, "a b": false, "ñ": false, + // Windows reserved: the name becomes files// in the manifest. + "con": false, "nul": false, "aux": false, "prn": false, "com1": false, "lpt9": false, + "com0": true, "console": true, "con-1": true, + } { + if got := ValidContextName(name); got != want { + t.Errorf("ValidContextName(%q) = %v", name, got) + } + } +} + +// Uniqueness is case-insensitive with Unicode full case folding of the NFC +// form: ß folds to ss, final sigma to sigma; the Turkish dotless ı has no +// default folding (only the Turkic one), so ı and I do not collide, and İ +// folds to i + U+0307, so it collides with neither i nor I. +func TestValidateTreeCaseCollisions(t *testing.T) { + for _, tc := range []struct { + name string + paths []string + want []PathError + }{ + {"distinct", []string{"AGENTS.md", "files/a.md", "files/b.md", "skills/x/SKILL.md"}, nil}, + {"ascii case", []string{"files/Readme.md", "files/README.md"}, []PathError{{"files/README.md", ReasonCaseCollision}}}, + {"eszett", []string{"files/Straße.md", "files/STRASSE.md"}, []PathError{{"files/STRASSE.md", ReasonCaseCollision}}}, + {"capital eszett", []string{"files/ẞ.md", "files/ss.md"}, []PathError{{"files/ss.md", ReasonCaseCollision}}}, + {"sigma", []string{"files/ΟΔΟΣ.md", "files/οδος.md"}, []PathError{{"files/οδος.md", ReasonCaseCollision}}}, + {"final sigma", []string{"files/οδος.md", "files/οδοσ.md"}, []PathError{{"files/οδοσ.md", ReasonCaseCollision}}}, + {"accented case", []string{"files/Ésta.md", "files/ésta.md"}, []PathError{{"files/ésta.md", ReasonCaseCollision}}}, + {"dotless i is distinct", []string{"files/ı.md", "files/I.md", "files/i.md"}, []PathError{{"files/i.md", ReasonCaseCollision}}}, + {"dotted capital I is distinct", []string{"files/İ.md", "files/i.md"}, nil}, + {"folder case", []string{"files/Docs/a.md", "files/docs/b.md"}, nil}, + {"file vs folder", []string{"files/docs", "files/Docs/a.md"}, []PathError{{"files/docs", ReasonCaseCollision}}}, + {"invalid reported too", []string{"files/.env", "files/a.md", "files/A.md"}, + []PathError{{"files/.env", ReasonDotfile}, {"files/A.md", ReasonCaseCollision}}}, + } { + got := ValidateTree(tc.paths) + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("%s: %v, want %v", tc.name, got, tc.want) + } + } +} + +func TestFoldKeyIsStable(t *testing.T) { + // Folding İ yields a combining sequence; the key is re-normalised so a + // second fold is a no-op. + for _, p := range []string{"files/İstanbul.md", "files/Straße.md", "files/ΟΔΟΣ.md"} { + k := FoldKey(p) + if FoldKey(k) != k { + t.Errorf("FoldKey(%q) not idempotent: %q vs %q", p, k, FoldKey(k)) + } + } +} + +// Names the API could never reach by name are refused: UUID-shaped names +// (always read as ids) and "blobs" (the upload routes). +func TestContextNameReachable(t *testing.T) { + for _, n := range []string{"123e4567-e89b-12d3-a456-426614174000", "blobs"} { + if ValidContextName(n) { + t.Errorf("%q accepted", n) + } + } + for _, n := range []string{"blobs-2", "my-blobs", "123e4567-e89b-12d3-a456-42661417400", "123e4567e89b12d3a456426614174000"} { + if !ValidContextName(n) { + t.Errorf("%q refused", n) + } + } +} + +// File names never carry a quote, a backslash or a percent sign (they would +// need escaping in Content-Disposition). +func TestStoredPathRefusesHeaderCharacters(t *testing.T) { + for _, p := range []string{`files/a"b.txt`, `files/a\b.txt`, "files/a%b.txt"} { + if ValidateStoredPath(p) == nil { + t.Errorf("%q accepted", p) + } + } +} diff --git a/internal/contexts/pins_test.go b/internal/contexts/pins_test.go new file mode 100644 index 0000000..dd01b5d --- /dev/null +++ b/internal/contexts/pins_test.go @@ -0,0 +1,47 @@ +package contexts + +import ( + "os" + "path/filepath" + "regexp" + "testing" + "unicode" +) + +// The path rules read Unicode tables from two places: the standard library +// (unicode.IsLetter, the categories) and golang.org/x/text (NFC, case +// folding). cloud-api accepts a name only when its own tables say so, so +// this CLI has to read the same tables or `nan context pull` could refuse a +// name the server stored (devops AGENT-CONTEXT.md §1.5). +// +// These are the pins both repositories must share. x/text is v0.39.0, the +// first release without GO-2026-5970 (an infinite loop in unicode/norm on +// invalid UTF-8, reached by these rules); cloud-api has to move to it too. +// When either side moves, scripts/check-cloud-api-pins.sh fails in CI until +// this record and go.mod match cloud-api again. +const ( + cloudAPIXText = "v0.39.0" + cloudAPIUnicode = "15.0.0" +) + +var reXText = regexp.MustCompile(`(?m)^\s*golang\.org/x/text\s+(v\S+)`) + +func TestXTextPinnedToCloudAPI(t *testing.T) { + b, err := os.ReadFile(filepath.Join("..", "..", "go.mod")) + if err != nil { + t.Fatal(err) + } + m := reXText.FindSubmatch(b) + if m == nil { + t.Fatal("go.mod does not require golang.org/x/text") + } + if got := string(m[1]); got != cloudAPIXText { + t.Errorf("go.mod pins golang.org/x/text %s, cloud-api builds with %s: the path rules would read other Unicode tables", got, cloudAPIXText) + } +} + +func TestUnicodeTablesMatchCloudAPI(t *testing.T) { + if unicode.Version != cloudAPIUnicode { + t.Errorf("this toolchain's unicode tables are %s, cloud-api's are %s", unicode.Version, cloudAPIUnicode) + } +} diff --git a/scripts/check-cloud-api-pins.sh b/scripts/check-cloud-api-pins.sh new file mode 100755 index 0000000..4d47e7c --- /dev/null +++ b/scripts/check-cloud-api-pins.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# Checks that `nan context` judges file names with the same rules and the +# same Unicode tables as cloud-api (helmcode/nan-devops AGENT-CONTEXT.md +# §1.5). The rules are a verbatim copy of cloud-api's +# internal/contexts/path.go, and they read Unicode from two places: the Go +# standard library (letters, marks, digits) and golang.org/x/text (NFC, case +# folding). If any of the three drifts, `nan context pull` can refuse a name +# the server stored, or push can send one the server refuses. +# +# Compared against cloud-api's default branch: +# 1. golang.org/x/text: the exact same version. +# 2. internal/contexts/path.go: byte-identical. +# 3. Go: the same minor as cloud-api's build image (Dockerfile) and go.mod; +# when the minors differ, the two toolchains must at least carry the same +# unicode.Version (the tables the rules read). A security patch can make +# this CLI move first; it may not change the tables without cloud-api. +# 4. The record in internal/contexts/pins_test.go matches cloud-api, so the +# test that runs everywhere (forks included) checks the right values. +# +# cloud-api is private: CLOUD_API_TOKEN must be a token that can read its +# contents (CI: the CLOUD_API_READ_TOKEN secret). Without it this fails; +# a check that passes because it could not look guards nothing. +set -euo pipefail + +REPO="${CLOUD_API_REPO:-helmcode/nan-cloud-api}" +REF="${CLOUD_API_REF:-main}" +TOKEN="${CLOUD_API_TOKEN:-}" +root="$(cd "$(dirname "$0")/.." && pwd)" + +fail() { + echo "::error::$*" >&2 + exit 1 +} + +if [ -z "$TOKEN" ]; then + fail "CLOUD_API_TOKEN is not set: it needs read access to $REPO (CI secret CLOUD_API_READ_TOKEN)" +fi + +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +fetch() { + curl -fsSL \ + -H "Authorization: Bearer $TOKEN" \ + -H "Accept: application/vnd.github.raw+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/$REPO/contents/$1?ref=$REF" -o "$2" || + fail "could not read $1 from $REPO@$REF" +} + +fetch go.mod "$tmp/go.mod" +fetch Dockerfile "$tmp/Dockerfile" +fetch internal/contexts/path.go "$tmp/path.go" + +# The version of golang.org/x/text a go.mod requires, in a require block or +# on a require line. +xtext_of() { + awk '{ for (i = 1; i < NF; i++) if ($i == "golang.org/x/text") { print $(i + 1); exit } }' "$1" +} +go_directive_of() { + awk '$1 == "go" { print $2; exit }' "$1" +} +minor_of() { + echo "$1" | cut -d. -f1,2 +} + +# 1. golang.org/x/text +up_xtext="$(xtext_of "$tmp/go.mod")" +my_xtext="$(xtext_of "$root/go.mod")" +[ -n "$up_xtext" ] || fail "cloud-api's go.mod does not require golang.org/x/text" +[ -n "$my_xtext" ] || fail "go.mod does not require golang.org/x/text" +if [ "$up_xtext" != "$my_xtext" ]; then + fail "golang.org/x/text: cloud-api builds with $up_xtext, nan-cli with $my_xtext. Both must use the same version: move the one that is behind (go get golang.org/x/text@VERSION) and update internal/contexts/pins_test.go" +fi +echo "golang.org/x/text $my_xtext on both sides" + +# 2. path.go +if ! cmp -s "$tmp/path.go" "$root/internal/contexts/path.go"; then + diff -u "$root/internal/contexts/path.go" "$tmp/path.go" >&2 || true + fail "internal/contexts/path.go differs from cloud-api's: copy it again, with its test table (internal/contexts/path_test.go)" +fi +echo "internal/contexts/path.go is byte-identical to cloud-api's" + +# 3. Go +up_image="$(sed -n 's/^FROM golang:\([0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' "$tmp/Dockerfile" | head -1)" +up_directive="$(go_directive_of "$tmp/go.mod")" +my_directive="$(go_directive_of "$root/go.mod")" +[ -n "$up_image" ] || fail "could not find the golang image in cloud-api's Dockerfile" +my_minor="$(minor_of "$my_directive")" + +# unicode.Version of a toolchain, from a module of its own so this repo's +# go directive does not pick the toolchain. +unicode_of() { + local dir="$tmp/uv-$1" + mkdir -p "$dir" + printf 'module uv\n\ngo 1.21\n' >"$dir/go.mod" + printf 'package main\n\nimport (\n\t"fmt"\n\t"unicode"\n)\n\nfunc main() { fmt.Println(unicode.Version) }\n' >"$dir/main.go" + (cd "$dir" && GOTOOLCHAIN="go$1" GOFLAGS=-mod=mod go run .) +} + +up_unicode="$(unicode_of "$up_image.0")" +my_unicode="$(unicode_of "$my_directive")" +for up in "$up_image" "$(minor_of "$up_directive")"; do + if [ "$up" != "$my_minor" ] && [ "$up_unicode" != "$my_unicode" ]; then + fail "Go: cloud-api uses $up (Unicode $up_unicode), nan-cli $my_directive (Unicode $my_unicode): the path rules would read other tables" + fi +done +if [ "$up_image" = "$my_minor" ]; then + echo "Go $my_minor on both sides (Unicode $my_unicode)" +else + echo "::notice::Go: cloud-api builds with $up_image, nan-cli with $my_directive; both carry Unicode $my_unicode" +fi + +# 4. The record the unit test checks. +rec_xtext="$(sed -n 's/^[[:space:]]*cloudAPIXText[[:space:]]*=[[:space:]]*"\(.*\)"/\1/p' "$root/internal/contexts/pins_test.go")" +rec_unicode="$(sed -n 's/^[[:space:]]*cloudAPIUnicode[[:space:]]*=[[:space:]]*"\(.*\)"/\1/p' "$root/internal/contexts/pins_test.go")" +if [ "$rec_xtext" != "$up_xtext" ] || [ "$rec_unicode" != "$up_unicode" ]; then + fail "internal/contexts/pins_test.go records x/text $rec_xtext and Unicode $rec_unicode; cloud-api has $up_xtext and $up_unicode" +fi +echo "internal/contexts/pins_test.go matches cloud-api" From b11725d8896386a8ca4adec0f5ed054f0891c866 Mon Sep 17 00:00:00 2001 From: barckcode Date: Mon, 26 Oct 2026 20:01:18 +0100 Subject: [PATCH 2/4] ci: move the cloud-api parity check to cloud-api's CI nan-cli is public and cloud-api private: cloud-api's CI reads this repository's go.mod and path.go without a token, so the check lives there and no secret is added here. The unit test keeps pinning golang.org/x/text v0.39.0 and the Unicode tables (15.0.0). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 26 ------- internal/contexts/path_test.go | 2 +- internal/contexts/pins_test.go | 8 ++- scripts/check-cloud-api-pins.sh | 120 -------------------------------- 4 files changed, 6 insertions(+), 150 deletions(-) delete mode 100755 scripts/check-cloud-api-pins.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8cab40..46f40c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -193,32 +193,6 @@ jobs: } Write-Host "installer parses, web requests basic-parsed, no unguarded exit, arch survives without .NET, no preference leaks" - # `nan context` refuses or accepts a file name with a copy of cloud-api's - # rules, and those rules read Unicode tables from the Go toolchain and from - # golang.org/x/text. A dependency bump on either side would make pull refuse - # a name the server stored, quietly, for some names only. This compares the - # x/text version, the Go toolchain's tables and the copied path.go with - # cloud-api's (AGENT-CONTEXT.md §1.5). cloud-api is private: a pull request - # from a fork gets no secret, and is left to the unit test that checks the - # recorded pins (internal/contexts/pins_test.go). - cloud-api-pins: - name: Context path rules match cloud-api - runs-on: ubuntu-latest - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository - - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - cache: true - - - name: Compare golang.org/x/text, Go and path.go with cloud-api - env: - CLOUD_API_TOKEN: ${{ secrets.CLOUD_API_READ_TOKEN }} - run: scripts/check-cloud-api-pins.sh - # Four vulnerabilities in the standard library this repo compiled against # went unnoticed until somebody ran this by hand. Nothing here was looking, # so nothing was going to say so - not on the next release either. diff --git a/internal/contexts/path_test.go b/internal/contexts/path_test.go index 67d2215..b9c4d82 100644 --- a/internal/contexts/path_test.go +++ b/internal/contexts/path_test.go @@ -9,7 +9,7 @@ import ( // Parity table: copied from cloud-api internal/contexts/path_test.go (the // middleware name-shape test is left out: nan-cli has no route table). The -// copy must stay in step with the server; see scripts/check-cloud-api-pins.sh. +// copy must stay in step with the server; cloud-api's CI checks that it does. // // Stored paths (devops AGENT-CONTEXT.md section 1.5): every refusal carries its // per-file reason; the table covers the Unicode rules and the exact maxima. diff --git a/internal/contexts/pins_test.go b/internal/contexts/pins_test.go index dd01b5d..1614167 100644 --- a/internal/contexts/pins_test.go +++ b/internal/contexts/pins_test.go @@ -16,9 +16,11 @@ import ( // // These are the pins both repositories must share. x/text is v0.39.0, the // first release without GO-2026-5970 (an infinite loop in unicode/norm on -// invalid UTF-8, reached by these rules); cloud-api has to move to it too. -// When either side moves, scripts/check-cloud-api-pins.sh fails in CI until -// this record and go.mod match cloud-api again. +// invalid UTF-8, reached by these rules). The cross-repo check lives in +// cloud-api's CI, which reads this repository's public go.mod and compares +// it (and its copy of path.go) with its own: this repository is public and +// cannot read cloud-api. When cloud-api moves x/text or its Go toolchain's +// Unicode tables, this record and go.mod move with it. const ( cloudAPIXText = "v0.39.0" cloudAPIUnicode = "15.0.0" diff --git a/scripts/check-cloud-api-pins.sh b/scripts/check-cloud-api-pins.sh deleted file mode 100755 index 4d47e7c..0000000 --- a/scripts/check-cloud-api-pins.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env bash -# Checks that `nan context` judges file names with the same rules and the -# same Unicode tables as cloud-api (helmcode/nan-devops AGENT-CONTEXT.md -# §1.5). The rules are a verbatim copy of cloud-api's -# internal/contexts/path.go, and they read Unicode from two places: the Go -# standard library (letters, marks, digits) and golang.org/x/text (NFC, case -# folding). If any of the three drifts, `nan context pull` can refuse a name -# the server stored, or push can send one the server refuses. -# -# Compared against cloud-api's default branch: -# 1. golang.org/x/text: the exact same version. -# 2. internal/contexts/path.go: byte-identical. -# 3. Go: the same minor as cloud-api's build image (Dockerfile) and go.mod; -# when the minors differ, the two toolchains must at least carry the same -# unicode.Version (the tables the rules read). A security patch can make -# this CLI move first; it may not change the tables without cloud-api. -# 4. The record in internal/contexts/pins_test.go matches cloud-api, so the -# test that runs everywhere (forks included) checks the right values. -# -# cloud-api is private: CLOUD_API_TOKEN must be a token that can read its -# contents (CI: the CLOUD_API_READ_TOKEN secret). Without it this fails; -# a check that passes because it could not look guards nothing. -set -euo pipefail - -REPO="${CLOUD_API_REPO:-helmcode/nan-cloud-api}" -REF="${CLOUD_API_REF:-main}" -TOKEN="${CLOUD_API_TOKEN:-}" -root="$(cd "$(dirname "$0")/.." && pwd)" - -fail() { - echo "::error::$*" >&2 - exit 1 -} - -if [ -z "$TOKEN" ]; then - fail "CLOUD_API_TOKEN is not set: it needs read access to $REPO (CI secret CLOUD_API_READ_TOKEN)" -fi - -tmp="$(mktemp -d)" -trap 'rm -rf "$tmp"' EXIT - -fetch() { - curl -fsSL \ - -H "Authorization: Bearer $TOKEN" \ - -H "Accept: application/vnd.github.raw+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/$REPO/contents/$1?ref=$REF" -o "$2" || - fail "could not read $1 from $REPO@$REF" -} - -fetch go.mod "$tmp/go.mod" -fetch Dockerfile "$tmp/Dockerfile" -fetch internal/contexts/path.go "$tmp/path.go" - -# The version of golang.org/x/text a go.mod requires, in a require block or -# on a require line. -xtext_of() { - awk '{ for (i = 1; i < NF; i++) if ($i == "golang.org/x/text") { print $(i + 1); exit } }' "$1" -} -go_directive_of() { - awk '$1 == "go" { print $2; exit }' "$1" -} -minor_of() { - echo "$1" | cut -d. -f1,2 -} - -# 1. golang.org/x/text -up_xtext="$(xtext_of "$tmp/go.mod")" -my_xtext="$(xtext_of "$root/go.mod")" -[ -n "$up_xtext" ] || fail "cloud-api's go.mod does not require golang.org/x/text" -[ -n "$my_xtext" ] || fail "go.mod does not require golang.org/x/text" -if [ "$up_xtext" != "$my_xtext" ]; then - fail "golang.org/x/text: cloud-api builds with $up_xtext, nan-cli with $my_xtext. Both must use the same version: move the one that is behind (go get golang.org/x/text@VERSION) and update internal/contexts/pins_test.go" -fi -echo "golang.org/x/text $my_xtext on both sides" - -# 2. path.go -if ! cmp -s "$tmp/path.go" "$root/internal/contexts/path.go"; then - diff -u "$root/internal/contexts/path.go" "$tmp/path.go" >&2 || true - fail "internal/contexts/path.go differs from cloud-api's: copy it again, with its test table (internal/contexts/path_test.go)" -fi -echo "internal/contexts/path.go is byte-identical to cloud-api's" - -# 3. Go -up_image="$(sed -n 's/^FROM golang:\([0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' "$tmp/Dockerfile" | head -1)" -up_directive="$(go_directive_of "$tmp/go.mod")" -my_directive="$(go_directive_of "$root/go.mod")" -[ -n "$up_image" ] || fail "could not find the golang image in cloud-api's Dockerfile" -my_minor="$(minor_of "$my_directive")" - -# unicode.Version of a toolchain, from a module of its own so this repo's -# go directive does not pick the toolchain. -unicode_of() { - local dir="$tmp/uv-$1" - mkdir -p "$dir" - printf 'module uv\n\ngo 1.21\n' >"$dir/go.mod" - printf 'package main\n\nimport (\n\t"fmt"\n\t"unicode"\n)\n\nfunc main() { fmt.Println(unicode.Version) }\n' >"$dir/main.go" - (cd "$dir" && GOTOOLCHAIN="go$1" GOFLAGS=-mod=mod go run .) -} - -up_unicode="$(unicode_of "$up_image.0")" -my_unicode="$(unicode_of "$my_directive")" -for up in "$up_image" "$(minor_of "$up_directive")"; do - if [ "$up" != "$my_minor" ] && [ "$up_unicode" != "$my_unicode" ]; then - fail "Go: cloud-api uses $up (Unicode $up_unicode), nan-cli $my_directive (Unicode $my_unicode): the path rules would read other tables" - fi -done -if [ "$up_image" = "$my_minor" ]; then - echo "Go $my_minor on both sides (Unicode $my_unicode)" -else - echo "::notice::Go: cloud-api builds with $up_image, nan-cli with $my_directive; both carry Unicode $my_unicode" -fi - -# 4. The record the unit test checks. -rec_xtext="$(sed -n 's/^[[:space:]]*cloudAPIXText[[:space:]]*=[[:space:]]*"\(.*\)"/\1/p' "$root/internal/contexts/pins_test.go")" -rec_unicode="$(sed -n 's/^[[:space:]]*cloudAPIUnicode[[:space:]]*=[[:space:]]*"\(.*\)"/\1/p' "$root/internal/contexts/pins_test.go")" -if [ "$rec_xtext" != "$up_xtext" ] || [ "$rec_unicode" != "$up_unicode" ]; then - fail "internal/contexts/pins_test.go records x/text $rec_xtext and Unicode $rec_unicode; cloud-api has $up_xtext and $up_unicode" -fi -echo "internal/contexts/pins_test.go matches cloud-api" From 96839fa472bea0f49e1c0df1908f3ac5e6a2664d Mon Sep 17 00:00:00 2001 From: barckcode Date: Mon, 26 Oct 2026 20:01:51 +0100 Subject: [PATCH 3/4] test(contexts): point the pins test at cloud-api's parity check Co-Authored-By: Claude Opus 5.5 --- internal/contexts/pins_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/contexts/pins_test.go b/internal/contexts/pins_test.go index 1614167..53b0227 100644 --- a/internal/contexts/pins_test.go +++ b/internal/contexts/pins_test.go @@ -17,9 +17,10 @@ import ( // These are the pins both repositories must share. x/text is v0.39.0, the // first release without GO-2026-5970 (an infinite loop in unicode/norm on // invalid UTF-8, reached by these rules). The cross-repo check lives in -// cloud-api's CI, which reads this repository's public go.mod and compares -// it (and its copy of path.go) with its own: this repository is public and -// cannot read cloud-api. When cloud-api moves x/text or its Go toolchain's +// cloud-api's CI (scripts/check-nan-cli-text-parity.sh, helmcode/nan-cloud-api +// #397): it reads this repository's public go.mod and compares the x/text +// version and unicode.Version under the go version declared here with its +// own. This repository is public and cannot read cloud-api. When cloud-api moves x/text or its Go toolchain's // Unicode tables, this record and go.mod move with it. const ( cloudAPIXText = "v0.39.0" From 4cefc877b2888a6c589e3533afbe6c6195f18f65 Mon Sep 17 00:00:00 2001 From: barckcode Date: Mon, 26 Oct 2026 20:16:19 +0100 Subject: [PATCH 4/4] fix(context): address the QA and Security review of #62 Security: - A DIR that is a symlink is resolved once and its target walked; WalkDir alone saw one non-directory and push published an empty tree. - push refuses to publish no files over a version that has some unless --allow-empty (also after a --force rebase). - Files are hashed and uploaded through os.OpenRoot on DIR and refused if they stopped being regular files. - pull downloads into a temporary name, verifies, sets the mode both ways and renames into place: a failed download never deletes a member's file, and a link or folder at a file's path is never replaced. The total of a version is capped at 250 MiB; a partial pull says how many files were written and points at --force. - NAME|id is checked locally (context name or id) before any request. - Remembered multipart uploads are keyed by API host and credential hash. QA: - Tests for a symlinked folder inside DIR and a symlinked file on pull, the blob_missing re-check body and single PUT, the version limits (31 skills, 1001 files, 250 MiB, 100 MiB per file, SKILL.md 64 KiB), aborting an unusable remembered upload, forgetting an upload dropped mid-parts, exit mappings (entitlement_required and others), the inherited executable bit, pull --force modes, the total cap and local ref checks. - Unchanged files keep the remote executable bit where the file system has none (Windows). - Dead ContextVersions/ContextVersionInfo removed. - The fake follows cloud-api's publish check order, decodes strictly and answers 411 without Content-Length. - Help says API keys are not used here; pins comment wrapped. Co-Authored-By: Claude Opus 5.5 --- README.md | 5 +- cmd/context.go | 258 +++++++++++++++++++----- cmd/context_fake_test.go | 95 +++++++-- cmd/context_hardening_test.go | 349 +++++++++++++++++++++++++++++++++ cmd/context_test.go | 32 ++- internal/api/contexts.go | 19 +- internal/contexts/pins_test.go | 17 +- 7 files changed, 675 insertions(+), 100 deletions(-) create mode 100644 cmd/context_hardening_test.go diff --git a/README.md b/README.md index 1810531..f4598ec 100644 --- a/README.md +++ b/README.md @@ -274,7 +274,7 @@ automations: nan context ls [--json] nan context show [--json] nan context create NAME [--description TEXT] [--from DIR] [--json] -nan context push DIR [--create] [--dry-run] [--force] [--allow-secrets] [--skip-invalid] [--json] +nan context push DIR [--create] [--dry-run] [--force] [--allow-secrets] [--skip-invalid] [--allow-empty] [--json] nan context pull DIR [--force] nan context edit [--name NEW] [--description TEXT] [--json] nan context rm [--yes] @@ -284,7 +284,8 @@ nan context rm [--yes] skipped; other refused names are listed with their reason and the push exits 65 unless `--skip-invalid`), uploads only what NaN does not already hold (large files in resumable 16 MiB parts) and publishes a new version. Text -that looks like a credential is refused unless `--allow-secrets`. `pull` +that looks like a credential is refused unless `--allow-secrets`, and an +empty folder never empties a context unless `--allow-empty`. `pull` writes into an empty folder only, after checking every path with the same rules. Without contexts on your account, these commands exit 64 and point at the portal. Docs: . diff --git a/cmd/context.go b/cmd/context.go index d1b0309..8f2d559 100644 --- a/cmd/context.go +++ b/cmd/context.go @@ -93,6 +93,8 @@ var ( ctxConcurrency = 4 // ctxSingleMax is the largest blob sent as one PUT. ctxSingleMax int64 = api.ContextPartSize + // ctxExecKnown is whether the file system carries an executable bit. + ctxExecKnown = runtime.GOOS != "windows" ) var ( @@ -106,6 +108,7 @@ var ( ctxPushAllowSecrets bool ctxPushDryRun bool ctxPushSkipInvalid bool + ctxPushAllowEmpty bool ctxPullForce bool @@ -123,8 +126,9 @@ var ctxVerbHints = map[string]string{ const ctxCredentialHelp = `Authentication, first match wins: --token-file, your session from nan auth login, NAN_TOKEN, a platform token saved with nan auth login --api-token. A platform token needs contexts:read to list, show and pull, -and contexts:write to create, push, edit and delete. Workspace keys never -read or write contexts.` +and contexts:write to create, push, edit and delete. API keys (sk-) are +not used by these commands, and workspace keys never read or write +contexts.` var contextCmd = &cobra.Command{ Use: "context", @@ -240,7 +244,10 @@ private keys) is refused with the file and line; keep credentials in nan secrets, or publish anyway with --allow-secrets (the agent can read and copy anything in a context). -With --create, a context that does not exist yet is created first. +With --create, a context that does not exist yet is created first. A DIR +that is a symbolic link is followed once (links inside it are refused). A +folder with no files is refused over a version that has some, since it +would remove them all; --allow-empty publishes it anyway. ` + ctxCredentialHelp + ` @@ -328,6 +335,7 @@ func init() { c.Flags().BoolVar(&ctxPushAllowSecrets, "allow-secrets", false, "Publish even if a text file looks like it holds a credential") c.Flags().BoolVar(&ctxPushSkipInvalid, "skip-invalid", false, "Leave refused files out instead of stopping") } + ctxPushCmd.Flags().BoolVar(&ctxPushAllowEmpty, "allow-empty", false, "Publish an empty folder over a version that has files (removes them all)") f := ctxPushCmd.Flags() f.BoolVar(&ctxPushCreate, "create", false, "Create the context if it does not exist") f.BoolVar(&ctxPushForce, "force", false, "Publish on top of a version someone else published meanwhile") @@ -339,12 +347,24 @@ func init() { } type pushOptions struct { - create, force, allowSecrets, dryRun, skipInvalid, json bool + create, force, allowSecrets, dryRun, skipInvalid, allowEmpty, json bool } func pushOptionsFromFlags() pushOptions { return pushOptions{create: ctxPushCreate, force: ctxPushForce, allowSecrets: ctxPushAllowSecrets, dryRun: ctxPushDryRun, - skipInvalid: ctxPushSkipInvalid, json: ctxJSON} + skipInvalid: ctxPushSkipInvalid, allowEmpty: ctxPushAllowEmpty, json: ctxJSON} +} + +var uuidPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + +// checkContextRef refuses a NAME|id that is neither a context name nor an +// id before it becomes part of a request path. +func checkContextRef(ref string) error { + if contexts.ValidContextName(ref) || uuidPattern.MatchString(ref) { + return nil + } + return usageErrorf("%q is neither a context name (lowercase letters, digits and dashes) nor a context id (see: nan context ls)", + runs.Truncate(runs.SanitizeLine(ref), 80)) } func contextNameRule(name string) string { @@ -464,6 +484,9 @@ func doContextLs(ctx context.Context, env *runsEnv) error { } func doContextShow(ctx context.Context, env *runsEnv, ref string) error { + if err := checkContextRef(ref); err != nil { + return err + } d, err := env.client.GetContext(ctx, ref) if err != nil { return ctxExit(err, env, scopeContextsRead) @@ -536,7 +559,7 @@ func doContextCreate(ctx context.Context, env *runsEnv, name string) error { if !contexts.ValidContextName(name) { return usageErrorf("%s", contextNameRule(name)) } - opts := pushOptions{allowSecrets: ctxPushAllowSecrets, skipInvalid: ctxPushSkipInvalid, json: ctxJSON} + opts := pushOptions{allowSecrets: ctxPushAllowSecrets, skipInvalid: ctxPushSkipInvalid, allowEmpty: ctxPushAllowEmpty, json: ctxJSON} var local *localTree if ctxCreateFrom != "" { // The folder is checked before the context exists, so a folder that @@ -545,6 +568,7 @@ func doContextCreate(ctx context.Context, env *runsEnv, name string) error { if local, err = scanLocal(env, ctxCreateFrom, opts.skipInvalid); err != nil { return err } + defer local.close() } idem := newIdempotencyKey() var c *api.Context @@ -569,6 +593,9 @@ func doContextCreate(ctx context.Context, env *runsEnv, name string) error { } func doContextEdit(ctx context.Context, env *runsEnv, ref string, patch api.ContextPatch) error { + if err := checkContextRef(ref); err != nil { + return err + } d, err := env.client.GetContext(ctx, ref) if err != nil { return ctxExit(err, env, scopeContextsRead) @@ -585,6 +612,9 @@ func doContextEdit(ctx context.Context, env *runsEnv, ref string, patch api.Cont } func doContextRm(ctx context.Context, env *runsEnv, ref string) error { + if err := checkContextRef(ref); err != nil { + return err + } d, err := env.client.GetContext(ctx, ref) if err != nil { return ctxExit(err, env, scopeContextsRead) @@ -605,10 +635,25 @@ func doContextRm(ctx context.Context, env *runsEnv, ref string) error { // localFile is one file of the folder being pushed. type localFile struct { path string // NFC, '/'-separated, relative to the folder - abs string + rel string // the name on disk, relative to root + root *os.Root size int64 exec bool - sha string + // execKnown is false where the file system has no executable bit + // (Windows): the bit then comes from the version being replaced. + execKnown bool + sha string +} + +// open reads the file through the folder's root: a name swapped for a +// symlink after the scan is refused instead of followed. +func (f localFile) open() (*os.File, error) { + if st, err := f.root.Lstat(f.rel); err != nil { + return nil, err + } else if !st.Mode().IsRegular() { + return nil, fmt.Errorf("it is no longer a regular file") + } + return f.root.Open(f.rel) } // refusedFile is one file push will not publish, with its reason. @@ -618,6 +663,13 @@ type refusedFile struct { type localTree struct { files []localFile + root *os.Root +} + +func (t *localTree) close() { + if t != nil && t.root != nil { + _ = t.root.Close() + } } func (t *localTree) totalBytes() int64 { @@ -633,6 +685,20 @@ func (t *localTree) totalBytes() int64 { // (exit 65) unless skipInvalid; dotfiles are always skipped with a notice. func scanLocal(env *runsEnv, dir string, skipInvalid bool) (*localTree, error) { shown := runs.SanitizeLine(dir) + lst, err := os.Lstat(dir) + if err != nil { + return nil, usageErrorf("cannot read %s: %v", shown, unwrapPath(err)) + } + if lst.Mode()&fs.ModeSymlink != 0 { + // The folder the member named is behind a link (a dotfile manager, + // ~/work -> /Volumes/...): walk what it points at. WalkDir would + // otherwise see one non-directory and report an empty tree. + target, err := filepath.EvalSymlinks(dir) + if err != nil { + return nil, usageErrorf("cannot read %s: %v", shown, unwrapPath(err)) + } + dir = target + } st, err := os.Stat(dir) if err != nil { return nil, usageErrorf("cannot read %s: %v", shown, unwrapPath(err)) @@ -640,12 +706,26 @@ func scanLocal(env *runsEnv, dir string, skipInvalid bool) (*localTree, error) { if !st.IsDir() { return nil, usageErrorf("%s is not a folder", shown) } + root, err := os.OpenRoot(dir) + if err != nil { + return nil, usageErrorf("cannot open %s: %v", shown, unwrapPath(err)) + } + tree, err := scanRoot(env, dir, root, skipInvalid) + if err != nil { + _ = root.Close() + return nil, err + } + return tree, nil +} + +func scanRoot(env *runsEnv, dir string, root *os.Root, skipInvalid bool) (*localTree, error) { + shown := runs.SanitizeLine(dir) var ( files []localFile refused []refusedFile dotted []string ) - err = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { if err != nil { return err } @@ -679,8 +759,8 @@ func scanLocal(env *runsEnv, dir string, skipInvalid bool) (*localTree, error) { if ierr != nil { return ierr } - files = append(files, localFile{path: name, abs: p, size: info.Size(), - exec: runtime.GOOS != "windows" && info.Mode().Perm()&0o111 != 0}) + files = append(files, localFile{path: name, rel: rel, root: root, size: info.Size(), + execKnown: ctxExecKnown, exec: ctxExecKnown && info.Mode().Perm()&0o111 != 0}) return nil }) if err != nil { @@ -730,13 +810,13 @@ func scanLocal(env *runsEnv, dir string, skipInvalid bool) (*localTree, error) { kept = append(kept, f) } } - tree := &localTree{files: kept} + tree := &localTree{files: kept, root: root} if err := checkTreeLimits(tree); err != nil { return nil, err } for i := range tree.files { f := &tree.files[i] - sum, size, err := hashFile(f.abs) + sum, size, err := hashFile(*f) if err != nil { return nil, exitf(exitFailed, "cannot read %s: %v", runs.SanitizeLine(f.path), unwrapPath(err)) } @@ -824,8 +904,8 @@ func checkTreeLimits(t *localTree) error { return nil } -func hashFile(p string) (string, int64, error) { - f, err := os.Open(p) +func hashFile(lf localFile) (string, int64, error) { + f, err := lf.open() if err != nil { return "", 0, err } @@ -841,10 +921,14 @@ func hashFile(p string) (string, int64, error) { // ─── push ────────────────────────────────────────────────────────────────── func doContextPush(ctx context.Context, env *runsEnv, ref, dir string, opts pushOptions) error { + if err := checkContextRef(ref); err != nil { + return err + } local, err := scanLocal(env, dir, opts.skipInvalid) if err != nil { return err } + defer local.close() d, err := env.client.GetContext(ctx, ref) if ctxCode(err) == "context_not_found" && opts.create { if !contexts.ValidContextName(ref) { @@ -926,6 +1010,33 @@ func printDiff(w io.Writer, d treeDiff, sizes map[string]int64) { } } +// inheritExec takes the executable bit of a file whose content did not +// change from the version being replaced, where the file system cannot say +// (Windows): pushing an unchanged pulled tree must not strip +x. +func inheritExec(remote []api.ContextFile, local *localTree) { + have := make(map[string]api.ContextFile, len(remote)) + for _, f := range remote { + have[f.Path] = f + } + for i := range local.files { + f := &local.files[i] + if r, ok := have[f.path]; ok && !f.execKnown && r.SHA256 == f.sha { + f.exec = r.Executable + } + } +} + +// checkNotEmptied refuses to publish no files over a version that has some +// unless --allow-empty: an empty or wrong folder would otherwise remove the +// whole context, and automations follow the latest version. +func checkNotEmptied(d *api.ContextDetail, local *localTree, opts pushOptions) error { + if len(local.files) > 0 || len(d.Files) == 0 || opts.allowEmpty { + return nil + } + return usageErrorf("the folder has no files to publish: that would remove all %d %s of context %s (automations use the latest version). "+ + "Check the folder, or add --allow-empty to empty the context", len(d.Files), pluralWord(len(d.Files), "file", "files"), runs.SanitizeLine(d.Name)) +} + // uploadPlan is the blobs a push sends: one local file per missing sha. type uploadPlan struct { files []localFile @@ -977,6 +1088,10 @@ func pushTree(ctx context.Context, env *runsEnv, d *api.ContextDetail, local *lo for _, f := range local.files { sizes[f.path] = f.size } + inheritExec(d.Files, local) + if err := checkNotEmptied(d, local, opts); err != nil { + return err + } diff := diffTree(d.Files, local) if diff.empty() && d.Version > 0 { if opts.json { @@ -1070,6 +1185,10 @@ func pushTree(ctx context.Context, env *runsEnv, d *api.ContextDetail, local *lo } fmt.Fprintf(env.stderr, "context %s changed to version %d meanwhile; publishing on top of it (--force). Against it, this push:\n", name, latest.Version) + inheritExec(latest.Files, local) + if err := checkNotEmptied(latest, local, opts); err != nil { + return err + } diff = diffTree(latest.Files, local) printDiff(env.stderr, diff, sizes) req.BaseVersion = latest.Version @@ -1172,7 +1291,7 @@ func uploadOne(ctx context.Context, env *runsEnv, f localFile) error { // openChecked opens a file being uploaded and makes sure it is still the // size it was hashed at (the platform verifies the sha256 too). func openChecked(f localFile) (*os.File, error) { - fh, err := os.Open(f.abs) + fh, err := f.open() if err != nil { return nil, exitf(exitFailed, "cannot read %s: %v", runs.SanitizeLine(f.path), unwrapPath(err)) } @@ -1205,7 +1324,16 @@ type uploadState struct { var shaPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) -func uploadStatePath(sha string) (string, error) { +// uploadAccount keys the remembered uploads by API host and credential (a +// hash, never the credential itself): an upload id is only ever sent back +// to the account and endpoint that issued it. +func uploadAccount(env *runsEnv) string { + h := sha256.New() + _, _ = io.WriteString(h, env.client.BaseURL()+"\x00"+env.cred.session+"\x00"+env.cred.bearer) + return hex.EncodeToString(h.Sum(nil))[:32] +} + +func uploadStatePath(env *runsEnv, sha string) (string, error) { if !shaPattern.MatchString(sha) { return "", fmt.Errorf("bad sha") } @@ -1213,11 +1341,11 @@ func uploadStatePath(sha string) (string, error) { if err != nil { return "", err } - return filepath.Join(dir, sha+".json"), nil + return filepath.Join(dir, uploadAccount(env), sha+".json"), nil } -func loadUploadState(sha string, size int64) *uploadState { - p, err := uploadStatePath(sha) +func loadUploadState(env *runsEnv, sha string, size int64) *uploadState { + p, err := uploadStatePath(env, sha) if err != nil { return nil } @@ -1232,8 +1360,8 @@ func loadUploadState(sha string, size int64) *uploadState { return &s } -func saveUploadState(sha string, s uploadState) { - p, err := uploadStatePath(sha) +func saveUploadState(env *runsEnv, sha string, s uploadState) { + p, err := uploadStatePath(env, sha) if err != nil { return } @@ -1244,8 +1372,8 @@ func saveUploadState(sha string, s uploadState) { _ = os.WriteFile(p, b, 0o600) } -func clearUploadState(sha string) { - if p, err := uploadStatePath(sha); err == nil { +func clearUploadState(env *runsEnv, sha string) { + if p, err := uploadStatePath(env, sha); err == nil { _ = os.Remove(p) } } @@ -1269,7 +1397,7 @@ func putMultipart(ctx context.Context, env *runsEnv, f localFile) error { shown := runs.SanitizeLine(f.path) var up *api.ContextUpload - if s := loadUploadState(f.sha, f.size); s != nil { + if s := loadUploadState(env, f.sha, f.size); s != nil { st, err := env.client.UploadStatus(ctx, f.sha, s.UploadID) switch { case err == nil && validUpload(st, f.size): @@ -1280,9 +1408,9 @@ func putMultipart(ctx context.Context, env *runsEnv, f localFile) error { // An upload this client cannot continue: drop it, or its marker // would refuse the new one (upload_in_progress). _ = env.client.AbortUpload(ctx, f.sha, s.UploadID) - clearUploadState(f.sha) + clearUploadState(env, f.sha) case ctxCode(err) == "upload_not_found", ctxCode(err) == "upload_expired": - clearUploadState(f.sha) + clearUploadState(env, f.sha) default: return err } @@ -1303,7 +1431,7 @@ func putMultipart(ctx context.Context, env *runsEnv, f localFile) error { if !validUpload(up, f.size) { return exitf(exitUnavailable, "nan.builders answered an upload of %s this CLI cannot send; update nan (nan --version)", shown) } - saveUploadState(f.sha, uploadState{UploadID: up.UploadID, Size: f.size}) + saveUploadState(env, f.sha, uploadState{UploadID: up.UploadID, Size: f.size}) } failures := 0 @@ -1321,7 +1449,7 @@ func putMultipart(ctx context.Context, env *runsEnv, f localFile) error { return ctx.Err() } if code := ctxCode(err); code == "upload_not_found" || code == "upload_expired" { - clearUploadState(f.sha) + clearUploadState(env, f.sha) return err } failures++ @@ -1356,17 +1484,20 @@ func putMultipart(ctx context.Context, env *runsEnv, f localFile) error { }) if err != nil { if code := ctxCode(err); code == "upload_not_found" || code == "upload_expired" || code == "blob_hash_mismatch" { - clearUploadState(f.sha) + clearUploadState(env, f.sha) } return err } - clearUploadState(f.sha) + clearUploadState(env, f.sha) return nil } // ─── pull ────────────────────────────────────────────────────────────────── func doContextPull(ctx context.Context, env *runsEnv, ref, dir string) error { + if err := checkContextRef(ref); err != nil { + return err + } shown := runs.SanitizeLine(dir) if st, err := os.Stat(dir); err == nil { if !st.IsDir() { @@ -1402,9 +1533,15 @@ func doContextPull(ctx context.Context, env *runsEnv, ref, dir string) error { files := append([]api.ContextFile(nil), d.Files...) sort.Slice(files, func(i, j int) bool { return files[i].Path < files[j].Path }) var total int64 - for _, f := range files { + for i, f := range files { if err := pullOne(ctx, env, root, d, f); err != nil { - return err + if i == 0 { + return err + } + var exit *ExitError + errors.As(err, &exit) + return &ExitError{Code: exit.Code, Err: fmt.Errorf("%v. %d of %d files were already written to %s; run the pull again with --force to finish it", + exit.Err, i, len(files), shown)} } total += f.Size } @@ -1415,18 +1552,25 @@ func doContextPull(ctx context.Context, env *runsEnv, ref, dir string) error { // checkServerTree refuses a file list a well-behaved platform never sends: // paths the shared validator refuses (absolute, .., case collisions, -// reserved names, ...), bad hashes or sizes. Nothing is written then. +// reserved names, ...), bad hashes or sizes, more than a version holds. +// Nothing is written then. func checkServerTree(env *runsEnv, files []api.ContextFile) error { if len(files) > ctxMaxFiles { return exitf(exitDataErr, "nan.builders sent %d files, more than a context holds; nothing was written", len(files)) } paths := make([]string, len(files)) var bad []string + var total int64 for i, f := range files { paths[i] = f.Path if !shaPattern.MatchString(f.SHA256) || f.Size < 0 || f.Size > api.ContextMaxFileBytes { bad = append(bad, fmt.Sprintf("%s: bad hash or size", runs.SanitizeLine(f.Path))) + continue } + total += f.Size + } + if total > ctxMaxTotalBytes { + return exitf(exitDataErr, "nan.builders sent %s of files, more than a context version holds (250 MiB); nothing was written", humanBytes(total)) } for _, pe := range contexts.ValidateTree(paths) { bad = append(bad, fmt.Sprintf("%s: %s", runs.SanitizeLine(pe.Path), pe.Reason)) @@ -1441,32 +1585,35 @@ func checkServerTree(env *runsEnv, files []api.ContextFile) error { return exitf(exitDataErr, "nan.builders sent %d %s this CLI will not write; nothing was written", len(bad), pluralWord(len(bad), "path", "paths")) } +// pullOne downloads one file into a temporary name next to it, checks it, +// sets its mode and renames it into place: a failed download never leaves +// a half file nor removes one the member had. An existing link or folder at +// the path is never replaced. func pullOne(ctx context.Context, env *runsEnv, root *os.Root, d *api.ContextDetail, f api.ContextFile) error { shown := runs.SanitizeLine(f.Path) local := filepath.FromSlash(f.Path) - if parent := filepath.Dir(local); parent != "." { + parent := filepath.Dir(local) + if parent != "." { if err := root.MkdirAll(parent, 0o755); err != nil { return exitf(exitFailed, "cannot create the folder of %s: %v", shown, unwrapPath(err)) } } - flags := os.O_WRONLY | os.O_CREATE | os.O_EXCL - if ctxPullForce { - flags = os.O_WRONLY | os.O_CREATE | os.O_TRUNC - } - mode := os.FileMode(0o644) - if f.Executable { - mode = 0o755 + switch st, err := root.Lstat(local); { + case err == nil && (st.Mode()&fs.ModeSymlink != 0 || !st.Mode().IsRegular()): + return exitf(exitFailed, "%s is a link or a folder in the target folder; it was left as it is and nothing was written there", shown) + case err == nil && !ctxPullForce: + return exitf(exitFailed, "%s already exists (use --force to replace files)", shown) + case err != nil && !errors.Is(err, fs.ErrNotExist): + return exitf(exitFailed, "cannot write %s: %v", shown, unwrapPath(err)) } - out, err := root.OpenFile(local, flags, mode) + tmp := filepath.Join(parent, ".nan-pull-"+newIdempotencyKey()[len("nan-cli-"):]) + out, err := root.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) if err != nil { - if errors.Is(err, fs.ErrExist) { - return exitf(exitFailed, "%s already exists; nothing more was written (use --force to replace files)", shown) - } return exitf(exitFailed, "cannot write %s: %v", shown, unwrapPath(err)) } fail := func(e *ExitError) error { out.Close() - _ = root.Remove(local) + _ = root.Remove(tmp) return e } var body io.ReadCloser @@ -1488,11 +1635,22 @@ func pullOne(ctx context.Context, env *runsEnv, root *os.Root, d *api.ContextDet return fail(exitf(exitUnavailable, "the download of %s does not match its sha256 and size; it was not kept", shown)) } if err := out.Close(); err != nil { + _ = root.Remove(tmp) return exitf(exitFailed, "cannot write %s: %v", shown, err) } - if f.Executable && runtime.GOOS != "windows" { - // The umask may have taken the bits off the mode OpenFile asked for. - _ = root.Chmod(local, 0o755) + mode := os.FileMode(0o644) + if f.Executable { + mode = 0o755 + } + // Both ways: a replaced file that was executable and no longer is + // loses the bit. + if err := root.Chmod(tmp, mode); err != nil && runtime.GOOS != "windows" { + _ = root.Remove(tmp) + return exitf(exitFailed, "cannot write %s: %v", shown, err) + } + if err := root.Rename(tmp, local); err != nil { + _ = root.Remove(tmp) + return exitf(exitFailed, "cannot write %s: %v", shown, unwrapPath(err)) } return nil } diff --git a/cmd/context_fake_test.go b/cmd/context_fake_test.go index 592e629..e25d0fe 100644 --- a/cmd/context_fake_test.go +++ b/cmd/context_fake_test.go @@ -10,6 +10,7 @@ import ( "io" "net/http" "net/http/httptest" + "path/filepath" "sort" "strconv" "strings" @@ -18,6 +19,7 @@ import ( "time" "github.com/nxssie/nan-cli/internal/api" + "github.com/nxssie/nan-cli/internal/contexts" ) // fakeCtx is an in-memory /v1/contexts with the shapes, codes and status @@ -74,6 +76,34 @@ type fakeCtx struct { corrupt map[string]bool // failPart answers 500 to the first PUT of these part numbers. failPart map[int]bool + srv *httptest.Server +} + +// server starts the fake once; every run against it sees the same URL, so +// the remembered uploads (keyed by API host and account) carry over. +func (f *fakeCtx) server(t *testing.T) *httptest.Server { + if f.srv == nil { + f.srv = httptest.NewServer(f) + t.Cleanup(f.srv.Close) + } + return f.srv +} + +// stateFile is where a push run against f remembers an upload of sha. +func (f *fakeCtx) stateFile(t *testing.T, stateDir, sha string) string { + h := sha256.Sum256([]byte(f.server(t).URL + "\x00" + "sess-cli" + "\x00")) + return filepath.Join(stateDir, hex.EncodeToString(h[:])[:32], sha+".json") +} + +// strict decodes a JSON body like cloud-api: unknown fields are a 400. +func strict(w http.ResponseWriter, body []byte, v any) bool { + dec := json.NewDecoder(bytes.NewReader(body)) + dec.DisallowUnknownFields() + if err := dec.Decode(v); err != nil { + ctxFail(w, 400, "invalid_request", "body has an unknown or malformed field", nil, 0) + return false + } + return true } func newFakeCtx() *fakeCtx { @@ -194,6 +224,10 @@ func (f *fakeCtx) blobView(sha string) map[string]any { func (f *fakeCtx) ServeHTTP(w http.ResponseWriter, r *http.Request) { isRaw := r.Header.Get("Content-Type") == "application/octet-stream" + if r.Method == http.MethodPut && r.ContentLength < 0 { + ctxFail(w, 411, "invalid_request", "Content-Length is required", nil, 0) + return + } body, _ := io.ReadAll(r.Body) f.mu.Lock() defer f.mu.Unlock() @@ -230,8 +264,13 @@ func (f *fakeCtx) ServeHTTP(w http.ResponseWriter, r *http.Request) { } ctxJSONResp(w, 200, map[string]any{"data": data, "usage": map[string]any{"stored_bytes": 1536, "quota_bytes": 500 << 20}}) case rest == "" && r.Method == http.MethodPost: - var in struct{ Name, Description string } - _ = json.Unmarshal(body, &in) + var in struct { + Name string `json:"name"` + Description string `json:"description"` + } + if !strict(w, body, &in) { + return + } if f.find(in.Name) != nil { ctxFail(w, 409, "context_name_taken", "You already have a context with this name.", nil, 0) return @@ -258,13 +297,18 @@ func (f *fakeCtx) ServeHTTP(w http.ResponseWriter, r *http.Request) { v["files"] = files ctxJSONResp(w, 200, v) case "PATCH ": - var in map[string]*string - _ = json.Unmarshal(body, &in) - if in["name"] != nil { - c.Name = *in["name"] + var in struct { + Name *string `json:"name"` + Description *string `json:"description"` + } + if !strict(w, body, &in) { + return + } + if in.Name != nil { + c.Name = *in.Name } - if in["description"] != nil { - c.Description = *in["description"] + if in.Description != nil { + c.Description = *in.Description } ctxJSONResp(w, 200, c.view()) case "DELETE ": @@ -299,10 +343,21 @@ func (f *fakeCtx) ServeHTTP(w http.ResponseWriter, r *http.Request) { f.onPublish(f, f.publishes) } var in api.PublishRequest - _ = json.Unmarshal(body, &in) - if in.BaseVersion != c.Version { - ctxFail(w, 409, "version_conflict", "The context changed since you loaded it; reload and publish again.", - map[string]any{"current_version": c.Version}, 0) + if !strict(w, body, &in) { + return + } + // cloud-api's order (service.Publish): the tree (paths, then + // blobs), then base_version. + paths := make([]string, len(in.Files)) + for i, fl := range in.Files { + paths[i] = fl.Path + } + if bad := contexts.ValidateTree(paths); len(bad) > 0 { + refused := []any{} + for _, b := range bad { + refused = append(refused, map[string]any{"path": b.Path, "reason": string(b.Reason)}) + } + ctxFail(w, 400, "invalid_path", "Some file names are not allowed.", map[string]any{"files": refused}, 0) return } var missing []string @@ -315,6 +370,11 @@ func (f *fakeCtx) ServeHTTP(w http.ResponseWriter, r *http.Request) { ctxFail(w, 409, "blob_missing", "Some files are no longer stored; upload them again and publish.", map[string]any{"sha256": missing}, 0) return } + if in.BaseVersion != c.Version { + ctxFail(w, 409, "version_conflict", "The context changed since you loaded it; reload and publish again.", + map[string]any{"current_version": c.Version}, 0) + return + } if !in.AllowSecretLike { var hits []any for _, fl := range in.Files { @@ -345,7 +405,9 @@ func (f *fakeCtx) serveBlobs(w http.ResponseWriter, r *http.Request, parts []str var in struct { SHA256 []string `json:"sha256"` } - _ = json.Unmarshal(body, &in) + if !strict(w, body, &in) { + return + } missing := []string{} for _, s := range in.SHA256 { if _, ok := f.blobs[s]; !ok { @@ -381,7 +443,9 @@ func (f *fakeCtx) serveBlobs(w http.ResponseWriter, r *http.Request, parts []str Size int64 `json:"size"` Filename string `json:"filename"` } - _ = json.Unmarshal(body, &in) + if !strict(w, body, &in) { + return + } id := fmt.Sprintf("u0000000-0000-4000-8000-%012d", len(f.uploads)+1) u := &fakeUpload{sha: sha, size: in.Size, partSize: f.partSize, next: 1} f.uploads[id] = u @@ -456,8 +520,7 @@ type ctxSetup struct { func runCtx(t *testing.T, fake *fakeCtx, setup ctxSetup, args ...string) ctxRun { t.Helper() - srv := httptest.NewServer(fake) - t.Cleanup(srv.Close) + srv := fake.server(t) fakeHome(t, cliSession) t.Setenv(tokenEnvVar, "") swap(t, &runsBaseURL, srv.URL) diff --git a/cmd/context_hardening_test.go b/cmd/context_hardening_test.go new file mode 100644 index 0000000..e9ace57 --- /dev/null +++ b/cmd/context_hardening_test.go @@ -0,0 +1,349 @@ +package cmd + +import ( + "fmt" + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/nxssie/nan-cli/internal/api" +) + +// The review findings of #62 (QA and Security), each as a test first. + +func skipOnWindows(t *testing.T) { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("needs symlinks and Unix modes") + } +} + +// A DIR that is itself a symlink is the folder the member named: push walks +// what it points at instead of publishing an empty tree. +func TestContextPushThroughASymlinkedFolder(t *testing.T) { + skipOnWindows(t) + fake := newFakeCtx() + fake.seed("acme-api", 2, map[string]string{"files/a.md": "a\n"}) + real := writeTree(t, map[string]string{"files/a.md": "a\n", "files/b.md": "b\n"}) + link := filepath.Join(t.TempDir(), "link") + if err := os.Symlink(real, link); err != nil { + t.Fatal(err) + } + r := runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", link) + mustCtxExit(t, r, 0) + pubs := fake.sent("POST", "/v1/contexts/"+fakeCtxID+"/versions") + if len(pubs) != 1 || strings.Join(publishPaths(publishBody(t, pubs[0])), ",") != "files/a.md,files/b.md" { + t.Errorf("published %+v", pubs) + } +} + +// An empty folder never silently empties a context. +func TestContextPushRefusesEmptyingAContext(t *testing.T) { + fake := newFakeCtx() + fake.seed("acme-api", 2, map[string]string{"files/a.md": "a\n"}) + empty := t.TempDir() + r := runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", empty) + mustCtxExit(t, r, exitUsage, "no files to publish", "remove all 1 file", "--allow-empty") + if len(fake.sent("POST", "/v1/contexts/"+fakeCtxID+"/versions")) != 0 { + t.Fatal("published an empty tree") + } + // Only dotfiles left is the same thing. + dotted := writeTree(t, map[string]string{".env": "S=1\n"}) + mustCtxExit(t, runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dotted), exitUsage, "--allow-empty") + + r = runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", empty, "--allow-empty") + mustCtxExit(t, r, 0) + pubs := fake.sent("POST", "/v1/contexts/"+fakeCtxID+"/versions") + if len(pubs) != 1 || len(publishBody(t, pubs[0]).Files) != 0 || fake.contexts[0].Version != 3 { + t.Errorf("publishes %+v", pubs) + } + // An empty context takes an empty folder without the flag. + r = runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", empty) + mustCtxExit(t, r, 0) + wantAll(t, "stdout", r.stdout, "is up to date") +} + +// Writes go through os.OpenRoot: a link already inside DIR never leads a +// pull outside it, and a link at a file's path is never replaced. +func TestContextPullNeverWritesThroughALink(t *testing.T) { + skipOnWindows(t) + t.Run("files is a link", func(t *testing.T) { + fake := newFakeCtx() + fake.seed("acme-api", 1, map[string]string{"files/a.md": "a\n"}) + outside := t.TempDir() + dir := t.TempDir() + if err := os.Symlink(outside, filepath.Join(dir, "files")); err != nil { + t.Fatal(err) + } + r := runCtx(t, fake, ctxSetup{}, "context", "pull", "acme-api", dir, "--force") + mustCtxExit(t, r, exitFailed) + if entries, _ := os.ReadDir(outside); len(entries) != 0 { + t.Errorf("wrote outside DIR: %v", entries) + } + }) + t.Run("files/a.md is a link", func(t *testing.T) { + fake := newFakeCtx() + fake.seed("acme-api", 1, map[string]string{"files/a.md": "a\n"}) + outside := filepath.Join(t.TempDir(), "victim") + if err := os.WriteFile(outside, []byte("keep\n"), 0o644); err != nil { + t.Fatal(err) + } + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "files"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(dir, "files", "a.md")); err != nil { + t.Fatal(err) + } + r := runCtx(t, fake, ctxSetup{}, "context", "pull", "acme-api", dir, "--force") + mustCtxExit(t, r, exitFailed, "link") + if b, _ := os.ReadFile(outside); string(b) != "keep\n" { + t.Errorf("wrote through the link: %q", b) + } + if st, err := os.Lstat(filepath.Join(dir, "files", "a.md")); err != nil || st.Mode()&os.ModeSymlink == 0 { + t.Errorf("replaced the link: %v", err) + } + }) +} + +// The limits of one version, checked before anything is hashed or sent. +func TestContextPushTreeLimits(t *testing.T) { + sparse := func(t *testing.T, dir, rel string, size int64) { + t.Helper() + p := filepath.Join(dir, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + f, err := os.Create(p) + if err != nil { + t.Fatal(err) + } + if err := f.Truncate(size); err != nil { + t.Fatal(err) + } + f.Close() + } + for name, tc := range map[string]struct { + build func(t *testing.T, dir string) + want string + }{ + "31 skills": {func(t *testing.T, dir string) { + for i := 0; i < 31; i++ { + sparse(t, dir, fmt.Sprintf("skills/s%d/SKILL.md", i), 10) + } + }, "31 skills: a context holds at most 30"}, + "1001 files": {func(t *testing.T, dir string) { + for i := 0; i < 1001; i++ { + sparse(t, dir, fmt.Sprintf("files/f%d.md", i), 0) + } + }, "1001 files: a context version holds at most 1000"}, + "250 MiB": {func(t *testing.T, dir string) { + for i := 0; i < 3; i++ { + sparse(t, dir, fmt.Sprintf("files/big%d.bin", i), 90<<20) + } + }, "270.0 MiB in total: a context version holds at most 250 MiB"}, + "100 MiB file": {func(t *testing.T, dir string) { + sparse(t, dir, "files/huge.bin", 100<<20+1) + }, ""}, + "64 KiB SKILL.md": {func(t *testing.T, dir string) { + sparse(t, dir, "skills/x/SKILL.md", 64<<10+1) + }, ""}, + } { + t.Run(name, func(t *testing.T) { + fake := newFakeCtx() + fake.seed("acme-api", 1, nil) + dir := t.TempDir() + tc.build(t, dir) + r := runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, exitDataErr, tc.want) + switch name { + case "100 MiB file": + wantAll(t, "stderr", r.stderr, "refused files/huge.bin: too_large (a file is at most 100 MiB)") + case "64 KiB SKILL.md": + wantAll(t, "stderr", r.stderr, "refused skills/x/SKILL.md: skill_invalid (SKILL.md is at most 64 KiB)") + } + if len(fake.requests()) != 0 { + t.Errorf("sent %d requests", len(fake.requests())) + } + }) + } + // At the limits exactly, nothing is refused locally. + fake := newFakeCtx() + fake.seed("acme-api", 1, nil) + dir := t.TempDir() + for i := 0; i < 30; i++ { + sparse(t, dir, fmt.Sprintf("skills/s%d/SKILL.md", i), 0) + } + sparse(t, dir, "skills/s0/SKILL.md", 64<<10) + r := runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir, "--dry-run") + mustCtxExit(t, r, 0) +} + +// A remembered upload the platform describes in a way this CLI cannot +// continue is aborted (its marker would refuse a new one), and a new one +// is started. +func TestContextPushAbortsAnUnusableRememberedUpload(t *testing.T) { + swap(t, &ctxSingleMax, int64(8)) + fake := newFakeCtx() + fake.partSize = 4 + fake.seed("acme-api", 1, nil) + big := "0123456789" + sha := shaOf([]byte(big)) + const id = "u0000000-0000-4000-8000-000000000042" + fake.uploads[id] = &fakeUpload{sha: sha, size: 10, partSize: 32 << 20, next: 1} + state := t.TempDir() + writeState(t, fake.stateFile(t, state, sha), id, 10) + dir := writeTree(t, map[string]string{"files/big.csv": big}) + r := runCtx(t, fake, ctxSetup{stateDir: state}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, 0) + if dels := fake.sent("DELETE", "/v1/contexts/blobs/"+sha+"/uploads/"+id); len(dels) != 1 { + t.Errorf("abort not sent: %v", fake.requests()) + } + if _, err := os.Stat(fake.stateFile(t, state, sha)); !os.IsNotExist(err) { + t.Errorf("state file left: %v", err) + } + if string(fake.blobs[sha]) != big { + t.Errorf("stored %q", fake.blobs[sha]) + } +} + +// An upload the platform dropped mid-way (expired, or gone) is forgotten: +// the next push starts a new one instead of asking for it again. +func TestContextPushForgetsAnUploadThePlatformDropped(t *testing.T) { + for code, status := range map[string]int{"upload_not_found": 404, "upload_expired": 409} { + t.Run(code, func(t *testing.T) { + swap(t, &ctxSingleMax, int64(8)) + fake := newFakeCtx() + fake.partSize = 4 + fake.seed("acme-api", 1, nil) + big := "0123456789" + sha := shaOf([]byte(big)) + const id = "u0000000-0000-4000-8000-000000000001" // the fake's first upload id + fake.queue("PUT /v1/contexts/blobs/"+sha+"/uploads/"+id+"/parts/2", fakeResp{status: status, body: map[string]any{ + "error": map[string]any{"message": "The upload expired before it finished; upload the file again.", "type": "invalid_request_error", "param": nil, "code": code}}}) + state := t.TempDir() + dir := writeTree(t, map[string]string{"files/big.csv": big}) + r := runCtx(t, fake, ctxSetup{stateDir: state}, "context", "push", "acme-api", dir) + if r.code == 0 { + t.Fatalf("push succeeded: %s", r.stdout) + } + wantAll(t, "message", r.msg, "upload of files/big.csv failed") + if _, err := os.Stat(fake.stateFile(t, state, sha)); !os.IsNotExist(err) { + t.Errorf("state file left: %v", err) + } + }) + } +} + +func TestContextExitMapping(t *testing.T) { + for _, tc := range []struct { + status int + code string + exit int + want string + }{ + {403, "entitlement_required", exitAuth, "membership does not include agent runs"}, + {503, "storage_unavailable", exitUnavailable, "unavailable"}, + {409, "context_name_taken", exitUsage, "already have a context"}, + {409, "too_many_contexts", exitDataErr, "at most 20"}, + } { + fake := newFakeCtx() + msg := "Context storage is unavailable right now; try again later." + if tc.code == "too_many_contexts" { + msg = "You can keep at most 20 contexts." + } + // Retryable refusals are retried (3 times): queue enough of them. + for i := 0; i < 4; i++ { + fake.queue("POST /v1/contexts", fakeResp{status: tc.status, body: map[string]any{ + "error": map[string]any{"message": msg, "type": "invalid_request_error", "param": nil, "code": tc.code}}}) + } + r := runCtx(t, fake, ctxSetup{}, "context", "create", "acme-api") + mustCtxExit(t, r, tc.exit, tc.want) + } +} + +// Where the file system has no executable bit, an unchanged file keeps the +// bit of the version it replaces. +func TestContextPushKeepsTheRemoteExecutableBit(t *testing.T) { + swap(t, &ctxExecKnown, false) + fake := newFakeCtx() + fake.seed("acme-api", 2, map[string]string{"skills/x/SKILL.md": skillMD, "skills/x/run.sh": "#!/bin/sh\n"}) + dir := writeTree(t, map[string]string{"skills/x/SKILL.md": skillMD, "skills/x/run.sh": "#!/bin/sh\n"}) + r := runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, 0) + wantAll(t, "stdout", r.stdout, "is up to date (version 2)") + + // A changed file has no remote bit to inherit. + dir = writeTree(t, map[string]string{"skills/x/SKILL.md": skillMD, "skills/x/run.sh": "#!/bin/sh\necho 2\n"}) + r = runCtx(t, fake, ctxSetup{}, "context", "push", "acme-api", dir) + mustCtxExit(t, r, 0) + pubs := fake.sent("POST", "/v1/contexts/"+fakeCtxID+"/versions") + for _, f := range publishBody(t, pubs[len(pubs)-1]).Files { + if f.Executable { + t.Errorf("%s published executable", f.Path) + } + } +} + +// pull --force replaces files whole: the mode follows the remote file both +// ways, and a failed download leaves the member's file as it was. +func TestContextPullForceReplacesFilesWhole(t *testing.T) { + skipOnWindows(t) + fake := newFakeCtx() + fake.seed("acme-api", 1, map[string]string{"files/a.md": "new\n", "files/b.md": "fine\n"}) + dir := writeTree(t, map[string]string{"files/a.md": "mine\n", "files/b.md": "keep me\n"}) + if err := os.Chmod(filepath.Join(dir, "files", "a.md"), 0o755); err != nil { + t.Fatal(err) + } + fake.corrupt["files/b.md"] = true + r := runCtx(t, fake, ctxSetup{}, "context", "pull", "acme-api", dir, "--force") + mustCtxExit(t, r, exitUnavailable, "does not match", "1 of 2 files were already written", "--force") + if b, _ := os.ReadFile(filepath.Join(dir, "files", "b.md")); string(b) != "keep me\n" { + t.Errorf("a failed download changed the member's file: %q", b) + } + st, _ := os.Stat(filepath.Join(dir, "files", "a.md")) + if b, _ := os.ReadFile(filepath.Join(dir, "files", "a.md")); string(b) != "new\n" || st.Mode().Perm() != 0o644 { + t.Errorf("a.md %q %v", b, st.Mode()) + } + entries, _ := os.ReadDir(filepath.Join(dir, "files")) + if len(entries) != 2 { + t.Errorf("left temporary files: %v", entries) + } +} + +func TestContextPullCapsTheTotal(t *testing.T) { + fake := newFakeCtx() + c := fake.seed("acme-api", 1, nil) + for i := 0; i < 3; i++ { + c.Files = append(c.Files, api.ContextFile{Path: fmt.Sprintf("files/big%d.bin", i), SHA256: shaOf([]byte{byte(i)}), Size: 90 << 20}) + } + dir := filepath.Join(t.TempDir(), "out") + r := runCtx(t, fake, ctxSetup{}, "context", "pull", "acme-api", dir) + mustCtxExit(t, r, exitDataErr, "more than a context version holds", "nothing was written") + if _, err := os.Stat(dir); !os.IsNotExist(err) { + t.Errorf("created %s", dir) + } +} + +func TestContextRefIsCheckedLocally(t *testing.T) { + for _, ref := range []string{"..", ".", "a/b", "Acme", "%2e%2e"} { + for _, args := range [][]string{{"show", ref}, {"pull", ref, t.TempDir()}, {"rm", ref, "--yes"}, {"edit", ref, "--name", "x"}} { + fake := newFakeCtx() + r := runCtx(t, fake, ctxSetup{}, append([]string{"context"}, args...)...) + mustCtxExit(t, r, exitUsage, "neither a context name") + if len(fake.requests()) != 0 { + t.Errorf("%v sent %v", args, fake.requests()) + } + } + } + // A context id is accepted as is. + fake := newFakeCtx() + fake.seed("acme-api", 1, nil) + mustCtxExit(t, runCtx(t, fake, ctxSetup{}, "context", "show", fakeCtxID), 0) + if got := fake.requests(); len(got) != 1 || got[0].Method != http.MethodGet { + t.Errorf("%v", got) + } +} diff --git a/cmd/context_test.go b/cmd/context_test.go index 1290ca1..6014e37 100644 --- a/cmd/context_test.go +++ b/cmd/context_test.go @@ -2,6 +2,7 @@ package cmd import ( "encoding/json" + "fmt" "net/http" "os" "path/filepath" @@ -366,8 +367,8 @@ func TestContextPushMultipartResumesAfterAFailedPart(t *testing.T) { if starts := fake.sent("POST", "/v1/contexts/blobs/"+sha+"/uploads"); len(starts) != 2 { // start + complete t.Errorf("start/complete %v", starts) } - if entries, _ := os.ReadDir(r.stateDir); len(entries) != 0 { - t.Errorf("upload state left behind: %v", entries) + if _, err := os.Stat(fake.stateFile(t, r.stateDir, sha)); !os.IsNotExist(err) { + t.Errorf("upload state left behind: %v", err) } } @@ -383,9 +384,7 @@ func TestContextPushMultipartResumesAcrossRuns(t *testing.T) { const id = "u0000000-0000-4000-8000-000000000042" fake.uploads[id] = &fakeUpload{sha: sha, size: int64(len(big)), partSize: 4, next: 2, data: []byte("0123")} state := t.TempDir() - if err := os.WriteFile(filepath.Join(state, sha+".json"), []byte(`{"upload_id":"`+id+`","size":10}`), 0o600); err != nil { - t.Fatal(err) - } + writeState(t, fake.stateFile(t, state, sha), id, 10) dir := writeTree(t, map[string]string{"files/big.csv": big}) r := runCtx(t, fake, ctxSetup{stateDir: state}, "context", "push", "acme-api", dir) mustCtxExit(t, r, 0) @@ -408,9 +407,7 @@ func TestContextPushMultipartResumesAcrossRuns(t *testing.T) { fake2 := newFakeCtx() fake2.partSize = 4 fake2.seed("acme-api", 1, nil) - if err := os.WriteFile(filepath.Join(state, sha+".json"), []byte(`{"upload_id":"`+id+`","size":10}`), 0o600); err != nil { - t.Fatal(err) - } + writeState(t, fake2.stateFile(t, state, sha), id, 10) r = runCtx(t, fake2, ctxSetup{stateDir: state}, "context", "push", "acme-api", dir) mustCtxExit(t, r, 0) if string(fake2.blobs[sha]) != big { @@ -418,6 +415,16 @@ func TestContextPushMultipartResumesAcrossRuns(t *testing.T) { } } +func writeState(t *testing.T, path, id string, size int) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(fmt.Sprintf(`{"upload_id":%q,"size":%d}`, id, size)), 0o600); err != nil { + t.Fatal(err) + } +} + func TestContextPushVersionConflict(t *testing.T) { newFake := func() *fakeCtx { fake := newFakeCtx() @@ -492,6 +499,15 @@ func TestContextPushBlobMissingReuploads(t *testing.T) { if puts := fake.sent("PUT", "/v1/contexts/blobs/"+gone); len(puts) != 2 { t.Errorf("uploads of the missing blob: %d", len(puts)) } + // It asks again, for exactly the blobs the publish named, and sends + // only those: the other file goes up once. + checks := fake.sent("POST", "/v1/contexts/blobs/check") + if len(checks) != 2 || checks[1].Body != `{"sha256":["`+gone+`"]}` { + t.Errorf("checks %+v", checks) + } + if puts := fake.sent("PUT", "/v1/contexts/blobs/"+shaOf([]byte("a\n"))); len(puts) != 1 { + t.Errorf("uploads of the file still stored: %d", len(puts)) + } if pubs := fake.sent("POST", "/v1/contexts/"+fakeCtxID+"/versions"); len(pubs) != 2 { t.Errorf("publishes %d", len(pubs)) } diff --git a/internal/api/contexts.go b/internal/api/contexts.go index e106541..4e159ac 100644 --- a/internal/api/contexts.go +++ b/internal/api/contexts.go @@ -77,15 +77,6 @@ type ContextList struct { Usage ContextUsage `json:"usage"` } -// ContextVersionInfo is one retained version. -type ContextVersionInfo struct { - Version int `json:"version"` - TreeSHA256 string `json:"tree_sha256"` - FileCount int `json:"file_count"` - TotalBytes int64 `json:"total_bytes"` - CreatedAt string `json:"created_at"` -} - // SecretFinding is where a text blob looks like it holds a credential. The // platform never returns the match itself. type SecretFinding struct { @@ -260,6 +251,9 @@ func (c *RunsClient) rawRequest(ctx context.Context, method, path string, body i return req, nil } +// BaseURL is where this client sends its requests. +func (c *RunsClient) BaseURL() string { return c.baseURL } + func contextPath(id string) string { return "/v1/contexts/" + url.PathEscape(id) } func blobPath(sha string) string { return "/v1/contexts/blobs/" + url.PathEscape(sha) } @@ -308,13 +302,6 @@ func (c *RunsClient) DeleteContext(ctx context.Context, id string) error { return err } -// ContextVersions is GET /v1/contexts/{id}/versions, newest first. -func (c *RunsClient) ContextVersions(ctx context.Context, id string) ([]ContextVersionInfo, error) { - var out []ContextVersionInfo - _, err := c.ctxJSON(ctx, http.MethodGet, contextPath(id)+"/versions", nil, &out, "") - return out, err -} - // CheckBlobs is POST /v1/contexts/blobs/check: which of shas the platform // does not hold for this member (asked in batches of the platform's cap). func (c *RunsClient) CheckBlobs(ctx context.Context, shas []string) ([]string, error) { diff --git a/internal/contexts/pins_test.go b/internal/contexts/pins_test.go index 53b0227..5938c29 100644 --- a/internal/contexts/pins_test.go +++ b/internal/contexts/pins_test.go @@ -14,14 +14,15 @@ import ( // this CLI has to read the same tables or `nan context pull` could refuse a // name the server stored (devops AGENT-CONTEXT.md §1.5). // -// These are the pins both repositories must share. x/text is v0.39.0, the -// first release without GO-2026-5970 (an infinite loop in unicode/norm on -// invalid UTF-8, reached by these rules). The cross-repo check lives in -// cloud-api's CI (scripts/check-nan-cli-text-parity.sh, helmcode/nan-cloud-api -// #397): it reads this repository's public go.mod and compares the x/text -// version and unicode.Version under the go version declared here with its -// own. This repository is public and cannot read cloud-api. When cloud-api moves x/text or its Go toolchain's -// Unicode tables, this record and go.mod move with it. +// These are the pins both repositories must share. x/text is v0.39.0, +// the first release without GO-2026-5970 (an infinite loop in +// unicode/norm on invalid UTF-8, reached by these rules). +// +// The cross-repo check lives in cloud-api's CI +// (scripts/check-nan-cli-text-parity.sh, helmcode/nan-cloud-api#397): it +// reads this repository's public go.mod and compares the x/text version +// and unicode.Version under the go version declared here with its own. +// This repository is public and cannot read cloud-api. const ( cloudAPIXText = "v0.39.0" cloudAPIUnicode = "15.0.0"