feat(evaluate): send a directory of policies as one bundle

--policy may name a directory, and every file below it travels keyed by
its path relative to that directory. Nothing is left out by name: what a
bundle may hold, and what its modules may import, is for the evaluator
that runs it to judge. The published file and byte caps, and an empty
directory, are still named here rather than sent to be refused.

A policy now comes from this machine only. Fetching one from a URL is on
its way out, so this command does not offer it.

Slice 6 of kosli-dev/server#6920.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: jumboduck

cmd/kosli/evaluateHelpers.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"net/http"
 	"net/url"
 	"os"
@@ -36,6 +37,10 @@ const policyMaxBytes = 5 << 20 // 5 MiB
 // whenever that schema changes.
 const maxServerSideTrails = 100
 
+// maxPolicyBundleFiles mirrors the ceiling the evaluations API publishes on
+// the entries in a policy bundle.
+const maxPolicyBundleFiles = 100
+
 // serverPolicyMaxBytes mirrors the cap the evaluations API publishes on a
 // policy bundle, which counts the names as well as the sources. It is a fifth
 // of what a remote --policy read allows, so a policy can be fetched in full
@@ -304,12 +309,7 @@ func runServerEvaluation(out io.Writer, spec serverEvaluation) error {
 		return err
 	}
 
-	policySource, err := loadPolicy(spec.policyRef)
-	if err != nil {
-		return err
-	}
-
-	files, err := policyBundle(spec.policyRef, policySource)
+	files, err := policyBundle(spec.policyRef)
 	if err != nil {
 		return err
 	}
@@ -437,16 +437,79 @@ func serverVerdict(result *evaluations.Result) *evaluate.Result {
 	return &evaluate.Result{Allow: result.Allow, Violations: violations}
 }
 
-// policyBundle wraps the policy source as the one-file bundle the API takes,
-// refusing one too large for it rather than letting the request be rejected.
-// The cap counts the names as well as the sources, exactly as the API counts.
-func policyBundle(ref string, source []byte) (map[string]string, error) {
-	key := policyBundleKey(ref)
-	if size := len(key) + len(source); size > serverPolicyMaxBytes {
+// policyBundle reads what --policy names as the bundle the API takes. The caps
+// are checked here so an oversized bundle is named as such rather than rejected
+// as an opaque 422, and the byte cap counts the names as well as the sources,
+// exactly as the API counts.
+func policyBundle(ref string) (map[string]string, error) {
+	files, err := policyBundleFiles(ref)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(files) > maxPolicyBundleFiles {
+		return nil, fmt.Errorf("policy bundle holds %d files, over the limit of %d",
+			len(files), maxPolicyBundleFiles)
+	}
+	size := 0
+	for name, source := range files {
+		size += len(name) + len(source)
+	}
+	if size > serverPolicyMaxBytes {
 		return nil, fmt.Errorf("policy bundle is %d bytes, over the %d byte limit",
 			size, serverPolicyMaxBytes)
 	}
-	return map[string]string{key: string(source)}, nil
+	return files, nil
+}
+
+func policyBundleFiles(ref string) (map[string]string, error) {
+	if !isRemotePolicyRef(ref) {
+		if info, err := os.Stat(ref); err == nil && info.IsDir() {
+			return policyDirectory(ref)
+		}
+	}
+
+	source, err := loadPolicy(ref)
+	if err != nil {
+		return nil, err
+	}
+	return map[string]string{policyBundleKey(ref): string(source)}, nil
+}
+
+// policyDirectory collects every file below root, keyed by its path relative
+// to it. Nothing is left out by name: a rule here would refuse bundles the
+// evaluator that runs them would have accepted.
+func policyDirectory(root string) (map[string]string, error) {
+	files := map[string]string{}
+	err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if entry.IsDir() {
+			return nil
+		}
+		source, err := os.ReadFile(path)
+		if err != nil {
+			return fmt.Errorf("failed to read policy file: %w", err)
+		}
+		name, err := filepath.Rel(root, path)
+		if err != nil {
+			return err
+		}
+		// Relative paths reach the API spelled one way, whatever this machine
+		// spells them with.
+		files[filepath.ToSlash(name)] = string(source)
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	if len(files) == 0 {
+		// The API takes at least one file, so an empty directory is named here
+		// rather than sent to be refused.
+		return nil, fmt.Errorf("no file found under %s", root)
+	}
+	return files, nil
 }
 
 // policyBundleKey names the policy inside the uploaded bundle. Only the base

cmd/kosli/evaluatePolicy.go

@@ -18,6 +18,9 @@ recorded it, and the verdict is printed here.
 Name what to evaluate with ` + "`--context trail=<flow>/<trail>`" + `, repeated once per
 trail. Trails named in one command are all evaluated at the same instant.
 
+` + "`--policy`" + ` takes a single Rego file or a directory. A directory travels as one
+bundle of every file below it, keyed by its path relative to that directory.
+
 Pass ` + "`--control`" + ` to record the outcome as a decision against that control, in the
 ` + "`--flow`" + ` and ` + "`--trail`" + ` given. The decision is recorded where the policy runs, so
 the verdict is never asserted from here. Without ` + "`--control`" + ` nothing is recorded.
@@ -103,7 +106,7 @@ func newEvaluatePolicyCmd(out io.Writer) *cobra.Command {
 	}
 
 	cmd.Flags().StringArrayVar(&o.contexts, "context", []string{}, policyContextFlag)
-	cmd.Flags().StringVarP(&o.policyRef, "policy", "p", "", "Path or http(s):// URL of a Rego policy to evaluate the trail against.")
+	cmd.Flags().StringVarP(&o.policyRef, "policy", "p", "", "Path of a Rego policy file, or of a directory sent as one bundle.")
 	cmd.Flags().StringVar(&o.params, "params", "", policyParamsFlag)
 	cmd.Flags().StringVarP(&o.output, "output", "o", "table", outputFlag)
 	cmd.Flags().BoolVar(&o.assert, "assert", false, policyAssertFlag)
@@ -122,6 +125,12 @@ func newEvaluatePolicyCmd(out io.Writer) *cobra.Command {
 }
 
 func (o *evaluatePolicyOptions) run(out io.Writer) error {
+	// Fetching a policy from a URL is on its way out, so this command does not
+	// offer it, though the older evaluate commands still do.
+	if isRemotePolicyRef(o.policyRef) {
+		return fmt.Errorf("--policy takes a file or a directory on this machine, not a URL")
+	}
+
 	trails, err := parseTrailContexts(o.contexts)
 	if err != nil {
 		return err

cmd/kosli/evaluatePolicy_test.go

@@ -2,6 +2,9 @@ package main
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
 	"testing"
 	"time"
 
@@ -456,6 +459,92 @@ func (suite *EvaluatePolicyCommandTestSuite) TestAClassifiedFailureIsNotADenial(
 	require.NotContains(suite.T(), combined, "DENIED")
 }
 
+func (suite *EvaluatePolicyCommandTestSuite) TestADirectoryTravelsAsOneBundle() {
+	server, fake := newFakeEvaluations(suite.T(), verdictAllowed)
+
+	_, _, _, _, err := executeCommandC(fmt.Sprintf(
+		"evaluate policy --context trail=my-flow/my-trail --policy testdata/policies/bundle "+
+			"--host %s --org test-org --api-token test-token --max-api-retries 0", server.URL))
+
+	require.NoError(suite.T(), err)
+	files := fake.created[0]["policy"].(map[string]interface{})["files"].(map[string]interface{})
+	require.Equal(suite.T(), []string{"README.md", "lib/helpers.rego", "policy.rego"}, sortedKeys(files),
+		"keyed by path relative to the directory, and nothing left out by name")
+	require.Contains(suite.T(), files["policy.rego"], "package policy")
+	require.Contains(suite.T(), files["lib/helpers.rego"], "package lib.helpers")
+}
+
+// The API takes at least one file.
+func (suite *EvaluatePolicyCommandTestSuite) TestAnEmptyDirectoryIsRefused() {
+	directory := suite.T().TempDir()
+	server, fake := newFakeEvaluations(suite.T(), verdictAllowed)
+
+	_, _, _, _, err := executeCommandC(fmt.Sprintf(
+		"evaluate policy --context trail=my-flow/my-trail --policy %s "+
+			"--host %s --org test-org --api-token test-token --max-api-retries 0", directory, server.URL))
+
+	require.Error(suite.T(), err)
+	require.Contains(suite.T(), err.Error(), directory)
+	require.Empty(suite.T(), fake.created)
+}
+
+func (suite *EvaluatePolicyCommandTestSuite) TestABundleOverTheCapsIsRefusedWithTheCapNamed() {
+	for _, test := range []struct {
+		name  string
+		build func(string)
+		says  string
+	}{
+		{"too many files", func(directory string) {
+			for i := 0; i <= maxPolicyBundleFiles; i++ {
+				require.NoError(suite.T(), os.WriteFile(
+					filepath.Join(directory, fmt.Sprintf("policy-%d.rego", i)),
+					[]byte("package policy\n"), 0644))
+			}
+		}, fmt.Sprintf("%d", maxPolicyBundleFiles)},
+		{"too many bytes", func(directory string) {
+			source := make([]byte, serverPolicyMaxBytes+1)
+			for i := range source {
+				source[i] = 'a'
+			}
+			require.NoError(suite.T(), os.WriteFile(
+				filepath.Join(directory, "policy.rego"), source, 0644))
+		}, fmt.Sprintf("%d", serverPolicyMaxBytes)},
+	} {
+		suite.Run(test.name, func() {
+			directory := suite.T().TempDir()
+			test.build(directory)
+			server, fake := newFakeEvaluations(suite.T(), verdictAllowed)
+
+			_, _, _, _, err := executeCommandC(fmt.Sprintf(
+				"evaluate policy --context trail=my-flow/my-trail --policy %s "+
+					"--host %s --org test-org --api-token test-token --max-api-retries 0",
+				directory, server.URL))
+
+			require.Error(suite.T(), err)
+			require.Contains(suite.T(), err.Error(), test.says)
+			require.Empty(suite.T(), fake.created, "nothing is sent")
+		})
+	}
+}
+
+// A policy comes from the machine that runs the command.
+func (suite *EvaluatePolicyCommandTestSuite) TestARemotePolicyIsRefused() {
+	for _, ref := range []string{"http://policies.example.com/pr.rego", "https://policies.example.com/pr.rego"} {
+		suite.Run(ref, func() {
+			server, fake := newFakeEvaluations(suite.T(), verdictAllowed)
+
+			_, combined, _, _, err := executeCommandC(fmt.Sprintf(
+				"evaluate policy --context trail=my-flow/my-trail --policy %s "+
+					"--host %s --org test-org --api-token test-token --max-api-retries 0", ref, server.URL))
+
+			require.Error(suite.T(), err)
+			require.Contains(suite.T(), err.Error(), "--policy")
+			require.Empty(suite.T(), fake.created, "nothing is sent")
+			require.NotContains(suite.T(), combined, "RESULT")
+		})
+	}
+}
+
 func (suite *EvaluatePolicyCommandTestSuite) TestADryRunSendsNothing() {
 	server, fake := newFakeEvaluations(suite.T(), verdictAllowed)
 
@@ -470,6 +559,15 @@ func (suite *EvaluatePolicyCommandTestSuite) TestADryRunSendsNothing() {
 	require.NotContains(suite.T(), combined, "RESULT")
 }
 
+func sortedKeys(files map[string]interface{}) []string {
+	keys := make([]string, 0, len(files))
+	for key := range files {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
 func TestEvaluatePolicyCommandTestSuite(t *testing.T) {
 	suite.Run(t, new(EvaluatePolicyCommandTestSuite))
 }

cmd/kosli/testdata/policies/bundle/README.md

@@ -0,0 +1 @@
+The bundle's own notes, which are not a policy and do not travel with it.

cmd/kosli/testdata/policies/bundle/lib/helpers.rego

@@ -0,0 +1,3 @@
+package lib.helpers
+
+always_true := true

cmd/kosli/testdata/policies/bundle/policy.rego

@@ -0,0 +1,5 @@
+package policy
+
+import data.lib.helpers
+
+allow := helpers.always_true

docs/handover/6920-evaluate-an-inline-policy-and-record-its-decis.md

@@ -22,7 +22,7 @@ Server-side evaluation reaches the CLI today only through the hidden `--server-s
 - A denial is a decision, recorded as non-compliant with its violations. A policy that cannot run is not: it records nothing and must never print a denial.
 - Destination flags without `--control` are refused, and a control or a destination the caller cannot write to is refused before the evaluation is queued.
 - The command always waits for a terminal status; `--assert` exits non-zero on denial and changes nothing else.
-- `--policy` accepts a single file or a directory, within the published bundle caps.
+- `--policy` accepts a single file or a directory on this machine, within the published bundle caps. A URL is refused.
 - Output and `--output json` match `kosli evaluate trail`, so a caller switching commands does not re-parse.
 - An organisation without the server-side evaluation entitlement is refused in words that name it.
 - `--name` defaults to `<control>-decision`, so the common case names only the control.
@@ -42,7 +42,7 @@ Transcribed from [docs/plans/6920-evaluate-policy.md](../plans/6920-evaluate-pol
 - [x] Slice 3 — `--context` names what is evaluated, and is always required.
 - [x] Slice 4 — `--control` records a decision, with `--flow` and `--trail` as its destination.
 - [x] Slice 5 — refusals travel in the API's own words, and a classified failure is never a denial.
-- [ ] Slice 6 — a directory of policy files as one bundle, with the caps refused here.
+- [x] Slice 6 — a directory of policy files as one bundle, with the caps refused here.
 - [ ] Slice 7 — help text, docs, changelog, lint, full test run, and a staging check against an entitled organisation.
 
 ---
@@ -60,6 +60,8 @@ Transcribed from [docs/plans/6920-evaluate-policy.md](../plans/6920-evaluate-pol
 - The command declares its own options rather than inheriting the evaluate commands' shared ones, because four of those flags have no meaning here and inheriting them only to hide them is how two commands drift apart.
 - Asserting is opt-in on this command and the default is silent, which is the reverse of `evaluate trail`. A command that records a decision should not fail a pipeline unless the caller asked it to, and the flag that asks is the one the tutorial already publishes.
 - What is evaluated and where a decision lands are named separately: `--context` is the only way to say what to evaluate and is always required, while `--flow` and `--trail` name the destination alone. Neither is refused for being present without `--control`, because a pipeline sets them as environment variables for every command it runs, and refusing them would refuse an ordinary run that asked for no decision. The ticket's example predates this split.
+- A policy comes from the machine that runs the command: this command does not fetch one from a URL, though the older evaluate commands do, because that way of naming a policy is on its way out and a new command should not take it on.
+- A directory of policy files travels whole, with nothing left out by name and nothing here reading the modules. What a bundle may hold, and what its modules may import, is for the evaluator that runs it to judge; a rule here would refuse bundles the evaluator would have accepted, and would go stale as the evaluator changes. Only the published caps and an empty directory are refused here, because those the caller can act on before sending.
 - Refusals are passed on as the API worded them rather than being classified here, and the slice that was to give each case a sentence of its own was cut back to two tests. A list of cases in the CLI would go stale against the server that writes them, and the one thing that must not vary — a failed policy never reading as a denial — is pinned by a test instead.
 - The destination is read as a resolved value rather than as a flag the caller typed, so `KOSLI_FLOW` and `KOSLI_TRAIL` satisfy `--control` exactly as the flags do.
 - The first cut of the command is synchronous only, and `--sync` is not offered: with nothing to opt into, the flag would name the one behaviour there is. A command whose purpose is recording a decision should not return before the decision exists. An asynchronous mode, and the `--sync` flag that would pair with it, belong to a later ticket if anyone asks for them.
@@ -76,5 +78,5 @@ Transcribed from [docs/plans/6920-evaluate-policy.md](../plans/6920-evaluate-pol
 
 ## Next Steps
 
-- [ ] Slice 6: a directory of policy files as one bundle, with the caps refused here.
+- [ ] Slice 7: help text, docs, changelog, the full integration run, and a staging check against an entitled organisation.
 - [ ] Check what the create endpoint refuses for each destination failure, against staging, so Slice 5's messages are written from real answers rather than guessed.

docs/plans/6920-evaluate-policy.md

@@ -76,7 +76,7 @@ The evaluation resource gains `decision_attestation_id`: the id of the decision
 | Flag | Required | Meaning |
 |---|---|---|
 | `--context` | yes | Repeatable `trail=<flow>/<trail>`. What is evaluated, all of it at one instant. |
-| `--policy`, `-p` | yes | A `.rego` file, a directory, or an `http(s)://` URL. |
+| `--policy`, `-p` | yes | A `.rego` file or a directory on this machine. |
 | `--params` | no | Inline JSON or `@file.json`, unchanged, read by the policy as `data.params`. |
 | `--control` | no | The control the decision answers. Present, a decision is recorded; absent, nothing is. |
 | `--flow`, `-f` | with `--control` | Flow the decision is recorded in. |
@@ -136,7 +136,7 @@ The ticket asks for denial, a broken policy and a fault of ours to be three dist
 
 ### 4.6 A directory of policy files
 
-`--policy` pointing at a directory uploads every file below it as one bundle, keyed by path relative to that directory, within the published 100-file and 1 MiB caps. Paths stay inside the bundle. A single file keeps today's behaviour: one entry named after the file, no extension imposed.
+`--policy` pointing at a directory uploads every file below it as one bundle, keyed by path relative to that directory, within the published 100-file and 1 MiB caps. Nothing is left out by name, and nothing here reads the modules: what a bundle may hold, and what its modules may import, is the evaluator's to judge, and a rule here would refuse bundles the evaluator would have accepted. An empty directory is named here, because the API takes at least one file. A single file keeps today's behaviour: one entry named after the file, no extension imposed. A URL is refused: fetching a policy from one is on its way out, so this command never offers it.
 
 ### 4.7 `--context`
 
@@ -200,7 +200,7 @@ Deliberately shallow. A refusal is reported as the status the API answered with
 
 ### Slice 6: a directory of policy files
 
-Relative keys, the file and byte caps refused here with the cap named, paths that would climb out of the bundle refused.
+Relative keys, the file and byte caps refused here with the cap named, an empty directory named here, the bundle's contents left for the evaluator to judge, and a URL refused.
 
 ### Slice 7: wrap-up