feat(evaluate): add --output-rule to add policy rules to the output

cmd/kosli/evaluate.go

@@ -22,6 +22,7 @@ Use ` + "`evaluate input`" + ` to evaluate a local JSON file or stdin without an
 
 The policy must use ` + "`package policy`" + ` and define an ` + "`allow`" + ` rule.
 An optional ` + "`violations`" + ` rule (a set of strings) can provide human-readable denial reasons.
+Use ` + "`--output-rule`" + ` to add other rules of the policy, like a report, to the JSON output.
 
 By default a deny exits with code 1 so the command can gate a pipeline.
 Pass ` + "`--no-assert`" + ` to use the command as a policy decision point: it prints

cmd/kosli/evaluateHelpers.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"slices"
 	"strings"
 	"time"
 
@@ -63,6 +64,7 @@ type commonEvaluateOptions struct {
 	assert       bool
 	noAssert     bool
 	serverSide   bool
+	outputRules  []string
 }
 
 func (o *commonEvaluateOptions) addFlags(cmd *cobra.Command, policyDesc string) {
@@ -74,6 +76,7 @@ func (o *commonEvaluateOptions) addFlags(cmd *cobra.Command, policyDesc string)
 	cmd.Flags().StringVar(&o.params, "params", "", policyParamsFlag)
 	cmd.Flags().BoolVar(&o.assert, "assert", false, "[optional] Exit with a non-zero status when the policy denies. This is the current default; pass --assert to lock it in across future releases.")
 	cmd.Flags().BoolVar(&o.noAssert, "no-assert", false, "[optional] Print the result and always exit 0, even when the policy denies. Use when this command feeds another tool as a policy decision point.")
+	cmd.Flags().StringSliceVar(&o.outputRules, "output-rule", nil, policyOutputRuleFlag)
 	cmd.MarkFlagsMutuallyExclusive("assert", "no-assert")
 }
 
@@ -254,16 +257,23 @@ func parseParams(raw string) (map[string]any, error) {
 	return params, nil
 }
 
-func evaluateAndPrintResult(out io.Writer, policyRef string, input map[string]any, outputFormat string, showInput bool, params map[string]any, assertOnDeny bool) error {
+func evaluateAndPrintResult(out io.Writer, policyRef string, input map[string]any, outputFormat string, showInput bool, params map[string]any, assertOnDeny bool, outputRules []string) error {
+	if err := validateOutputRules(outputRules); err != nil {
+		return err
+	}
+
 	policySource, err := loadPolicy(policyRef)
 	if err != nil {
 		return err
 	}
 
-	result, err := evaluate.Evaluate(string(policySource), input, params)
+	result, err := evaluate.Evaluate(string(policySource), input, params, outputRules...)
 	if err != nil {
 		return err
 	}
+	if len(outputRules) > 0 && outputFormat == "table" {
+		logger.Warn("--output-rule values are only shown with --output json")
+	}
 
 	return printEvaluateResult(out, result, input, outputFormat, showInput, params, assertOnDeny, "")
 }
@@ -371,6 +381,11 @@ func (o *commonEvaluateOptions) refuseWhatTheServerCannotDo() error {
 			"--show-input is not supported with --server-side; " +
 				"the server does not return the input it evaluated")
 	}
+	if len(o.outputRules) > 0 {
+		return fmt.Errorf(
+			"--output-rule is not supported with --server-side; " +
+				"the server only returns allow and violations")
+	}
 	return nil
 }
 
@@ -531,13 +546,27 @@ func policyBundleKey(ref string) string {
 	return base
 }
 
+var evaluateResultKeys = []string{"allow", "violations", "input", "params", "decision_attestation_id"}
+
+func validateOutputRules(rules []string) error {
+	for _, rule := range rules {
+		if slices.Contains(evaluateResultKeys, rule) {
+			return fmt.Errorf("--output-rule cannot be '%s', it is already part of the output", rule)
+		}
+	}
+	return nil
+}
+
 // printEvaluateResult renders a verdict, whatever produced it, so that every
 // evaluation path prints the same bytes for the same verdict.
 func printEvaluateResult(out io.Writer, result *evaluate.Result, input map[string]any, outputFormat string, showInput bool, params map[string]any, assertOnDeny bool, decisionID string) error {
 	auditResult := map[string]any{
 		"allow":      result.Allow,
 		"violations": result.Violations,
 	}
+	for rule, value := range result.Outputs {
+		auditResult[rule] = value
+	}
 	// Absent everywhere else, so a caller reading a verdict alone parses the
 	// same page as before.
 	if decisionID != "" {

cmd/kosli/evaluateHelpers_test.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"testing"
+
+	"github.com/kosli-dev/cli/internal/evaluate"
+	"github.com/stretchr/testify/require"
+)
+
+func TestEvaluateResultKeysListsEveryKeyTheOutputCanHave(t *testing.T) {
+	var out bytes.Buffer
+	result := &evaluate.Result{Allow: true}
+	err := printEvaluateResult(&out, result, map[string]any{}, "json", true, map[string]any{}, false, "decision-id")
+	require.NoError(t, err)
+
+	var printed map[string]any
+	require.NoError(t, json.Unmarshal(out.Bytes(), &printed))
+	for key := range printed {
+		require.Contains(t, evaluateResultKeys, key)
+	}
+}

cmd/kosli/evaluateInput.go

@@ -25,6 +25,7 @@ the policy input from a ` + "`--show-input --output json`" + ` capture.
 
 The policy must use ` + "`package policy`" + ` and define an ` + "`allow`" + ` rule.
 An optional ` + "`violations`" + ` rule (a set of strings) can provide human-readable denial reasons.
+Use ` + "`--output-rule`" + ` to add other rules of the policy, like a report, to the JSON output.
 
 By default a deny exits with code 1. Pass ` + "`--no-assert`" + ` to print the verdict
 and exit 0 even on deny, when this command is feeding another tool as a
@@ -78,7 +79,14 @@ kosli evaluate input \
 kosli evaluate input \
 	--input-file trail-data.json \
 	--policy policy.rego \
-	--no-assert`
+	--no-assert
+
+# add the policy's report rule to the JSON output:
+kosli evaluate input \
+	--input-file trail-data.json \
+	--policy policy.rego \
+	--output-rule report \
+	--output json`
 
 func newEvaluateInputCmd(out io.Writer) *cobra.Command {
 	o := new(evaluateInputOptions)
@@ -128,7 +136,7 @@ func (o *evaluateInputOptions) run(out io.Writer, in io.Reader) error {
 		return err
 	}
 
-	return evaluateAndPrintResult(out, o.policyRef, input, o.output, o.showInput, params, o.assertOnDeny())
+	return evaluateAndPrintResult(out, o.policyRef, input, o.output, o.showInput, params, o.assertOnDeny(), o.outputRules)
 }
 
 func loadInputFromFile(filePath string) (result map[string]any, err error) {

cmd/kosli/evaluateInput_test.go

@@ -144,6 +144,57 @@ func (suite *EvaluateInputCommandTestSuite) TestEvaluateInputCmd() {
 				{"allow", false},
 			},
 		},
+		{
+			name: "--output-rule adds the rule to the JSON output",
+			cmd:  "evaluate input --input-file testdata/evaluate/trail-input.json --policy testdata/policies/allow-with-report.rego --output-rule report --output json",
+			goldenJson: []jsonCheck{
+				{"allow", true},
+				{"report.compliant", true},
+			},
+		},
+		{
+			name: "--output-rule adds the rule to the JSON output when the policy denies",
+			cmd:  "evaluate input --input-file testdata/evaluate/trail-input.json --policy testdata/policies/deny-with-report.rego --output-rule report --output json --no-assert",
+			goldenJson: []jsonCheck{
+				{"allow", false},
+				{"report.compliant", false},
+			},
+		},
+		{
+			name: "--output-rule can be repeated",
+			cmd:  "evaluate input --input-file testdata/evaluate/trail-input.json --policy testdata/policies/allow-with-report.rego --output-rule report --output-rule summary --output json",
+			goldenJson: []jsonCheck{
+				{"report.compliant", true},
+				{"summary", "all good"},
+			},
+		},
+		{
+			name: "--output-rule takes a comma-separated list",
+			cmd:  "evaluate input --input-file testdata/evaluate/trail-input.json --policy testdata/policies/allow-with-report.rego --output-rule report,summary --output json",
+			goldenJson: []jsonCheck{
+				{"report.compliant", true},
+				{"summary", "all good"},
+			},
+		},
+		{
+			wantError:   true,
+			name:        "--output-rule naming a rule the policy does not declare fails",
+			cmd:         "evaluate input --input-file testdata/evaluate/trail-input.json --policy testdata/policies/allow-all.rego --output-rule report --output json",
+			goldenRegex: `policy does not declare a 'report' rule`,
+		},
+		{
+			name: "--output-rule prints null for a rule with no value",
+			cmd:  "evaluate input --input-file testdata/evaluate/trail-input.json --policy testdata/policies/undefined-report.rego --output-rule report --output json",
+			goldenJson: []jsonCheck{
+				{"report", nil},
+			},
+		},
+		{
+			name:         "--output-rule is left out of table output, with a hint",
+			cmd:          "evaluate input --input-file testdata/evaluate/trail-input.json --policy testdata/policies/allow-with-report.rego --output-rule report",
+			goldenStdout: "RESULT:  ALLOWED\n",
+			goldenStderr: "[warning] --output-rule values are only shown with --output json\n",
+		},
 	}
 	runTestCmd(suite.T(), tests)
 }
@@ -377,6 +428,19 @@ func TestLoadPolicyHonorsHTTPProxy(t *testing.T) {
 	require.True(t, sawProxyStyleRequest, "expected proxy to receive an absolute-URL request")
 }
 
+func (suite *EvaluateInputCommandTestSuite) TestEvaluateInputCmdRefusesOutputRulesClashingWithOutputKeys() {
+	tests := []cmdTestCase{}
+	for _, name := range []string{"allow", "violations", "input", "params", "decision_attestation_id"} {
+		tests = append(tests, cmdTestCase{
+			wantError:   true,
+			name:        "--output-rule " + name + " is refused",
+			cmd:         "evaluate input --input-file testdata/evaluate/trail-input.json --policy testdata/policies/allow-all.rego --output-rule " + name,
+			goldenRegex: `--output-rule cannot be '` + name + `', it is already part of the output`,
+		})
+	}
+	runTestCmd(suite.T(), tests)
+}
+
 func TestEvaluateInputCommandTestSuite(t *testing.T) {
 	suite.Run(t, new(EvaluateInputCommandTestSuite))
 }

cmd/kosli/evaluateServerSide_test.go

@@ -370,6 +370,11 @@ func (suite *EvaluateServerSideTestSuite) TestItRefusesWhatTheServerCannotDo() {
 			extra:   "--show-input",
 			message: "--show-input is not supported with --server-side",
 		},
+		{
+			name:    "adding policy rules to the output",
+			extra:   "--output-rule report",
+			message: "--output-rule is not supported with --server-side",
+		},
 	} {
 		suite.Run(test.name, func() {
 			server, fake := newFakeEvaluations(suite.T(), verdictAllowed)

cmd/kosli/evaluateTrail.go

@@ -15,7 +15,8 @@ The trail data is passed to the policy as ` + "`input.trail`" + `.
 
 Use ` + "`--attestations`" + ` to enrich the input with detailed attestation data
 (e.g. pull request approvers, scan results). Use ` + "`--show-input`" + ` to inspect the
-full data structure available to the policy. Use ` + "`--output json`" + ` for structured output.`
+full data structure available to the policy. Use ` + "`--output json`" + ` for structured output,
+and ` + "`--output-rule`" + ` to add other rules of the policy, like a report, to it.`
 
 const evaluateTrailExample = `
 # evaluate a trail against a policy:
@@ -71,6 +72,15 @@ kosli evaluate trail yourTrailName \
 	--flow yourFlowName \
 	--no-assert \
 	--api-token yourAPIToken \
+	--org yourOrgName
+
+# add the policy's report rule to the JSON output:
+kosli evaluate trail yourTrailName \
+	--policy yourPolicyFile.rego \
+	--flow yourFlowName \
+	--output-rule report \
+	--output json \
+	--api-token yourAPIToken \
 	--org yourOrgName`
 
 type evaluateTrailOptions struct {
@@ -128,5 +138,5 @@ func (o *evaluateTrailOptions) run(out io.Writer, args []string) error {
 		"trail": trailData,
 	}
 
-	return evaluateAndPrintResult(out, o.policyRef, input, o.output, o.showInput, params, o.assertOnDeny())
+	return evaluateAndPrintResult(out, o.policyRef, input, o.output, o.showInput, params, o.assertOnDeny(), o.outputRules)
 }

cmd/kosli/evaluateTrail_test.go

@@ -150,6 +150,19 @@ func (suite *EvaluateTrailCommandTestSuite) TestEvaluateTrailCmd() {
 			cmd:         fmt.Sprintf(`evaluate trail %s --flow %s --policy testdata/policies/allow-all.rego --assert --no-assert %s`, suite.trailName, suite.flowName, suite.defaultKosliArguments),
 			goldenRegex: `none of the others can be.*\[assert no-assert\] were all set`,
 		},
+		{
+			name: "--output-rule adds the rule to the JSON output",
+			cmd:  fmt.Sprintf(`evaluate trail %s --flow %s --policy testdata/policies/trail-report.rego --output-rule report --output json %s`, suite.trailName, suite.flowName, suite.defaultKosliArguments),
+			goldenJson: []jsonCheck{
+				{"report.trail", suite.trailName},
+			},
+		},
+		{
+			wantError:   true,
+			name:        "--output-rule clashing with an output key is refused",
+			cmd:         fmt.Sprintf(`evaluate trail %s --flow %s --policy testdata/policies/allow-all.rego --output-rule allow %s`, suite.trailName, suite.flowName, suite.defaultKosliArguments),
+			goldenRegex: `--output-rule cannot be 'allow', it is already part of the output`,
+		},
 	}
 
 	runTestCmd(suite.T(), tests)

cmd/kosli/evaluateTrails.go

@@ -16,7 +16,8 @@ The trail data is passed to the policy as ` + "`input.trails`" + ` (an array), u
 
 Use ` + "`--attestations`" + ` to enrich the input with detailed attestation data
 (e.g. pull request approvers, scan results). Use ` + "`--show-input`" + ` to inspect the
-full data structure available to the policy. Use ` + "`--output json`" + ` for structured output.`
+full data structure available to the policy. Use ` + "`--output json`" + ` for structured output,
+and ` + "`--output-rule`" + ` to add other rules of the policy, like a report, to it.`
 
 const evaluateTrailsExample = `
 # evaluate multiple trails against a policy:
@@ -64,6 +65,15 @@ kosli evaluate trails yourTrailName1 yourTrailName2 \
 	--flow yourFlowName \
 	--no-assert \
 	--api-token yourAPIToken \
+	--org yourOrgName
+
+# add the policy's report rule to the JSON output:
+kosli evaluate trails yourTrailName1 yourTrailName2 \
+	--policy yourPolicyFile.rego \
+	--flow yourFlowName \
+	--output-rule report \
+	--output json \
+	--api-token yourAPIToken \
 	--org yourOrgName`
 
 type evaluateTrailsOptions struct {
@@ -128,5 +138,5 @@ func (o *evaluateTrailsOptions) run(out io.Writer, args []string) error {
 		"trails": trails,
 	}
 
-	return evaluateAndPrintResult(out, o.policyRef, input, o.output, o.showInput, params, o.assertOnDeny())
+	return evaluateAndPrintResult(out, o.policyRef, input, o.output, o.showInput, params, o.assertOnDeny(), o.outputRules)
 }

cmd/kosli/evaluateTrails_test.go

@@ -109,6 +109,19 @@ func (suite *EvaluateTrailsCommandTestSuite) TestEvaluateTrailsCmd() {
 			cmd:         fmt.Sprintf(`evaluate trails %s --flow %s --policy testdata/policies/allow-all.rego --assert --no-assert %s`, suite.trailName, suite.flowName, suite.defaultKosliArguments),
 			goldenRegex: `none of the others can be.*\[assert no-assert\] were all set`,
 		},
+		{
+			name: "--output-rule adds one rule value covering all the trails",
+			cmd:  fmt.Sprintf(`evaluate trails %s %s --flow %s --policy testdata/policies/trails-report.rego --output-rule report --output json %s`, suite.trailName, suite.trailName2, suite.flowName, suite.defaultKosliArguments),
+			goldenJson: []jsonCheck{
+				{"report.trails", 2.0},
+			},
+		},
+		{
+			wantError:   true,
+			name:        "--output-rule clashing with an output key is refused",
+			cmd:         fmt.Sprintf(`evaluate trails %s --flow %s --policy testdata/policies/allow-all.rego --output-rule allow %s`, suite.trailName, suite.flowName, suite.defaultKosliArguments),
+			goldenRegex: `--output-rule cannot be 'allow', it is already part of the output`,
+		},
 	}
 
 	runTestCmd(suite.T(), tests)

cmd/kosli/root.go

@@ -148,6 +148,7 @@ Paths the list already matches stay excluded whatever is later added there, so k
 	outputFlag                      = "[defaulted] The format of the output. Valid formats are: [table, json]."
 	serverSideFlag                  = "[hidden] Evaluate the policy on the Kosli server rather than on this machine. Unsupported and subject to change."
 	policyParamsFlag                = "[optional] Policy parameters as inline JSON or @file.json. Available in policies as data.params."
+	policyOutputRuleFlag            = "[optional] Name of a policy rule to add to the JSON output, next to allow and violations. Can be repeated, or given as a comma-separated list."
 	policyAssertFlag                = "[optional] Exit with a non-zero status when the policy denies. Without it the verdict is printed and the command exits 0."
 	policyContextFlag               = "What to evaluate, as trail=<flow>/<trail>. Repeat it to evaluate several trails at one instant."
 	policyControlFlag               = "[optional] Record the outcome as a decision against this control. Without it nothing is recorded."

cmd/kosli/testdata/empty-flag-audit-coverage.json

@@ -594,6 +594,7 @@
   "input-file": "string",
   "no-assert": "bool",
   "output": "string",
+  "output-rule": "stringSlice",
   "params": "string",
   "policy": "string",
   "show-input": "bool"
@@ -616,6 +617,7 @@
   "flow": "string",
   "no-assert": "bool",
   "output": "string",
+  "output-rule": "stringSlice",
   "params": "string",
   "policy": "string",
   "server-side": "bool",
@@ -627,6 +629,7 @@
   "flow": "string",
   "no-assert": "bool",
   "output": "string",
+  "output-rule": "stringSlice",
   "params": "string",
   "policy": "string",
   "server-side": "bool",

cmd/kosli/testdata/policies/allow-with-report.rego

@@ -0,0 +1,7 @@
+package policy
+
+allow = true
+
+report := {"compliant": true}
+
+summary := "all good"

cmd/kosli/testdata/policies/deny-with-report.rego

@@ -0,0 +1,5 @@
+package policy
+
+allow = false
+
+report := {"compliant": false}