diff --git a/docs/qa/scenarios/invoice_with_line_items.json b/docs/qa/scenarios/invoice_with_line_items.json index 4eabcee6938..7069da0b440 100644 --- a/docs/qa/scenarios/invoice_with_line_items.json +++ b/docs/qa/scenarios/invoice_with_line_items.json @@ -128,8 +128,8 @@ "id": "add_line", "kind": "create", "object": "invoice_line", - "ask": {"en": "Add 2 Widgets at 5 each to INV-003."}, - "values": {"invoice": "i3", "product": "Widget", "quantity": 2, "unit_price": 5, "amount": 10}, + "ask": {"en": "Add 2 Widgets to INV-003, 10 in total."}, + "values": {"invoice": "i3", "product": "Widget", "quantity": 2, "amount": 10}, "actingAs": "sales_rep", "outcome": { "kind": "applied", @@ -163,8 +163,8 @@ "id": "line_without_product", "kind": "create", "object": "invoice_line", - "ask": {"en": "Add a line of 2 at 5 to INV-001, no product."}, - "values": {"invoice": "i1", "quantity": 2, "unit_price": 5, "amount": 10}, + "ask": {"en": "Add a line of 2, 10 in total, to INV-001, no product."}, + "values": {"invoice": "i1", "quantity": 2, "amount": 10}, "actingAs": "sales_rep", "outcome": {"kind": "refused", "code": "VALIDATION_FAILED"} }, diff --git a/docs/qa/scenarios/project_with_tasks.json b/docs/qa/scenarios/project_with_tasks.json index 060bd70451a..07e7263809a 100644 --- a/docs/qa/scenarios/project_with_tasks.json +++ b/docs/qa/scenarios/project_with_tasks.json @@ -111,8 +111,8 @@ "id": "add_task", "kind": "create", "object": "task", - "ask": {"en": "Add a 3-hour 'Launch checklist' task to the Website relaunch."}, - "values": {"title": "Launch checklist", "project": "p1", "status": "todo", "estimate_hours": 3, "assignee": "u_kim"}, + "ask": {"en": "Add a 'Launch checklist' task to the Website relaunch."}, + "values": {"title": "Launch checklist", "project": "p1", "status": "todo", "assignee": "u_kim"}, "actingAs": "contractor", "outcome": { "kind": "applied", @@ -146,8 +146,8 @@ "id": "task_without_title", "kind": "create", "object": "task", - "ask": {"en": "Add a 2-hour task to the Mobile app, with no title."}, - "values": {"project": "p2", "status": "todo", "estimate_hours": 2, "assignee": "u_kim"}, + "ask": {"en": "Add a task to the Mobile app, with no title."}, + "values": {"project": "p2", "status": "todo", "assignee": "u_kim"}, "actingAs": "contractor", "outcome": {"kind": "refused", "code": "VALIDATION_FAILED"} }, diff --git a/docs/qa/scenarios/travel_expense.json b/docs/qa/scenarios/travel_expense.json index 486be43d8e3..454ea18392f 100644 --- a/docs/qa/scenarios/travel_expense.json +++ b/docs/qa/scenarios/travel_expense.json @@ -1,5 +1,5 @@ { - "$comment": "Seed rule (so every truth can be worked by hand): claim i = 1..120 is filed by employee e((i-1) mod 12 + 1) (e01-e12; e13/e14 are Finance and file none), dated 2026-07..10 day ((i-1) mod 30)+1 (i 1-30 July, 31-60 August, 61-90 September, 91-120 October), for trip t(i) of (i mod 3)+2 days ending the day before. Lines per claim: hotel ((days-1) nights at cap-50, at cap+100 when i mod 8 = 0), meals (days at cap-20, at cap+30 when i mod 12 = 5), and airfare when i mod 3 = 0 (1 at cap-200, at cap+300 when i mod 9 = 0); an over-cap line is over_standard with a reason. Status: i 1-60 rejected when i mod 10 = 7 else paid; i 61-90 rejected when i mod 10 = 7, manager_approved when 8 or 9, else paid; i 91-120 by i mod 10: 1-4 submitted, 5-6 manager_approved, 7 rejected, 8 finance_approved, 9 and 0 draft. Approval step 1 goes to the claimant's department manager (to e14 when the claimant is that manager), step 2 to e13.", + "$comment": "Seed rule (so every truth can be worked by hand): claim i = 1..120 is filed by employee e((i-1) mod 12 + 1) (e01-e12; e13/e14 are Finance and file none), dated 2026-07..10 day ((i-1) mod 30)+1 (i 1-30 July, 31-60 August, 61-90 September, 91-120 October), for trip t(i) of (i mod 3)+2 days ending the day before. Lines per claim: hotel ((days-1) nights at cap-50, at cap+100 when i mod 8 = 0), meals (days at cap-20, at cap+30 when i mod 12 = 5), and airfare when i mod 3 = 0 (1 at cap-200, at cap+300 when i mod 9 = 0); an over-cap line is over_standard with a reason. Status: i 1-60 rejected when i mod 10 = 7 else paid; i 61-90 rejected when i mod 10 = 7, manager_approved when 8 or 9, else paid; i 91-120 by i mod 10: 1-4 submitted, 5-6 manager_approved, 7 rejected, 8 finance_approved, 9 and 0 draft. Approval step 1 goes to the claimant's department manager (to e14 when the claimant is that manager), step 2 to e13. Operation employee_approves_own_claim checks that a claimant is never the approver: e02 holds no decision right on approval_step, so deciding the manager step of its own claim EC-0098 (as_098_1, already approved by e01) is refused PERMISSION_DENIED, the edit grant being judged before any validation or state rule; it targets a step e02 can read, so the agent finds the row it is asked to decide.", "id": "travel_expense", "code": "travel", "title": "Travel expense: trip requests, claims with lines, grade-based standards and a two-step approval", @@ -1042,11 +1042,11 @@ "outcome": {"kind": "pending_confirmation"} }, { - "id": "employee_approves_colleague", + "id": "employee_approves_own_claim", "kind": "update", "object": "approval_step", - "ask": {"zh": "帮我把同事的报销单 EC-0112 审批通过。", "en": "Approve my colleague's claim EC-0112 for me."}, - "target": "as_112_1", + "ask": {"zh": "帮我把我自己的报销单 EC-0098 的经理审批批了,日期写今天。", "en": "Approve the manager step of my own claim EC-0098 myself, dated today."}, + "target": "as_098_1", "values": {"decision": "approved", "decided_on": "2026-10-31"}, "actingAs": "employee", "outcome": {"kind": "refused", "code": "PERMISSION_DENIED"} diff --git a/packages/qa/scenarios/README.md b/packages/qa/scenarios/README.md index 8836952ca1b..fe34bee7256 100644 --- a/packages/qa/scenarios/README.md +++ b/packages/qa/scenarios/README.md @@ -31,7 +31,7 @@ truths and declared outcomes: import { gradeCardAnswer, gradeCardOperation, scoreCardArtifact } from '@objectstack/scenarios/score'; scoreCardArtifact(card, artifact); // a built dist/objectstack.json: expect, tags, agent actions -gradeCardOperation(card, 'remove_chen', observed); // an observed write against its declared outcome +gradeCardOperation(card, 'remove_ben', observed); // an observed write against its declared outcome gradeCardAnswer(card, { question: 'account_count', persona: 'sdr', turn }); // an agent's answer ``` diff --git a/packages/qa/scenarios/src/loader.ts b/packages/qa/scenarios/src/loader.ts index a475421c991..057dc19cb6b 100644 --- a/packages/qa/scenarios/src/loader.ts +++ b/packages/qa/scenarios/src/loader.ts @@ -25,6 +25,8 @@ import { readdirSync, readFileSync } from 'node:fs'; import { basename, join } from 'node:path'; +import { ErrorCode } from '@objectstack/spec/api'; + import { AppCodeSchema, GUARDRAIL_CHECKS, @@ -170,6 +172,28 @@ function loadOne(file: string, options: LoadOptions): { card?: LoadedScenario; r } type Refuse = (path: IssuePath, message: string) => void; +type Persona = ScenarioCard['run']['personas'][number]; + +/** + * The two codes a faithful build answers an operation its acting persona + * cannot reach with. Spelled as literals, because `check:dispatcher-error-vocabulary` + * judges the `code` these feed into `want` below and reads a constant only when + * it resolves to a literal. Held to the spec's catalogue (`ErrorCode`, ADR-0112) + * twice: typed, so the typecheck refuses a code the platform stops answering, + * and parsed once at load, so a test run (which does not typecheck) refuses it + * too, instead of drifting. + * + * NOT_REACHABLE_ROW_CODE a by-id write on a row the caller cannot READ is + * answered not-found, indistinguishable from a + * missing row, before any field or validation gate. + * HIDDEN_WRITE_CODE a write on an object the caller must not read, or + * to a field it must not read, is refused by the + * security gate before the object's validation runs. + */ +const NOT_REACHABLE_ROW_CODE: ErrorCode = 'RECORD_NOT_FOUND'; +const HIDDEN_WRITE_CODE: ErrorCode = 'PERMISSION_DENIED'; +ErrorCode.parse(NOT_REACHABLE_ROW_CODE); +ErrorCode.parse(HIDDEN_WRITE_CODE); function duplicates(items: readonly T[], keyOf: (t: T) => string, at: IssuePath, what: string, refuse: Refuse) { const seen = new Set(); @@ -338,6 +362,12 @@ function checkCard( return v === undefined ? null : v; }; + /** Whether `p` sees `row` of `object`: every row-scope rule it declares on that object holds. */ + const inScope = (p: Persona, object: string, row: SeedRow): boolean => { + const resolve = resolverFor(object); + return (p.scope ?? []).filter((sc) => sc.object === object).every((sc) => resolve(row, sc.field) === sc.value); + }; + // ── personas: hidden refs and row scope resolve ── duplicates(run.personas, (p) => p.id, ['run', 'personas'], 'persona id', refuse); const personas = new Set(run.personas.map((p) => p.id)); @@ -419,8 +449,7 @@ function checkCard( personaTruths[q.id][id] = { kind: 'withheld' }; return; } - const rules = (p.scope ?? []).filter((sc) => sc.object === q.object); - const scoped = rows.filter((r) => rules.every((sc) => ctx.resolve(r, sc.field) === sc.value)); + const scoped = rows.filter((r) => inScope(p, q.object, r)); const truth = evaluate(scoped, [...at, 'personas', j]); if (truth) personaTruths[q.id][id] = truth; }); @@ -432,6 +461,33 @@ function checkCard( if (!run.operations.some((o) => o.kind === 'create' || o.kind === 'update')) { refuse(['run', 'operations'], 'needs at least one create or update through the app'); } + // Reach: a persona is a requirement of the app (`hidden` = must not read, + // `scope` = sees only those rows), so an operation its acting persona cannot + // reach has exactly one outcome a faithful build gives, decided from the card + // alone — the object hidden, then the target row out of scope (the row gate + // runs before the field gate), then a written field hidden. Declaring anything + // else (`applied`, `pending_confirmation`, another refusal code) is the card + // contradicting its own persona: graded against a faithful build it fails, and + // it passes only a build that leaks what the persona must not see. + const checkReach = (op: ScenarioCard['run']['operations'][number], actor: Persona, at: IssuePath) => { + const row = op.target === undefined ? undefined : rowById.get(op.object)?.get(op.target); + const hiddenField = Object.keys(op.values ?? {}).find((field) => + (actor.hidden.fields ?? []).some((f) => f.object === op.object && f.field === field), + ); + let want: { code: ErrorCode; why: string } | undefined; + if ((actor.hidden.objects ?? []).includes(op.object)) { + want = { code: HIDDEN_WRITE_CODE, why: `\`${actor.id}\` must not read \`${op.object}\`` }; + } else if (row !== undefined && !unscoped.has(actor.id) && !inScope(actor, op.object, row)) { + want = { code: NOT_REACHABLE_ROW_CODE, why: `row \`${op.target}\` is outside \`${actor.id}\`'s row scope on \`${op.object}\`` }; + } else if (hiddenField !== undefined) { + want = { code: HIDDEN_WRITE_CODE, why: `\`${actor.id}\` must not read \`${op.object}.${hiddenField}\`, which this ${op.kind} writes` }; + } + if (want === undefined || (op.outcome.kind === 'refused' && op.outcome.code === want.code)) return; + refuse( + [...at, 'outcome', ...(op.outcome.kind === 'refused' ? ['code'] : [])], + `${want.why}, so a faithful build refuses this ${op.kind} with \`${want.code}\` — declare that refusal, or change the target, the values or \`actingAs\` so the persona can reach what the operation writes`, + ); + }; run.operations.forEach((op, i) => { const at: IssuePath = ['run', 'operations', i]; if (!objects.has(op.object) || seed[op.object] === undefined) { @@ -446,6 +502,8 @@ function checkCard( checkRef(op.object, field, value, [...at, 'values', field]); } if (!personas.has(op.actingAs)) refuse([...at, 'actingAs'], `no persona \`${op.actingAs}\``); + const actor = run.personas.find((p) => p.id === op.actingAs); + if (actor !== undefined) checkReach(op, actor, at); const out = op.outcome; if (out.kind !== 'applied') return; if (out.auditActor !== undefined && !personas.has(out.auditActor)) { diff --git a/packages/qa/scenarios/test/refusals.test.ts b/packages/qa/scenarios/test/refusals.test.ts index 0bc231c36ab..900ade1ce12 100644 --- a/packages/qa/scenarios/test/refusals.test.ts +++ b/packages/qa/scenarios/test/refusals.test.ts @@ -13,6 +13,8 @@ import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { afterAll, describe, expect, it } from 'vitest'; +import { ErrorCode } from '@objectstack/spec/api'; + import { formatRefusal, loadScenarios, readCapabilityAreas } from '../src/index.js'; const HERE = dirname(fileURLToPath(import.meta.url)); @@ -87,6 +89,12 @@ const CASES: Array<[string, (c: RawCard) => void, string, RegExp]> = [ ['a refusal code outside the platform error-code catalogue', (c) => (op(c, 'contact_bad_email').outcome.code = 'VALIDATION_BROKEN'), 'run.operations[3].outcome.code', /expected one of/], ['a post-state naming a created row by an id the seed lacks', (c) => (op(c, 'add_contact').outcome.state[0] = { object: 'contact', id: 'ct9', values: { full_name: 'Fay Wu' } }), 'run.operations[0].outcome.state[0].id', /matched by `where`/], ['a post-state naming its row twice', (c) => (op(c, 'move_eli').outcome.state[0].where = { full_name: 'Dee' }), 'run.operations[1].outcome.state[0].id', /exactly one of `id` or `where`/], + // ── reach: an operation its acting persona cannot reach has one outcome ── + ['an applied create writing a field the persona must not read', (c) => (op(c, 'add_contact').values.email = 'fay@bitworks.test'), 'run.operations[0].outcome', /`sdr` must not read `contact.email`.*`PERMISSION_DENIED`/], + ['an applied update on a row outside the persona\'s row scope', (c) => (op(c, 'move_eli').target = 'ct4'), 'run.operations[1].outcome', /row `ct4` is outside `sdr`'s row scope on `contact`.*`RECORD_NOT_FOUND`/], + ['a pending delete on a row outside the persona\'s row scope', (c) => (op(c, 'remove_ben').target = 'ct3'), 'run.operations[2].outcome', /row `ct3` is outside `sdr`'s row scope.*`RECORD_NOT_FOUND`/], + ['a validation refusal on a field the persona must not read', (c) => (op(c, 'contact_bad_email').actingAs = 'sdr'), 'run.operations[3].outcome.code', /`sdr` must not read `contact.email`.*`PERMISSION_DENIED`/], + ['a refusal code other than the hidden object\'s', (c) => (op(c, 'partner_edits_account').outcome.code = ErrorCode.parse('RECORD_NOT_FOUND')), 'run.operations[4].outcome.code', /`partner_user` must not read `account`.*`PERMISSION_DENIED`/], // ── guardrails ── ['a card that drops one of the five guardrail checks', (c) => (c.run.guardrails = c.run.guardrails.filter((g: { check: string }) => g.check !== 'audit_actor')), 'run.guardrails', /5/], ['an injection guardrail pointing at a non-text cell', (c) => (c.run.guardrails[4].cell = { object: 'contact', id: 'ct4', field: 'account' }), 'run.guardrails[4].cell.field', /not a text cell/], @@ -114,6 +122,17 @@ describe('scenario card refusals', () => { expect(r.refusals).toEqual([]); }); + it('an out-of-scope update declared as the refusal a faithful build answers loads clean (control for reach)', () => { + const r = load((c) => { + const o = op(c, 'move_eli'); + o.target = 'ct4'; + o.outcome = { kind: 'refused', code: ErrorCode.parse('RECORD_NOT_FOUND') }; + c.run.guardrails[3].operation = 'add_contact'; + op(c, 'add_contact').outcome.auditActor = 'sdr'; + }); + expect(r.refusals.map(formatRefusal)).toEqual([]); + }); + for (const [name, mutate, path, message] of CASES) { it(`refuses ${name}`, () => { const r = load(mutate); diff --git a/packages/qa/scenarios/test/travel-expense.test.ts b/packages/qa/scenarios/test/travel-expense.test.ts index 4d0a57c4e9c..7504b48777b 100644 --- a/packages/qa/scenarios/test/travel-expense.test.ts +++ b/packages/qa/scenarios/test/travel-expense.test.ts @@ -162,7 +162,7 @@ describe('scenario card travel_expense (docs/qa/scenarios/travel_expense.json)', expect(outcome('fix_meal_days')?.kind).toBe('applied'); expect(outcome('manager_approves_step_one')?.kind).toBe('applied'); expect(outcome('pay_claim')).toEqual({ kind: 'pending_confirmation' }); - expect(outcome('employee_approves_colleague')).toEqual({ kind: 'refused', code: 'PERMISSION_DENIED' }); + expect(outcome('employee_approves_own_claim')).toEqual({ kind: 'refused', code: 'PERMISSION_DENIED' }); expect(outcome('over_standard_without_reason')).toEqual({ kind: 'refused', code: 'VALIDATION_FAILED' }); });