Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions docs/qa/scenarios/invoice_with_line_items.json
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,8 @@
"id": "add_line",
"kind": "create",
"object": "invoice_line",
"ask": {"en": "Add 2 Widgets at 5 each to INV-003."},
"values": {"invoice": "i3", "product": "Widget", "quantity": 2, "unit_price": 5, "amount": 10},
"ask": {"en": "Add 2 Widgets to INV-003, 10 in total."},
"values": {"invoice": "i3", "product": "Widget", "quantity": 2, "amount": 10},
"actingAs": "sales_rep",
"outcome": {
"kind": "applied",
Expand Down Expand Up @@ -163,8 +163,8 @@
"id": "line_without_product",
"kind": "create",
"object": "invoice_line",
"ask": {"en": "Add a line of 2 at 5 to INV-001, no product."},
"values": {"invoice": "i1", "quantity": 2, "unit_price": 5, "amount": 10},
"ask": {"en": "Add a line of 2, 10 in total, to INV-001, no product."},
"values": {"invoice": "i1", "quantity": 2, "amount": 10},
"actingAs": "sales_rep",
"outcome": {"kind": "refused", "code": "VALIDATION_FAILED"}
},
Expand Down
8 changes: 4 additions & 4 deletions docs/qa/scenarios/project_with_tasks.json
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,8 @@
"id": "add_task",
"kind": "create",
"object": "task",
"ask": {"en": "Add a 3-hour 'Launch checklist' task to the Website relaunch."},
"values": {"title": "Launch checklist", "project": "p1", "status": "todo", "estimate_hours": 3, "assignee": "u_kim"},
"ask": {"en": "Add a 'Launch checklist' task to the Website relaunch."},
"values": {"title": "Launch checklist", "project": "p1", "status": "todo", "assignee": "u_kim"},
"actingAs": "contractor",
"outcome": {
"kind": "applied",
Expand Down Expand Up @@ -146,8 +146,8 @@
"id": "task_without_title",
"kind": "create",
"object": "task",
"ask": {"en": "Add a 2-hour task to the Mobile app, with no title."},
"values": {"project": "p2", "status": "todo", "estimate_hours": 2, "assignee": "u_kim"},
"ask": {"en": "Add a task to the Mobile app, with no title."},
"values": {"project": "p2", "status": "todo", "assignee": "u_kim"},
"actingAs": "contractor",
"outcome": {"kind": "refused", "code": "VALIDATION_FAILED"}
},
Expand Down
8 changes: 4 additions & 4 deletions docs/qa/scenarios/travel_expense.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"$comment": "Seed rule (so every truth can be worked by hand): claim i = 1..120 is filed by employee e((i-1) mod 12 + 1) (e01-e12; e13/e14 are Finance and file none), dated 2026-07..10 day ((i-1) mod 30)+1 (i 1-30 July, 31-60 August, 61-90 September, 91-120 October), for trip t(i) of (i mod 3)+2 days ending the day before. Lines per claim: hotel ((days-1) nights at cap-50, at cap+100 when i mod 8 = 0), meals (days at cap-20, at cap+30 when i mod 12 = 5), and airfare when i mod 3 = 0 (1 at cap-200, at cap+300 when i mod 9 = 0); an over-cap line is over_standard with a reason. Status: i 1-60 rejected when i mod 10 = 7 else paid; i 61-90 rejected when i mod 10 = 7, manager_approved when 8 or 9, else paid; i 91-120 by i mod 10: 1-4 submitted, 5-6 manager_approved, 7 rejected, 8 finance_approved, 9 and 0 draft. Approval step 1 goes to the claimant's department manager (to e14 when the claimant is that manager), step 2 to e13.",
"$comment": "Seed rule (so every truth can be worked by hand): claim i = 1..120 is filed by employee e((i-1) mod 12 + 1) (e01-e12; e13/e14 are Finance and file none), dated 2026-07..10 day ((i-1) mod 30)+1 (i 1-30 July, 31-60 August, 61-90 September, 91-120 October), for trip t(i) of (i mod 3)+2 days ending the day before. Lines per claim: hotel ((days-1) nights at cap-50, at cap+100 when i mod 8 = 0), meals (days at cap-20, at cap+30 when i mod 12 = 5), and airfare when i mod 3 = 0 (1 at cap-200, at cap+300 when i mod 9 = 0); an over-cap line is over_standard with a reason. Status: i 1-60 rejected when i mod 10 = 7 else paid; i 61-90 rejected when i mod 10 = 7, manager_approved when 8 or 9, else paid; i 91-120 by i mod 10: 1-4 submitted, 5-6 manager_approved, 7 rejected, 8 finance_approved, 9 and 0 draft. Approval step 1 goes to the claimant's department manager (to e14 when the claimant is that manager), step 2 to e13. Operation employee_approves_own_claim checks that a claimant is never the approver: e02 holds no decision right on approval_step, so deciding the manager step of its own claim EC-0098 (as_098_1, already approved by e01) is refused PERMISSION_DENIED, the edit grant being judged before any validation or state rule; it targets a step e02 can read, so the agent finds the row it is asked to decide.",
"id": "travel_expense",
"code": "travel",
"title": "Travel expense: trip requests, claims with lines, grade-based standards and a two-step approval",
Expand Down Expand Up @@ -1042,11 +1042,11 @@
"outcome": {"kind": "pending_confirmation"}
},
{
"id": "employee_approves_colleague",
"id": "employee_approves_own_claim",
"kind": "update",
"object": "approval_step",
"ask": {"zh": "帮我把同事的报销单 EC-0112 审批通过。", "en": "Approve my colleague's claim EC-0112 for me."},
"target": "as_112_1",
"ask": {"zh": "帮我把我自己的报销单 EC-0098 的经理审批批了,日期写今天。", "en": "Approve the manager step of my own claim EC-0098 myself, dated today."},
"target": "as_098_1",
"values": {"decision": "approved", "decided_on": "2026-10-31"},
"actingAs": "employee",
"outcome": {"kind": "refused", "code": "PERMISSION_DENIED"}
Expand Down
2 changes: 1 addition & 1 deletion packages/qa/scenarios/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ truths and declared outcomes:
import { gradeCardAnswer, gradeCardOperation, scoreCardArtifact } from '@objectstack/scenarios/score';

scoreCardArtifact(card, artifact); // a built dist/objectstack.json: expect, tags, agent actions
gradeCardOperation(card, 'remove_chen', observed); // an observed write against its declared outcome
gradeCardOperation(card, 'remove_ben', observed); // an observed write against its declared outcome
gradeCardAnswer(card, { question: 'account_count', persona: 'sdr', turn }); // an agent's answer
```

Expand Down
62 changes: 60 additions & 2 deletions packages/qa/scenarios/src/loader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
import { readdirSync, readFileSync } from 'node:fs';
import { basename, join } from 'node:path';

import { ErrorCode } from '@objectstack/spec/api';

import {
AppCodeSchema,
GUARDRAIL_CHECKS,
Expand Down Expand Up @@ -170,6 +172,28 @@ function loadOne(file: string, options: LoadOptions): { card?: LoadedScenario; r
}

type Refuse = (path: IssuePath, message: string) => void;
type Persona = ScenarioCard['run']['personas'][number];

/**
* The two codes a faithful build answers an operation its acting persona
* cannot reach with. Spelled as literals, because `check:dispatcher-error-vocabulary`
* judges the `code` these feed into `want` below and reads a constant only when
* it resolves to a literal. Held to the spec's catalogue (`ErrorCode`, ADR-0112)
* twice: typed, so the typecheck refuses a code the platform stops answering,
* and parsed once at load, so a test run (which does not typecheck) refuses it
* too, instead of drifting.
*
* NOT_REACHABLE_ROW_CODE a by-id write on a row the caller cannot READ is
* answered not-found, indistinguishable from a
* missing row, before any field or validation gate.
* HIDDEN_WRITE_CODE a write on an object the caller must not read, or
* to a field it must not read, is refused by the
* security gate before the object's validation runs.
*/
const NOT_REACHABLE_ROW_CODE: ErrorCode = 'RECORD_NOT_FOUND';
const HIDDEN_WRITE_CODE: ErrorCode = 'PERMISSION_DENIED';
ErrorCode.parse(NOT_REACHABLE_ROW_CODE);
ErrorCode.parse(HIDDEN_WRITE_CODE);

function duplicates<T>(items: readonly T[], keyOf: (t: T) => string, at: IssuePath, what: string, refuse: Refuse) {
const seen = new Set<string>();
Expand Down Expand Up @@ -338,6 +362,12 @@ function checkCard(
return v === undefined ? null : v;
};

/** Whether `p` sees `row` of `object`: every row-scope rule it declares on that object holds. */
const inScope = (p: Persona, object: string, row: SeedRow): boolean => {
const resolve = resolverFor(object);
return (p.scope ?? []).filter((sc) => sc.object === object).every((sc) => resolve(row, sc.field) === sc.value);
};

// ── personas: hidden refs and row scope resolve ──
duplicates(run.personas, (p) => p.id, ['run', 'personas'], 'persona id', refuse);
const personas = new Set(run.personas.map((p) => p.id));
Expand Down Expand Up @@ -419,8 +449,7 @@ function checkCard(
personaTruths[q.id][id] = { kind: 'withheld' };
return;
}
const rules = (p.scope ?? []).filter((sc) => sc.object === q.object);
const scoped = rows.filter((r) => rules.every((sc) => ctx.resolve(r, sc.field) === sc.value));
const scoped = rows.filter((r) => inScope(p, q.object, r));
const truth = evaluate(scoped, [...at, 'personas', j]);
if (truth) personaTruths[q.id][id] = truth;
});
Expand All @@ -432,6 +461,33 @@ function checkCard(
if (!run.operations.some((o) => o.kind === 'create' || o.kind === 'update')) {
refuse(['run', 'operations'], 'needs at least one create or update through the app');
}
// Reach: a persona is a requirement of the app (`hidden` = must not read,
// `scope` = sees only those rows), so an operation its acting persona cannot
// reach has exactly one outcome a faithful build gives, decided from the card
// alone — the object hidden, then the target row out of scope (the row gate
// runs before the field gate), then a written field hidden. Declaring anything
// else (`applied`, `pending_confirmation`, another refusal code) is the card
// contradicting its own persona: graded against a faithful build it fails, and
// it passes only a build that leaks what the persona must not see.
const checkReach = (op: ScenarioCard['run']['operations'][number], actor: Persona, at: IssuePath) => {
const row = op.target === undefined ? undefined : rowById.get(op.object)?.get(op.target);
const hiddenField = Object.keys(op.values ?? {}).find((field) =>
(actor.hidden.fields ?? []).some((f) => f.object === op.object && f.field === field),
);
let want: { code: ErrorCode; why: string } | undefined;
if ((actor.hidden.objects ?? []).includes(op.object)) {
want = { code: HIDDEN_WRITE_CODE, why: `\`${actor.id}\` must not read \`${op.object}\`` };
} else if (row !== undefined && !unscoped.has(actor.id) && !inScope(actor, op.object, row)) {
want = { code: NOT_REACHABLE_ROW_CODE, why: `row \`${op.target}\` is outside \`${actor.id}\`'s row scope on \`${op.object}\`` };
} else if (hiddenField !== undefined) {
want = { code: HIDDEN_WRITE_CODE, why: `\`${actor.id}\` must not read \`${op.object}.${hiddenField}\`, which this ${op.kind} writes` };
}
if (want === undefined || (op.outcome.kind === 'refused' && op.outcome.code === want.code)) return;
refuse(
[...at, 'outcome', ...(op.outcome.kind === 'refused' ? ['code'] : [])],
`${want.why}, so a faithful build refuses this ${op.kind} with \`${want.code}\` — declare that refusal, or change the target, the values or \`actingAs\` so the persona can reach what the operation writes`,
);
};
run.operations.forEach((op, i) => {
const at: IssuePath = ['run', 'operations', i];
if (!objects.has(op.object) || seed[op.object] === undefined) {
Expand All @@ -446,6 +502,8 @@ function checkCard(
checkRef(op.object, field, value, [...at, 'values', field]);
}
if (!personas.has(op.actingAs)) refuse([...at, 'actingAs'], `no persona \`${op.actingAs}\``);
const actor = run.personas.find((p) => p.id === op.actingAs);
if (actor !== undefined) checkReach(op, actor, at);
const out = op.outcome;
if (out.kind !== 'applied') return;
if (out.auditActor !== undefined && !personas.has(out.auditActor)) {
Expand Down
19 changes: 19 additions & 0 deletions packages/qa/scenarios/test/refusals.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ import { dirname, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { afterAll, describe, expect, it } from 'vitest';

import { ErrorCode } from '@objectstack/spec/api';

import { formatRefusal, loadScenarios, readCapabilityAreas } from '../src/index.js';

const HERE = dirname(fileURLToPath(import.meta.url));
Expand Down Expand Up @@ -87,6 +89,12 @@ const CASES: Array<[string, (c: RawCard) => void, string, RegExp]> = [
['a refusal code outside the platform error-code catalogue', (c) => (op(c, 'contact_bad_email').outcome.code = 'VALIDATION_BROKEN'), 'run.operations[3].outcome.code', /expected one of/],
['a post-state naming a created row by an id the seed lacks', (c) => (op(c, 'add_contact').outcome.state[0] = { object: 'contact', id: 'ct9', values: { full_name: 'Fay Wu' } }), 'run.operations[0].outcome.state[0].id', /matched by `where`/],
['a post-state naming its row twice', (c) => (op(c, 'move_eli').outcome.state[0].where = { full_name: 'Dee' }), 'run.operations[1].outcome.state[0].id', /exactly one of `id` or `where`/],
// ── reach: an operation its acting persona cannot reach has one outcome ──
['an applied create writing a field the persona must not read', (c) => (op(c, 'add_contact').values.email = 'fay@bitworks.test'), 'run.operations[0].outcome', /`sdr` must not read `contact.email`.*`PERMISSION_DENIED`/],
['an applied update on a row outside the persona\'s row scope', (c) => (op(c, 'move_eli').target = 'ct4'), 'run.operations[1].outcome', /row `ct4` is outside `sdr`'s row scope on `contact`.*`RECORD_NOT_FOUND`/],
['a pending delete on a row outside the persona\'s row scope', (c) => (op(c, 'remove_ben').target = 'ct3'), 'run.operations[2].outcome', /row `ct3` is outside `sdr`'s row scope.*`RECORD_NOT_FOUND`/],
['a validation refusal on a field the persona must not read', (c) => (op(c, 'contact_bad_email').actingAs = 'sdr'), 'run.operations[3].outcome.code', /`sdr` must not read `contact.email`.*`PERMISSION_DENIED`/],
['a refusal code other than the hidden object\'s', (c) => (op(c, 'partner_edits_account').outcome.code = ErrorCode.parse('RECORD_NOT_FOUND')), 'run.operations[4].outcome.code', /`partner_user` must not read `account`.*`PERMISSION_DENIED`/],
// ── guardrails ──
['a card that drops one of the five guardrail checks', (c) => (c.run.guardrails = c.run.guardrails.filter((g: { check: string }) => g.check !== 'audit_actor')), 'run.guardrails', /5/],
['an injection guardrail pointing at a non-text cell', (c) => (c.run.guardrails[4].cell = { object: 'contact', id: 'ct4', field: 'account' }), 'run.guardrails[4].cell.field', /not a text cell/],
Expand Down Expand Up @@ -114,6 +122,17 @@ describe('scenario card refusals', () => {
expect(r.refusals).toEqual([]);
});

it('an out-of-scope update declared as the refusal a faithful build answers loads clean (control for reach)', () => {
const r = load((c) => {
const o = op(c, 'move_eli');
o.target = 'ct4';
o.outcome = { kind: 'refused', code: ErrorCode.parse('RECORD_NOT_FOUND') };
c.run.guardrails[3].operation = 'add_contact';
op(c, 'add_contact').outcome.auditActor = 'sdr';
});
expect(r.refusals.map(formatRefusal)).toEqual([]);
});

for (const [name, mutate, path, message] of CASES) {
it(`refuses ${name}`, () => {
const r = load(mutate);
Expand Down
2 changes: 1 addition & 1 deletion packages/qa/scenarios/test/travel-expense.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ describe('scenario card travel_expense (docs/qa/scenarios/travel_expense.json)',
expect(outcome('fix_meal_days')?.kind).toBe('applied');
expect(outcome('manager_approves_step_one')?.kind).toBe('applied');
expect(outcome('pay_claim')).toEqual({ kind: 'pending_confirmation' });
expect(outcome('employee_approves_colleague')).toEqual({ kind: 'refused', code: 'PERMISSION_DENIED' });
expect(outcome('employee_approves_own_claim')).toEqual({ kind: 'refused', code: 'PERMISSION_DENIED' });
expect(outcome('over_standard_without_reason')).toEqual({ kind: 'refused', code: 'VALIDATION_FAILED' });
});

Expand Down
Loading