feat(cli): port the deterministic scenario scorer — shape score, artifact graph reader and Run grader — with a discrimination self-test

.changeset/30290-scenario-scorer-port.md

@@ -0,0 +1,14 @@
+---
+'@objectstack/cli': patch
+---
+
+feat(cli): a deterministic scenario scorer under `src/lint` — the structural shape score, an artifact-to-graph reader and a Run-half grader, ported from the cloud eval harness
+
+Clause-②: no
+Metadata-protocol: none
+
+- **Structural scorer** (`src/lint/scenario-score.ts`): `scoreGraph` matches a scenario's `expect` block by exact name, `scoreGraphShape` scores the build's shape regardless of names, and `deriveShape` turns the first into the second. Two checks that let a degraded build score as high as an intact one are closed: an expected nav target now passes only when an app navigates to it through an entry that resolves (every app nav entry is also scored for resolving), and a dashboard widget bound to a dataset the build does not carry now fails its measure check.
+- **Artifact reader** (`src/lint/scenario-graph.ts`): reads a built `dist/objectstack.json` into the scorer's graph, single-package or with its collections only under `packages[]` (ADR-0130), through the shared `resolveStackCollection` rule, reading each key in its one spec shape and refusing an unreadable object instead of scoring it as an empty one.
+- **Run grader** (`src/lint/scenario-run-grade.ts`): grades one agent turn against a question's computed truth as `{ correct, cited, leaks, problems }` — figures matched (with `万` / `亿` / `k` / `M` read), each figure's object cited, and hidden values, hidden aggregates and tool calls answered over a hidden field or object reported as leaks, in the reply or in any tool output.
+
+No model call anywhere, no new CLI flag and no new package-entry export: nothing calls these yet; the `os lint --eval` wiring is a later change.

packages/cli/src/lint/scenario-graph.test.ts

@@ -0,0 +1,150 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+import { describe, expect, it } from 'vitest';
+
+import { ObjectStackDefinitionSchema, normalizeStackInput } from '@objectstack/spec';
+
+import { DEFAULT_METADATA_EVAL_CORPUS } from './corpus.js';
+import { artifactToEvalGraph } from './scenario-graph.js';
+
+type Bag = Record<string, any>;
+
+const manifest = { id: 'com.example.library', namespace: 'library', version: '1.0.0', name: 'Library', type: 'app' };
+
+const objects = [
+  {
+    name: 'loan',
+    label: 'Loan',
+    fields: {
+      book: { type: 'lookup', label: 'Book', reference: 'book' },
+      // As `Field.user()` writes it: the carrier states the type's fixed target.
+      borrower: { type: 'user', label: 'Borrower', reference: 'sys_user' },
+      status: {
+        type: 'select',
+        label: 'Status',
+        required: true,
+        options: [{ label: 'Out', value: 'out' }, { label: 'Back', value: 'back' }],
+      },
+    },
+  },
+  { name: 'book', label: 'Book', fields: { title: { type: 'text', label: 'Title', required: true } } },
+];
+const datasets = [{
+  name: 'loan_metrics',
+  label: 'Loan Metrics',
+  object: 'loan',
+  dimensions: [{ name: 'status', label: 'Status', field: 'status', type: 'string' }],
+  measures: [{ name: 'loan_count', label: 'Loans', aggregate: 'count' }],
+}];
+const dashboards = [{
+  name: 'library_overview',
+  label: 'Overview',
+  widgets: [{ id: 'by_status', title: 'By status', type: 'bar', dataset: 'loan_metrics', dimensions: ['status'], values: ['loan_count'] }],
+}];
+const apps = [{
+  name: 'library',
+  label: 'Library',
+  navigation: [
+    { id: 'nav_home', type: 'dashboard', label: 'Home', dashboardName: 'library_overview' },
+    {
+      id: 'nav_data',
+      type: 'group',
+      label: 'Data',
+      children: [
+        { id: 'nav_loan', type: 'object', label: 'Loans', objectName: 'loan' },
+        { id: 'nav_book', type: 'object', label: 'Books', objectName: 'book' },
+        { id: 'nav_users', type: 'object', label: 'Users', objectName: 'sys_user' },
+        { id: 'nav_gone', type: 'object', label: 'Gone', objectName: 'member' },
+      ],
+    },
+    { id: 'nav_docs', type: 'url', label: 'Docs', url: 'https://example.com' },
+  ],
+}];
+
+const EXPECTED_GRAPH = {
+  objects: [
+    {
+      name: 'loan',
+      fields: [
+        { name: 'book', type: 'lookup', reference: 'book' },
+        { name: 'borrower', type: 'user' },
+        { name: 'status', type: 'select', required: true, options: [{ value: 'out', label: 'Out' }, { value: 'back', label: 'Back' }] },
+      ],
+    },
+    { name: 'book', fields: [{ name: 'title', type: 'text', required: true }] },
+  ],
+  dashboards: [{ name: 'library_overview', widgets: [{ dataset: 'loan_metrics', measure: 'loan_count' }] }],
+  apps: [{
+    name: 'library',
+    nav: [
+      { target: 'library_overview', resolves: true },
+      { target: 'loan', resolves: true },
+      { target: 'book', resolves: true },
+      { target: 'sys_user', resolves: true },
+      { target: 'member', resolves: false },
+    ],
+  }],
+  datasets: [{ name: 'loan_metrics', measures: ['loan_count'] }],
+};
+
+/** The stack parsed as `os build` parses it, round-tripped through JSON as the artifact file is. */
+function parsedArtifact(stack: Bag): Bag {
+  const parsed = ObjectStackDefinitionSchema.safeParse(normalizeStackInput(stack));
+  if (!parsed.success) throw new Error(parsed.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; '));
+  return JSON.parse(JSON.stringify(parsed.data)) as Bag;
+}
+
+describe('artifactToEvalGraph', () => {
+  it('reads a single-package artifact: fields, options, authored references, nested nav, widget measures', () => {
+    const artifact = parsedArtifact({ manifest, objects, datasets, dashboards, apps });
+    // The artifact carries `user`'s fixed target; the graph still leaves it out.
+    expect(artifact.objects[0].fields.borrower.reference).toBe('sys_user');
+    expect(artifactToEvalGraph(artifact)).toEqual(EXPECTED_GRAPH);
+  });
+
+  it('reads a multi-package artifact whose collections live only under packages[] (ADR-0130 option B)', () => {
+    const artifact = {
+      manifest,
+      packages: [
+        { manifest: { id: 'com.example.library.core', name: 'core', version: '1.0.0', type: 'app', objects } },
+        {
+          manifest: {
+            id: 'com.example.library.ui',
+            name: 'ui',
+            version: '1.0.0',
+            type: 'app',
+            dependencies: { 'com.example.library.core': '1.0.0' },
+            datasets,
+            dashboards,
+            apps,
+          },
+        },
+      ],
+    };
+    expect(artifact).not.toHaveProperty('objects');
+    expect(artifactToEvalGraph(artifact)).toEqual(EXPECTED_GRAPH);
+  });
+
+  it('prefers the top-level collection over packages[] when the artifact carries both (the additive shape)', () => {
+    const artifact = {
+      manifest,
+      objects: [objects[1]],
+      packages: [{ manifest: { id: 'com.example.library.core', name: 'core', version: '1.0.0', type: 'app', objects } }],
+    };
+    expect(artifactToEvalGraph(artifact).objects.map((o) => o.name)).toEqual(['book']);
+  });
+
+  it('reads every golden corpus build', () => {
+    for (const c of DEFAULT_METADATA_EVAL_CORPUS) {
+      const graph = artifactToEvalGraph(parsedArtifact(c.fixture as Bag));
+      expect(graph.objects.length, c.id).toBe((c.fixture as Bag).objects.length);
+      expect(graph.objects.flatMap((o) => o.fields).some((f) => f.reference), c.id).toBe(true);
+    }
+  });
+
+  it('refuses an object with no name, and fields that are not a name-keyed record', () => {
+    expect(() => artifactToEvalGraph({ objects: [{ fields: {} }] })).toThrow(TypeError);
+    expect(() => artifactToEvalGraph({ objects: [{ name: 'a', fields: [{ name: 'x', type: 'text' }] }] }))
+      .toThrow(/not a name-keyed record/);
+  });
+});

packages/cli/src/lint/scenario-graph.ts

@@ -0,0 +1,168 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+// Ported from the cloud eval harness under the cloud#3600 D2 ruling (the graph reader in agent-e2e.ts).
+
+/**
+ * A built artifact (`dist/objectstack.json`, as `os build` writes it) read into
+ * the {@link EvalGraph} the scenario scorers judge.
+ *
+ * ## One reader of `packages[]`, not a second one
+ *
+ * A multi-package artifact may carry its collections only inside
+ * `packages[i].manifest` (ADR-0130 D4, option B). Every collection here is read
+ * through `resolveStackCollection` — the rule `@objectstack/cli` and
+ * `@objectstack/verify` already share (`utils/stack-collections.ts`): the
+ * top-level array when present, otherwise every package body's, in the
+ * dependency order `resolveArtifactPackageOrder` decides. ⛔ No local walk of
+ * `packages[]`.
+ *
+ * ## The spec's shapes, and only those
+ *
+ * The cloud reader this replaces read a live environment's metadata API and
+ * tolerated several spellings per key (`navigation ?? nav ?? tabs ?? …`,
+ * `reference_to ?? referenceTo ?? reference`). An artifact has been parsed by
+ * the spec before it was written, so each key is read in its ONE declared
+ * shape: object `fields` is a name-keyed record, a field's target is
+ * `reference` (through `referenceTargetOf`), an app's nav is `navigation` with
+ * nested `children`, a widget's measures are `values` and its binding is
+ * `dataset` (ADR-0021). A scorer that quietly accepted a second spelling would
+ * score an off-spec build as if it were on spec.
+ *
+ * A value that cannot be read in that shape — an object with no name, a
+ * `fields` that is not a record — is refused with a thrown `TypeError` naming
+ * where, never read as an empty object: an empty object is a DEGRADED build to
+ * the scorer, and an unreadable artifact must not score as one.
+ */
+
+import { REFERENCE_VALUE_TYPES, referenceTargetOf } from '@objectstack/spec/data';
+
+import { resolveStackCollection } from '../utils/stack-collections.js';
+import { PLATFORM_OBJECT_PREFIX, type EvalGraph, type EvalGraphField, type EvalGraphNavEntry } from './scenario-score.js';
+
+type Bag = Record<string, unknown>;
+
+const isBag = (value: unknown): value is Bag => !!value && typeof value === 'object' && !Array.isArray(value);
+
+function named(item: unknown, where: string): Bag & { name: string } {
+  if (!isBag(item) || typeof item.name !== 'string' || item.name === '') {
+    throw new TypeError(`scenario graph: ${where} has no \`name\` — the artifact cannot be scored.`);
+  }
+  return item as Bag & { name: string };
+}
+
+/**
+ * The author-chosen target of a field, or `undefined`.
+ *
+ * A type whose target the TYPE fixes (`user` → `sys_user`) has none: whether a
+ * type fixes its target is asked of the spec (`referenceTargetOf` on the bare
+ * type answers only for those), never transcribed here.
+ */
+function authoredReference(def: Bag): string | undefined {
+  if (typeof def.type !== 'string' || !REFERENCE_VALUE_TYPES.has(def.type)) return undefined;
+  if (referenceTargetOf({ type: def.type }) !== undefined) return undefined;
+  return referenceTargetOf(def);
+}
+
+function toField(name: string, def: unknown, where: string): EvalGraphField {
+  if (!isBag(def)) throw new TypeError(`scenario graph: ${where} is not a field definition.`);
+  const field: EvalGraphField = { name, type: typeof def.type === 'string' ? def.type : '' };
+  if (def.required === true) field.required = true;
+  if (Array.isArray(def.options)) {
+    field.options = def.options.filter(isBag).map((o) => ({
+      value: String(o.value),
+      ...(typeof o.label === 'string' ? { label: o.label } : {}),
+    }));
+  }
+  const reference = authoredReference(def);
+  if (reference !== undefined) field.reference = reference;
+  return field;
+}
+
+/** `[kind, targetKey]` per nav item type that targets a named build member. */
+const NAV_TARGET_KEYS: Readonly<Record<string, readonly [kind: string, key: string]>> = {
+  object: ['object', 'objectName'],
+  dashboard: ['dashboard', 'dashboardName'],
+  page: ['page', 'pageName'],
+  report: ['report', 'reportName'],
+};
+
+function navEntries(items: unknown, members: Readonly<Record<string, Set<string>>>, out: EvalGraphNavEntry[]): void {
+  if (!Array.isArray(items)) return;
+  for (const item of items) {
+    if (!isBag(item)) continue;
+    const spec = typeof item.type === 'string' ? NAV_TARGET_KEYS[item.type] : undefined;
+    if (spec) {
+      const [kind, key] = spec;
+      const target = item[key];
+      if (typeof target === 'string') {
+        const resolves = members[kind]!.has(target) || (kind === 'object' && PLATFORM_OBJECT_PREFIX.test(target));
+        out.push({ target, resolves });
+      }
+    }
+    navEntries(item.children, members, out);
+  }
+}
+
+/**
+ * Read a built artifact into an {@link EvalGraph}.
+ *
+ * @param artifact - The artifact exactly as `JSON.parse` returns
+ *   `dist/objectstack.json` — single-package, or multi-package with its
+ *   collections under `packages[]`.
+ * @throws TypeError when a collection member cannot be read in its spec shape;
+ *   the ADR-0112 refusal of `resolveArtifactPackageOrder` for a malformed
+ *   `packages` array.
+ */
+export function artifactToEvalGraph(artifact: unknown): EvalGraph {
+  const collection = (key: string) => resolveStackCollection(artifact, key);
+
+  const objects = collection('objects').map((raw, i) => {
+    const o = named(raw, `objects[${i}]`);
+    const fieldsRaw = o.fields ?? {};
+    if (!isBag(fieldsRaw)) {
+      throw new TypeError(`scenario graph: object ${o.name} has a \`fields\` that is not a name-keyed record.`);
+    }
+    return {
+      name: o.name,
+      fields: Object.entries(fieldsRaw).map(([name, def]) => toField(name, def, `${o.name}.${name}`)),
+    };
+  });
+
+  const dashboardsRaw = collection('dashboards').map((raw, i) => named(raw, `dashboards[${i}]`));
+  const dashboards = dashboardsRaw.map((d) => ({
+    name: d.name,
+    widgets: (Array.isArray(d.widgets) ? d.widgets : []).filter(isBag).flatMap((w) => {
+      const dataset = typeof w.dataset === 'string' ? w.dataset : undefined;
+      const values = Array.isArray(w.values) ? w.values.filter((v): v is string => typeof v === 'string') : [];
+      return values.length ? values.map((measure) => ({ dataset, measure })) : [{ dataset }];
+    }),
+  }));
+
+  const datasets = collection('datasets').map((raw, i) => {
+    const d = named(raw, `datasets[${i}]`);
+    return {
+      name: d.name,
+      measures: (Array.isArray(d.measures) ? d.measures : [])
+        .filter(isBag)
+        .map((m) => m.name)
+        .filter((n): n is string => typeof n === 'string'),
+    };
+  });
+
+  const nameSet = (key: string) =>
+    new Set(collection(key).map((raw) => (isBag(raw) && typeof raw.name === 'string' ? raw.name : '')).filter(Boolean));
+  const members: Record<string, Set<string>> = {
+    object: new Set(objects.map((o) => o.name)),
+    dashboard: new Set(dashboards.map((d) => d.name)),
+    page: nameSet('pages'),
+    report: nameSet('reports'),
+  };
+
+  const apps = collection('apps').map((raw, i) => {
+    const a = named(raw, `apps[${i}]`);
+    const nav: EvalGraphNavEntry[] = [];
+    navEntries(a.navigation, members, nav);
+    return { name: a.name, nav };
+  });
+
+  return { objects, dashboards, apps, datasets };
+}