diff --git a/docs/qa/scenarios/README.md b/docs/qa/scenarios/README.md index 828a283692e..7b56ad33546 100644 --- a/docs/qa/scenarios/README.md +++ b/docs/qa/scenarios/README.md @@ -165,6 +165,6 @@ meet this floor: ## Running the cards `pnpm --filter @objectstack/scenarios run-scenarios` builds, runs and iterates every card here with a local -Claude Code session, and writes one run record per card. Grading uses `@objectstack/scenarios/score`; no +Claude Code session, and writes one run record per card. Grading uses `@objectstack/scenarios/score/full`; no API-billed model is used. The runner reads a card's names relative to the card's namespace (`account` → `democrm_account`), and a build that names any item without the prefix fails its Build half. Flags and modes: [the package README](../../../packages/qa/scenarios/README.md#running-the-cards--objectstackscenariosrunner). diff --git a/packages/qa/scenarios/README.md b/packages/qa/scenarios/README.md index fe34bee7256..ff432528cad 100644 --- a/packages/qa/scenarios/README.md +++ b/packages/qa/scenarios/README.md @@ -18,17 +18,28 @@ for (const r of refusals) console.error(formatRefusal(r)); // FILE: FIELD.PATH: seed does not determine. - `src/loader.ts` — reads a directory of cards, validates shape and cross-references, and returns typed cards or located refusals. -- `src/score/` — the deterministic scorer, entry `@objectstack/scenarios/score` (below). +- `src/score/` — the deterministic scorer, entries `@objectstack/scenarios/score` and + `@objectstack/scenarios/score/full` (below). -No build step (`exports` points at `src`), and nothing here calls a model or the network. +Not published. `pnpm --filter @objectstack/scenarios build` (tsup) writes `dist/` as ESM, CJS and +`.d.ts`, and every `exports` subpath points there, so a consumer that links the framework tree +and bundles its own entry can load the scorer under plain Node. The suites and `run-scenarios` +read `src/` directly. Nothing here calls a model or the network. ## Scoring a run against a card — `@objectstack/scenarios/score` -Every function takes a card the loader returned, so it judges against the loader's computed -truths and declared outcomes: +`@objectstack/scenarios/score` is the narrow entry (`src/score/narrow.ts`): the graph-shape +scorer (`scoreGraphShape`, `artifactToEvalGraph`), the item-name check and the run and +operation graders. Its import graph reads `@objectstack/core` and `@objectstack/spec` only +(pinned by `test/build/`). `@objectstack/scenarios/score/full` (`src/score/index.ts`) adds the +capability-tag checks, which read `@objectstack/objectql` and `@objectstack/plugin-sharing`, and +the card-level functions below. + +Every card-level function takes a card the loader returned, so it judges against the loader's +computed truths and declared outcomes: ```ts -import { gradeCardAnswer, gradeCardOperation, scoreCardArtifact } from '@objectstack/scenarios/score'; +import { gradeCardAnswer, gradeCardOperation, scoreCardArtifact } from '@objectstack/scenarios/score/full'; scoreCardArtifact(card, artifact); // a built dist/objectstack.json: expect, tags, agent actions gradeCardOperation(card, 'remove_ben', observed); // an observed write against its declared outcome diff --git a/packages/qa/scenarios/package.json b/packages/qa/scenarios/package.json index 26cb5356422..d7f57365a80 100644 --- a/packages/qa/scenarios/package.json +++ b/packages/qa/scenarios/package.json @@ -3,15 +3,64 @@ "version": "0.1.0", "private": true, "license": "Apache-2.0", - "description": "Scenario-card schema and loader for the open scenario library under docs/qa/scenarios/. A card states an authoring prompt, the structural expectation a generated app must meet, seed rows, data questions whose truth the loader COMPUTES from a declarative spec over those rows, personas, write operations with their guardrail checks, and one iteration request \u2014 plus the deterministic scorer that judges a built artifact, an observed write or an agent's answer against a loaded card (`@objectstack/scenarios/score`). Not published, no build step, no model calls: a devx instrument, not a protocol (packages/spec carries no obligation for this format).", + "description": "Scenario-card schema and loader for the open scenario library under docs/qa/scenarios/. A card states an authoring prompt, the structural expectation a generated app must meet, seed rows, data questions whose truth the loader COMPUTES from a declarative spec over those rows, personas, write operations with their guardrail checks, and one iteration request \u2014 plus the deterministic scorer that judges a built artifact, an observed write or an agent's answer against a loaded card (`@objectstack/scenarios/score`). Not published (a minimal tsup build to `dist/` so a linked consumer's bundled entry can load it under plain Node; `./score` is the narrow scorer, `./score/full` adds the capability-tag checks), no model calls: a devx instrument, not a protocol (packages/spec carries no obligation for this format).", "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", "exports": { - ".": "./src/index.ts", - "./score": "./src/score/index.ts", - "./runner": "./src/runner/index.ts", - "./export": "./src/export/index.ts" + ".": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + } + }, + "./score": { + "import": { + "types": "./dist/score.d.ts", + "default": "./dist/score.js" + }, + "require": { + "types": "./dist/score.d.cts", + "default": "./dist/score.cjs" + } + }, + "./score/full": { + "import": { + "types": "./dist/score-full.d.ts", + "default": "./dist/score-full.js" + }, + "require": { + "types": "./dist/score-full.d.cts", + "default": "./dist/score-full.cjs" + } + }, + "./runner": { + "import": { + "types": "./dist/runner.d.ts", + "default": "./dist/runner.js" + }, + "require": { + "types": "./dist/runner.d.cts", + "default": "./dist/runner.cjs" + } + }, + "./export": { + "import": { + "types": "./dist/export.d.ts", + "default": "./dist/export.js" + }, + "require": { + "types": "./dist/export.d.cts", + "default": "./dist/export.cjs" + } + } }, "scripts": { + "build": "tsup && node ../../../scripts/check-dts-emitted.mjs", "test": "vitest run", "typecheck": "tsc --noEmit", "run-scenarios": "tsx src/runner/cli.ts", diff --git a/packages/qa/scenarios/src/score/index.ts b/packages/qa/scenarios/src/score/index.ts index 5f6257a9ca6..8fb45e85721 100644 --- a/packages/qa/scenarios/src/score/index.ts +++ b/packages/qa/scenarios/src/score/index.ts @@ -1,10 +1,12 @@ // Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. /** - * `@objectstack/scenarios/score` — score a LOADED card against what a run - * produced: a built artifact (`dist/objectstack.json`), an observed write, or - * an agent's answer. The entry the scenario runner calls; private, like the - * rest of this package. + * `@objectstack/scenarios/score/full` — score a LOADED card against what a + * run produced: a built artifact (`dist/objectstack.json`), an observed write, + * or an agent's answer. The entry the scenario runner calls; private, like the + * rest of this package. The narrow `@objectstack/scenarios/score` is + * `./narrow.ts`: everything here except the capability-tag checks and the + * card-level functions below. * * Every function takes a {@link LoadedScenario} (what `loadScenarios` / * `loadScenarioFile` return), so the card it judges against has already passed diff --git a/packages/qa/scenarios/src/score/narrow.ts b/packages/qa/scenarios/src/score/narrow.ts new file mode 100644 index 00000000000..adde011e979 --- /dev/null +++ b/packages/qa/scenarios/src/score/narrow.ts @@ -0,0 +1,24 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * `@objectstack/scenarios/score` — the NARROW scorer entry (#30915): the + * graph-shape scorer (`scoreGraphShape`, `scoreGraph`, `artifactToEvalGraph`), + * the item-name check, the operation grader and the run graders + * (`gradeRun`, `gradePersonaRun`). + * + * Its import graph reaches `@objectstack/core` and `@objectstack/spec` and no + * other runtime package. The capability-tag checks (`scenario-tags.ts`) read + * `@objectstack/objectql` and `@objectstack/plugin-sharing`, so they and the + * card-level wrappers beside them (`scoreCardArtifact`, `gradeCardOperation`, + * `gradeCardAnswer`) are `@objectstack/scenarios/score/full` + * (`./index.ts`), which re-exports everything here as well. + * + * Re-exports only: the modules stay where they are. Pinned by + * test/build/narrow-score-graph.test.ts. + */ + +export * from './scenario-graph.js'; +export * from './scenario-naming.js'; +export * from './scenario-operation-grade.js'; +export * from './scenario-run-grade.js'; +export * from './scenario-score.js'; diff --git a/packages/qa/scenarios/test/build/built-consumer.test.ts b/packages/qa/scenarios/test/build/built-consumer.test.ts new file mode 100644 index 00000000000..be8ef2ce8e2 --- /dev/null +++ b/packages/qa/scenarios/test/build/built-consumer.test.ts @@ -0,0 +1,131 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * A BUILT consumer of `@objectstack/scenarios/score` (#30915): a throwaway + * package that depends on this one, bundles its own entry with tsup (which + * leaves a `dependencies` entry external), and is then loaded by plain Node, + * the way a linked framework tree's consumer loads it at boot. Before the + * package had a build step its exports named `.ts` files whose `.js` + * specifiers only a TypeScript resolver follows, so this load failed with + * ERR_MODULE_NOT_FOUND under both module kinds. + * + * The child process records every module it resolves (`module.registerHooks`), + * so the narrow entry's runtime graph is read, not inferred: no + * `@objectstack/objectql` or `@objectstack/plugin-sharing` file loads. + * + * Reads `dist/`: turbo runs this package's own `build` before its `test`. + */ + +import { execFileSync } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const PKG = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const REPO = resolve(PKG, '../../..'); +// The JS is what plain Node loads, so it is required. The declarations are +// not built under `OS_SKIP_DTS` (CI's build sets it); when they are built, +// the package's own `build` script already fails without them +// (`check-dts-emitted.mjs`), so this test does not ask for them. +const BUILT = ['score.js', 'score.cjs'].map((f) => join(PKG, 'dist', f)); + +const TSUP_PKG = createRequire(join(REPO, 'package.json')).resolve('tsup/package.json'); +const TSUP_BIN = join(dirname(TSUP_PKG), (JSON.parse(readFileSync(TSUP_PKG, 'utf8')) as { bin: { tsup: string } }).bin.tsup); + +const ENTRY = ` +import { scoreGraphShape } from '@objectstack/scenarios/score'; + +export const shape = () => + scoreGraphShape({ objects: [{ name: 'loan', fields: [{ name: 'title', type: 'text' }] }] }, { minObjects: 1 }); + +export const loadScore = () => import('@objectstack/scenarios/score'); +`; + +// Preloaded into the child: every URL Node resolves, written to the file +// named by SCENARIOS_RESOLVE_RECORD. A FILE, written synchronously: the record +// is ~140 KB, larger than a pipe's buffer, and a stdout write from an `exit` +// handler is not flushed past what the pipe takes before the process ends +// (a truncated record on a loaded CI runner). +const RECORDER = ` +const { writeFileSync } = require('node:fs'); +const { registerHooks } = require('node:module'); +const seen = []; +registerHooks({ resolve(specifier, context, next) { const r = next(specifier, context); seen.push(r.url); return r; } }); +process.on('exit', () => writeFileSync(process.env.SCENARIOS_RESOLVE_RECORD, JSON.stringify(seen))); +`; + +let fixture = ''; + +function runNode(code: string, kind: 'module' | 'commonjs'): { out: Record; resolved: string[] } { + const record = join(fixture, `resolved-${kind}.json`); + rmSync(record, { force: true }); + const stdout = execFileSync(process.execPath, ['--require', join(fixture, 'recorder.cjs'), `--input-type=${kind}`, '-e', code], { + cwd: fixture, + encoding: 'utf8', + env: { ...process.env, SCENARIOS_RESOLVE_RECORD: record }, + }); + return { out: JSON.parse(stdout.trim()) as Record, resolved: JSON.parse(readFileSync(record, 'utf8')) as string[] }; +} + +const probe = (mod: string) => ` + const m = ${mod}; + const s = await m.loadScore(); + const r = m.shape(); + console.log(JSON.stringify({ shape: typeof s.scoreGraphShape, run: typeof s.gradePersonaRun, tags: typeof s.scoreScenarioArtifact, score: r.score, total: r.total })); +`; + +describe('a tsup-built consumer of @objectstack/scenarios/score', () => { + beforeAll(() => { + const missing = BUILT.filter((f) => !existsSync(f)); + if (missing.length > 0) throw new Error(`build @objectstack/scenarios first (pnpm --filter @objectstack/scenarios build): missing ${missing.join(', ')}`); + + fixture = mkdtempSync(join(tmpdir(), 'scenarios-built-consumer-')); + writeFileSync( + join(fixture, 'package.json'), + JSON.stringify({ name: 'scenarios-built-consumer', private: true, type: 'module', dependencies: { '@objectstack/scenarios': 'workspace:*' } }), + ); + mkdirSync(join(fixture, 'src')); + writeFileSync(join(fixture, 'src/index.ts'), ENTRY); + writeFileSync(join(fixture, 'recorder.cjs'), RECORDER); + mkdirSync(join(fixture, 'node_modules/@objectstack'), { recursive: true }); + symlinkSync(PKG, join(fixture, 'node_modules/@objectstack/scenarios'), 'dir'); + + execFileSync(process.execPath, [TSUP_BIN, 'src/index.ts', '--format', 'esm,cjs', '--target', 'es2020', '--silent'], { + cwd: fixture, + stdio: 'pipe', + }); + }, 120_000); + + afterAll(() => { + if (fixture) rmSync(fixture, { recursive: true, force: true }); + }); + + it('leaves the scorer external: the bundle imports it, it does not carry it', () => { + for (const out of ['dist/index.js', 'dist/index.cjs']) { + const text = readFileSync(join(fixture, out), 'utf8'); + expect(text, out).toContain('@objectstack/scenarios/score'); + expect(text, out).not.toContain('PLATFORM_OBJECT_PREFIX'); + } + }); + + it.each([ + ['ESM', 'module', `await import(${JSON.stringify('PLACEHOLDER')})`, 'dist/index.js'], + ['CJS', 'commonjs', `require(${JSON.stringify('PLACEHOLDER')})`, 'dist/index.cjs'], + ] as const)('loads under plain Node (%s) and reaches neither objectql nor plugin-sharing', (_kind, inputType, load, file) => { + const target = inputType === 'module' ? pathToFileURL(join(fixture, file)).href : join(fixture, file); + const body = probe(load.replace('PLACEHOLDER', target)); + const { out, resolved } = runNode(inputType === 'module' ? body : `(async () => { ${body} })()`, inputType); + + expect(out).toEqual({ shape: 'function', run: 'function', tags: 'undefined', score: 1, total: 1 }); + const scorer = resolved.filter((u) => u.includes('/qa/scenarios/dist/')); + expect(scorer.length, 'the scorer is loaded from its dist').toBeGreaterThan(0); + expect(resolved.filter((u) => u.includes('/qa/scenarios/src/'))).toEqual([]); + expect(resolved.filter((u) => /objectql|plugin-sharing/.test(u))).toEqual([]); + // Control: the recorder sees the scorer's own runtime imports. + expect(resolved.some((u) => u.includes('/packages/core/'))).toBe(true); + }); +}); diff --git a/packages/qa/scenarios/test/build/narrow-score-graph.test.ts b/packages/qa/scenarios/test/build/narrow-score-graph.test.ts new file mode 100644 index 00000000000..db7646a07c0 --- /dev/null +++ b/packages/qa/scenarios/test/build/narrow-score-graph.test.ts @@ -0,0 +1,66 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The narrow scorer entry's SOURCE import graph (#30915): everything + * `src/score/narrow.ts` (`@objectstack/scenarios/score`) reaches through + * relative imports, type-only ones included, names no `@objectstack/objectql` + * or `@objectstack/plugin-sharing` specifier. Read from the checkout, so it + * holds without a build; test/build/built-consumer.test.ts reads the same + * fact off the built entry at runtime. + * + * The control leg walks the full entry (`src/score/index.ts`, `./score/full`) + * with the same walker and must find both packages: a walker that sees no + * import at all would pass the narrow leg too. + */ + +import { readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { describe, expect, it } from 'vitest'; + +const SRC = resolve(dirname(fileURLToPath(import.meta.url)), '../../src'); +const FORBIDDEN = ['@objectstack/objectql', '@objectstack/plugin-sharing']; + +// `import … from 'x'`, `export … from 'x'`, `import 'x'`, `import('x')`. +const SPECIFIER = /(?:\bfrom\s*|\bimport\s*\(?\s*)['"]([^'"]+)['"]/g; + +/** Bare specifiers reached from `entry`, each with the first file that imports it. */ +function bareImports(entry: string): Map { + const bare = new Map(); + const seen = new Set(); + const queue = [resolve(SRC, entry)]; + while (queue.length > 0) { + const file = queue.shift()!; + if (seen.has(file)) continue; + seen.add(file); + const text = readFileSync(file, 'utf8').replace(/\/\*[\s\S]*?\*\/|\/\/[^\n]*/g, ''); + for (const [, spec] of text.matchAll(SPECIFIER)) { + if (spec.startsWith('.')) queue.push(resolve(dirname(file), spec.replace(/\.js$/, '.ts'))); + else if (!bare.has(spec)) bare.set(spec, file.slice(SRC.length + 1)); + } + } + return bare; +} + +const reaches = (bare: Map, pkg: string) => + [...bare.keys()].filter((s) => s === pkg || s.startsWith(`${pkg}/`)); + +describe('the ./score entry import graph', () => { + it('reaches neither objectql nor plugin-sharing', () => { + const bare = bareImports('score/narrow.ts'); + for (const pkg of FORBIDDEN) expect(reaches(bare, pkg), `${pkg} via ${bare.get(pkg)}`).toEqual([]); + // What it does read: the core stack-collection rule and the spec. + expect([...bare.keys()].filter((s) => s.startsWith('@objectstack/')).sort()).toEqual([ + '@objectstack/core', + '@objectstack/spec/api', + '@objectstack/spec/data', + '@objectstack/spec/meta-spelling', + ]); + }); + + it('control: the same walker finds both packages from ./score/full', () => { + const bare = bareImports('score/index.ts'); + for (const pkg of FORBIDDEN) expect(reaches(bare, pkg)).toEqual([pkg]); + }); +}); diff --git a/packages/qa/scenarios/tsconfig.build.json b/packages/qa/scenarios/tsconfig.build.json new file mode 100644 index 00000000000..ecf41e27879 --- /dev/null +++ b/packages/qa/scenarios/tsconfig.build.json @@ -0,0 +1,17 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src", + "types": [ + "node" + ] + }, + "include": [ + "src/**/*" + ], + "exclude": [ + "node_modules", + "dist" + ] +} diff --git a/packages/qa/scenarios/tsup.config.ts b/packages/qa/scenarios/tsup.config.ts new file mode 100644 index 00000000000..f8c9db847a2 --- /dev/null +++ b/packages/qa/scenarios/tsup.config.ts @@ -0,0 +1,42 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { defineConfig } from 'tsup'; + +import { dropSourcesContent } from '../../../scripts/tsup-drop-sources-content.mjs'; + +/** + * A minimal build for a PRIVATE package (#30915): it is never published, but a + * consumer that links the framework tree and bundles its own entry with tsup + * leaves `@objectstack/scenarios` external, so plain Node must be able to load + * it. The `.ts` sources import with `.js` specifiers, which only a TypeScript + * resolver follows: `dist/` is what Node loads (ESM `.js`, CJS `.cjs`). + * + * One entry per export-map subpath. `score` is the narrow scorer + * (`src/score/narrow.ts`): it reads `@objectstack/core` and `@objectstack/spec` + * only. `score-full` is the whole scorer (`src/score/index.ts`), whose + * capability-tag checks read `@objectstack/objectql` and + * `@objectstack/plugin-sharing`. `splitting: false` keeps each entry one file, + * so an entry's external imports are exactly the ones its own graph reaches. + * + * Dependencies stay external (tsup's default for `dependencies`; `export`'s + * `playwright-core` included, which it loads with a dynamic `import()`); the source + * suites and `run-scenarios` keep reading `src/` and never this output. + */ +export default defineConfig({ + entry: { + index: 'src/index.ts', + score: 'src/score/narrow.ts', + 'score-full': 'src/score/index.ts', + runner: 'src/runner/index.ts', + export: 'src/export/index.ts', + }, + tsconfig: 'tsconfig.build.json', + splitting: false, + sourcemap: true, + clean: true, + dts: !process.env.OS_SKIP_DTS, + format: ['esm', 'cjs'], + target: 'es2020', + shims: true, + esbuildOptions: dropSourcesContent, +}); diff --git a/turbo.json b/turbo.json index 3ee6edd0eef..e8653b4b9aa 100644 --- a/turbo.json +++ b/turbo.json @@ -701,7 +701,7 @@ ] }, "@objectstack/scenarios#test": { - "dependsOn": ["^build"], + "dependsOn": ["build"], "outputs": [], "env": ["OS_TEST_TIERS"], "inputs": [