diff --git a/packages/qa/scenarios/package.json b/packages/qa/scenarios/package.json index 578fef407fd..26cb5356422 100644 --- a/packages/qa/scenarios/package.json +++ b/packages/qa/scenarios/package.json @@ -8,18 +8,21 @@ "exports": { ".": "./src/index.ts", "./score": "./src/score/index.ts", - "./runner": "./src/runner/index.ts" + "./runner": "./src/runner/index.ts", + "./export": "./src/export/index.ts" }, "scripts": { "test": "vitest run", "typecheck": "tsc --noEmit", - "run-scenarios": "tsx src/runner/cli.ts" + "run-scenarios": "tsx src/runner/cli.ts", + "export-gallery": "tsx src/export/cli.ts" }, "dependencies": { "@objectstack/core": "workspace:*", "@objectstack/objectql": "workspace:*", "@objectstack/plugin-sharing": "workspace:*", "@objectstack/spec": "workspace:*", + "playwright-core": "1.63.0", "zod": "^4.6.6" }, "devDependencies": { diff --git a/packages/qa/scenarios/src/export/cli.ts b/packages/qa/scenarios/src/export/cli.ts new file mode 100644 index 00000000000..b6233d79a60 --- /dev/null +++ b/packages/qa/scenarios/src/export/cli.ts @@ -0,0 +1,124 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * `pnpm --filter @objectstack/scenarios export-gallery --runs DIR --out DIR [flags]` + * — write every verified scenario app the runner's records prove into a + * local directory, in the layout of objectstack-ai/gallery (`exporter.ts`). + * + * --runs DIR the run records (`run-scenarios --out`; default docs/qa/platform-checklist/runs) + * --out DIR where `apps/` and `index.json` go (required) + * --recording F a run's recording (`run-scenarios --record F`, or the + * file a `--driver recorded:F` run replayed); repeatable + * --cards DIR the card directory (default: the repo's docs/qa/scenarios) + * --work DIR read the projects here, not at each record's `evidenceDir` + * (both directories are redacted to `$WORK` in the output) + * --no-screenshots export without screenshots (no app is booted); an app + * whose previous export had screenshots is kept as it was + * + * A runs directory with no scenario record, over an index that lists apps, + * exits non-zero and touches nothing (a wrong `--runs`, never "every app gone"). + * --chromium PATH the Chromium executable for the screenshots + * + * It writes only under `--out`; it pushes nothing anywhere. ⛔ It exports only + * cards marked generic, and the card schema has no generic marker yet, so + * today it refuses every card (`generic.ts`) — and offers no flag to do + * otherwise. + */ + +import { existsSync, readFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { formatRefusal, loadScenarios, readCapabilityAreas } from '../loader.js'; +import { HttpApp } from '../runner/app.js'; +import { ADMIN, findUp } from '../runner/cli.js'; +import type { Recording } from '../runner/driver.js'; +import { bootDev } from '../runner/project.js'; +import { exportGallery, type ExportSummary } from './exporter.js'; +import { captureScreenshots, playwrightBrowser, type CaptureScreenshots } from './screenshots.js'; + +export interface ExportCliArgs { + runs?: string; + out?: string; + recordings: string[]; + cards?: string; + work?: string; + screenshots: boolean; + chromium?: string; +} + +export function parseExportArgs(argv: readonly string[]): ExportCliArgs { + const a: ExportCliArgs = { recordings: [], screenshots: true }; + for (let i = 0; i < argv.length; i++) { + const flag = argv[i]!; + const val = () => { + const v = argv[++i]; + if (v === undefined) throw new Error(`${flag} needs a value`); + return v; + }; + if (flag === '--runs') a.runs = val(); + else if (flag === '--out') a.out = val(); + else if (flag === '--recording') a.recordings.push(val()); + else if (flag === '--cards') a.cards = val(); + else if (flag === '--work') a.work = val(); + else if (flag === '--no-screenshots') a.screenshots = false; + else if (flag === '--chromium') a.chromium = val(); + else throw new Error(`unknown flag \`${flag}\``); + } + if (!a.out) throw new Error('--out DIR is required: the directory `apps/` and `index.json` are written into'); + return a; +} + +/** The summary, one line per app, as the CLI prints it. */ +export function renderSummary(s: ExportSummary): string[] { + const lines: string[] = []; + for (const e of s.exported) lines.push(`exported ${e.id} · ${e.screenshots.length} screenshot(s)${e.skippedShots.map((k) => ` · no ${k.kind}: ${k.reason}`).join('')}`); + for (const r of s.refused) lines.push(`refused ${r.id} · ${r.reasons.join(' · ')}`); + for (const r of s.removed) lines.push(`removed ${r.id} · ${r.why}`); + for (const k of s.kept) lines.push(`kept ${k.id} · ${k.why}`); + for (const u of s.unowned) lines.push(`left apps/${u} · not in the previous index, so not the exporter's`); + for (const f of s.ignoredFiles) lines.push(`ignored ${f} · not a scenario-runner record`); + lines.push(`index.json ${s.indexChanged ? 'written' : 'unchanged'} · ${s.exported.length} exported, ${s.refused.length} refused, ${s.removed.length} removed, ${s.kept.length} kept`); + return lines; +} + +/** Test seam: replace the screenshot step (the CLI's is the real browser). */ +export interface ExportCliDeps { + screenshots?: CaptureScreenshots; +} + +export async function main(argv: readonly string[], deps: ExportCliDeps = {}): Promise { + const args = parseExportArgs(argv); + const here = dirname(fileURLToPath(import.meta.url)); + const repo = findUp(here, (d) => existsSync(join(d, 'pnpm-workspace.yaml'))); + const cardsDir = resolve(args.cards ?? join(repo, 'docs/qa/scenarios')); + const runsDir = resolve(args.runs ?? join(repo, 'docs/qa/platform-checklist/runs')); + const { cards, refusals } = loadScenarios(cardsDir, { areas: readCapabilityAreas(join(repo, 'docs/qa/platform-checklist/areas')) }); + for (const r of refusals) console.error(formatRefusal(r)); + const recordings = args.recordings.map((f) => JSON.parse(readFileSync(resolve(f), 'utf8')) as Recording); + const browser = playwrightBrowser(args.chromium ? { executablePath: args.chromium } : {}); + const screenshots: CaptureScreenshots | false = !args.screenshots + ? false + : (deps.screenshots ?? + ((input) => captureScreenshots(input, { boot: (dir) => bootDev(dir), harness: (url) => new HttpApp({ baseUrl: url, adminEmail: ADMIN.email, adminPassword: ADMIN.password }), browser, admin: ADMIN }))); + const summary = await exportGallery({ + runsDir, + outDir: resolve(args.out!), + cards, + recordings, + ...(args.work ? { workDir: resolve(args.work) } : {}), + screenshots, + }); + for (const line of renderSummary(summary)) (/^(refused|removed|kept) /.test(line) ? console.error : console.log)(line); + return 0; +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main(process.argv.slice(2)).then( + (code) => process.exit(code), + (e) => { + console.error(e instanceof Error ? e.message : e); + process.exit(1); + }, + ); +} diff --git a/packages/qa/scenarios/src/export/exporter.ts b/packages/qa/scenarios/src/export/exporter.ts new file mode 100644 index 00000000000..f8118c2f6ba --- /dev/null +++ b/packages/qa/scenarios/src/export/exporter.ts @@ -0,0 +1,324 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The gallery exporter: verified scenario apps, written to a local directory + * in the layout of objectstack-ai/gallery. It writes nothing anywhere else — + * how the output reaches the gallery repository is a separate decision. + * + * For each card the runs directory holds a record of, the NEWEST record (by + * date) decides, in this order, each a refusal with its reason: + * + * 1. the card is loaded and marked generic ({@link GenericGuard}); + * 2. the record passed every half ({@link judgeRecord}); + * 3. the run's scratch project is still on disk, with its build; + * 4. its transcript joins with the recording ({@link buildTranscript}); + * 5. its screenshots were taken, unless the caller turned them off. + * + * Output (`--out`), per exported app, written whole into a staging directory + * and then swapped in, so a re-export first deletes the old directory and + * nothing stale survives (see {@link APP_LAYOUT}): + * + * apps//README.md what the app is, its scorecard, the files + * apps//source/… the authored metadata source (`source.ts`) + * apps//run-record.json the run record, the work directories redacted (`workDirsOf`) + * apps//scorecard.json|.md the per-half scorecard + * apps//transcript.json|.md the per-persona 问数 transcript + * apps//screenshots/.png list, kanban/calendar, record, dashboard + * index.json the manifest (`manifest.ts`) + * + * What the exporter owns is what the previous `index.json` lists. An app it + * listed that does not export this time — its newest record fails, its card + * is gone or not generic, its project is gone — is deleted and leaves the + * index; the summary names why. A directory under `apps/` the index never + * listed is not the exporter's and is left alone (and named). Nothing is + * touched when the previous index does not validate, or when the runs + * directory holds no scenario record while the index lists apps (a wrong + * `--runs`, never "every app gone"). + * + * An export without screenshots (`screenshots: false`) never replaces an app + * whose previous export had them: the old directory and entry are kept, and + * the summary says so (`kept`). It still removes an app that no longer passes. + */ + +import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; + +import type { LoadedScenario } from '../loader.js'; +import { scenarioNamespace } from '../schema.js'; +import type { Recording } from '../runner/driver.js'; +import { modeDir } from '../runner/orchestrate.js'; +import { readArtifact } from '../runner/project.js'; +import { renderScorecard, type CardScorecard } from '../runner/scorecard.js'; +import { NO_GENERIC_MARKER, type GenericGuard } from './generic.js'; +import { parseIndex, serializeIndex, type GalleryApp, type GalleryIndex } from './manifest.js'; +import { judgeRecord, type PassJudgement } from './pass.js'; +import { readRunRecords, redactWorkDir, type RecordFile } from './record.js'; +import type { CaptureScreenshots, PlannedShot, SkippedShot } from './screenshots.js'; +import { projectFrameworkVersion, readAuthoredSource } from './source.js'; +import { buildTranscript, mergeRecordings, renderTranscript } from './transcript.js'; + +/** The files of one exported app, relative to `apps//`. */ +export const APP_LAYOUT = { + readme: 'README.md', + source: 'source', + runRecord: 'run-record.json', + scorecardJson: 'scorecard.json', + scorecardMd: 'scorecard.md', + transcriptJson: 'transcript.json', + transcriptMd: 'transcript.md', + screenshots: 'screenshots', +} as const; + +/** + * The run's scratch project, where the runner CLI scaffolds it: under the + * record's work directory, `-scratch/` (`modeDir` + + * the name `ProjectOps.scaffold` is given). After the run it holds the app as + * the change request left it, which is the app the iterate halves graded. + */ +export function scratchProjectDir(workDir: string, card: { id: string; code: string }): string { + return join(workDir, modeDir(card, 'scratch'), scenarioNamespace(card)); +} + +export interface ExportOptions { + /** Where the runner wrote its records. */ + runsDir: string; + /** The gallery checkout (or any directory) to write into. */ + outDir: string; + /** The loaded cards (`loadScenarios`). */ + cards: readonly LoadedScenario[]; + /** The run's recordings: the transcript's reply text. */ + recordings: readonly Recording[]; + /** Read the projects here instead of each record's `evidenceDir` (records moved off the machine that ran them). */ + workDir?: string; + /** Decides "generic". Default {@link NO_GENERIC_MARKER}, which refuses every card. */ + isGeneric?: GenericGuard; + /** Take screenshots with this; `false` exports without them. */ + screenshots: CaptureScreenshots | false; +} + +export interface ExportSummary { + exported: Array<{ id: string; screenshots: PlannedShot[]; skippedShots: SkippedShot[] }>; + refused: Array<{ id: string; reasons: string[] }>; + /** Apps the previous index listed that this export deleted. */ + removed: Array<{ id: string; why: string }>; + /** + * Apps that would have exported, kept as the previous export left them + * instead: an export without screenshots never replaces one that had them. + */ + kept: Array<{ id: string; why: string }>; + /** Directories under `apps/` the exporter does not own, left as they were. */ + unowned: string[]; + /** JSON files in the runs directory that are not scenario-runner records. */ + ignoredFiles: string[]; + /** True when the index bytes on disk changed. */ + indexChanged: boolean; +} + +class Refused extends Error { + constructor(readonly reasons: string[]) { + super(reasons.join('; ')); + } +} + +const json = (v: unknown) => `${JSON.stringify(v, null, 2)}\n`; + +function appReadme(scenario: LoadedScenario, entry: GalleryApp, judgement: PassJudgement, scorecard: CardScorecard, shots: PlannedShot[], skipped: SkippedShot[]): string { + const c = scenario.card; + const lines = [ + `# ${c.title}`, + '', + `Generated by the ObjectStack scenario pipeline — do not edit by hand. Fix the card, the platform or the authoring skills instead; the next release rebuild corrects this directory.`, + '', + `- Scenario: \`${c.id}\` · function \`${c.category}\` · namespace \`${entry.namespace}\``, + `- Framework: @objectstack/spec ${entry.framework.version} (runner checkout ${entry.framework.runner}) · run ${entry.run} on ${entry.runDate}`, + `- 问数: ${entry.qa.questions} questions, ${entry.qa.turns} persona turns · ${entry.operations} write operations`, + '', + '## The requirement', + '', + ...(c.prompt.short.zh ? [`- ${c.prompt.short.zh}`] : []), + ...(c.prompt.short.en ? [`- ${c.prompt.short.en}`] : []), + '', + '## What passed', + '', + ...judgement.halves.map((h) => `- \`${h.id}\`: ${h.verdict}`), + '', + renderScorecard(scorecard), + '', + '## Files', + '', + `- [\`${APP_LAYOUT.source}/\`](./${APP_LAYOUT.source}/) — the app's metadata source, as the AI wrote it (after the change request)`, + `- [\`${APP_LAYOUT.transcriptMd}\`](./${APP_LAYOUT.transcriptMd}) — every data question, each persona's answer and its grade ([JSON](./${APP_LAYOUT.transcriptJson}))`, + `- [\`${APP_LAYOUT.scorecardMd}\`](./${APP_LAYOUT.scorecardMd}) — the scorecard ([JSON](./${APP_LAYOUT.scorecardJson}))`, + `- [\`${APP_LAYOUT.runRecord}\`](./${APP_LAYOUT.runRecord}) — the full run record`, + ...shots.map((s) => `- [\`${s.file}\`](./${s.file}) — ${s.kind}${s.view ? ` (${s.view})` : s.object ? ` (${s.object})` : ''}`), + ...skipped.map((s) => `- no ${s.kind} screenshot: ${s.reason}`), + '', + ]; + return lines.join('\n'); +} + +/** Write `files` (relative path → bytes) under `dir`. */ +function writeTree(dir: string, files: Iterable<[string, string | Buffer]>): void { + for (const [rel, data] of files) { + const p = join(dir, rel); + mkdirSync(dirname(p), { recursive: true }); + writeFileSync(p, data); + } +} + +/** + * Every absolute directory that names this run's work directory: the record's + * own `evidenceDir` (the machine that ran it — every build tail and error text + * names it), `--work` when the projects are read elsewhere, and the real path + * of each that exists here (a symbolic-linked temp directory). Each is redacted + * from everything the gallery receives. + */ +export function workDirsOf(recordEvidenceDir: string, workDir: string | undefined): string[] { + const dirs = [recordEvidenceDir, ...(workDir !== undefined ? [workDir] : [])]; + for (const d of [...dirs]) { + try { + dirs.push(realpathSync(d)); + } catch { + // not on this machine: the record's directory, moved off it + } + } + return [...new Set(dirs)]; +} + +export async function exportGallery(opts: ExportOptions): Promise { + const isGeneric = opts.isGeneric ?? NO_GENERIC_MARKER; + const appsDir = join(opts.outDir, 'apps'); + const indexFile = join(opts.outDir, 'index.json'); + + // Read everything that can refuse the whole export before touching the output. + const before = existsSync(indexFile) ? readFileSync(indexFile, 'utf8') : undefined; + let previous: GalleryIndex | undefined; + if (before !== undefined) { + try { + previous = parseIndex(before); + } catch (e) { + throw new Error(`${indexFile} is not a gallery index this exporter wrote, so nothing is touched: ${e instanceof Error ? e.message : String(e)}`); + } + } + const { newest, ignored } = readRunRecords(opts.runsDir); + // An empty or wrong `--runs` would otherwise read as "every app lost its record" and delete the gallery. + if (newest.size === 0 && (previous?.apps.length ?? 0) > 0) { + throw new Error( + `${opts.runsDir} holds no scenario run record, and ${indexFile} lists ${previous!.apps.length} app(s), so nothing is touched: a runs directory with no record is read as a wrong --runs, never as every app gone`, + ); + } + const previousById = new Map((previous?.apps ?? []).map((a) => [a.id, a])); + const turns = mergeRecordings(opts.recordings); + const cards = new Map(opts.cards.map((s) => [s.card.id, s])); + + const summary: ExportSummary = { exported: [], refused: [], removed: [], kept: [], unowned: [], ignoredFiles: ignored, indexChanged: false }; + const entries: GalleryApp[] = []; + mkdirSync(appsDir, { recursive: true }); + + for (const id of [...newest.keys()].sort()) { + const rf = newest.get(id)!; + const staging = join(appsDir, `.staging-${id}`); + rmSync(staging, { recursive: true, force: true }); + try { + const done = await exportOne(id, rf, cards.get(id), staging, isGeneric, turns, opts); + const old = previousById.get(id); + if (opts.screenshots === false && old && old.screenshots.length > 0 && existsSync(join(appsDir, id))) { + // It still passes, but this export has no screenshots: the old directory and entry stand, shots and all. + rmSync(staging, { recursive: true, force: true }); + entries.push(old); + summary.kept.push({ id, why: `exported without screenshots, and the previous export has ${old.screenshots.length}: it is kept as it was, and not replaced by an entry with none` }); + continue; + } + rmSync(join(appsDir, id), { recursive: true, force: true }); + renameSync(staging, join(appsDir, id)); + entries.push(done.entry); + summary.exported.push({ id, screenshots: done.shots, skippedShots: done.skipped }); + } catch (e) { + rmSync(staging, { recursive: true, force: true }); + summary.refused.push({ id, reasons: e instanceof Refused ? e.reasons : [e instanceof Error ? e.message : String(e)] }); + } + } + + const exported = new Set(entries.map((e) => e.id)); // written or kept + const refusedWhy = new Map(summary.refused.map((r) => [r.id, r.reasons.join('; ')])); + for (const old of previous?.apps ?? []) { + if (exported.has(old.id)) continue; + rmSync(join(appsDir, old.id), { recursive: true, force: true }); + summary.removed.push({ id: old.id, why: refusedWhy.get(old.id) ?? 'the runs directory holds no record of it' }); + } + const owned = new Set([...(previous?.apps ?? []).map((a) => a.id), ...exported]); + summary.unowned = readdirSync(appsDir, { withFileTypes: true }) + .filter((e) => e.isDirectory() && !owned.has(e.name)) + .map((e) => e.name) + .sort(); + + const after = serializeIndex(entries); + if (after !== before) writeFileSync(indexFile, after); + summary.indexChanged = after !== before; + return summary; +} + +async function exportOne( + id: string, + rf: RecordFile, + scenario: LoadedScenario | undefined, + staging: string, + isGeneric: GenericGuard, + turns: ReturnType, + opts: ExportOptions, +): Promise<{ entry: GalleryApp; shots: PlannedShot[]; skipped: SkippedShot[] }> { + if (!scenario) throw new Refused([`no loaded card \`${id}\` (removed, renamed, or refused by the loader)`]); + const generic = isGeneric(scenario); + if (!generic.generic) throw new Refused([generic.reason]); + const { record } = rf; + const judgement = judgeRecord(record); + if (!judgement.passed) throw new Refused([`${rf.file} did not pass every half`, ...judgement.reasons]); + + const card = scenario.card; + const workDir = opts.workDir ?? record.evidenceDir; + const projectDir = scratchProjectDir(workDir, card); + if (!existsSync(projectDir)) throw new Refused([`the run's scratch project is gone: ${projectDir}`]); + const artifact = readArtifact(projectDir); + if (artifact === undefined) throw new Refused([`${projectDir} holds no dist/objectstack.json`]); + const source = readAuthoredSource(projectDir); + const frameworkVersion = projectFrameworkVersion(projectDir); + const redact = (v: T): T => redactWorkDir(v, workDirsOf(record.evidenceDir, opts.workDir)); + const transcript = redact(buildTranscript(scenario, record, turns)); + + let shots: PlannedShot[] = []; + let skipped: SkippedShot[] = []; + if (opts.screenshots) { + const taken = await opts.screenshots({ scenario, projectDir, artifact, dir: staging }); + shots = taken.shots; + skipped = taken.skipped; + } + + const scorecard = redact(record.scorecard as unknown as CardScorecard); + const entry: GalleryApp = { + id, + code: card.code, + namespace: scenarioNamespace(card), + path: `apps/${id}`, + function: card.category, + title: { en: card.title, zh: null }, + summary: { ...(card.prompt.short.en ? { en: card.prompt.short.en } : {}), ...(card.prompt.short.zh ? { zh: card.prompt.short.zh } : {}) }, + capabilities: [...card.capabilities].sort(), + screenshots: shots.map((s) => ({ kind: s.kind, path: `apps/${id}/${s.file}`, ...(s.object ? { object: s.object } : {}), ...(s.view ? { view: s.view } : {}) })), + qa: { questions: record.cardRun.counts.questions, turns: record.cardRun.counts.questionTurns, personas: record.cardRun.counts.personas }, + operations: record.cardRun.counts.operations, + framework: { version: frameworkVersion, runner: record.env.framework }, + runDate: record.date, + run: record.run, + }; + + writeTree(join(staging, APP_LAYOUT.source), source.files); + writeTree(staging, [ + [APP_LAYOUT.runRecord, json(redact(rf.raw))], + [APP_LAYOUT.scorecardJson, json(scorecard)], + [APP_LAYOUT.scorecardMd, `${renderScorecard(scorecard)}\n`], + [APP_LAYOUT.transcriptJson, json(transcript)], + [APP_LAYOUT.transcriptMd, renderTranscript(transcript, card.title)], + [APP_LAYOUT.readme, redact(appReadme(scenario, entry, judgement, scorecard, shots, skipped))], + ]); + return { entry, shots, skipped }; +} diff --git a/packages/qa/scenarios/src/export/generic.ts b/packages/qa/scenarios/src/export/generic.ts new file mode 100644 index 00000000000..4ca7c7285fa --- /dev/null +++ b/packages/qa/scenarios/src/export/generic.ts @@ -0,0 +1,38 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * Generic only. ⛔ No customer-derived or held-out scenario is ever exported to + * the gallery (the D1 ruling of the scenario-library epic, and the gallery's + * own AGENTS.md: "No customer-derived content"). + * + * A card is exported only when it is EXPLICITLY marked generic. Where it lives + * is not a marker: the open library's README says every card in it is + * generic, but the loader reads whatever directory it is handed, the private + * overlay included, and a card file carries nothing that says which one it + * came from. Its `provenance` is free prose, and prose is not a marker either. + * + * The card schema has no such marker today, so {@link NO_GENERIC_MARKER} — the + * guard the CLI uses — refuses every card, and the gallery stays empty until + * the marker is decided and added to the card schema. That guard is the one + * place to change then; the exporter takes the guard as a parameter so its + * own tests can run the export path with a double, and the CLI offers no flag + * to pass another one. + */ + +import type { LoadedScenario } from '../loader.js'; + +export interface GenericVerdict { + generic: boolean; + /** Why, in one line — printed with every refusal. */ + reason: string; +} + +/** Decides whether a card may be exported. Must refuse whatever it cannot show is generic. */ +export type GenericGuard = (scenario: LoadedScenario) => GenericVerdict; + +/** The guard the CLI uses until the card schema carries an explicit generic marker: it refuses every card. */ +export const NO_GENERIC_MARKER: GenericGuard = () => ({ + generic: false, + reason: + 'not marked generic: the card schema carries no explicit generic marker, so no card can be shown not to be customer-derived or held out, and none is exported', +}); diff --git a/packages/qa/scenarios/src/export/index.ts b/packages/qa/scenarios/src/export/index.ts new file mode 100644 index 00000000000..341e8ed6f8c --- /dev/null +++ b/packages/qa/scenarios/src/export/index.ts @@ -0,0 +1,17 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * `@objectstack/scenarios/export` — the gallery exporter: verified scenario + * apps written to a local directory in the layout of objectstack-ai/gallery. + * Private, like the rest of this package. CLI: + * `pnpm --filter @objectstack/scenarios export-gallery --runs DIR --out DIR`. + */ + +export * from './exporter.js'; +export * from './generic.js'; +export * from './manifest.js'; +export * from './pass.js'; +export * from './record.js'; +export * from './screenshots.js'; +export * from './source.js'; +export * from './transcript.js'; diff --git a/packages/qa/scenarios/src/export/manifest.ts b/packages/qa/scenarios/src/export/manifest.ts new file mode 100644 index 00000000000..dd332d1a58c --- /dev/null +++ b/packages/qa/scenarios/src/export/manifest.ts @@ -0,0 +1,90 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * `index.json`, the gallery's one machine-readable manifest. The gallery's + * README promises it carries each app's function, title, screenshots, question + * count, framework version and run date; the landing pages on objectos.ai are + * built from it. + * + * Validated before it is written ({@link GalleryIndexSchema}, strict at every + * level), and written so that an export with no change in its inputs is a + * no-op diff: apps sorted by id, every key in schema order, no export + * timestamp, a trailing newline. + */ + +import { z } from 'zod'; + +import { CategorySchema, MachineNameSchema } from '../schema.js'; + +export const GALLERY_INDEX_SCHEMA_VERSION = 1; + +export const SHOT_KINDS = ['list', 'kanban', 'calendar', 'record', 'dashboard'] as const; +export const ShotKindSchema = z.enum(SHOT_KINDS); +export type ShotKind = z.infer; + +export const GalleryScreenshotSchema = z.strictObject({ + kind: ShotKindSchema, + /** Relative to the gallery root, e.g. `apps/order_to_cash/screenshots/list.png`. */ + path: z.string().regex(/^apps\/[a-z][a-z0-9_]*\/screenshots\/[a-z]+\.png$/), + /** The object the view shows (`demoorder_invoice`); absent for a dashboard. */ + object: z.string().min(1).optional(), + /** The view (`demoorder_invoice.board`) or dashboard name shown. */ + view: z.string().min(1).optional(), +}); + +export const GalleryAppSchema = z.strictObject({ + /** The scenario card's id, which is also the app's directory under `apps/`. */ + id: MachineNameSchema, + /** The card's app code; the app lives in namespace `demo` + code. */ + code: z.string().regex(/^[a-z]{2,8}$/), + namespace: z.string().regex(/^demo[a-z]{2,8}$/), + /** `apps/`. */ + path: z.string().regex(/^apps\/[a-z][a-z0-9_]*$/), + /** The business function: the card's `category` (the marketplace taxonomy). */ + function: CategorySchema, + /** `en` is the card's `title`; the card schema has no Chinese title, so `zh` is `null` until it does. */ + title: z.strictObject({ en: z.string().min(1), zh: z.string().min(1).nullable() }), + /** The card's one-sentence requirement (`prompt.short`), in the languages it has. */ + summary: z.strictObject({ en: z.string().min(1).optional(), zh: z.string().min(1).optional() }), + /** The card's capability tags, sorted. */ + capabilities: z.array(z.string().min(1)), + screenshots: z.array(GalleryScreenshotSchema), + /** The 问数 count: the card's questions, and the turns they are asked in (one per persona a question names). */ + qa: z.strictObject({ questions: z.number().int().min(0), turns: z.number().int().min(0), personas: z.number().int().min(0) }), + /** Write operations the run graded. */ + operations: z.number().int().min(0), + framework: z.strictObject({ + /** The `@objectstack/spec` version the generated project pins. */ + version: z.string().min(1), + /** The objectstack checkout the runner ran from (`git rev-parse --short HEAD`), or `unknown`. */ + runner: z.string().min(1), + }), + /** The run record's date (`YYYY-MM-DD`). */ + runDate: z.iso.date(), + /** The run record's id. */ + run: z.string().min(1), +}); +export type GalleryApp = z.infer; + +export const GalleryIndexSchema = z + .strictObject({ + schemaVersion: z.literal(GALLERY_INDEX_SCHEMA_VERSION), + apps: z.array(GalleryAppSchema), + }) + .refine((ix) => ix.apps.every((a, i) => i === 0 || ix.apps[i - 1]!.id < a.id), { message: 'apps are sorted by id, each id once' }) + .refine((ix) => ix.apps.every((a) => a.path === `apps/${a.id}` && a.screenshots.every((s) => s.path.startsWith(`${a.path}/`))), { + message: 'an app lives at apps/, and its screenshots under it', + }); +export type GalleryIndex = z.infer; + +/** The index for these apps, validated, as the bytes to write. Throws a ZodError on an invalid index. */ +export function serializeIndex(apps: readonly GalleryApp[]): string { + const sorted = [...apps].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)); + const index = GalleryIndexSchema.parse({ schemaVersion: GALLERY_INDEX_SCHEMA_VERSION, apps: sorted }); + return `${JSON.stringify(index, null, 2)}\n`; +} + +/** Read an existing index. Throws on one that does not validate. */ +export function parseIndex(text: string): GalleryIndex { + return GalleryIndexSchema.parse(JSON.parse(text)); +} diff --git a/packages/qa/scenarios/src/export/pass.ts b/packages/qa/scenarios/src/export/pass.ts new file mode 100644 index 00000000000..b6e2592febd --- /dev/null +++ b/packages/qa/scenarios/src/export/pass.ts @@ -0,0 +1,83 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * "Passed every half", read from a run record's `results` — the per-half + * verdicts `runRecord` (`runner/scorecard.ts`) writes, one row per + * `..`. Nothing here re-scores: the verdict the record holds + * is the verdict. + * + * - The verdict must be `pass`. `partial`, `fail`, `blocked` and `not-run` + * are not a pass (a half with nothing measured is `not-run` in the record, + * never `pass`), and a row the record does not hold is not a pass either. + * - The record must hold the scratch mode: the gallery ships the scratch + * project's source, so that is the run the verdicts are about. + * - Scratch mode owes every half: Build, Run-问数, Run-operations, and the + * three iterate halves (the iterate build and its re-run). A record run + * with `--no-iterate-run` leaves the iterate Run halves `not-run`, so it + * does not pass. + * - Template mode, when the record holds it, owes its iterate halves. Its + * first Build and Run rows are `not-run` by construction (the mode installs + * a verified build and builds nothing first), so they are not asked. A + * template mode that was skipped holds `not-run` in every row and fails. + * - No turn may be NOT MEASURED, in any Run half the record holds (the + * first Run and the iterate re-run, both modes), questions and operations + * alike. The scorecard leaves a not-measured turn (one that reads a seed + * row the app refused) out of both passed and total, so its half can still + * read `pass`; the gallery is trust-first, and a turn nobody measured is a + * claim nobody proved. Each one is a reason of its own. + */ + +import type { ScenarioRunRecord, Verdict } from './record.js'; + +/** The halves each mode owes, in the record's row order. */ +export const OWED_HALVES = { + scratch: ['build', 'run-ask', 'run-operations', 'iterate-build', 'iterate-run-ask', 'iterate-run-operations'], + template: ['iterate-build', 'iterate-run-ask', 'iterate-run-operations'], +} as const satisfies Record<'scratch' | 'template', readonly string[]>; + +export interface HalfVerdict { + /** `..`, the record's row id. */ + id: string; + /** `missing` when the record holds no row for it. */ + verdict: Verdict | 'missing'; +} + +export interface PassJudgement { + passed: boolean; + /** Every owed half with its verdict, in order. */ + halves: HalfVerdict[]; + /** Why it did not pass, one line per reason (a failing half, or a not-measured turn); empty when it passed. */ + reasons: string[]; +} + +export function judgeRecord(record: ScenarioRunRecord): PassJudgement { + const card = record.cardRun.card; + const byId = new Map(record.results.map((r) => [r.id, r.verdict])); + const modes = record.cardRun.modes.map((m) => m.mode); + const reasons: string[] = []; + if (!modes.includes('scratch')) reasons.push('the record holds no scratch mode, and the scratch project is what the gallery ships'); + const halves: HalfVerdict[] = []; + for (const mode of (['scratch', 'template'] as const).filter((m) => modes.includes(m))) { + for (const half of OWED_HALVES[mode]) { + const id = `${card}.${mode}.${half}`; + const verdict = byId.get(id) ?? 'missing'; + halves.push({ id, verdict }); + if (verdict !== 'pass') reasons.push(`${id}: ${verdict}`); + } + } + for (const m of record.cardRun.modes) { + const runs = [ + ['run', m.run], + ['iterate-run', m.iterate?.run], + ] as const; + for (const [phase, half] of runs) { + for (const q of half?.questions ?? []) { + if (q.notMeasured) reasons.push(`${card}.${m.mode}.${phase} question ${q.question}${q.persona ? ` as ${q.persona}` : ''}: not measured — ${q.notMeasured}`); + } + for (const o of half?.operations ?? []) { + if (o.notMeasured) reasons.push(`${card}.${m.mode}.${phase} operation ${o.operation} as ${o.persona}: not measured — ${o.notMeasured}`); + } + } + } + return { passed: reasons.length === 0, halves, reasons }; +} diff --git a/packages/qa/scenarios/src/export/record.ts b/packages/qa/scenarios/src/export/record.ts new file mode 100644 index 00000000000..ec60b0169be --- /dev/null +++ b/packages/qa/scenarios/src/export/record.ts @@ -0,0 +1,158 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The run records the local runner writes (`run-scenarios --out DIR`, one + * `-scenario-.json` per card and day, `runner/scorecard.ts` + * `runRecord`), read back for the gallery exporter. + * + * Only what the exporter reads is checked, and checked loudly: a file that + * says it is a scenario-runner record and does not parse stops the whole + * export before anything in the output is touched. Every other key passes + * through (`looseObject`), and the record is written to the gallery from the + * file as it was read, never from this parse. + */ + +import { readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import { z } from 'zod'; + +import type { CardRun } from '../runner/orchestrate.js'; + +/** The verdict of one half, as `runRecord` writes it. */ +export const VerdictSchema = z.enum(['pass', 'partial', 'fail', 'blocked', 'not-run']); +export type Verdict = z.infer; + +const QuestionResultSchema = z.looseObject({ + question: z.string().min(1), + persona: z.string().min(1).nullable(), + notMeasured: z.string().optional(), + grade: z.looseObject({ correct: z.boolean(), cited: z.boolean(), leaks: z.array(z.string()) }).optional(), + error: z.string().optional(), +}); + +const OperationResultSchema = z.looseObject({ + operation: z.string().min(1), + persona: z.string().min(1), + notMeasured: z.string().optional(), +}); + +const RunHalfSchema = z.looseObject({ questions: z.array(QuestionResultSchema), operations: z.array(OperationResultSchema) }); + +const ModeRunSchema = z.looseObject({ + mode: z.enum(['scratch', 'template']), + run: RunHalfSchema.optional(), + iterate: z.looseObject({ run: RunHalfSchema.optional() }).optional(), + skipped: z.string().optional(), +}); + +/** The part of a run record the exporter reads. */ +export const ScenarioRunRecordSchema = z.looseObject({ + run: z.string().min(1), + date: z.iso.date(), + app: z.string().min(1), + runner: z.literal('scenario-runner'), + env: z.looseObject({ framework: z.string().min(1), driver: z.looseObject({ driver: z.string().min(1) }) }), + evidenceDir: z.string().min(1), + results: z.array(z.looseObject({ id: z.string().min(1), verdict: VerdictSchema })), + scorecard: z.looseObject({ card: z.string().min(1), modes: z.array(z.unknown()) }), + cardRun: z.looseObject({ + card: z.string().min(1), + counts: z.strictObject({ questions: z.number().int(), questionTurns: z.number().int(), operations: z.number().int(), personas: z.number().int() }), + modes: z.array(ModeRunSchema).min(1), + }), +}); + +export interface ScenarioRunRecord extends z.infer { + cardRun: z.infer['cardRun'] & CardRun; +} + +/** One record file, parsed, with the bytes it was read from. */ +export interface RecordFile { + file: string; + record: ScenarioRunRecord; + /** The file's JSON as read — what the gallery receives (less the work directory, see `redactWorkDir`). */ + raw: unknown; +} + +export interface ReadRecordsResult { + /** The newest record of each card, by `date` (then by file name, so a tie is still decided one way). */ + newest: Map; + /** JSON files in the directory that are not scenario-runner records (platform-checklist runs, …). */ + ignored: string[]; +} + +export class RunRecordError extends Error { + constructor(file: string, detail: string) { + super(`${file}: not a readable scenario run record: ${detail}`); + this.name = 'RunRecordError'; + } +} + +/** Read every `*.json` in `runsDir` (not recursive); refuse the whole directory on one unreadable scenario record. */ +export function readRunRecords(runsDir: string): ReadRecordsResult { + const newest = new Map(); + const ignored: string[] = []; + const files = readdirSync(runsDir, { withFileTypes: true }) + .filter((e) => e.isFile() && e.name.endsWith('.json')) + .map((e) => e.name) + .sort(); + for (const name of files) { + const file = join(runsDir, name); + let raw: unknown; + try { + raw = JSON.parse(readFileSync(file, 'utf8')); + } catch (e) { + throw new RunRecordError(file, e instanceof Error ? e.message : String(e)); + } + if ((raw as { runner?: unknown } | null)?.runner !== 'scenario-runner') { + ignored.push(file); + continue; + } + const parsed = ScenarioRunRecordSchema.safeParse(raw); + if (!parsed.success) { + const first = parsed.error.issues[0]!; + throw new RunRecordError(file, `${first.path.join('.') || '(root)'}: ${first.message}`); + } + const record = parsed.data as ScenarioRunRecord; + if (record.cardRun.card !== record.app) throw new RunRecordError(file, `\`app\` is \`${record.app}\` but \`cardRun.card\` is \`${record.cardRun.card}\``); + const held = newest.get(record.app); + // Files are visited in name order, so on an equal date the later name wins. + if (!held || record.date >= held.record.date) newest.set(record.app, { file, record, raw }); + } + return { newest, ignored }; +} + +/** + * A work directory is only that directory where it is not part of a longer + * name: not preceded by a path-name character or `/` (`/x/tmp` is not `/tmp`), + * and not followed by one (`/tmpfoo`) — a `.` that ends a sentence does not + * lengthen it, a `.` that starts a suffix (`/tmp.d`) does. + */ +const BEFORE = '(?(value: T, workDirs: string | readonly string[]): T { + const dirs = [...new Set((typeof workDirs === 'string' ? [workDirs] : workDirs).map((d) => d.replace(/\/+$/, '')).filter((d) => d !== ''))].sort((x, y) => y.length - x.length); + if (dirs.length === 0) return value; + const escape = (d: string) => d.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const re = new RegExp(`${BEFORE}(?:${dirs.map(escape).join('|')})${AFTER}`, 'g'); + const walk = (v: unknown): unknown => { + if (typeof v === 'string') return v.replace(re, '$$WORK'); + if (Array.isArray(v)) return v.map(walk); + if (typeof v === 'object' && v !== null) return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, walk(x)])); + return v; + }; + return walk(value) as T; +} diff --git a/packages/qa/scenarios/src/export/screenshots.ts b/packages/qa/scenarios/src/export/screenshots.ts new file mode 100644 index 00000000000..433a2f30392 --- /dev/null +++ b/packages/qa/scenarios/src/export/screenshots.ts @@ -0,0 +1,324 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * Screenshots of the exported app, taken by a real browser against the app + * booted the way the runner boots it (`bootDev`, `objectstack dev --fresh`), + * seeded the way the runner seeds it (`seedApp`, as the admin), in the Console + * at `/_console/`. + * + * Which views, decided by {@link planShots} from the card and the build alone + * (pure, so it is tested without a browser): + * + * - `list`: the app's first object the card puts in the app's navigation + * (`expect.appNavTargets`), else the card's first object — + * `apps//`; + * - `kanban` / `calendar`: only where the card TAGS one (a capability tag + * whose name says `kanban` or `calendar`, e.g. + * `records-forms.kanban-drag-persistence`), showing a view of that type the + * build declares (`apps///view/.`), the list + * object's own first; a tagged view the build does not declare is a + * skipped shot with the reason, never a silent omission; + * - `record`: the list object's first seed row — `apps///record/` + * (the seed keeps the card's row ids verbatim); skipped when the app + * refused that row; + * - `dashboard`: a dashboard the card expects, else the build's first — + * `apps//dashboard/`; skipped when the build declares none. + * + * A page the Console did not show is never proof: a navigation that answers + * no response or a status outside 2xx (304, a revalidation, is the page), or a page that draws one of the + * Console's own not-found views ({@link CONSOLE_NOT_FOUND}), refuses that shot + * ({@link pageRefusal}). It is reported as a skipped shot with the reason, and + * no file of it is kept. + * + * The browser is a seam ({@link ScreenshotBrowser}): {@link playwrightBrowser} + * drives Chromium through `playwright-core`; the exporter's tests pass a + * double. The exporter runs this step only when asked (`--no-screenshots` + * skips it), so everything else is testable without a browser. + */ + +import { existsSync, mkdirSync, rmSync } from 'node:fs'; +import { dirname, join } from 'node:path'; + +import { resolveStackCollection } from '@objectstack/core'; +import { deriveViewContainerObject, expandViewContainer } from '@objectstack/spec/ui'; + +import type { LoadedScenario } from '../loader.js'; +import { scenarioNamespace } from '../schema.js'; +import type { AppHarness } from '../runner/app.js'; +import { namespacedScenario } from '../runner/namespace.js'; +import { seedApp, type BootedApp, type RunHalf } from '../runner/orchestrate.js'; +import type { ShotKind } from './manifest.js'; + +/** Where `objectstack dev` serves the Console. */ +export const CONSOLE_BASE = '/_console'; + +export interface PlannedShot { + kind: ShotKind; + /** Relative to the app's export directory: `screenshots/.png`. */ + file: string; + /** The Console path to open, under the booted app's origin. */ + path: string; + object?: string; + view?: string; + /** For a record shot: the seed row it shows. */ + row?: string; +} + +export interface SkippedShot { + kind: ShotKind; + reason: string; +} + +export interface ShotPlan { + /** The app the Console opens, `undefined` when the build declares none. */ + app?: string; + shots: PlannedShot[]; + skipped: SkippedShot[]; +} + +/** The view types a card can tag, and the tag names that tag them. */ +export const TAGGED_VIEW_KINDS = { kanban: /kanban/, calendar: /calendar/ } as const satisfies Partial>; + +/** The view kinds the card's capability tags name, in {@link TAGGED_VIEW_KINDS} order. */ +export function taggedViewKinds(card: Pick): Array { + const names = card.capabilities.map((t) => t.slice(t.indexOf('.') + 1)); + return (Object.keys(TAGGED_VIEW_KINDS) as Array).filter((k) => names.some((n) => TAGGED_VIEW_KINDS[k].test(n))); +} + +const nameOf = (v: unknown): string | undefined => (typeof v === 'object' && v !== null && typeof (v as { name?: unknown }).name === 'string' ? (v as { name: string }).name : undefined); + +/** Every list view the build declares: `{ object, name: '.', type }`, in declaration order, containers expanded by the spec's own composer. */ +export function declaredListViews(artifact: unknown): Array<{ object: string; name: string; type: string }> { + const out: Array<{ object: string; name: string; type: string }> = []; + for (const raw of resolveStackCollection(artifact, 'views')) { + if (typeof raw !== 'object' || raw === null) continue; + const item = raw as Record; + if ('list' in item || 'listViews' in item) { + // The platform's one spelling of a container's binding (a built container usually carries no `object` of its own). + const object = deriveViewContainerObject(item); + if (!object) continue; + for (const v of expandViewContainer(object, item)) { + if (v.viewKind !== 'list') continue; + const type = (v.config as { type?: unknown } | undefined)?.type; + out.push({ object, name: v.name, type: typeof type === 'string' ? type : 'grid' }); + } + } else if (typeof item.type === 'string' && typeof item.name === 'string' && typeof item.object === 'string') { + out.push({ object: item.object, name: item.name.includes('.') ? item.name : `${item.object}.${item.name}`, type: item.type }); + } + } + return out; +} + +const seg = (s: string) => encodeURIComponent(s); + +/** Decide the screenshots for one app. Pure over the card and the build. */ +export function planShots(original: LoadedScenario, artifact: unknown): ShotPlan { + const ns = scenarioNamespace(original.card); + const card = namespacedScenario(original).card; + const apps = resolveStackCollection(artifact, 'apps').map(nameOf).filter((n): n is string => !!n); + const app = apps.find((a) => a.startsWith(`${ns}_`)) ?? apps[0]; + const tagged = taggedViewKinds(original.card); + const kinds: ShotKind[] = ['list', ...tagged, 'record', 'dashboard']; + if (!app) return { shots: [], skipped: kinds.map((kind) => ({ kind, reason: 'the build declares no app for the Console to open' })) }; + + const shots: PlannedShot[] = []; + const skipped: SkippedShot[] = []; + const base = `${CONSOLE_BASE}/apps/${seg(app)}`; + const objects = new Set(resolveStackCollection(artifact, 'objects').map(nameOf).filter((n): n is string => !!n)); + const expected = [...(card.expect.appNavTargets ?? []).filter((t) => card.expect.objects.includes(t)), ...card.expect.objects]; + const object = expected.find((o) => objects.has(o)); + + if (object) shots.push({ kind: 'list', file: 'screenshots/list.png', path: `${base}/${seg(object)}`, object }); + else skipped.push({ kind: 'list', reason: 'the build declares none of the objects the card expects' }); + + const views = declaredListViews(artifact); + for (const kind of tagged) { + const ofType = views.filter((v) => v.type === kind); + const view = ofType.find((v) => v.object === object) ?? ofType[0]; + if (view) shots.push({ kind, file: `screenshots/${kind}.png`, path: `${base}/${seg(view.object)}/view/${seg(view.name)}`, object: view.object, view: view.name }); + else skipped.push({ kind, reason: `the card tags ${kind}, and the build declares no ${kind} view` }); + } + + const row = object ? card.run.seed[object]?.[0]?.id : undefined; + if (object && row !== undefined) shots.push({ kind: 'record', file: 'screenshots/record.png', path: `${base}/${seg(object)}/record/${seg(row)}`, object, row }); + else skipped.push({ kind: 'record', reason: object ? `the card seeds no ${object} row to open` : 'no list object to open a record of' }); + + const dashboards = resolveStackCollection(artifact, 'dashboards').map(nameOf).filter((n): n is string => !!n); + const wanted = [...(card.expect.dashboards ?? []), ...(card.iterate.expectDelta.dashboards ?? [])]; + const dashboard = wanted.find((d) => dashboards.includes(d)) ?? dashboards[0]; + if (dashboard) shots.push({ kind: 'dashboard', file: 'screenshots/dashboard.png', path: `${base}/dashboard/${seg(dashboard)}`, view: dashboard }); + else skipped.push({ kind: 'dashboard', reason: 'the build declares no dashboard' }); + + return { app, shots, skipped }; +} + +// ── Capture ───────────────────────────────────────────────────────────────── + +/** + * The Console's own "nothing here" views, as a page shows them: the + * `data-testid`s of the object, record and form not-found states, and the + * title the app router's catch-all route draws (in the `en-US` locale the + * browser runs in). Read from the Console source (`ui/packages/app-shell`, + * `ui/apps/console`); a Console that drew a new one under another name would + * pass this check, so the HTTP status is checked first and alone suffices for + * a server error. + */ +export const CONSOLE_NOT_FOUND = { + testIds: ['object-not-found', 'object-package-disabled', 'record-page-not-found', 'form-not-found'], + titles: ['Page not found'], +} as const; + +/** What a page showed, as {@link pageRefusal} judges it. */ +export interface PageReading { + /** The navigation's HTTP status; `null` when the navigation answered no response. */ + status: number | null; + /** Which of {@link CONSOLE_NOT_FOUND}'s test ids and titles the page draws. */ + notFound: string[]; +} + +/** Why a page is not a screenshot of the app, or `undefined` when it is one. */ +export function pageRefusal(page: PageReading): string | undefined { + if (page.status === null) return 'the navigation answered no response'; + // 304: the browser revalidated a page it holds (each shot's page is visited twice), which is the page shown. + if ((page.status < 200 || page.status > 299) && page.status !== 304) return `the navigation answered HTTP ${page.status}`; + if (page.notFound.length > 0) return `the Console drew its not-found view (${page.notFound.join(', ')})`; + return undefined; +} + +/** A capture's outcome: the page was shot, or it was not the app and nothing of it is proof. */ +export type CaptureOutcome = { shown: true } | { shown: false; reason: string }; + +/** One signed-in browser page against a booted app. */ +export interface ScreenshotSession { + /** Open `path` under the app's origin; when the page is the app ({@link pageRefusal}), write a PNG to `file`. */ + capture(path: string, file: string): Promise; + close(): Promise; +} + +export interface ScreenshotBrowser { + open(baseUrl: string, admin: { email: string; password: string }): Promise; +} + +export interface CaptureInput { + scenario: LoadedScenario; + /** The run's scratch project, booted as the runner boots it. */ + projectDir: string; + /** Its `dist/objectstack.json`. */ + artifact: unknown; + /** The app's export directory; shots land at `/`. */ + dir: string; +} + +export interface CaptureDeps { + boot(dir: string): Promise; + harness(url: string): AppHarness; + browser: ScreenshotBrowser; + admin: { email: string; password: string }; +} + +export interface ScreenshotResult { + shots: PlannedShot[]; + skipped: SkippedShot[]; + /** What the seed loaded, when the app was booted. */ + seed?: RunHalf['seed']; +} + +export type CaptureScreenshots = (input: CaptureInput) => Promise; + +/** Boot, seed, sign in, and take every planned shot. Throws when a shot writes no file. */ +export async function captureScreenshots(input: CaptureInput, deps: CaptureDeps): Promise { + const plan = planShots(input.scenario, input.artifact); + if (plan.shots.length === 0) return { shots: [], skipped: plan.skipped }; + const booted = await deps.boot(input.projectDir); + try { + const app = deps.harness(booted.url); + await app.signInAdmin(); + const seed = await seedApp(namespacedScenario(input.scenario), app, []); + const refused = new Set(seed.refused.map((r) => `${r.object}/${r.id}`)); + const shots: PlannedShot[] = []; + const skipped = [...plan.skipped]; + for (const shot of plan.shots) { + if (shot.kind === 'record' && refused.has(`${shot.object}/${shot.row}`)) skipped.push({ kind: 'record', reason: `the app refused seed row ${shot.object} ${shot.row}` }); + else shots.push(shot); + } + const session = await deps.browser.open(booted.url, deps.admin); + try { + for (const shot of [...shots]) { + const file = join(input.dir, shot.file); + mkdirSync(dirname(file), { recursive: true }); + const outcome = await session.capture(shot.path, file); + if (!outcome.shown) { + rmSync(file, { force: true }); + shots.splice(shots.indexOf(shot), 1); + skipped.push({ kind: shot.kind, reason: `the Console did not show ${shot.path}: ${outcome.reason}` }); + continue; + } + if (!existsSync(file)) throw new Error(`the browser wrote no screenshot for ${shot.kind} (${shot.path})`); + } + } finally { + await session.close(); + } + return { shots, skipped, seed }; + } finally { + await booted.stop(); + } +} + +// ── The real browser ──────────────────────────────────────────────────────── + +export interface PlaywrightBrowserOptions { + /** A Chromium executable; default `$PLAYWRIGHT_BROWSERS_PATH/chromium` when it exists, else playwright-core's own. */ + executablePath?: string; + viewport?: { width: number; height: number }; + /** How long a page may take to go quiet before it is shot anyway; default 30 s. */ + settleTimeoutMs?: number; +} + +/** Chromium through `playwright-core`, signed in by the Console's own sign-in door so the session cookie is the browser's. */ +export function playwrightBrowser(opts: PlaywrightBrowserOptions = {}): ScreenshotBrowser { + return { + async open(baseUrl, admin) { + const { chromium } = await import('playwright-core'); + const fallback = process.env.PLAYWRIGHT_BROWSERS_PATH ? join(process.env.PLAYWRIGHT_BROWSERS_PATH, 'chromium') : undefined; + const executablePath = opts.executablePath ?? (fallback && existsSync(fallback) ? fallback : undefined); + const browser = await chromium.launch({ headless: true, ...(executablePath ? { executablePath } : {}) }); + try { + const context = await browser.newContext({ viewport: opts.viewport ?? { width: 1440, height: 900 }, locale: 'en-US' }); + const signIn = await context.request.post(`${baseUrl}/api/v1/auth/sign-in/email`, { data: admin }); + if (!signIn.ok()) throw new Error(`the browser's admin sign-in was refused: ${signIn.status()}`); + const page = await context.newPage(); + const visit = async (path: string): Promise => { + const response = await page.goto(`${baseUrl}${path}`, { waitUntil: 'load' }); + // A page that never goes quiet (a stream, a poll) is shot after the budget, not skipped. + await page.waitForLoadState('networkidle', { timeout: opts.settleTimeoutMs ?? 30_000 }).catch(() => undefined); + await page.waitForTimeout(1_000); + const notFound: string[] = []; + for (const id of CONSOLE_NOT_FOUND.testIds) if ((await page.locator(`[data-testid="${id}"]`).count()) > 0) notFound.push(id); + for (const title of CONSOLE_NOT_FOUND.titles) if ((await page.getByText(title, { exact: true }).count()) > 0) notFound.push(`"${title}"`); + return { status: response ? response.status() : null, notFound }; + }; + // An administrator's first visit to an app page can open first-run UI over it (measured on + // 18.3: a default-currency dialog over the first app page, once per browser session; the + // Console's landing page does not take it). So the first shot's page is visited once + // unshot, and nothing is shot under that dialog. + let warmed = false; + return { + async capture(path, file) { + if (!warmed) { + await visit(path); + warmed = true; + } + const refusal = pageRefusal(await visit(path)); + if (refusal !== undefined) return { shown: false, reason: refusal }; + await page.screenshot({ path: file }); + return { shown: true }; + }, + close: () => browser.close(), + }; + } catch (e) { + await browser.close(); + throw e; + } + }, + }; +} diff --git a/packages/qa/scenarios/src/export/source.ts b/packages/qa/scenarios/src/export/source.ts new file mode 100644 index 00000000000..9482c205f3a --- /dev/null +++ b/packages/qa/scenarios/src/export/source.ts @@ -0,0 +1,74 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The generated app's metadata source: what the AI wrote, as a reader reviews + * it — never `node_modules`, the build output, the installed skills or the + * scaffold's deployment boilerplate. + * + * It is an allow-list, so nothing reaches the gallery by default: the four + * project files that declare the app ({@link SOURCE_ROOT_FILES}) and the whole + * `src/` tree, regular files only. A symbolic link anywhere in it refuses the + * export — what it points at is not the project's own text. + */ + +import { existsSync, lstatSync, readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +/** The project files at the root that declare the app; `objectstack.config.ts` is required. */ +export const SOURCE_ROOT_FILES = ['objectstack.config.ts', 'objectstack.manifest.json', 'package.json', 'tsconfig.json'] as const; + +/** The directory whose whole tree is the app's metadata. */ +export const SOURCE_DIR = 'src'; + +/** Trees never copied, even under `src/`. */ +const NEVER = new Set(['node_modules', 'dist', '.objectstack', '.git']); + +export interface AuthoredSource { + /** Relative path (forward slashes) → bytes, sorted by path. */ + files: Map; +} + +export class SourceRefusal extends Error { + constructor(message: string) { + super(message); + this.name = 'SourceRefusal'; + } +} + +export function readAuthoredSource(projectDir: string): AuthoredSource { + if (!existsSync(join(projectDir, 'objectstack.config.ts'))) throw new SourceRefusal(`no objectstack.config.ts in ${projectDir}: not the run's project`); + const found: Array<[string, Buffer]> = []; + const take = (rel: string) => { + const p = join(projectDir, rel); + const st = lstatSync(p); + if (st.isSymbolicLink()) throw new SourceRefusal(`${rel} is a symbolic link: only the project's own files are exported`); + if (st.isFile()) found.push([rel, readFileSync(p)]); + }; + for (const f of SOURCE_ROOT_FILES) if (existsSync(join(projectDir, f))) take(f); + const walk = (rel: string) => { + for (const ent of readdirSync(join(projectDir, rel), { withFileTypes: true })) { + const child = `${rel}/${ent.name}`; + if (ent.isSymbolicLink()) throw new SourceRefusal(`${child} is a symbolic link: only the project's own files are exported`); + if (ent.isDirectory()) { + if (!NEVER.has(ent.name)) walk(child); + } else if (ent.isFile()) take(child); + } + }; + if (existsSync(join(projectDir, SOURCE_DIR))) walk(SOURCE_DIR); + found.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + return { files: new Map(found) }; +} + +/** + * The framework version the app was built on: the `@objectstack/spec` the + * project pins (`create-objectstack` pins every `@objectstack/*` package to + * its own version, exactly). Refused when the project names none. + */ +export function projectFrameworkVersion(projectDir: string): string { + const p = join(projectDir, 'package.json'); + if (!existsSync(p)) throw new SourceRefusal(`no package.json in ${projectDir}: the framework version cannot be stated`); + const pkg = JSON.parse(readFileSync(p, 'utf8')) as { dependencies?: Record; devDependencies?: Record }; + const v = pkg.dependencies?.['@objectstack/spec'] ?? pkg.devDependencies?.['@objectstack/spec']; + if (typeof v !== 'string' || v === '') throw new SourceRefusal(`the project's package.json names no @objectstack/spec: the framework version cannot be stated`); + return v; +} diff --git a/packages/qa/scenarios/src/export/transcript.ts b/packages/qa/scenarios/src/export/transcript.ts new file mode 100644 index 00000000000..d3cc7a85983 --- /dev/null +++ b/packages/qa/scenarios/src/export/transcript.ts @@ -0,0 +1,195 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The 问数 transcript a gallery reader sees: every data question, each + * persona's answer, and its grade — so a reader sees the permission boundary + * hold, one persona answered and the next one declined. + * + * The run record keeps the grade of each turn but not the reply: the reply + * text is in the run's recording (`run-scenarios --record F`, or the file a + * `--driver recorded:F` run replayed), keyed by the step the runner asked it + * under (`stepKey`). So the transcript is the record joined with the + * recording, and the join is proven: every recorded answer is graded again by + * `@objectstack/scenarios/score` exactly as the runner graded it, and a grade + * that differs from the record's refuses the export — a recording from some + * other run never reaches the gallery as this run's answers. An asked turn + * with no recorded reply refuses it too: a transcript with holes in it is not + * a proof. + */ + +import type { LoadedScenario } from '../loader.js'; +import { gradeCardAnswer, type PersonaRunGrade } from '../score/index.js'; +import type { Truth } from '../truth.js'; +import { stepKey, type Recording, type StepRef } from '../runner/driver.js'; +import { citationLabels, namespacedScenario } from '../runner/namespace.js'; +import { hiddenValues, type QuestionResult } from '../runner/orchestrate.js'; +import type { ParsedTurn } from '../runner/transcript.js'; +import type { ScenarioRunRecord } from './record.js'; + +export interface TranscriptTurn { + question: string; + /** `null`: asked with no persona (as the admin). */ + persona: string | null; + /** What the seed determines for this asker. */ + truth: Truth; + answer: { replyText: string; toolCalls: Array<{ name: string; input: unknown; output: unknown; isError?: boolean }> } | null; + grade: PersonaRunGrade | null; + notMeasured?: string; + error?: string; +} + +export interface TranscriptHalf { + mode: 'scratch' | 'template'; + /** `run`: the first Run half; `iterate-run`: the Run half after the change request. */ + phase: 'run' | 'iterate-run'; + turns: TranscriptTurn[]; +} + +export interface Transcript { + card: string; + questions: Array<{ id: string; ask: { zh?: string; en?: string }; object: string; personas: string[] }>; + personas: Array<{ id: string; role: string }>; + halves: TranscriptHalf[]; +} + +export class TranscriptRefusal extends Error { + constructor(message: string) { + super(message); + this.name = 'TranscriptRefusal'; + } +} + +/** Several recordings as one; a step recorded twice with different turns is refused. */ +export function mergeRecordings(recordings: readonly Recording[]): Map { + const out = new Map(); + for (const r of recordings) { + for (const [key, turn] of Object.entries(r.turns)) { + const held = out.get(key); + if (held && JSON.stringify(held) !== JSON.stringify(turn)) throw new TranscriptRefusal(`step \`${key}\` is recorded twice with different turns`); + out.set(key, turn); + } + } + return out; +} + +const canon = (v: unknown): unknown => + Array.isArray(v) ? v.map(canon) : typeof v === 'object' && v !== null ? Object.fromEntries(Object.keys(v).sort().map((k) => [k, canon((v as Record)[k])])) : v; +const same = (a: unknown, b: unknown) => JSON.stringify(canon(a)) === JSON.stringify(canon(b)); + +export function buildTranscript(original: LoadedScenario, record: ScenarioRunRecord, turns: ReadonlyMap): Transcript { + const scenario = namespacedScenario(original); + const labels = citationLabels(original); + const card = original.card; + const halves: TranscriptHalf[] = []; + for (const mode of record.cardRun.modes) { + const runs: Array<[TranscriptHalf['phase'], { questions: QuestionResult[] } | undefined]> = [ + ['run', mode.run], + ['iterate-run', mode.iterate?.run], + ]; + for (const [phase, half] of runs) { + if (!half) continue; + const out: TranscriptHalf = { mode: mode.mode, phase, turns: [] }; + for (const q of half.questions) { + const truth = q.persona === null ? original.truths[q.question] : original.personaTruths[q.question]?.[q.persona]; + if (!truth) throw new TranscriptRefusal(`${mode.mode} ${phase}: the record asks \`${q.question}\`${q.persona ? ` as ${q.persona}` : ''}, which the card does not`); + const turn: TranscriptTurn = { question: q.question, persona: q.persona, truth, answer: null, grade: q.grade ?? null }; + if (q.notMeasured) turn.notMeasured = q.notMeasured; + if (q.error) turn.error = q.error; + if (q.grade) { + const ref: StepRef = { card: card.id, mode: mode.mode, phase, step: q.question, ...(q.persona ? { persona: q.persona } : {}) }; + const key = stepKey(ref); + const recorded = turns.get(key); + if (!recorded) throw new TranscriptRefusal(`no recorded reply for \`${key}\`: the transcript would show a grade without its answer`); + const regrade = gradeCardAnswer(scenario, { + question: q.question, + ...(q.persona ? { persona: q.persona, hidden: { values: hiddenValues(scenario, q.persona) } } : {}), + objectLabels: labels, + turn: recorded, + }); + if (!same(regrade, q.grade)) throw new TranscriptRefusal(`the recorded reply for \`${key}\` grades differently from the run record: the recording is not this run's`); + turn.answer = { + replyText: recorded.replyText, + toolCalls: recorded.toolCalls.map((c) => ({ name: c.name, input: c.input, output: c.output, ...(c.isError ? { isError: true } : {}) })), + }; + } + out.turns.push(turn); + } + halves.push(out); + } + } + return { + card: card.id, + questions: card.run.questions.map((q) => ({ id: q.id, ask: q.ask, object: q.object, personas: q.personas ?? [] })), + personas: card.run.personas.map((p) => ({ id: p.id, role: p.role })), + halves, + }; +} + +// ── Markdown ──────────────────────────────────────────────────────────────── + +const truthText = (t: Truth): string => { + switch (t.kind) { + case 'scalar': + return String(t.value); + case 'groups': + return t.groups.map((g) => `${String(g.key)}: ${g.value}`).join(', '); + case 'ranking': + return t.rows.map((g, i) => `${i + 1}. ${String(g.key)} (${g.value})`).join(', '); + case 'unanswerable': + return 'the data cannot answer this; a correct reply says so'; + case 'withheld': + return 'withheld from this persona; a correct reply declines'; + } +}; + +const verdictText = (t: TranscriptTurn): string => { + if (t.notMeasured) return `not measured — ${t.notMeasured}`; + if (t.error) return `error — ${t.error}`; + const g = t.grade; + if (!g) return 'not graded'; + const ok = g.correct && g.cited && g.leaks.length === 0; + const parts = [g.correct ? 'correct' : 'not correct', g.cited ? 'cited' : 'not cited', g.leaks.length ? `LEAKED: ${g.leaks.join('; ')}` : 'no leak']; + if (g.declined) parts.push('declined'); + return `${ok ? '✅ pass' : '❌ fail'} — ${parts.join(', ')}${g.problems.length ? ` (${g.problems.join('; ')})` : ''}`; +}; + +const quote = (text: string) => + text + .trim() + .split('\n') + .map((l) => `> ${l}`.trimEnd()) + .join('\n'); + +const HALF_TITLE: Record = { + 'scratch/run': 'Built from the requirement — Run', + 'scratch/iterate-run': 'After the change request — Run', + 'template/iterate-run': 'Installed as a template, then changed — Run', +}; + +/** The transcript as Markdown: per Run half, per question, every asker's answer and grade. */ +export function renderTranscript(t: Transcript, title: string): string { + const personaRole = new Map(t.personas.map((p) => [p.id, p.role])); + const lines = [`# 问数 transcript — ${title}`, '', `Every data question of \`${t.card}\`, asked of the app over its MCP door as each persona it names, with what the seed determines and the grade the scorer gave.`, '']; + lines.push('## Personas', ''); + for (const p of t.personas) lines.push(`- **${p.id}** — ${p.role}`); + for (const half of t.halves) { + lines.push('', `## ${HALF_TITLE[`${half.mode}/${half.phase}`] ?? `${half.mode} · ${half.phase}`}`); + for (const q of t.questions) { + const turns = half.turns.filter((x) => x.question === q.id); + if (!turns.length) continue; + lines.push('', `### \`${q.id}\``, ''); + if (q.ask.zh) lines.push(`- 问:${q.ask.zh}`); + if (q.ask.en) lines.push(`- Q: ${q.ask.en}`); + for (const turn of turns) { + const who = turn.persona ?? 'admin (no persona)'; + lines.push('', `#### As ${who}${turn.persona && personaRole.has(turn.persona) ? ` — ${personaRole.get(turn.persona)}` : ''}`, ''); + lines.push(`- Expected: ${truthText(turn.truth)}`, `- Grade: ${verdictText(turn)}`); + if (turn.answer) { + lines.push('', quote(turn.answer.replyText || '(empty reply)')); + if (turn.answer.toolCalls.length) lines.push('', `Tool calls: ${turn.answer.toolCalls.map((c) => `\`${c.name}\`${c.isError ? ' (error)' : ''}`).join(', ')}`); + } + } + } + } + return `${lines.join('\n')}\n`; +} diff --git a/packages/qa/scenarios/src/runner/cli.ts b/packages/qa/scenarios/src/runner/cli.ts index 0be462fde74..8081ec1ce45 100644 --- a/packages/qa/scenarios/src/runner/cli.ts +++ b/packages/qa/scenarios/src/runner/cli.ts @@ -33,9 +33,10 @@ import { runCard, type ProjectOps } from './orchestrate.js'; import { bootDev, os, readArtifact, readNamespace, scaffold } from './project.js'; import { cardScorecard, renderScorecard, runRecord } from './scorecard.js'; -const ADMIN = { email: 'admin@objectos.ai', password: 'admin123' }; +/** The dev admin `objectstack dev --fresh` seeds; the runner and the gallery exporter sign in as it. */ +export const ADMIN = { email: 'admin@objectos.ai', password: 'admin123' }; -function findUp(start: string, test: (dir: string) => boolean): string { +export function findUp(start: string, test: (dir: string) => boolean): string { let dir = start; for (;;) { if (test(dir)) return dir; diff --git a/packages/qa/scenarios/src/runner/orchestrate.ts b/packages/qa/scenarios/src/runner/orchestrate.ts index 3e1ac32b479..27191ad9d67 100644 --- a/packages/qa/scenarios/src/runner/orchestrate.ts +++ b/packages/qa/scenarios/src/runner/orchestrate.ts @@ -371,6 +371,57 @@ async function buildHalf( return half; } +/** + * Load the card's seed into a booted app, as the admin, in lookup order. + * + * Row ownership is mapped from the persona handles. A row whose direct scope + * field names a persona's handle is owned (`owner_id`, the platform's record + * owner) by that persona's user. A user-handle field (`account.owner`, + * `userHandleFields`) is itself loaded with a user id — the persona's, or the + * admin's for a handle no persona holds (`u_tia`), and every handle is the + * admin's when no persona account is passed (the gallery's screenshot boot) — + * because a faithful build models it as a user lookup, which refuses the + * handle text. Any other scope field names a row or an option and is sent + * verbatim. + * + * The tally is kept in `seed` as each row is loaded, and `seed` is what is + * returned: when an insert throws, the caller's tally already holds what was + * inserted and refused before it (the record shows the partial seed next to the + * half's error). + */ +export async function seedApp( + scenario: LoadedScenario, + app: AppHarness, + accounts: readonly PersonaAccount[], + seed: RunHalf['seed'] = { inserted: 0, refused: [] }, +): Promise { + const { card } = scenario; + const handles = personaHandles(card); + const ownerFields = new Set(card.run.personas.flatMap((p) => (p.scope ?? []).filter((s) => !s.field.includes('.')).map((s) => `${s.object}.${s.field}`))); + const userFields = userHandleFields(card); + for (const object of seedOrder(scenario)) { + for (const row of card.run.seed[object] ?? []) { + const body: Record = { ...row }; + for (const [k, v] of Object.entries(row)) { + if (typeof v !== 'string' || !ownerFields.has(`${object}.${k}`)) continue; + const persona = handles.get(v); + const acct = persona ? accounts.find((a) => a.persona === persona) : undefined; + if (acct) body.owner_id = acct.userId; + if (userFields.has(`${object}.${k}`)) { + const userId = acct?.userId ?? app.adminUserId(); + body[k] = userId; + body.owner_id = userId; + if (!acct) seed.handlesAsAdmin = [...new Set([...(seed.handlesAsAdmin ?? []), v])].sort(); + } + } + const r = await app.insertRow(object, body); + if (r.ok) seed.inserted++; + else seed.refused.push({ object, id: row.id, ...(r.code ? { code: r.code } : {}), ...(r.message ? { message: r.message.slice(0, 300) } : {}) }); + } + } + return seed; +} + /** Seed, personas, every question and every operation, against one booted app. */ export async function runHalf( scenario: LoadedScenario, @@ -392,37 +443,7 @@ export async function runHalf( } const adminKey = await app.adminApiKey(); - // Seed, with row ownership mapped from the persona handles. A row whose - // direct scope field names a persona's handle is owned (`owner_id`, the - // platform's record owner) by that persona's user. A user-handle field - // (`account.owner`, `userHandleFields`) is itself loaded with a user id — - // the persona's, or the admin's for a handle no persona holds (`u_tia`) — - // because a faithful build models it as a user lookup, which refuses the - // handle text. Any other scope field names a row or an option and is sent - // verbatim. - const handles = personaHandles(card); - const ownerFields = new Set(card.run.personas.flatMap((p) => (p.scope ?? []).filter((s) => !s.field.includes('.')).map((s) => `${s.object}.${s.field}`))); - const userFields = userHandleFields(card); - for (const object of seedOrder(scenario)) { - for (const row of card.run.seed[object] ?? []) { - const body: Record = { ...row }; - for (const [k, v] of Object.entries(row)) { - if (typeof v !== 'string' || !ownerFields.has(`${object}.${k}`)) continue; - const persona = handles.get(v); - const acct = persona ? accounts.find((a) => a.persona === persona) : undefined; - if (acct) body.owner_id = acct.userId; - if (userFields.has(`${object}.${k}`)) { - const userId = acct?.userId ?? app.adminUserId(); - body[k] = userId; - body.owner_id = userId; - if (!acct) half.seed.handlesAsAdmin = [...new Set([...(half.seed.handlesAsAdmin ?? []), v])].sort(); - } - } - const r = await app.insertRow(object, body); - if (r.ok) half.seed.inserted++; - else half.seed.refused.push({ object, id: row.id, ...(r.code ? { code: r.code } : {}), ...(r.message ? { message: r.message.slice(0, 300) } : {}) }); - } - } + await seedApp(scenario, app, accounts, half.seed); // tallied in place, so a throw keeps the partial seed const keyOf = (persona: string | null) => (persona === null ? adminKey : accounts.find((a) => a.persona === persona)!.apiKey); // A refused seed row makes what reads it not measured — never a fail of the app's answer. @@ -502,6 +523,13 @@ async function withApp(ops: ProjectOps, dir: string, findings: string[], use: // ── The modes ─────────────────────────────────────────────────────────────── +/** + * The directory, under the run's work directory, a mode's project is + * scaffolded in (`ProjectOps.scaffold`'s `under`): one per card and mode. The + * gallery exporter finds a run's scratch project by it. + */ +export const modeDir = (card: { id: string }, mode: 'scratch' | 'template'): string => `${card.id.replace(/_/g, '-')}-${mode}`; + /** Did the build pass — every counted check of the card's `expect`, and no bare item name? */ export const buildPassed = (b: BuildHalf | undefined): boolean => !!b?.score && b.score.expect.checks.length > 0 && [...b.score.expect.checks, ...b.score.naming].every((c) => c.pass); @@ -510,7 +538,7 @@ async function scratchMode(original: LoadedScenario, driver: AgentDriver, ops: P const mode: ModeRun = { mode: 'scratch', findings: [] }; // The project is named after the card's namespace, which is what the scaffolder derives its namespace from. const ns = scenarioNamespace(original.card); - const { dir, result } = await ops.scaffold(ns, `${original.card.id.replace(/_/g, '-')}-scratch`); + const { dir, result } = await ops.scaffold(ns, modeDir(original.card, 'scratch')); if (result.exit !== 0) { mode.skipped = `scaffold failed: ${result.tail.slice(-500)}`; return { mode }; @@ -556,7 +584,7 @@ async function templateMode(original: LoadedScenario, driver: AgentDriver, ops: return mode; } // The blank host gets a namespace of its own (`templateHostName` says why). - const { dir, result } = await ops.scaffold(templateHostName(original.card), `${card.id.replace(/_/g, '-')}-template`); + const { dir, result } = await ops.scaffold(templateHostName(original.card), modeDir(card, 'template')); if (result.exit !== 0) { mode.skipped = `scaffold failed: ${result.tail.slice(-500)}`; return mode; diff --git a/packages/qa/scenarios/test/export/exporter.test.ts b/packages/qa/scenarios/test/export/exporter.test.ts new file mode 100644 index 00000000000..fa9747ac7c8 --- /dev/null +++ b/packages/qa/scenarios/test/export/exporter.test.ts @@ -0,0 +1,487 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The gallery exporter on a real run of the real crm card (`fixture.ts`): the + * layout it writes, the pass rule it reads from the record, the generic-only + * guard, re-export replacing what was there, an app that stops passing + * leaving the gallery, the transcript joined with (and proven against) the + * recording, and `index.json` validated and stable. + */ + +import { createHash } from 'node:crypto'; +import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, statSync, writeFileSync } from 'node:fs'; +import { join, relative } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +import { exportGallery, scratchProjectDir, type ExportOptions } from '../../src/export/exporter.js'; +import { NO_GENERIC_MARKER, type GenericGuard } from '../../src/export/generic.js'; +import { GalleryIndexSchema, parseIndex } from '../../src/export/manifest.js'; +import { judgeRecord } from '../../src/export/pass.js'; +import { readRunRecords, redactWorkDir, ScenarioRunRecordSchema, type ScenarioRunRecord } from '../../src/export/record.js'; +import type { CaptureScreenshots } from '../../src/export/screenshots.js'; +import { main, parseExportArgs, renderSummary } from '../../src/export/cli.js'; +import type { Recording } from '../../src/runner/driver.js'; +import { CARD_ID, loadAllCards, loadCrm, runCrm, tempDir, writeRecord, type FixtureRun } from './fixture.js'; + +/** The test's stand-in for the generic marker the card schema does not have yet. */ +const GENERIC_DOUBLE: GenericGuard = () => ({ generic: true, reason: 'test double' }); + +const asRecord = (r: Record) => ScenarioRunRecordSchema.parse(r) as ScenarioRunRecord; + +/** Every file under `dir`, relative path → sha256. */ +function tree(dir: string): Record { + const out: Record = {}; + const walk = (d: string) => { + for (const e of readdirSync(d, { withFileTypes: true })) { + const p = join(d, e.name); + if (e.isDirectory()) walk(p); + else out[relative(dir, p)] = createHash('sha256').update(readFileSync(p)).digest('hex'); + } + }; + walk(dir); + return out; +} + +async function setup(run?: FixtureRun) { + const fx = run ?? (await runCrm()); + const runs = tempDir('runs'); + const out = tempDir('out'); + writeRecord(runs, fx.record); + const opts = (over: Partial = {}): ExportOptions => ({ + runsDir: runs, + outDir: out, + cards: [fx.scenario], + recordings: [fx.recording], + isGeneric: GENERIC_DOUBLE, + screenshots: false, + ...over, + }); + return { fx, runs, out, opts }; +} + +describe('judgeRecord — "passed every half", read from the record', () => { + it('a clean run passes: all six scratch halves and the three template iterate halves', async () => { + const { record } = await runCrm(); + const j = judgeRecord(asRecord(record)); + expect(j.reasons).toEqual([]); + expect(j.passed).toBe(true); + expect(j.halves.map((h) => h.id)).toEqual([ + `${CARD_ID}.scratch.build`, + `${CARD_ID}.scratch.run-ask`, + `${CARD_ID}.scratch.run-operations`, + `${CARD_ID}.scratch.iterate-build`, + `${CARD_ID}.scratch.iterate-run-ask`, + `${CARD_ID}.scratch.iterate-run-operations`, + `${CARD_ID}.template.iterate-build`, + `${CARD_ID}.template.iterate-run-ask`, + `${CARD_ID}.template.iterate-run-operations`, + ]); + // Template mode's first rows are not-run by construction and are not asked. + const rows = (record.results as Array<{ id: string; verdict: string }>).filter((r) => r.id === `${CARD_ID}.template.build`); + expect(rows[0]!.verdict).toBe('not-run'); + }); + + it('a not-run half is not a pass: a run without the iterate re-run fails', async () => { + const { record } = await runCrm({ iterateRun: false, modes: ['scratch'] }); + const j = judgeRecord(asRecord(record)); + expect(j.passed).toBe(false); + expect(j.reasons).toEqual([`${CARD_ID}.scratch.iterate-run-ask: not-run`, `${CARD_ID}.scratch.iterate-run-operations: not-run`]); + }); + + it('partial, fail and a missing row are not a pass; a record without the scratch mode is refused', async () => { + const { record } = await runCrm(); + const results = (record.results as Array<{ id: string; verdict: string }>).map((r) => + r.id.endsWith('scratch.run-ask') ? { ...r, verdict: 'partial' } : r.id.endsWith('template.iterate-build') ? { ...r, verdict: 'fail' } : r, + ); + const j = judgeRecord(asRecord({ ...record, results: results.filter((r) => !r.id.endsWith('scratch.run-operations')) })); + expect(j.reasons).toEqual([`${CARD_ID}.scratch.run-ask: partial`, `${CARD_ID}.scratch.run-operations: missing`, `${CARD_ID}.template.iterate-build: fail`]); + + const cardRun = record.cardRun as { modes: Array<{ mode: string }> }; + const templateOnly = { ...record, cardRun: { ...cardRun, modes: cardRun.modes.filter((m) => m.mode === 'template') } }; + expect(judgeRecord(asRecord(templateOnly)).reasons[0]).toMatch(/no scratch mode/); + }); +}); + +describe('judgeRecord — ⛔ a not-measured turn is not a pass, even under a `pass` verdict', () => { + it('a refused seed row leaves every half `pass` in the record, and the record still does not pass', async () => { + // ct4 is a contact: every question on contacts, and every operation that names it, is not measured. + const { record } = await runCrm({ refuse: ['democrm_contact/ct4'] }); + const verdicts = (record.results as Array<{ id: string; verdict: string }>).filter((r) => /scratch\.(run-ask|run-operations)$/.test(r.id)).map((r) => r.verdict); + expect(verdicts).toEqual(['pass', 'pass']); // the scorecard counts not-measured turns in neither passed nor total + const j = judgeRecord(asRecord(record)); + expect(j.passed).toBe(false); + expect(j.halves.every((h) => h.verdict === 'pass')).toBe(true); + expect(j.reasons.length).toBeGreaterThan(0); + for (const r of j.reasons) expect(r).toMatch(/^crm_account_with_contacts\.(scratch|template)\.(run|iterate-run) (question|operation) \S+.*: not measured — seed: /); + // Every Run half the record holds is read: scratch first run, scratch re-run, template re-run. + expect(new Set(j.reasons.map((r) => r.split(' ')[0]))).toEqual( + new Set([`${CARD_ID}.scratch.run`, `${CARD_ID}.scratch.iterate-run`, `${CARD_ID}.template.iterate-run`]), + ); + }); + + it('an operation marked not measured is a reason of its own', async () => { + const { record } = await runCrm(); + const cardRun = record.cardRun as { modes: Array<{ mode: string; run: { operations: Array> } }> }; + const scratch = cardRun.modes.find((m) => m.mode === 'scratch')!; + const [first, ...rest] = scratch.run.operations; + const edited = { ...record, cardRun: { ...cardRun, modes: cardRun.modes.map((m) => (m === scratch ? { ...m, run: { ...m.run, operations: [{ ...first, notMeasured: 'seed: needs democrm_account ac1, which the app refused' }, ...rest] } } : m)) } }; + expect(judgeRecord(asRecord(edited)).reasons).toEqual([ + `${CARD_ID}.scratch.run operation ${String(first!.operation)} as ${String(first!.persona)}: not measured — seed: needs democrm_account ac1, which the app refused`, + ]); + }); +}); + +describe('exportGallery — one passing, generic app', () => { + it('writes the apps// layout: source, run record, scorecard, transcript, README, and the index', async () => { + const { out, fx, opts } = await setup(); + const s = await exportGallery(opts()); + expect(s.refused).toEqual([]); + expect(s.exported.map((e) => e.id)).toEqual([CARD_ID]); + expect(s.indexChanged).toBe(true); + + const files = Object.keys(tree(join(out, 'apps', CARD_ID))).sort(); + expect(files).toEqual([ + 'README.md', + 'run-record.json', + 'scorecard.json', + 'scorecard.md', + 'source/objectstack.config.ts', + 'source/objectstack.manifest.json', + 'source/package.json', + 'source/src/objects/account.object.ts', + 'source/src/objects/contact.object.ts', + 'source/tsconfig.json', + 'transcript.json', + 'transcript.md', + ]); + + // The run record, verbatim but for the local work directory. + const recordText = readFileSync(join(out, 'apps', CARD_ID, 'run-record.json'), 'utf8'); + expect(recordText).not.toContain(fx.work); + expect(recordText).toContain('$WORK'); + expect(JSON.parse(recordText).results).toEqual(fx.record.results); + + const index = parseIndex(readFileSync(join(out, 'index.json'), 'utf8')); + expect(index.apps).toEqual([ + { + id: CARD_ID, + code: 'crm', + namespace: 'democrm', + path: `apps/${CARD_ID}`, + function: 'crm', + title: { en: fx.scenario.card.title, zh: null }, + summary: { ...(fx.scenario.card.prompt.short.en ? { en: fx.scenario.card.prompt.short.en } : {}), ...(fx.scenario.card.prompt.short.zh ? { zh: fx.scenario.card.prompt.short.zh } : {}) }, + capabilities: [...fx.scenario.card.capabilities].sort(), + screenshots: [], + qa: { questions: 8, turns: 9, personas: 2 }, + operations: 5, + framework: { version: '18.3.0', runner: 'abc1234' }, + runDate: '2026-11-06', + run: `2026-11-06-scenario-${CARD_ID}`, + }, + ]); + }); + + it('the transcript carries every persona answer with its grade, as JSON and as Markdown', async () => { + const { out, opts } = await setup(); + await exportGallery(opts()); + const t = JSON.parse(readFileSync(join(out, 'apps', CARD_ID, 'transcript.json'), 'utf8')); + expect(t.halves.map((h: { mode: string; phase: string }) => `${h.mode}/${h.phase}`)).toEqual(['scratch/run', 'scratch/iterate-run', 'template/iterate-run']); + const first = t.halves[0].turns; + expect(first).toHaveLength(9); // 8 questions, one of them asked as both personas + for (const turn of first) { + expect(turn.answer.replyText.length).toBeGreaterThan(0); + expect(turn.grade).toMatchObject({ correct: true, cited: true, leaks: [] }); + } + // Asked as the admin where a question names no persona, and as each persona it names. + const askers = new Set(first.map((x: { persona: string | null }) => x.persona ?? 'admin')); + expect([...askers].sort()).toEqual(['admin', 'partner_user', 'sdr']); + const md = readFileSync(join(out, 'apps', CARD_ID, 'transcript.md'), 'utf8'); + expect(md).toContain('#### As sdr'); + expect(md).toContain('#### As partner_user'); + expect(md).toContain('✅ pass'); + }); + + it('a re-export with no change is a no-op: the same bytes, file for file, and the index is not rewritten', async () => { + const { out, opts } = await setup(); + await exportGallery(opts()); + const first = tree(out); + const again = await exportGallery(opts()); + expect(again.indexChanged).toBe(false); + expect(tree(out)).toEqual(first); + }); + + it('a re-export deletes the old directory first: nothing stale survives', async () => { + const { out, opts } = await setup(); + await exportGallery(opts()); + writeFileSync(join(out, 'apps', CARD_ID, 'stale.txt'), 'left from an old release'); + mkdirSync(join(out, 'apps', CARD_ID, 'screenshots'), { recursive: true }); + writeFileSync(join(out, 'apps', CARD_ID, 'screenshots', 'old.png'), 'x'); + await exportGallery(opts()); + expect(existsSync(join(out, 'apps', CARD_ID, 'stale.txt'))).toBe(false); + expect(existsSync(join(out, 'apps', CARD_ID, 'screenshots'))).toBe(false); + expect(readdirSync(join(out, 'apps')).filter((n) => n.startsWith('.staging'))).toEqual([]); + }); + + it('an app whose newest record no longer passes leaves apps/ and the index; a directory the index never listed is left alone', async () => { + const { out, runs, opts } = await setup(); + await exportGallery(opts()); + mkdirSync(join(out, 'apps', 'hand_made')); + // The next release's run: the iterate re-run was skipped, so two halves are not-run. + const later = await runCrm({ date: '2026-12-01', iterateRun: false }); + writeRecord(runs, later.record); + const s = await exportGallery(opts({ recordings: [later.recording] })); + expect(s.exported).toEqual([]); + expect(s.removed).toEqual([{ id: CARD_ID, why: expect.stringContaining(`${CARD_ID}.scratch.iterate-run-ask: not-run`) }]); + expect(existsSync(join(out, 'apps', CARD_ID))).toBe(false); + expect(parseIndex(readFileSync(join(out, 'index.json'), 'utf8')).apps).toEqual([]); + expect(s.unowned).toEqual(['hand_made']); + expect(existsSync(join(out, 'apps', 'hand_made'))).toBe(true); + }); + + it('an app whose newest record has a not-measured turn is refused, and a previous export of it removed', async () => { + const { out, runs, opts } = await setup(); + await exportGallery(opts()); + const later = await runCrm({ date: '2026-12-01', refuse: ['democrm_contact/ct4'] }); + writeRecord(runs, later.record); + const s = await exportGallery(opts({ recordings: [later.recording] })); + expect(s.exported).toEqual([]); + expect(s.refused[0]!.reasons[0]).toMatch(/did not pass every half/); + expect(s.refused[0]!.reasons.slice(1).every((r) => r.includes(': not measured — seed: '))).toBe(true); + expect(s.removed).toEqual([{ id: CARD_ID, why: expect.stringContaining('not measured') }]); + expect(existsSync(join(out, 'apps', CARD_ID))).toBe(false); + expect(parseIndex(readFileSync(join(out, 'index.json'), 'utf8')).apps).toEqual([]); + }); + + it('the newest record decides, even when an older one passed', async () => { + const { runs } = await setup(); + const later = await runCrm({ date: '2026-12-01', iterateRun: false }); + writeRecord(runs, later.record); + const { newest } = readRunRecords(runs); + expect(newest.get(CARD_ID)!.record.date).toBe('2026-12-01'); + }); + + it('refuses, and touches nothing, when the previous index.json is not one it wrote', async () => { + const { out, opts } = await setup(); + await exportGallery(opts()); + writeFileSync(join(out, 'index.json'), JSON.stringify({ apps: 'hand edited' })); + const before = tree(out); + await expect(exportGallery(opts())).rejects.toThrow(/not a gallery index this exporter wrote/); + expect(tree(out)).toEqual(before); + }); + + it('⛔ a runs directory with no scenario record, over an index that lists apps, is a wrong --runs: it refuses and removes nothing', async () => { + const { out, opts } = await setup(); + await exportGallery(opts()); + const before = tree(out); + const empty = tempDir('empty-runs'); + writeFileSync(join(empty, 'checklist-run.json'), JSON.stringify({ runner: 'platform-checklist', results: [] })); + await expect(exportGallery(opts({ runsDir: empty }))).rejects.toThrow(/holds no scenario run record, and .* lists 1 app\(s\), so nothing is touched/); + expect(tree(out)).toEqual(before); + // Through the CLI: the export rejects, so the process exits non-zero. + await expect(main(['--runs', empty, '--out', out, '--no-screenshots'])).rejects.toThrow(/holds no scenario run record/); + expect(tree(out)).toEqual(before); + // With no previous index there is nothing to lose: an empty runs directory writes an empty index. + const fresh = tempDir('out-fresh'); + expect((await exportGallery(opts({ runsDir: empty, outDir: fresh }))).exported).toEqual([]); + expect(parseIndex(readFileSync(join(fresh, 'index.json'), 'utf8')).apps).toEqual([]); + }); + + it('refuses the whole export on an unreadable scenario-runner record, and ignores other JSON', async () => { + const { runs, opts } = await setup(); + writeFileSync(join(runs, 'checklist-run.json'), JSON.stringify({ runner: 'platform-checklist', results: [] })); + const s = await exportGallery(opts()); + expect(s.ignoredFiles.map((f) => f.slice(f.lastIndexOf('/') + 1))).toEqual(['checklist-run.json']); + writeFileSync(join(runs, '2026-11-07-scenario-x.json'), JSON.stringify({ runner: 'scenario-runner', app: 'x' })); + await expect(exportGallery(opts())).rejects.toThrow(/not a readable scenario run record/); + }); +}); + +describe('⛔ no local path: every work directory that names the run is redacted', () => { + /** Every file under `dir`, relative path → its text. */ + function texts(dir: string): Record { + const out: Record = {}; + const walk = (d: string) => { + for (const e of readdirSync(d, { withFileTypes: true })) { + const p = join(d, e.name); + if (e.isDirectory()) walk(p); + else out[relative(dir, p)] = readFileSync(p, 'utf8'); + } + }; + walk(dir); + return out; + } + + it('under --work, a record from another machine: neither its evidenceDir nor --work appears anywhere in --out', async () => { + const fx = await runCrm(); + const other = '/srv/ci-runner-7/scenario-work'; + // The record as the machine that ran it wrote it: every string that named its work directory names `other`. + const moved = JSON.parse(JSON.stringify(fx.record).split(fx.work).join(other)) as Record; + expect(moved.evidenceDir).toBe(other); + expect(JSON.stringify(moved).split(other).length - 1).toBeGreaterThan(1); // the build tails name it too + const { out, opts } = await setup({ ...fx, record: moved }); + const s = await exportGallery(opts({ workDir: fx.work })); + expect(s.exported.map((e) => e.id)).toEqual([CARD_ID]); + + const files = texts(out); + for (const name of ['run-record.json', 'scorecard.json', 'scorecard.md', 'transcript.json', 'transcript.md', 'README.md']) expect(Object.keys(files)).toContain(`apps/${CARD_ID}/${name}`); + for (const [name, text] of Object.entries(files)) { + for (const dir of [other, fx.work, realpathSync(fx.work)]) expect(text.includes(dir), `${name} names ${dir}`).toBe(false); + } + expect(JSON.parse(files[`apps/${CARD_ID}/run-record.json`]!).evidenceDir).toBe('$WORK'); + }); + + it('a directory is replaced only whole: `/tmp` never rewrites `/tmpfoo`, and the longer of two nested directories wins', () => { + expect(redactWorkDir({ a: ['/tmp', '/tmp/x', 'ran in /tmp.', '"/tmp"', '/tmpfoo', '/tmp.d/x', '/var/tmp/y', 'x/tmp'] }, '/tmp/')).toEqual({ + a: ['$WORK', '$WORK/x', 'ran in $WORK.', '"$WORK"', '/tmpfoo', '/tmp.d/x', '/var/tmp/y', 'x/tmp'], + }); + expect(redactWorkDir('/w/inner/p and /w/q', ['/w', '/w/inner'])).toBe('$WORK/p and $WORK/q'); + expect(redactWorkDir('/a/b', ['/', ''])).toBe('/a/b'); + }); +}); + +describe('exportGallery — what it refuses', () => { + it('⛔ generic only: with the default guard every card is refused and nothing is written but an empty index', async () => { + const { out, opts } = await setup(); + const s = await exportGallery({ ...opts(), isGeneric: undefined }); + expect(s.exported).toEqual([]); + expect(s.refused).toEqual([{ id: CARD_ID, reasons: [expect.stringMatching(/^not marked generic: the card schema carries no explicit generic marker/)] }]); + expect(readdirSync(join(out, 'apps'))).toEqual([]); + expect(GalleryIndexSchema.parse(JSON.parse(readFileSync(join(out, 'index.json'), 'utf8'))).apps).toEqual([]); + }); + + it('the default guard refuses every card of the open library: none carries a generic marker', () => { + const cards = loadAllCards(); + expect(cards.length).toBeGreaterThanOrEqual(9); + for (const c of cards) expect(NO_GENERIC_MARKER(c)).toMatchObject({ generic: false }); + }); + + it('an asked turn with no recorded reply is refused: a grade is never shown without its answer', async () => { + const { out, opts } = await setup(); + const s = await exportGallery(opts({ recordings: [] })); + expect(s.refused[0]!.reasons[0]).toMatch(/^no recorded reply for `crm_account_with_contacts\/scratch\/run\//); + expect(existsSync(join(out, 'apps', CARD_ID))).toBe(false); + }); + + it("a recording that is not this run's is refused: its replies grade differently from the record", async () => { + const { fx, opts } = await setup(); + const turns = { ...fx.recording.turns }; + const key = Object.keys(turns).find((k) => k.startsWith(`${CARD_ID}/scratch/run/`) && !k.includes('/author/'))!; + turns[key] = { ...turns[key]!, replyText: 'No idea.' }; + const other: Recording = { ...fx.recording, turns }; + const s = await exportGallery(opts({ recordings: [other] })); + expect(s.refused[0]!.reasons[0]).toBe(`the recorded reply for \`${key}\` grades differently from the run record: the recording is not this run's`); + }); + + it("the run's project gone, or a card no longer loaded, is a refusal", async () => { + const { fx, opts } = await setup(); + expect((await exportGallery(opts({ cards: [] }))).refused[0]!.reasons[0]).toMatch(/^no loaded card `crm_account_with_contacts`/); + const elsewhere = tempDir('moved'); + expect((await exportGallery(opts({ workDir: elsewhere }))).refused[0]!.reasons[0]).toBe(`the run's scratch project is gone: ${scratchProjectDir(elsewhere, fx.scenario.card)}`); + }); +}); + +describe('exportGallery — screenshots', () => { + it('runs the screenshot step against the scratch project and lists every shot in the index', async () => { + const { out, fx, opts } = await setup(); + const seen: string[] = []; + const shoot: CaptureScreenshots = async (input) => { + seen.push(input.projectDir); + mkdirSync(join(input.dir, 'screenshots'), { recursive: true }); + writeFileSync(join(input.dir, 'screenshots', 'list.png'), 'png'); + return { shots: [{ kind: 'list', file: 'screenshots/list.png', path: '/_console/apps/democrm_crm/democrm_account', object: 'democrm_account' }], skipped: [{ kind: 'dashboard', reason: 'the build declares no dashboard' }] }; + }; + const s = await exportGallery(opts({ screenshots: shoot })); + expect(seen).toEqual([scratchProjectDir(fx.work, fx.scenario.card)]); + expect(s.exported[0]!.skippedShots).toEqual([{ kind: 'dashboard', reason: 'the build declares no dashboard' }]); + expect(statSync(join(out, 'apps', CARD_ID, 'screenshots', 'list.png')).isFile()).toBe(true); + const index = parseIndex(readFileSync(join(out, 'index.json'), 'utf8')); + expect(index.apps[0]!.screenshots).toEqual([{ kind: 'list', path: `apps/${CARD_ID}/screenshots/list.png`, object: 'democrm_account' }]); + expect(readFileSync(join(out, 'apps', CARD_ID, 'README.md'), 'utf8')).toContain('no dashboard screenshot: the build declares no dashboard'); + }); + + it('⛔ an export without screenshots never replaces one that had them: the old directory and entry are kept, and the summary says so', async () => { + const { out, runs, opts } = await setup(); + const shoot: CaptureScreenshots = async (input) => { + mkdirSync(join(input.dir, 'screenshots'), { recursive: true }); + writeFileSync(join(input.dir, 'screenshots', 'list.png'), 'png'); + return { shots: [{ kind: 'list', file: 'screenshots/list.png', path: '/_console/apps/democrm_crm/democrm_account', object: 'democrm_account' }], skipped: [] }; + }; + await exportGallery(opts({ screenshots: shoot })); + const before = tree(out); + const s = await exportGallery(opts({ screenshots: false })); + expect(s.exported).toEqual([]); + expect(s.removed).toEqual([]); + expect(s.kept).toEqual([{ id: CARD_ID, why: expect.stringContaining('the previous export has 1: it is kept as it was') }]); + expect(s.indexChanged).toBe(false); + expect(tree(out)).toEqual(before); + expect(renderSummary(s)).toContain(`kept ${CARD_ID} · ${s.kept[0]!.why}`); + expect(renderSummary(s).slice(-1)[0]).toBe('index.json unchanged · 0 exported, 0 refused, 0 removed, 1 kept'); + // It keeps nothing that no longer passes: a newer failing record still removes the app. + const later = await runCrm({ date: '2026-12-01', iterateRun: false }); + writeRecord(runs, later.record); + const t = await exportGallery(opts({ screenshots: false, recordings: [later.recording] })); + expect(t.kept).toEqual([]); + expect(t.removed.map((r) => r.id)).toEqual([CARD_ID]); + expect(existsSync(join(out, 'apps', CARD_ID))).toBe(false); + }); + + it('a screenshot step that fails refuses the app instead of exporting it without its proof', async () => { + const { out, opts } = await setup(); + const s = await exportGallery(opts({ screenshots: async () => Promise.reject(new Error('objectstack dev exited before it was ready')) })); + expect(s.refused).toEqual([{ id: CARD_ID, reasons: ['objectstack dev exited before it was ready'] }]); + expect(existsSync(join(out, 'apps', CARD_ID))).toBe(false); + }); +}); + +describe('index.json — the schema', () => { + const app = (id: string) => ({ + id, + code: 'crm', + namespace: 'democrm', + path: `apps/${id}`, + function: 'crm', + title: { en: 'CRM', zh: null }, + summary: { en: 'Build a CRM.' }, + capabilities: [], + screenshots: [], + qa: { questions: 8, turns: 9, personas: 2 }, + operations: 5, + framework: { version: '18.3.0', runner: 'abc1234' }, + runDate: '2026-11-06', + run: 'r', + }); + it('accepts a sorted index and refuses an unsorted one, a duplicate, a wrong path and an unknown key', () => { + expect(GalleryIndexSchema.safeParse({ schemaVersion: 1, apps: [app('a_card'), app('b_card')] }).success).toBe(true); + expect(GalleryIndexSchema.safeParse({ schemaVersion: 1, apps: [app('b_card'), app('a_card')] }).success).toBe(false); + expect(GalleryIndexSchema.safeParse({ schemaVersion: 1, apps: [app('a_card'), app('a_card')] }).success).toBe(false); + expect(GalleryIndexSchema.safeParse({ schemaVersion: 1, apps: [{ ...app('a_card'), path: 'apps/other' }] }).success).toBe(false); + expect(GalleryIndexSchema.safeParse({ schemaVersion: 1, apps: [{ ...app('a_card'), generatedAt: 'now' }] }).success).toBe(false); + }); +}); + +describe('export-gallery CLI', () => { + it('requires --out and refuses an unknown flag', () => { + expect(() => parseExportArgs(['--runs', 'r'])).toThrow(/--out DIR is required/); + expect(() => parseExportArgs(['--out', 'o', '--generic'])).toThrow(/unknown flag `--generic`/); + expect(parseExportArgs(['--out', 'o', '--recording', 'a', '--recording', 'b', '--no-screenshots'])).toMatchObject({ out: 'o', recordings: ['a', 'b'], screenshots: false }); + }); + + it('end to end over the real card directory: today it refuses every card, so the gallery stays empty', async () => { + const { fx, runs, out } = await setup(); + const recording = join(tempDir('rec'), 'recording.json'); + writeFileSync(recording, JSON.stringify(fx.recording)); + const errors: string[] = []; + const log = console.error; + console.error = (...a: unknown[]) => void errors.push(a.join(' ')); + try { + expect(await main(['--runs', runs, '--out', out, '--recording', recording, '--no-screenshots'])).toBe(0); + } finally { + console.error = log; + } + expect(errors.some((l) => l.startsWith(`refused ${CARD_ID} · not marked generic`))).toBe(true); + expect(parseIndex(readFileSync(join(out, 'index.json'), 'utf8')).apps).toEqual([]); + expect(loadCrm().card.id).toBe(CARD_ID); + }); +}); diff --git a/packages/qa/scenarios/test/export/fixture.ts b/packages/qa/scenarios/test/export/fixture.ts new file mode 100644 index 00000000000..a595bc8e5e1 --- /dev/null +++ b/packages/qa/scenarios/test/export/fixture.ts @@ -0,0 +1,270 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * A real run of the real `crm_account_with_contacts` card, for the gallery + * exporter's tests: `runCard` end to end (both modes) with its three outside + * worlds replaced — a project lifecycle that writes a REAL project directory + * under a temp work directory (the layout the runner CLI scaffolds into), an + * in-memory app, and a scripted model wrapped in the runner's own + * `RecordingDriver` — then the run record exactly as the CLI writes it. + * + * No model, no network, no browser: what booting a generated app needs (a + * model to write it, npm to scaffold it) is not available here, so the + * project is a fixture and the screenshots are a double. + */ + +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { formatRefusal, loadScenarioFile, loadScenarios, readCapabilityAreas, type LoadedScenario } from '../../src/loader.js'; +import { truthFigures } from '../../src/score/index.js'; +import type { AppHarness, HttpAnswer, PersonaAccount } from '../../src/runner/app.js'; +import { RecordingDriver, type AgentDriver, type AskTask, type AuthorTask, type DriverInfo, type Recording } from '../../src/runner/driver.js'; +import { namespacedScenario } from '../../src/runner/namespace.js'; +import { runCard, type CardRun, type ProjectOps } from '../../src/runner/orchestrate.js'; +import { runRecord } from '../../src/runner/scorecard.js'; +import type { ParsedTurn } from '../../src/runner/transcript.js'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +export const CARDS_DIR = resolve(HERE, '../../../../../docs/qa/scenarios'); +export const AREAS_DIR = resolve(HERE, '../../../../../docs/qa/platform-checklist/areas'); +export const CARD_ID = 'crm_account_with_contacts'; +export const NS = 'democrm'; + +export function loadCrm(): LoadedScenario { + const { card, refusals } = loadScenarioFile(join(CARDS_DIR, `${CARD_ID}.json`), { areas: readCapabilityAreas(AREAS_DIR) }); + if (!card) throw new Error(refusals.map(formatRefusal).join('\n')); + return card; +} + +export function loadAllCards(): LoadedScenario[] { + return loadScenarios(CARDS_DIR, { areas: readCapabilityAreas(AREAS_DIR) }).cards; +} + +export const tempDir = (prefix: string) => mkdtempSync(join(tmpdir(), `gallery-${prefix}-`)); + +// ── The build the fake `objectstack build` writes ─────────────────────────── + +const select = (values: string[]) => ({ type: 'select', options: values.map((value) => ({ value, label: value })) }); +const n = (name: string) => `${NS}_${name}`; + +export function crmArtifact(iterated: boolean) { + const action = (name: string, confirm = false) => ({ name, ai: { exposed: true, ...(confirm ? { requiresConfirmation: true } : {}) } }); + const objects: unknown[] = [ + { name: n('account'), fields: { name: { type: 'text' }, industry: select(['tech', 'retail']), owner: { type: 'text' } } }, + { + name: n('contact'), + fields: { full_name: { type: 'text' }, email: { type: 'email' }, account: { type: 'lookup', reference: n('account') } }, + actions: [action(n('create_contact')), action(n('reassign_contact')), action(n('delete_contact'), true)], + }, + ]; + if (iterated) objects.push({ name: n('opportunity'), fields: { stage: select(['a', 'b', 'c', 'd']), account: { type: 'lookup', reference: n('account') } } }); + return { + manifest: { id: `com.example.${NS}`, namespace: NS }, + objects, + permissions: [ + { name: n('sdr'), objects: { [n('account')]: { allowRead: true }, [n('contact')]: { allowRead: true, allowCreate: true, allowEdit: true } }, fields: { [`${n('contact')}.email`]: { readable: false } } }, + { name: n('partner_user'), objects: { [n('contact')]: { allowRead: true, allowCreate: true } } }, + ], + }; +} + +// ── The project lifecycle: a real directory per mode ──────────────────────── + +const FIXTURE_FILES: Record = { + 'objectstack.config.ts': `import { defineStack } from '@objectstack/spec';\nexport default defineStack({ manifest: { id: 'com.example.${NS}', namespace: '${NS}' } });\n`, + 'objectstack.manifest.json': '{ "id": "com.example.democrm" }\n', + 'package.json': JSON.stringify({ name: NS, private: true, dependencies: { '@objectstack/spec': '18.3.0', '@objectstack/runtime': '18.3.0' } }, null, 2), + 'tsconfig.json': '{}\n', + 'src/objects/account.object.ts': "export const account = { name: 'democrm_account' };\n", + 'src/objects/contact.object.ts': "export const contact = { name: 'democrm_contact' };\n", + // What the gallery must NOT receive: dependencies, build output, installed skills, scaffold boilerplate, local state. + 'node_modules/@objectstack/spec/package.json': '{ "version": "18.3.0" }\n', + '.agents/skills/objectstack-data/SKILL.md': '# skill\n', + '.claude/settings.json': '{}\n', + '.objectstack/dev.db': 'local state\n', + 'README.md': '# scaffold readme\n', + 'Dockerfile': 'FROM node\n', + '.env': 'SECRET=never-exported\n', +}; + +function fakeOps(work: string, app: () => FakeApp): ProjectOps & { phase: Map } { + const phase = new Map(); + const ok = (command: string) => ({ command, exit: 0, tail: '', ms: 1 }); + return { + phase, + async scaffold(name, under) { + const dir = join(work, under, name); + for (const [rel, text] of Object.entries(FIXTURE_FILES)) { + mkdirSync(dirname(join(dir, rel)), { recursive: true }); + writeFileSync(join(dir, rel), text); + } + phase.set(dir, 'scaffolded'); + return { dir, result: ok(`create-objectstack ${name}`) }; + }, + async os(dir, args) { + const p = phase.get(dir)!; + if (args[0] === 'build' && p !== 'scaffolded') { + mkdirSync(join(dir, 'dist'), { recursive: true }); + writeFileSync(join(dir, 'dist', 'objectstack.json'), JSON.stringify(crmArtifact(p === 'iterated'))); + } + // A command tail naming the work directory, as a real refusal would: the export must not carry it. + return { command: `npx objectstack ${args.join(' ')}`, exit: 0, tail: `ran in ${dir}`, ms: 1 }; + }, + readArtifact(dir) { + const p = phase.get(dir)!; + return p === 'scaffolded' ? undefined : crmArtifact(p === 'iterated'); + }, + async boot() { + return { url: 'http://fake', stop: async () => {}, log: () => '' }; + }, + harness: () => app(), + namespace: (dir) => dir.slice(dir.lastIndexOf('/') + 1), + }; +} + +// ── The in-memory app ─────────────────────────────────────────────────────── + +export class FakeApp implements AppHarness { + readonly baseUrl = 'http://fake'; + readonly mcpUrl = 'http://fake/api/v1/mcp'; + rows: Record>> = {}; + audit: Record = {}; + installed: Record | undefined; + /** Seed rows this app refuses, `object/id`. */ + refuse = new Set(); + signedIn = 0; + async signInAdmin() { + this.signedIn++; + } + adminUserId() { + return 'u-admin'; + } + organizationId() { + return 'org'; + } + async createPersona(persona: string, permissionSet: string): Promise { + return { persona, userId: `u-${persona}`, email: `${persona}@scenario.test`, apiKey: `key-${persona}`, permissionSet: { name: permissionSet, assigned: true } }; + } + async adminApiKey() { + return 'key-admin'; + } + async insertRow(object: string, row: Record): Promise { + if (this.refuse.has(`${object}/${String(row.id)}`)) return { ok: false, status: 400, body: {}, code: 'VALIDATION_FAILED', message: 'refused by the double' }; + (this.rows[object] ??= []).push({ ...row }); + return { ok: true, status: 200, body: {} }; + } + async listRows(object: string) { + return { ok: true, status: 200, body: {}, rows: (this.rows[object] ?? []).map((r) => ({ ...r })) }; + } + async auditActorUserId(_object: string, recordId: string) { + return this.audit[recordId] ?? null; + } + async installUnmanaged(manifest: Record) { + this.installed = manifest; + return { ok: true, status: 200, body: {} }; + } + async metadataAsArtifact(collections: readonly string[]) { + const src = (this.installed ?? {}) as Record; + const out: Record = {}; + for (const c of collections) out[c] = [...(src[c] ?? [])]; + return out; + } +} + +// ── The scripted model ────────────────────────────────────────────────────── + +const turn = (replyText: string, toolCalls: ParsedTurn['toolCalls'] = []): ParsedTurn => ({ replyText, toolCalls, meta: { resultSubtype: 'success' } }); + +/** A truthful answer, built from the loader's computed truth. */ +function truthfulAnswer(scenario: LoadedScenario, question: string, persona?: string): string { + const q = scenario.card.run.questions.find((x) => x.id === question)!; + const truth = persona ? scenario.personaTruths[q.id]![persona]! : scenario.truths[q.id]!; + if (truth.kind === 'withheld' || truth.kind === 'unanswerable') return 'I cannot answer that: it is not available to you.'; + const figures = truthFigures(truth, q.object, 'x').map((f) => `${f.label.replace(/^x\[?|\]$/g, '') || 'total'}: ${f.value}`); + return `${figures.join(', ')}\nSource: ${q.object}`; +} + +class ScriptedDriver implements AgentDriver { + private readonly scenario: LoadedScenario; + constructor( + scenario: LoadedScenario, + private readonly ops: ReturnType, + private readonly app: () => FakeApp, + ) { + this.scenario = namespacedScenario(scenario); + } + info(): DriverInfo { + return { driver: 'recorded', version: 'scripted test replay', apiBilled: false }; + } + async author(task: AuthorTask) { + if (task.mode === 'scratch') this.ops.phase.set(task.cwd, task.phase === 'build' ? 'built' : 'iterated'); + if (task.mode === 'template') { + const app = this.app(); + const objects = (app.installed!.objects as unknown[]) ?? []; + app.installed = { ...app.installed!, objects: [...objects, ...crmArtifact(true).objects.slice(2)] }; + } + return { turn: turn('Done.') }; + } + async ask(task: AskTask): Promise { + const app = this.app(); + const op = this.scenario.card.run.operations.find((o) => o.id === task.step); + if (!op) return turn(truthfulAnswer(this.scenario, task.step, task.persona), [{ name: 'query_records', rawName: 'mcp__app__query_records', input: { objectName: 'x' }, output: '{"records":[]}' }]); + const rows = (app.rows[op.object] ??= []); + if (op.outcome.kind === 'applied') { + if (op.kind === 'create') rows.push({ id: `new-${op.id}`, ...op.values }); + if (op.kind === 'update') Object.assign(rows.find((r) => r.id === op.target)!, op.values); + app.audit[op.target ?? `new-${op.id}`] = `u-${op.actingAs}`; + return turn('Done.', [{ name: `${op.kind}_record`, rawName: 'x', input: { objectName: op.object }, output: '{"ok":true}' }]); + } + if (op.outcome.kind === 'refused') { + return turn('That was refused.', [{ name: `${op.kind}_record`, rawName: 'x', input: { objectName: op.object }, output: JSON.stringify({ error: { code: op.outcome.code } }), isError: true }]); + } + return turn('This cannot be undone — do you want me to delete it? Please confirm.'); + } +} + +export const ADMIN = { email: 'admin@objectos.ai', password: 'admin123' }; + +export interface FixtureRun { + scenario: LoadedScenario; + work: string; + cardRun: CardRun; + record: Record; + recording: Recording; +} + +/** One run of the crm card, both modes; `iterateRun: false` leaves the iterate Run halves not run; `refuse` makes every booted app refuse those seed rows. */ +export async function runCrm( + opts: { work?: string; date?: string; iterateRun?: boolean; modes?: Array<'scratch' | 'template'>; /** Seed rows every booted app refuses, `object/id`. */ refuse?: string[] } = {}, +): Promise { + const scenario = loadCrm(); + const work = opts.work ?? tempDir('work'); + const fresh = () => { + const app = new FakeApp(); + for (const r of opts.refuse ?? []) app.refuse.add(r); + return app; + }; + let current = fresh(); + const ops = fakeOps(work, () => current); + const boot = ops.boot.bind(ops); + ops.boot = async (dir) => { + current = fresh(); + return boot(dir); + }; + const driver = new RecordingDriver(new ScriptedDriver(scenario, ops, () => current)); + const cardRun = await runCard(scenario, driver, ops, { modes: opts.modes ?? ['scratch', 'template'], admin: ADMIN, ...(opts.iterateRun === false ? { iterateRun: false } : {}) }); + const date = opts.date ?? '2026-11-06'; + const record = runRecord({ run: `${date}-scenario-${CARD_ID}`, date, framework: 'abc1234', driver: driver.info(), runner: 'scenario-runner', evidenceDir: work }, cardRun); + return { scenario, work, cardRun, record, recording: driver.recording }; +} + +/** Write a record where the runner CLI writes it: `/-scenario-.json`. */ +export function writeRecord(runs: string, record: Record): string { + mkdirSync(runs, { recursive: true }); + const file = join(runs, `${String(record.date)}-scenario-${String(record.app)}.json`); + writeFileSync(file, JSON.stringify(record, null, 2)); + return file; +} diff --git a/packages/qa/scenarios/test/export/screenshots.test.ts b/packages/qa/scenarios/test/export/screenshots.test.ts new file mode 100644 index 00000000000..b22527f9849 --- /dev/null +++ b/packages/qa/scenarios/test/export/screenshots.test.ts @@ -0,0 +1,231 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The screenshot step without a browser: which views {@link planShots} picks + * from the card's tags and the build, and {@link captureScreenshots} driving a + * browser double against a booted-app double — booted the runner's way, + * seeded the runner's way, stopped afterwards. + * + * NOT MEASURED here: a real Console page. Booting a generated app locally + * needs the app (a model writes it) and `create-objectstack` from npm; the + * Playwright adapter (`playwrightBrowser`) is exercised by the PR's manual + * reading, not by this suite. + */ + +import { existsSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +import { formatRefusal, loadScenarioFile, readCapabilityAreas, type LoadedScenario } from '../../src/loader.js'; +import { captureScreenshots, CONSOLE_NOT_FOUND, declaredListViews, pageRefusal, planShots, taggedViewKinds, type CaptureOutcome, type PageReading, type ScreenshotBrowser } from '../../src/export/screenshots.js'; +import { AREAS_DIR, CARDS_DIR, FakeApp, tempDir } from './fixture.js'; + +function load(id: string): LoadedScenario { + const { card, refusals } = loadScenarioFile(join(CARDS_DIR, `${id}.json`), { areas: readCapabilityAreas(AREAS_DIR) }); + if (!card) throw new Error(refusals.map(formatRefusal).join('\n')); + return card; +} + +/** The order-to-cash card, with extra capability tags (a card that TAGS a view kind). */ +const orderCard = (extraTags: string[] = []): LoadedScenario => { + const s = load('order_to_cash'); + return { ...s, card: { ...s.card, capabilities: [...s.card.capabilities, ...extraTags] } }; +}; + +const KANBAN_TAG = 'records-forms.kanban-drag-persistence'; +const CALENDAR_TAG = 'records-forms.calendar-interactions'; + +const orderBuild = (over: { views?: unknown[]; dashboards?: unknown[]; apps?: unknown[] } = {}) => ({ + manifest: { id: 'com.example.demoorder', namespace: 'demoorder' }, + apps: over.apps ?? [{ name: 'demoorder_order_to_cash', label: 'Order to cash' }], + objects: [{ name: 'demoorder_employee' }, { name: 'demoorder_account' }, { name: 'demoorder_invoice' }, { name: 'demoorder_activity' }], + views: over.views ?? [ + { object: 'demoorder_invoice', list: { type: 'grid', columns: ['name'] }, listViews: { board: { type: 'kanban', columns: ['name'], kanban: { groupByField: 'status' } } } }, + { object: 'demoorder_activity', listViews: { schedule: { type: 'calendar', columns: ['subject'] } } }, + ], + dashboards: over.dashboards ?? [{ name: 'demoorder_overview' }, { name: 'demoorder_receivables_aging_report' }], +}); + +describe('taggedViewKinds — kanban and calendar only where the card tags them', () => { + it('reads the tag names, not the area', () => { + expect(taggedViewKinds({ capabilities: ['records-forms.lookup'] })).toEqual([]); + expect(taggedViewKinds({ capabilities: [KANBAN_TAG] })).toEqual(['kanban']); + expect(taggedViewKinds({ capabilities: [CALENDAR_TAG, KANBAN_TAG] })).toEqual(['kanban', 'calendar']); + }); + + it('no card of the open library tags either today', () => { + for (const id of ['order_to_cash', 'service_desk', 'crm_account_with_contacts']) expect(taggedViewKinds(load(id).card)).toEqual([]); + }); +}); + +describe('planShots', () => { + it('an untagged card: list, record and dashboard — never a kanban or calendar the build happens to declare', () => { + const plan = planShots(orderCard(), orderBuild()); + expect(plan.app).toBe('demoorder_order_to_cash'); + expect(plan.shots).toEqual([ + { kind: 'list', file: 'screenshots/list.png', path: '/_console/apps/demoorder_order_to_cash/demoorder_account', object: 'demoorder_account' }, + { kind: 'record', file: 'screenshots/record.png', path: '/_console/apps/demoorder_order_to_cash/demoorder_account/record/a01', object: 'demoorder_account', row: 'a01' }, + // The card expects `receivables_aging_report`; that one is chosen over the build's first. + { kind: 'dashboard', file: 'screenshots/dashboard.png', path: '/_console/apps/demoorder_order_to_cash/dashboard/demoorder_receivables_aging_report', view: 'demoorder_receivables_aging_report' }, + ]); + expect(plan.skipped).toEqual([]); + }); + + it('a card tagging kanban and calendar gets a shot of each view of that type the build declares', () => { + const plan = planShots(orderCard([KANBAN_TAG, CALENDAR_TAG]), orderBuild()); + expect(plan.shots.map((s) => s.kind)).toEqual(['list', 'kanban', 'calendar', 'record', 'dashboard']); + expect(plan.shots.find((s) => s.kind === 'kanban')).toMatchObject({ path: '/_console/apps/demoorder_order_to_cash/demoorder_invoice/view/demoorder_invoice.board', view: 'demoorder_invoice.board' }); + expect(plan.shots.find((s) => s.kind === 'calendar')).toMatchObject({ object: 'demoorder_activity', view: 'demoorder_activity.schedule' }); + }); + + it('a tagged view the build does not declare, and a build with no dashboard, are skipped with the reason', () => { + const plan = planShots(orderCard([KANBAN_TAG]), orderBuild({ views: [], dashboards: [] })); + expect(plan.shots.map((s) => s.kind)).toEqual(['list', 'record']); + expect(plan.skipped).toEqual([ + { kind: 'kanban', reason: 'the card tags kanban, and the build declares no kanban view' }, + { kind: 'dashboard', reason: 'the build declares no dashboard' }, + ]); + }); + + it('a build with no app has nothing for the Console to open', () => { + const plan = planShots(orderCard(), orderBuild({ apps: [] })); + expect(plan.shots).toEqual([]); + expect(plan.skipped.map((s) => s.kind)).toEqual(['list', 'record', 'dashboard']); + }); + + it('reads a container as `os build` writes it: no `object` of its own, the binding in `list.data.object`', () => { + // The shape measured from a create-objectstack 18.3 project's dist/objectstack.json. + const built = { + list: { label: 'All accounts', type: 'grid', data: { provider: 'object', object: 'demoorder_account' }, columns: [{ field: 'name' }] }, + listViews: { board: { label: 'By industry', type: 'kanban', data: { provider: 'object', object: 'demoorder_account' }, columns: ['name'], kanban: { groupByField: 'industry' } } }, + }; + expect(declaredListViews(orderBuild({ views: [built] }))).toEqual([ + { object: 'demoorder_account', name: 'demoorder_account.board', type: 'kanban' }, + { object: 'demoorder_account', name: 'demoorder_account.default', type: 'grid' }, + ]); + }); + + it('reads a container through the spec composer: the default list is `.default`, a standalone view keeps its name', () => { + expect(declaredListViews(orderBuild({ views: [{ object: 'demoorder_invoice', list: { type: 'grid' } }, { name: 'demoorder_account.board', object: 'demoorder_account', type: 'kanban' }] }))).toEqual([ + { object: 'demoorder_invoice', name: 'demoorder_invoice.default', type: 'grid' }, + { object: 'demoorder_account', name: 'demoorder_account.board', type: 'kanban' }, + ]); + }); +}); + +describe('pageRefusal — what counts as the page the shot shows', () => { + it('a 2xx, or a 304 revalidation, with no not-found view is the page', () => { + for (const status of [200, 204, 304]) expect(pageRefusal({ status, notFound: [] })).toBeUndefined(); + }); + + it('no response, any other status, or one of the Console’s not-found views refuses it', () => { + expect(pageRefusal({ status: null, notFound: [] })).toBe('the navigation answered no response'); + for (const status of [301, 401, 403, 404, 500, 502]) expect(pageRefusal({ status, notFound: [] })).toBe(`the navigation answered HTTP ${status}`); + expect(pageRefusal({ status: 200, notFound: ['object-not-found', '"Page not found"'] })).toBe('the Console drew its not-found view (object-not-found, "Page not found")'); + }); + + it('the not-found views it looks for are the Console’s own', () => { + expect(CONSOLE_NOT_FOUND.testIds).toEqual(['object-not-found', 'object-package-disabled', 'record-page-not-found', 'form-not-found']); + expect(CONSOLE_NOT_FOUND.titles).toEqual(['Page not found']); + }); +}); + +describe('captureScreenshots — boot, seed, shoot, stop', () => { + /** `pages`: what the browser reads at a path (default a 200 with no not-found view); the double writes the file either way, as a real browser would have nothing to stop it. */ + function doubles(app: FakeApp, pages: Record = {}) { + const events: string[] = []; + const browser: ScreenshotBrowser = { + async open(baseUrl, admin) { + events.push(`open ${baseUrl} as ${admin.email}`); + return { + async capture(path, file): Promise { + events.push(`shoot ${path}`); + writeFileSync(file, 'png'); + const refusal = pageRefusal(pages[path] ?? { status: 200, notFound: [] }); + return refusal === undefined ? { shown: true } : { shown: false, reason: refusal }; + }, + async close() { + events.push('close'); + }, + }; + }, + }; + return { + events, + deps: { + boot: async (dir: string) => { + events.push(`boot ${dir}`); + return { url: 'http://localhost:4567', stop: async () => void events.push('stop'), log: () => '' }; + }, + harness: () => app, + browser, + admin: { email: 'admin@objectos.ai', password: 'admin123' }, + }, + }; + } + + it('boots the project, seeds it as the admin, takes every planned shot, then closes and stops', async () => { + const app = new FakeApp(); + const { events, deps } = doubles(app); + const dir = tempDir('shots'); + const r = await captureScreenshots({ scenario: orderCard(), projectDir: '/work/order-to-cash-scratch/demoorder', artifact: orderBuild(), dir }, deps); + expect(events).toEqual([ + 'boot /work/order-to-cash-scratch/demoorder', + 'open http://localhost:4567 as admin@objectos.ai', + 'shoot /_console/apps/demoorder_order_to_cash/demoorder_account', + 'shoot /_console/apps/demoorder_order_to_cash/demoorder_account/record/a01', + 'shoot /_console/apps/demoorder_order_to_cash/dashboard/demoorder_receivables_aging_report', + 'close', + 'stop', + ]); + expect(app.signedIn).toBe(1); + expect(app.rows.demoorder_account!.map((row) => row.id)).toContain('a01'); + expect(r.seed!.refused).toEqual([]); + for (const s of r.shots) expect(existsSync(join(dir, s.file))).toBe(true); + }); + + it('a record shot whose seed row the app refused is skipped with the reason', async () => { + const app = new FakeApp(); + app.refuse.add('demoorder_account/a01'); + const { deps } = doubles(app); + const r = await captureScreenshots({ scenario: orderCard(), projectDir: '/p', artifact: orderBuild(), dir: tempDir('shots') }, deps); + expect(r.shots.map((s) => s.kind)).toEqual(['list', 'dashboard']); + expect(r.skipped).toEqual([{ kind: 'record', reason: 'the app refused seed row demoorder_account a01' }]); + }); + + it('a browser that writes no file is an error, and the app is still stopped', async () => { + const app = new FakeApp(); + const { events, deps } = doubles(app); + deps.browser = { open: async () => ({ capture: async () => ({ shown: true }), close: async () => void events.push('close') }) }; + await expect(captureScreenshots({ scenario: orderCard(), projectDir: '/p', artifact: orderBuild(), dir: tempDir('shots') }, deps)).rejects.toThrow(/the browser wrote no screenshot for list/); + expect(events.slice(-2)).toEqual(['close', 'stop']); + }); + + it('⛔ a page the Console did not show is no proof: a non-2xx or a not-found view is skipped with its reason, and its file is gone', async () => { + const app = new FakeApp(); + const base = '/_console/apps/demoorder_order_to_cash'; + const { events, deps } = doubles(app, { + [`${base}/demoorder_account`]: { status: 500, notFound: [] }, + [`${base}/demoorder_account/record/a01`]: { status: 200, notFound: ['record-page-not-found'] }, + }); + const dir = tempDir('shots'); + const r = await captureScreenshots({ scenario: orderCard(), projectDir: '/p', artifact: orderBuild(), dir }, deps); + expect(r.shots.map((s) => s.kind)).toEqual(['dashboard']); + expect(r.skipped).toEqual([ + { kind: 'list', reason: `the Console did not show ${base}/demoorder_account: the navigation answered HTTP 500` }, + { kind: 'record', reason: `the Console did not show ${base}/demoorder_account/record/a01: the Console drew its not-found view (record-page-not-found)` }, + ]); + expect(existsSync(join(dir, 'screenshots/list.png'))).toBe(false); + expect(existsSync(join(dir, 'screenshots/record.png'))).toBe(false); + expect(existsSync(join(dir, 'screenshots/dashboard.png'))).toBe(true); + expect(events.slice(-2)).toEqual(['close', 'stop']); + }); + + it('nothing planned: nothing is booted', async () => { + const { events, deps } = doubles(new FakeApp()); + const r = await captureScreenshots({ scenario: orderCard(), projectDir: '/p', artifact: orderBuild({ apps: [] }), dir: tempDir('shots') }, deps); + expect(r.shots).toEqual([]); + expect(events).toEqual([]); + }); +}); diff --git a/packages/qa/scenarios/test/runner/orchestrate.test.ts b/packages/qa/scenarios/test/runner/orchestrate.test.ts index 16ad9f8f2a5..8c9937b69fb 100644 --- a/packages/qa/scenarios/test/runner/orchestrate.test.ts +++ b/packages/qa/scenarios/test/runner/orchestrate.test.ts @@ -437,6 +437,28 @@ class RefusingApp extends FakeApp { } } +class ThrowingApp extends RefusingApp { + constructor(private readonly throwAt: string) { + super(new Set(['democrm_account/ac2'])); + } + override async insertRow(object: string, row: Record): Promise { + if (`${object}/${String(row.id)}` === this.throwAt) throw new Error('connection reset while seeding'); + return super.insertRow(object, row); + } +} + +describe('runCard — a seed insert that throws keeps the partial tally in the record', () => { + it('ac1 and ac3 inserted, ac2 refused, then ct1 throws: the half records exactly that next to its error', async () => { + const { scenario, ops, driver } = setup({ app: () => new ThrowingApp('democrm_contact/ct1') }); + const run = await runCard(scenario, driver, ops, { modes: ['scratch'], admin: ADMIN, iterateRun: false }); + const half = run.modes[0]!.run!; + expect(half.error).toContain('connection reset while seeding'); + expect(half.seed).toEqual({ inserted: 2, refused: [{ object: 'democrm_account', id: 'ac2', code: 'VALIDATION_FAILED', message: 'democrm_account ac2 refused' }], handlesAsAdmin: ['u_tia'] }); + expect(half.questions).toEqual([]); + expect(cardScorecard(run).modes[0]!.first.seed).toEqual({ inserted: 2, total: 3, refused: 1 }); // total counts the rows the tally reached + }); +}); + describe('runCard — a refused seed row makes what reads it not measured, never a fail', () => { it('shows ⚠ seed: 1 refused, leaves the questions on that object unasked and unscored, and grades the rest as usual', async () => { const { scenario, ops, driver } = setup({ app: () => new RefusingApp(new Set(['democrm_contact/ct4'])) }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3c6519037e4..2f4797c3091 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2493,6 +2493,9 @@ importers: '@objectstack/spec': specifier: workspace:* version: link:../../spec + playwright-core: + specifier: 1.63.0 + version: 1.63.0 zod: specifier: ^4.6.6 version: 4.6.7