feat: publish native iOS and macOS benchmark pilots

scripts/agent-benchmark/NATIVE.md

@@ -64,3 +64,22 @@ comparison, not a broad performance claim. Native Xcode projects can benefit
 from Stim's reusable artifacts and compiler cache. Native Gradle and macOS
 SwiftPM do not have reusable Stim artifact caching; measure any benefit in
 workspace/device preparation, incremental compilation, launch, and verification.
+
+## Website publication
+
+Publish only paired runs with accepted independent audits and completed cleanup:
+
+```bash
+node scripts/export-native-benchmarks.mjs "$STIM_BENCH_ROOT" website/src/data/benchmarks/native-luna-1.20.2.json website/static/benchmarks/native-luna-1.20.2
+```
+
+Each platform's `comparison.json` selects its Stim and control run IDs. The
+exporter verifies matching fixture, release, model, and environment metadata;
+turn timestamps; required audit checks; and all retained evidence hashes. A
+platform without a comparison is omitted. A selected rejected or incomplete
+pair refuses publication before writing output. Review every exported proof
+image for private content before committing it.
+
+The benchmark overview displays these pilots separately as total task time,
+including retries and evidence capture. Keep the one-pair limitation visible.
+Native Android joins that section only after an accepted comparison is exported.

scripts/export-native-benchmarks.mjs

@@ -0,0 +1,188 @@
+import { createHash } from 'node:crypto';
+import { copyFileSync, existsSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs';
+import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const digest = (path) => createHash('sha256').update(readFileSync(path)).digest('hex');
+const readJson = (path) => JSON.parse(readFileSync(path, 'utf8'));
+const platforms = ['ios', 'android', 'macos'];
+const requiredChecks = [
+  'sourceExact',
+  'runnerCompleted',
+  'buildCompleted',
+  'commandAudit',
+  'processUnchanged',
+  'revisionAndCountVisuallyVerified',
+];
+const requiredEvidence = [
+  'events.jsonl',
+  'commands.json',
+  'source.diff',
+  'proof/target.json',
+  'proof/screen.png',
+  'proof/coordinator-snapshot.txt',
+  'proof/coordinator-process-before.json',
+  'proof/coordinator-process-after.json',
+];
+
+function acceptedRun(directory, platform, arm) {
+  const meta = readJson(join(directory, 'meta.json'));
+  const record = readJson(join(directory, 'run.json'));
+  const audit = readJson(join(directory, 'audit.json'));
+  const cleanup = readJson(join(directory, 'cleanup.json'));
+  if (
+    record.cohort !== 'standalone-native-v1' ||
+    meta.cohort !== record.cohort ||
+    record.runId !== meta.runId ||
+    record.runId !== basename(directory) ||
+    record.platform !== platform ||
+    meta.platform !== platform ||
+    record.arm !== arm ||
+    meta.arm !== arm ||
+    record.source !== true ||
+    record.invalidReasons?.length !== 0 ||
+    record.runnerResult?.code !== 0 ||
+    record.runnerResult?.timedOut !== false ||
+    meta.runnerResult?.code !== 0 ||
+    meta.runnerResult?.timedOut !== false ||
+    audit.status !== 'accepted' ||
+    audit.valid !== true ||
+    cleanup.status !== 'complete' ||
+    requiredChecks.some((name) => audit.checks?.[name] !== true)
+  ) {
+    throw new Error(`Native ${platform} ${arm} needs an accepted audit and completed cleanup`);
+  }
+  const seconds = (Date.parse(meta.finishedAt) - Date.parse(meta.dispatchAt)) / 1000;
+  if (
+    !Number.isFinite(seconds) ||
+    seconds <= 0 ||
+    Math.abs(record.elapsedSeconds - seconds) > 0.001 ||
+    !Number.isFinite(record.elapsedSeconds)
+  )
+    throw new Error(`Native ${platform} ${arm} timing does not match its recorded turn`);
+  if (requiredEvidence.some((name) => !audit.hashes?.[name])) throw new Error('Native audit evidence is incomplete');
+  const root = realpathSync(directory);
+  for (const [name, expected] of Object.entries(audit.hashes)) {
+    const path = realpathSync(join(root, name));
+    const rel = relative(root, path);
+    if (
+      isAbsolute(name) ||
+      rel === '..' ||
+      rel.startsWith(`..${sep}`) ||
+      isAbsolute(rel) ||
+      !/^[a-f0-9]{64}$/.test(expected) ||
+      digest(path) !== expected
+    )
+      throw new Error(`Native audit evidence changed: ${name}`);
+  }
+  if (
+    record.eventsSha256 !== audit.hashes['events.jsonl'] ||
+    record.screenshotSha256 !== audit.hashes['proof/screen.png']
+  )
+    throw new Error('Native run evidence does not match its audit');
+  return { directory, meta, record, audit, seconds };
+}
+
+export function exportNativeBenchmarks(campaignRoot, outputPath, proofDirectory) {
+  const pairs = [];
+  const copies = [];
+  for (const platform of platforms) {
+    const comparisonPath = join(campaignRoot, 'native', platform, 'comparison.json');
+    if (!existsSync(comparisonPath)) continue;
+    const comparison = readJson(comparisonPath);
+    const runs = ['stim', 'control'].map((arm) => {
+      const id = comparison.results?.[arm]?.runId;
+      if (typeof id !== 'string' || !/^[a-zA-Z0-9_-]+$/.test(id))
+        throw new Error(`Missing native ${platform} ${arm} run`);
+      return acceptedRun(join(campaignRoot, 'native', platform, 'results', id), platform, arm);
+    });
+    const [stim, control] = runs;
+    for (const key of ['model', 'reasoningEffort', 'serviceTier']) {
+      if (typeof stim.meta[key] !== 'string' || stim.meta[key] !== control.meta[key])
+        throw new Error(`Native pair differs in ${key}`);
+    }
+    for (const key of [
+      'fixtureCommit',
+      'stimVersion',
+      'stimCliSha256',
+      'nodeVersion',
+      'macosVersion',
+      'macosBuild',
+      'xcodeVersion',
+      'xcodeBuild',
+      'codexVersion',
+    ]) {
+      if (
+        typeof stim.meta.provenance?.[key] !== 'string' ||
+        stim.meta.provenance[key] !== control.meta.provenance?.[key]
+      )
+        throw new Error(`Native pair differs in ${key}`);
+    }
+    if (
+      stim.meta.fixture.framework !== control.meta.fixture.framework ||
+      stim.meta.compatibility.manifestSha256 !== control.meta.compatibility.manifestSha256
+    )
+      throw new Error('Native pair uses different fixture or compatibility tools');
+    pairs.push({
+      platform,
+      backend: stim.meta.fixture.framework,
+      model: stim.meta.model,
+      reasoningEffort: stim.meta.reasoningEffort,
+      requestedServiceTier: stim.meta.serviceTier,
+      returnedServiceTier: 'unverified',
+      samplePairs: 1,
+      provenance: Object.fromEntries(
+        [
+          'stimVersion',
+          'nodeVersion',
+          'macosVersion',
+          'macosBuild',
+          'xcodeVersion',
+          'xcodeBuild',
+          'codexVersion',
+          'fixtureCommit',
+        ].map((key) => [key, stim.meta.provenance[key]]),
+      ),
+      timeReductionPercent: 100 * (1 - stim.seconds / control.seconds),
+      runs: runs.map(({ directory, meta, record, audit, seconds }) => {
+        const imageName = `${platform}-${record.arm}.png`;
+        copies.push([join(directory, 'proof/screen.png'), join(proofDirectory, imageName)]);
+        return {
+          arm: record.arm,
+          recordedOn: meta.dispatchAt.slice(0, 10),
+          taskSeconds: seconds,
+          usage: record.usage ?? null,
+          proof: imageName,
+          evidence: {
+            recordSha256: digest(join(directory, 'run.json')),
+            metadataSha256: digest(join(directory, 'meta.json')),
+            auditSha256: digest(join(directory, 'audit.json')),
+            cleanupSha256: digest(join(directory, 'cleanup.json')),
+            eventsSha256: audit.hashes['events.jsonl'],
+            proofSha256: audit.hashes['proof/screen.png'],
+          },
+        };
+      }),
+    });
+  }
+  if (!pairs.length) throw new Error('No accepted native benchmark pairs');
+  const result = {
+    schemaVersion: 1,
+    cohort: 'standalone-native-v1',
+    metric: 'dispatch-to-completed-task',
+    machine: readJson(join(campaignRoot, 'targets.json')).machine,
+    pairs,
+  };
+  mkdirSync(proofDirectory, { recursive: true });
+  mkdirSync(dirname(outputPath), { recursive: true });
+  for (const [source, target] of copies) copyFileSync(source, target);
+  writeFileSync(outputPath, `${JSON.stringify(result, null, 2)}\n`);
+  return result;
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  const [campaignRoot, outputPath, proofDirectory] = process.argv.slice(2);
+  if (!campaignRoot || !outputPath || !proofDirectory)
+    throw new Error('Usage: export-native-benchmarks.mjs <campaign-root> <output-json> <proof-directory>');
+  exportNativeBenchmarks(campaignRoot, outputPath, proofDirectory);
+}

scripts/export-native-benchmarks.test.mjs

@@ -0,0 +1,140 @@
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { afterEach, beforeEach, expect, it } from 'vitest';
+import { exportNativeBenchmarks } from './export-native-benchmarks.mjs';
+
+let root;
+const write = (path, value) => {
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, JSON.stringify(value));
+};
+const hash = (path) => createHash('sha256').update(readFileSync(path)).digest('hex');
+const files = [
+  'events.jsonl',
+  'commands.json',
+  'source.diff',
+  'proof/target.json',
+  'proof/screen.png',
+  'proof/coordinator-snapshot.txt',
+  'proof/coordinator-process-before.json',
+  'proof/coordinator-process-after.json',
+];
+
+function pair(platform) {
+  for (const arm of ['stim', 'control']) {
+    const id = `${platform}-${arm}`;
+    const dir = join(root, 'native', platform, 'results', id);
+    for (const name of files) {
+      mkdirSync(dirname(join(dir, name)), { recursive: true });
+      writeFileSync(join(dir, name), `${id}: ${name}`);
+    }
+    const runnerResult = { code: 0, timedOut: false };
+    write(join(dir, 'meta.json'), {
+      cohort: 'standalone-native-v1',
+      runId: id,
+      platform,
+      arm,
+      model: 'gpt-5.6-luna',
+      reasoningEffort: 'high',
+      serviceTier: 'priority',
+      dispatchAt: '2026-10-13T00:00:00.000Z',
+      finishedAt: `2026-10-13T00:0${arm === 'stim' ? 1 : 2}:00.000Z`,
+      runnerResult,
+      fixture: { framework: `native-${platform}` },
+      compatibility: { manifestSha256: 'compat' },
+      provenance: Object.fromEntries(
+        [
+          'fixtureCommit',
+          'stimVersion',
+          'stimCliSha256',
+          'nodeVersion',
+          'macosVersion',
+          'macosBuild',
+          'xcodeVersion',
+          'xcodeBuild',
+          'codexVersion',
+        ].map((key) => [key, 'same']),
+      ),
+    });
+    write(join(dir, 'run.json'), {
+      cohort: 'standalone-native-v1',
+      runId: id,
+      platform,
+      arm,
+      source: true,
+      invalidReasons: [],
+      runnerResult,
+      elapsedSeconds: arm === 'stim' ? 60 : 120,
+      eventsSha256: hash(join(dir, 'events.jsonl')),
+      screenshotSha256: hash(join(dir, 'proof/screen.png')),
+    });
+    write(join(dir, 'audit.json'), {
+      status: 'accepted',
+      valid: true,
+      checks: Object.fromEntries(
+        [
+          'sourceExact',
+          'runnerCompleted',
+          'buildCompleted',
+          'commandAudit',
+          'processUnchanged',
+          'revisionAndCountVisuallyVerified',
+        ].map((key) => [key, true]),
+      ),
+      hashes: Object.fromEntries(files.map((name) => [name, hash(join(dir, name))])),
+    });
+    write(join(dir, 'cleanup.json'), { status: 'complete' });
+  }
+  write(join(root, 'native', platform, 'comparison.json'), {
+    results: { stim: { runId: `${platform}-stim` }, control: { runId: `${platform}-control` } },
+  });
+}
+
+beforeEach(() => {
+  root = mkdtempSync(join(tmpdir(), 'stim-native-export-'));
+  write(join(root, 'targets.json'), { machine: 'Test machine' });
+  pair('ios');
+});
+afterEach(() => rmSync(root, { recursive: true, force: true }));
+const publish = () => exportNativeBenchmarks(root, join(root, 'published/data.json'), join(root, 'published/proof'));
+
+it('publishes accepted native pairs without local paths and adds Android only when a pair exists', () => {
+  pair('macos');
+  const first = publish();
+  expect(first.pairs.map((entry) => entry.platform)).toEqual(['ios', 'macos']);
+  expect(first.pairs[0]).toMatchObject({
+    timeReductionPercent: 50,
+    samplePairs: 1,
+    runs: [{ taskSeconds: 60 }, { taskSeconds: 120 }],
+  });
+  expect(JSON.stringify(first)).not.toContain(root);
+  expect(hash(join(root, 'published/proof/ios-stim.png'))).toBe(first.pairs[0].runs[0].evidence.proofSha256);
+  pair('android');
+  expect(publish().pairs.map((entry) => entry.platform)).toEqual(['ios', 'android', 'macos']);
+});
+
+it.each(['proof', 'missing evidence', 'audit', 'cleanup', 'timing', 'model', 'version', 'identity', 'timeout'])(
+  'refuses %s problems before publishing any partial comparison',
+  (failure) => {
+    const dir = join(root, 'native/ios/results/ios-control');
+    const update = (name, change) => {
+      const path = join(dir, name);
+      const value = JSON.parse(readFileSync(path));
+      change(value);
+      write(path, value);
+    };
+    if (failure === 'proof') writeFileSync(join(dir, 'proof/screen.png'), 'changed');
+    if (failure === 'missing evidence') update('audit.json', (value) => delete value.hashes['source.diff']);
+    if (failure === 'audit') update('audit.json', (value) => (value.valid = false));
+    if (failure === 'cleanup') update('cleanup.json', (value) => (value.status = 'failed'));
+    if (failure === 'timing') update('run.json', (value) => (value.elapsedSeconds = 1));
+    if (failure === 'model') update('meta.json', (value) => (value.model = 'different'));
+    if (failure === 'version') update('meta.json', (value) => (value.provenance.stimVersion = 'different'));
+    if (failure === 'identity') update('audit.json', (value) => (value.checks.processUnchanged = false));
+    if (failure === 'timeout') update('run.json', (value) => (value.runnerResult.timedOut = true));
+    expect(publish).toThrow(/Native/);
+    expect(existsSync(join(root, 'published'))).toBe(false);
+  },
+);