diff --git a/scripts/agent-benchmark/NATIVE.md b/scripts/agent-benchmark/NATIVE.md index ba2444d3d..e7fbde360 100644 --- a/scripts/agent-benchmark/NATIVE.md +++ b/scripts/agent-benchmark/NATIVE.md @@ -64,3 +64,22 @@ comparison, not a broad performance claim. Native Xcode projects can benefit from Stim's reusable artifacts and compiler cache. Native Gradle and macOS SwiftPM do not have reusable Stim artifact caching; measure any benefit in workspace/device preparation, incremental compilation, launch, and verification. + +## Website publication + +Publish only paired runs with accepted independent audits and completed cleanup: + +```bash +node scripts/export-native-benchmarks.mjs "$STIM_BENCH_ROOT" website/src/data/benchmarks/native-luna-1.20.2.json website/static/benchmarks/native-luna-1.20.2 +``` + +Each platform's `comparison.json` selects its Stim and control run IDs. The +exporter verifies matching fixture, release, model, and environment metadata; +turn timestamps; required audit checks; and all retained evidence hashes. A +platform without a comparison is omitted. A selected rejected or incomplete +pair refuses publication before writing output. Review every exported proof +image for private content before committing it. + +The benchmark overview displays these pilots separately as total task time, +including retries and evidence capture. Keep the one-pair limitation visible. +Native Android joins that section only after an accepted comparison is exported. diff --git a/scripts/export-native-benchmarks.mjs b/scripts/export-native-benchmarks.mjs new file mode 100644 index 000000000..d10f20da0 --- /dev/null +++ b/scripts/export-native-benchmarks.mjs @@ -0,0 +1,188 @@ +import { createHash } from 'node:crypto'; +import { copyFileSync, existsSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs'; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const digest = (path) => createHash('sha256').update(readFileSync(path)).digest('hex'); +const readJson = (path) => JSON.parse(readFileSync(path, 'utf8')); +const platforms = ['ios', 'android', 'macos']; +const requiredChecks = [ + 'sourceExact', + 'runnerCompleted', + 'buildCompleted', + 'commandAudit', + 'processUnchanged', + 'revisionAndCountVisuallyVerified', +]; +const requiredEvidence = [ + 'events.jsonl', + 'commands.json', + 'source.diff', + 'proof/target.json', + 'proof/screen.png', + 'proof/coordinator-snapshot.txt', + 'proof/coordinator-process-before.json', + 'proof/coordinator-process-after.json', +]; + +function acceptedRun(directory, platform, arm) { + const meta = readJson(join(directory, 'meta.json')); + const record = readJson(join(directory, 'run.json')); + const audit = readJson(join(directory, 'audit.json')); + const cleanup = readJson(join(directory, 'cleanup.json')); + if ( + record.cohort !== 'standalone-native-v1' || + meta.cohort !== record.cohort || + record.runId !== meta.runId || + record.runId !== basename(directory) || + record.platform !== platform || + meta.platform !== platform || + record.arm !== arm || + meta.arm !== arm || + record.source !== true || + record.invalidReasons?.length !== 0 || + record.runnerResult?.code !== 0 || + record.runnerResult?.timedOut !== false || + meta.runnerResult?.code !== 0 || + meta.runnerResult?.timedOut !== false || + audit.status !== 'accepted' || + audit.valid !== true || + cleanup.status !== 'complete' || + requiredChecks.some((name) => audit.checks?.[name] !== true) + ) { + throw new Error(`Native ${platform} ${arm} needs an accepted audit and completed cleanup`); + } + const seconds = (Date.parse(meta.finishedAt) - Date.parse(meta.dispatchAt)) / 1000; + if ( + !Number.isFinite(seconds) || + seconds <= 0 || + Math.abs(record.elapsedSeconds - seconds) > 0.001 || + !Number.isFinite(record.elapsedSeconds) + ) + throw new Error(`Native ${platform} ${arm} timing does not match its recorded turn`); + if (requiredEvidence.some((name) => !audit.hashes?.[name])) throw new Error('Native audit evidence is incomplete'); + const root = realpathSync(directory); + for (const [name, expected] of Object.entries(audit.hashes)) { + const path = realpathSync(join(root, name)); + const rel = relative(root, path); + if ( + isAbsolute(name) || + rel === '..' || + rel.startsWith(`..${sep}`) || + isAbsolute(rel) || + !/^[a-f0-9]{64}$/.test(expected) || + digest(path) !== expected + ) + throw new Error(`Native audit evidence changed: ${name}`); + } + if ( + record.eventsSha256 !== audit.hashes['events.jsonl'] || + record.screenshotSha256 !== audit.hashes['proof/screen.png'] + ) + throw new Error('Native run evidence does not match its audit'); + return { directory, meta, record, audit, seconds }; +} + +export function exportNativeBenchmarks(campaignRoot, outputPath, proofDirectory) { + const pairs = []; + const copies = []; + for (const platform of platforms) { + const comparisonPath = join(campaignRoot, 'native', platform, 'comparison.json'); + if (!existsSync(comparisonPath)) continue; + const comparison = readJson(comparisonPath); + const runs = ['stim', 'control'].map((arm) => { + const id = comparison.results?.[arm]?.runId; + if (typeof id !== 'string' || !/^[a-zA-Z0-9_-]+$/.test(id)) + throw new Error(`Missing native ${platform} ${arm} run`); + return acceptedRun(join(campaignRoot, 'native', platform, 'results', id), platform, arm); + }); + const [stim, control] = runs; + for (const key of ['model', 'reasoningEffort', 'serviceTier']) { + if (typeof stim.meta[key] !== 'string' || stim.meta[key] !== control.meta[key]) + throw new Error(`Native pair differs in ${key}`); + } + for (const key of [ + 'fixtureCommit', + 'stimVersion', + 'stimCliSha256', + 'nodeVersion', + 'macosVersion', + 'macosBuild', + 'xcodeVersion', + 'xcodeBuild', + 'codexVersion', + ]) { + if ( + typeof stim.meta.provenance?.[key] !== 'string' || + stim.meta.provenance[key] !== control.meta.provenance?.[key] + ) + throw new Error(`Native pair differs in ${key}`); + } + if ( + stim.meta.fixture.framework !== control.meta.fixture.framework || + stim.meta.compatibility.manifestSha256 !== control.meta.compatibility.manifestSha256 + ) + throw new Error('Native pair uses different fixture or compatibility tools'); + pairs.push({ + platform, + backend: stim.meta.fixture.framework, + model: stim.meta.model, + reasoningEffort: stim.meta.reasoningEffort, + requestedServiceTier: stim.meta.serviceTier, + returnedServiceTier: 'unverified', + samplePairs: 1, + provenance: Object.fromEntries( + [ + 'stimVersion', + 'nodeVersion', + 'macosVersion', + 'macosBuild', + 'xcodeVersion', + 'xcodeBuild', + 'codexVersion', + 'fixtureCommit', + ].map((key) => [key, stim.meta.provenance[key]]), + ), + timeReductionPercent: 100 * (1 - stim.seconds / control.seconds), + runs: runs.map(({ directory, meta, record, audit, seconds }) => { + const imageName = `${platform}-${record.arm}.png`; + copies.push([join(directory, 'proof/screen.png'), join(proofDirectory, imageName)]); + return { + arm: record.arm, + recordedOn: meta.dispatchAt.slice(0, 10), + taskSeconds: seconds, + usage: record.usage ?? null, + proof: imageName, + evidence: { + recordSha256: digest(join(directory, 'run.json')), + metadataSha256: digest(join(directory, 'meta.json')), + auditSha256: digest(join(directory, 'audit.json')), + cleanupSha256: digest(join(directory, 'cleanup.json')), + eventsSha256: audit.hashes['events.jsonl'], + proofSha256: audit.hashes['proof/screen.png'], + }, + }; + }), + }); + } + if (!pairs.length) throw new Error('No accepted native benchmark pairs'); + const result = { + schemaVersion: 1, + cohort: 'standalone-native-v1', + metric: 'dispatch-to-completed-task', + machine: readJson(join(campaignRoot, 'targets.json')).machine, + pairs, + }; + mkdirSync(proofDirectory, { recursive: true }); + mkdirSync(dirname(outputPath), { recursive: true }); + for (const [source, target] of copies) copyFileSync(source, target); + writeFileSync(outputPath, `${JSON.stringify(result, null, 2)}\n`); + return result; +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const [campaignRoot, outputPath, proofDirectory] = process.argv.slice(2); + if (!campaignRoot || !outputPath || !proofDirectory) + throw new Error('Usage: export-native-benchmarks.mjs '); + exportNativeBenchmarks(campaignRoot, outputPath, proofDirectory); +} diff --git a/scripts/export-native-benchmarks.test.mjs b/scripts/export-native-benchmarks.test.mjs new file mode 100644 index 000000000..9f719c992 --- /dev/null +++ b/scripts/export-native-benchmarks.test.mjs @@ -0,0 +1,140 @@ +import { createHash } from 'node:crypto'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { afterEach, beforeEach, expect, it } from 'vitest'; +import { exportNativeBenchmarks } from './export-native-benchmarks.mjs'; + +let root; +const write = (path, value) => { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, JSON.stringify(value)); +}; +const hash = (path) => createHash('sha256').update(readFileSync(path)).digest('hex'); +const files = [ + 'events.jsonl', + 'commands.json', + 'source.diff', + 'proof/target.json', + 'proof/screen.png', + 'proof/coordinator-snapshot.txt', + 'proof/coordinator-process-before.json', + 'proof/coordinator-process-after.json', +]; + +function pair(platform) { + for (const arm of ['stim', 'control']) { + const id = `${platform}-${arm}`; + const dir = join(root, 'native', platform, 'results', id); + for (const name of files) { + mkdirSync(dirname(join(dir, name)), { recursive: true }); + writeFileSync(join(dir, name), `${id}: ${name}`); + } + const runnerResult = { code: 0, timedOut: false }; + write(join(dir, 'meta.json'), { + cohort: 'standalone-native-v1', + runId: id, + platform, + arm, + model: 'gpt-5.6-luna', + reasoningEffort: 'high', + serviceTier: 'priority', + dispatchAt: '2026-10-13T00:00:00.000Z', + finishedAt: `2026-10-13T00:0${arm === 'stim' ? 1 : 2}:00.000Z`, + runnerResult, + fixture: { framework: `native-${platform}` }, + compatibility: { manifestSha256: 'compat' }, + provenance: Object.fromEntries( + [ + 'fixtureCommit', + 'stimVersion', + 'stimCliSha256', + 'nodeVersion', + 'macosVersion', + 'macosBuild', + 'xcodeVersion', + 'xcodeBuild', + 'codexVersion', + ].map((key) => [key, 'same']), + ), + }); + write(join(dir, 'run.json'), { + cohort: 'standalone-native-v1', + runId: id, + platform, + arm, + source: true, + invalidReasons: [], + runnerResult, + elapsedSeconds: arm === 'stim' ? 60 : 120, + eventsSha256: hash(join(dir, 'events.jsonl')), + screenshotSha256: hash(join(dir, 'proof/screen.png')), + }); + write(join(dir, 'audit.json'), { + status: 'accepted', + valid: true, + checks: Object.fromEntries( + [ + 'sourceExact', + 'runnerCompleted', + 'buildCompleted', + 'commandAudit', + 'processUnchanged', + 'revisionAndCountVisuallyVerified', + ].map((key) => [key, true]), + ), + hashes: Object.fromEntries(files.map((name) => [name, hash(join(dir, name))])), + }); + write(join(dir, 'cleanup.json'), { status: 'complete' }); + } + write(join(root, 'native', platform, 'comparison.json'), { + results: { stim: { runId: `${platform}-stim` }, control: { runId: `${platform}-control` } }, + }); +} + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'stim-native-export-')); + write(join(root, 'targets.json'), { machine: 'Test machine' }); + pair('ios'); +}); +afterEach(() => rmSync(root, { recursive: true, force: true })); +const publish = () => exportNativeBenchmarks(root, join(root, 'published/data.json'), join(root, 'published/proof')); + +it('publishes accepted native pairs without local paths and adds Android only when a pair exists', () => { + pair('macos'); + const first = publish(); + expect(first.pairs.map((entry) => entry.platform)).toEqual(['ios', 'macos']); + expect(first.pairs[0]).toMatchObject({ + timeReductionPercent: 50, + samplePairs: 1, + runs: [{ taskSeconds: 60 }, { taskSeconds: 120 }], + }); + expect(JSON.stringify(first)).not.toContain(root); + expect(hash(join(root, 'published/proof/ios-stim.png'))).toBe(first.pairs[0].runs[0].evidence.proofSha256); + pair('android'); + expect(publish().pairs.map((entry) => entry.platform)).toEqual(['ios', 'android', 'macos']); +}); + +it.each(['proof', 'missing evidence', 'audit', 'cleanup', 'timing', 'model', 'version', 'identity', 'timeout'])( + 'refuses %s problems before publishing any partial comparison', + (failure) => { + const dir = join(root, 'native/ios/results/ios-control'); + const update = (name, change) => { + const path = join(dir, name); + const value = JSON.parse(readFileSync(path)); + change(value); + write(path, value); + }; + if (failure === 'proof') writeFileSync(join(dir, 'proof/screen.png'), 'changed'); + if (failure === 'missing evidence') update('audit.json', (value) => delete value.hashes['source.diff']); + if (failure === 'audit') update('audit.json', (value) => (value.valid = false)); + if (failure === 'cleanup') update('cleanup.json', (value) => (value.status = 'failed')); + if (failure === 'timing') update('run.json', (value) => (value.elapsedSeconds = 1)); + if (failure === 'model') update('meta.json', (value) => (value.model = 'different')); + if (failure === 'version') update('meta.json', (value) => (value.provenance.stimVersion = 'different')); + if (failure === 'identity') update('audit.json', (value) => (value.checks.processUnchanged = false)); + if (failure === 'timeout') update('run.json', (value) => (value.runnerResult.timedOut = true)); + expect(publish).toThrow(/Native/); + expect(existsSync(join(root, 'published'))).toBe(false); + }, +); diff --git a/website/src/components/NativeBenchmarkResults.module.css b/website/src/components/NativeBenchmarkResults.module.css new file mode 100644 index 000000000..b22852a72 --- /dev/null +++ b/website/src/components/NativeBenchmarkResults.module.css @@ -0,0 +1,113 @@ +.section { + margin: 3rem 0; +} +.heading { + max-width: 58rem; +} +.eyebrow { + color: var(--ifm-color-primary); + font-size: 0.8rem; + font-weight: 700; +} +.heading h2 { + margin-top: 0.5rem; +} +.grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 1rem; + margin: 1.5rem 0; +} +.card { + padding: 1.4rem; + border: 1px solid var(--ifm-color-emphasis-300); + border-radius: 1rem; + background: var(--ifm-background-surface-color); + min-width: 0; +} +.cardHeading { + display: flex; + justify-content: space-between; + gap: 1rem; + align-items: flex-start; +} +.cardHeading h3 { + margin-bottom: 0.2rem; +} +.cardHeading > strong { + color: var(--ifm-color-primary); + white-space: nowrap; +} +.cardHeading span, +.caption, +.note { + color: var(--ifm-color-emphasis-700); +} +.caption { + font-size: 0.85rem; + margin: 1rem 0; +} +.row { + display: grid; + grid-template-columns: 3.6rem minmax(0, 1fr) 4.6rem; + gap: 0.7rem; + align-items: center; + margin: 0.7rem 0; + font-size: 0.9rem; +} +.row > strong { + text-align: right; +} +.track { + height: 0.8rem; + background: var(--ifm-color-emphasis-100); + border-radius: 1rem; + overflow: hidden; +} +.stim, +.control { + height: 100%; + border-radius: inherit; +} +.stim { + background: var(--ifm-color-primary); +} +.control { + background: var(--ifm-color-emphasis-500); +} +.card summary { + cursor: pointer; + color: var(--ifm-color-primary); + font-weight: 600; +} +.card details p { + font-size: 0.85rem; + margin-top: 1rem; +} +.proofs { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 0.7rem; +} +.proofs figure { + margin: 0; + min-width: 0; +} +.proofs img { + width: 100%; + height: 16rem; + object-fit: contain; + border: 1px solid var(--ifm-color-emphasis-300); + border-radius: 0.5rem; + background: var(--ifm-color-emphasis-100); +} +.proofs figcaption { + font-size: 0.8rem; + text-align: center; + margin-top: 0.4rem; +} +@media (max-width: 760px) { + .grid { + grid-template-columns: 1fr; + } +} diff --git a/website/src/components/NativeBenchmarkResults.test.ts b/website/src/components/NativeBenchmarkResults.test.ts new file mode 100644 index 000000000..d688c9576 --- /dev/null +++ b/website/src/components/NativeBenchmarkResults.test.ts @@ -0,0 +1,19 @@ +import { createElement } from 'react'; +import { renderToStaticMarkup } from 'react-dom/server'; +import { expect, it, vi } from 'vitest'; +import NativeBenchmarkResults from './NativeBenchmarkResults'; +import data from '../data/benchmarks/native-luna-1.20.2.json'; +vi.mock('@docusaurus/useBaseUrl', () => ({ default: (path: string) => path })); + +it('keeps native task time, pilot scope, and verified proof separate from readiness results', () => { + const html = renderToStaticMarkup(createElement(NativeBenchmarkResults)); + expect(html).toContain('Total task time'); + expect(html).toContain('One pair'); + for (const pair of data.pairs) { + for (const run of pair.runs) expect(html).toContain(`/benchmarks/native-luna-1.20.2/${run.proof}`); + } + expect(html).toContain('Native iOS'); + expect(html).toContain('Native macOS'); + expect(html).not.toContain('android-stim.png'); + expect(html).toContain('returned tier unverified'); +}); diff --git a/website/src/components/NativeBenchmarkResults.tsx b/website/src/components/NativeBenchmarkResults.tsx new file mode 100644 index 000000000..86be89f28 --- /dev/null +++ b/website/src/components/NativeBenchmarkResults.tsx @@ -0,0 +1,106 @@ +import type { ReactNode } from 'react'; +import useBaseUrl from '@docusaurus/useBaseUrl'; +import data from '../data/benchmarks/native-luna-1.20.2.json'; +import { formatSeconds } from './benchmarkData'; +import styles from './NativeBenchmarkResults.module.css'; + +const names: Record = { ios: 'Native iOS', android: 'Native Android', macos: 'Native macOS' }; +const backends: Record = { + 'native-xcode': 'SwiftUI / Xcode', + 'native-gradle': 'Kotlin / Gradle', + 'native-swiftpm': 'SwiftUI / SwiftPM', +}; + +export default function NativeBenchmarkResults(): ReactNode { + const proofRoot = useBaseUrl('/benchmarks/native-luna-1.20.2/'); + return ( +
+
+ Standalone native apps / Pilot results +

Beyond React Native

+

+ Change a native app's revision text, build and launch it, press Increment, and verify the new revision and + count. Each result below is one accepted Stim/control pair on {data.machine}. +

+

+ Total task time runs from dispatch to the completed agent turn, including reasoning, retries, + builds, and proof capture. It differs from the React Native Settings-ready metric above. These small pilots + show workflow results, not a general compile-speed claim. +

+
+
+ {data.pairs.map((pair) => { + const maximum = Math.max(...pair.runs.map((run) => run.taskSeconds)); + return ( +
+
+
+

{names[pair.platform] ?? pair.platform}

+ {backends[pair.backend] ?? pair.backend} +
+ + {Math.abs(pair.timeReductionPercent).toFixed(0)}%{' '} + {pair.timeReductionPercent >= 0 ? 'sooner' : 'longer'} + +
+

One pair / Total task time / Lower is better

+ {pair.runs.map((run) => ( +
+ {run.arm === 'stim' ? 'Stim' : 'Control'} + + ))} +

+ {pair.model} / {pair.reasoningEffort} reasoning / Stim {pair.provenance.stimVersion} +

+
+ Proof and environment +

+ Recorded {pair.runs[0].recordedOn}. macOS {pair.provenance.macosVersion}, Xcode{' '} + {pair.provenance.xcodeVersion}, Node {pair.provenance.nodeVersion}, Codex{' '} + {pair.provenance.codexVersion}. Requested service tier: {pair.requestedServiceTier}; returned tier + unverified. +

+

+ Both runs passed independent source, command, process-identity, and visual checks. Cleanup completed + before publication. Screenshots show the changed revision and Count: 1. +

+
+ {pair.runs.map((run) => ( +
+ + {`${names[pair.platform]} + +
+ {run.arm === 'stim' ? 'Stim' : 'Control'} / {run.taskSeconds.toFixed(3)} s +
+
+ ))} +
+
+
+ ); + })} +
+

+ Native Xcode builds can reuse Stim artifacts and compiler cache. Native Gradle and macOS SwiftPM do not use + Stim's artifact cache; their potential benefit is in workspace preparation, launch, and verification. + {!data.pairs.some((pair) => pair.platform === 'android') && + ' Native Android results will appear after an accepted paired comparison.'} +

+ + Read the native benchmark protocol + +
+ ); +} diff --git a/website/src/data/benchmarks/native-luna-1.20.2.json b/website/src/data/benchmarks/native-luna-1.20.2.json new file mode 100644 index 000000000..ea3b8cd59 --- /dev/null +++ b/website/src/data/benchmarks/native-luna-1.20.2.json @@ -0,0 +1,136 @@ +{ + "schemaVersion": 1, + "cohort": "standalone-native-v1", + "metric": "dispatch-to-completed-task", + "machine": "Mac mini, Apple M4, 16 GB", + "pairs": [ + { + "platform": "ios", + "backend": "native-xcode", + "model": "gpt-5.6-luna", + "reasoningEffort": "high", + "requestedServiceTier": "priority", + "returnedServiceTier": "unverified", + "samplePairs": 1, + "provenance": { + "stimVersion": "1.20.2", + "nodeVersion": "26.7.0", + "macosVersion": "27.0", + "macosBuild": "26A428", + "xcodeVersion": "27.0", + "xcodeBuild": "27A266a", + "codexVersion": "0.154.0", + "fixtureCommit": "cb1c4cae00fd77849242ec326585631da46ab462" + }, + "timeReductionPercent": 44.83326123653063, + "runs": [ + { + "arm": "stim", + "recordedOn": "2026-10-13", + "taskSeconds": 169.614, + "usage": { + "input_tokens": 1625913, + "cached_input_tokens": 1523200, + "cache_write_input_tokens": 0, + "output_tokens": 11057, + "reasoning_output_tokens": 3856 + }, + "proof": "ios-stim.png", + "evidence": { + "recordSha256": "0215ad002610f4de2b4e58d6bdbf4c5e575cd6e966c0884347534901b8d6a8f6", + "metadataSha256": "f52529a0dbbf18792edc6ac8a46164e9ebee98ff303ec46f058bacffc029fb29", + "auditSha256": "63514949091040a58df75ae160134bb058626cfb85933ab6ca073c0584516334", + "cleanupSha256": "a217922b9c82ac6d9bbc672540800fc0a8b9d2f185603a786e2bef1c7d20414e", + "eventsSha256": "7eb10be9c13802df0fa215565fe25cbfcd8c984c764ffe8327cb6978d6e702dd", + "proofSha256": "ed1a3625d806e7734865f38a927678218b80d20e13e3f41a8d0974eeacc93170" + } + }, + { + "arm": "control", + "recordedOn": "2026-10-13", + "taskSeconds": 307.457, + "usage": { + "input_tokens": 3303869, + "cached_input_tokens": 3137920, + "cache_write_input_tokens": 0, + "output_tokens": 18085, + "reasoning_output_tokens": 6031 + }, + "proof": "ios-control.png", + "evidence": { + "recordSha256": "f71aa61f8a9a92e69700182e68ab8023ddc7f033107f0c8f5df940889eead981", + "metadataSha256": "de667ab177fa36aedebe47fe0d9f5b0f9427a6c838bd238e5359b52c29f87bf5", + "auditSha256": "5c6c6bb6321859b9e8d647d46c9b4920eec8b7aff2660096ff0efdd918a73e41", + "cleanupSha256": "0ce547e287a1a40d928ddec1c6138e655e004d0793992db9f0a490b05f395146", + "eventsSha256": "6e9eb1481fb252939c0ef6820f8d1109a7cd287011b8bb602b7fd8b168c48622", + "proofSha256": "6f83706fff9d2638eaa9c4cc6bbf0c05490f10b95dcb67b912ac0481e5df8bfa" + } + } + ] + }, + { + "platform": "macos", + "backend": "native-swiftpm", + "model": "gpt-5.6-luna", + "reasoningEffort": "high", + "requestedServiceTier": "priority", + "returnedServiceTier": "unverified", + "samplePairs": 1, + "provenance": { + "stimVersion": "1.20.2", + "nodeVersion": "26.7.0", + "macosVersion": "27.0", + "macosBuild": "26A428", + "xcodeVersion": "27.0", + "xcodeBuild": "27A266a", + "codexVersion": "0.154.0", + "fixtureCommit": "4b3e8b17e84e2d597ca28b0ee0bcdfae038adabd" + }, + "timeReductionPercent": 51.31305341724391, + "runs": [ + { + "arm": "stim", + "recordedOn": "2026-10-13", + "taskSeconds": 70.71, + "usage": { + "input_tokens": 675950, + "cached_input_tokens": 637696, + "cache_write_input_tokens": 0, + "output_tokens": 7647, + "reasoning_output_tokens": 2912 + }, + "proof": "macos-stim.png", + "evidence": { + "recordSha256": "82261d94e9a01ff0b9a8aa3e365529acca8685e080d27ff896f359aa1b1ed1f6", + "metadataSha256": "fb7de9c67307423798dc9402c02127114b6f84a8499e53e4358cf276908006db", + "auditSha256": "6733da7f1c406cf3a34b6ded03881f4b976e2e6e4e5d95aea5e6e6bcaac5cc20", + "cleanupSha256": "3d2e3994a49fb3b414455f271e1a5832f3bd5f4ba73e2c5f2f64f82bf8fc97c4", + "eventsSha256": "6b7f93f5888ada1ed0f183b2a18d91238895f88f3c2b7626d92c26ef7ee46784", + "proofSha256": "67352593d49023a5d7768b358913f3a1da67f4c7d9fca68ca0fa9feb58182734" + } + }, + { + "arm": "control", + "recordedOn": "2026-10-13", + "taskSeconds": 145.234, + "usage": { + "input_tokens": 1132184, + "cached_input_tokens": 1072640, + "cache_write_input_tokens": 0, + "output_tokens": 14710, + "reasoning_output_tokens": 6750 + }, + "proof": "macos-control.png", + "evidence": { + "recordSha256": "bdd10ae3c8e094acb6ed0ccf7877ed1b07e7db5004ce65a340e3d381b3377cab", + "metadataSha256": "0caa486404af12a7e55343863a083781ae5f8ebb847e1e277eda10ce53668cc9", + "auditSha256": "a79c8585f7f9ab367b99c4b6a64755a28e51e9178ccfd3978e9e3ffaaea5b864", + "cleanupSha256": "18b97cdcd516a53000bb62e77e9553144416d220a91dddc78de464427f6bce1f", + "eventsSha256": "f3a58f61ac345b8dc3f164accd032b7d698c9c68063816f85b83e5863da1c5db", + "proofSha256": "0ed6265f17b9cdb02b92934a1a90faa594d447c4704d804674bc88a87703265f" + } + } + ] + } + ] +} diff --git a/website/src/pages/benchmarks.tsx b/website/src/pages/benchmarks.tsx index a433bc702..9d93d568e 100644 --- a/website/src/pages/benchmarks.tsx +++ b/website/src/pages/benchmarks.tsx @@ -4,6 +4,7 @@ import Layout from '@theme/Layout'; import Heading from '@theme/Heading'; import { benchmarks, displayVariant } from '@site/src/components/benchmarkCatalog'; import BenchmarkVideo from '@site/src/components/BenchmarkVideo'; +import NativeBenchmarkResults from '@site/src/components/NativeBenchmarkResults'; import { benchmarkModelLabel } from '@site/src/components/benchmarkSelection'; import { benchmarkDisplayTitle, @@ -94,7 +95,7 @@ export default function Benchmarks(): ReactNode { return (
@@ -102,9 +103,9 @@ export default function Benchmarks(): ReactNode {
Agent benchmark
Stim agent benchmarks

- Compare how coding agents launch the same React Native app with Stim and the local Expo/native toolchain. - Platforms and JavaScript/native tasks are measured separately, and every published time links to its - command-level audit and Settings-screen proof. + Compare coding agents using Stim and local platform tools. React Native results measure Settings-ready + time and link to command-level audits. Standalone native pilots below measure complete task time and + include verified screen proof. Platforms and tasks are measured separately.

@@ -155,11 +156,13 @@ export default function Benchmarks(): ReactNode { ) : null} + +
Methodology - How the comparisons work + How the React Native comparisons work

We compare Stim with standard Expo and native build tools on the same app. The goal is to measure how diff --git a/website/static/benchmarks/native-luna-1.20.2/ios-control.png b/website/static/benchmarks/native-luna-1.20.2/ios-control.png new file mode 100644 index 000000000..154b4c6db Binary files /dev/null and b/website/static/benchmarks/native-luna-1.20.2/ios-control.png differ diff --git a/website/static/benchmarks/native-luna-1.20.2/ios-stim.png b/website/static/benchmarks/native-luna-1.20.2/ios-stim.png new file mode 100644 index 000000000..d1acbf4d7 Binary files /dev/null and b/website/static/benchmarks/native-luna-1.20.2/ios-stim.png differ diff --git a/website/static/benchmarks/native-luna-1.20.2/macos-control.png b/website/static/benchmarks/native-luna-1.20.2/macos-control.png new file mode 100644 index 000000000..d4afbe1a6 Binary files /dev/null and b/website/static/benchmarks/native-luna-1.20.2/macos-control.png differ diff --git a/website/static/benchmarks/native-luna-1.20.2/macos-stim.png b/website/static/benchmarks/native-luna-1.20.2/macos-stim.png new file mode 100644 index 000000000..db77526c5 Binary files /dev/null and b/website/static/benchmarks/native-luna-1.20.2/macos-stim.png differ