From f6987d2b85230b5762abfe952cbdf466b7974433 Mon Sep 17 00:00:00 2001 From: Martin Date: Thu, 22 Oct 2026 17:52:35 +0200 Subject: [PATCH 1/2] bench: compare superjson base, PR branch and devalue, interleaved Adds a /compare page that runs origin/main, perf/faster-serialize and devalue in the same interleaved rounds. devalue is adapted to the SuperJSON call shape and checked by round-trip fidelity, because its text differs from superjson's. Cases devalue refuses (classes, symbols, custom types, Errors, Promises) are recorded as refused instead of timed. Also adds benchmarks/devalue-features.ts, a per-feature round-trip table, and devalue as a dev dependency. --- benchmarks/compare-app.ts | 101 +++++++++++++++++++++++++++++++++ benchmarks/compare.html | 16 ++++++ benchmarks/devalue-features.ts | 83 +++++++++++++++++++++++++++ benchmarks/devalue-lib.ts | 27 +++++++++ benchmarks/ledger-run.ts | 6 +- benchmarks/server.ts | 30 ++++++++++ benchmarks/suite.ts | 24 +++++++- package-lock.json | 11 ++++ package.json | 1 + 9 files changed, 295 insertions(+), 4 deletions(-) create mode 100644 benchmarks/compare-app.ts create mode 100644 benchmarks/compare.html create mode 100644 benchmarks/devalue-features.ts create mode 100644 benchmarks/devalue-lib.ts diff --git a/benchmarks/compare-app.ts b/benchmarks/compare-app.ts new file mode 100644 index 0000000..39ac7dd --- /dev/null +++ b/benchmarks/compare-app.ts @@ -0,0 +1,101 @@ +import type { SuperJSONCtor } from './fixtures.js'; +import { isDevalue, prepare, runSteps, selectCases, type Case, type Op } from './suite.js'; + +interface LibMeta { + id: string; + label: string; + url: string; + sha: string; + ref: string; + kind: string; + subject: string; +} + +const $ = (id: string) => { + const el = document.getElementById(id); + if (!el) throw new Error(`missing #${id}`); + + return el; +}; + +const params = new URLSearchParams(location.search); + +function calibrate() { + const start = performance.now(); + let x = 0; + for (let i = 0; i < 3e7; i++) { + x = (x + (i & 7)) | 0; + } + (globalThis as { __sink?: number }).__sink = x; + + return performance.now() - start; +} + +/** null when the library can run the case, otherwise the first line of its error. */ +function refusal(Lib: SuperJSONCtor, c: Case): string | null { + try { + prepare(Lib, c)(); + + return null; + } catch (error) { + return (error as Error).message.split('\n')[0]; + } +} + +async function runCompare(opts: { filter: string; ops: Op[]; timeMs: number; throttle: number; series?: string; run?: string }) { + $('status').textContent = 'loading versions…'; + const libs: LibMeta[] = await (await fetch('/api/compare/libs')).json(); + const versions: SuperJSONCtor[] = await Promise.all( + libs.map(async l => (await import(/* @vite-ignore */ l.url)).default) + ); + const selected = selectCases(opts.filter, opts.ops); + const calibrationMs = calibrate(); + const started = performance.now(); + const cases = []; + for (const [i, c] of selected.entries()) { + $('status').textContent = `${i + 1}/${selected.length} ${c.id}`; + const refused = versions.map(v => (isDevalue(v) ? refusal(v, c) : null)); + const timed = versions.flatMap((_, k) => (refused[k] === null ? [k] : [])); + const { bytes, results } = await runSteps( + timed.map(k => versions[k]), + 0, + c, + { timeMs: opts.timeMs, minSampleMs: 25 } + ); + const full = versions.map((_, k) => { + const at = timed.indexOf(k); + if (at < 0) { + return { ms: null, rme: null, samples: 0, identical: false, refused: refused[k] }; + } + + return { ...results[at], refused: null }; + }); + cases.push({ id: c.id, fixture: c.fixture.name, group: c.fixture.group, op: c.op, bytes, results: full }); + } + + const next = { + createdAt: new Date().toISOString(), + engine: navigator.userAgent, + throttle: opts.throttle, + calibrationMs, + crossOriginIsolated, + timeMs: opts.timeMs, + series: opts.series, + run: opts.run, + steps: libs.map(l => ({ id: l.id, sha: l.sha, ref: l.ref, kind: l.kind, subject: l.subject, body: '', parent: null, files: [], diff: '' })), + cases, + }; + const saved = await (await fetch('/api/compare', { method: 'POST', body: JSON.stringify(next) })).json(); + $('status').textContent = `done in ${((performance.now() - started) / 1000).toFixed(0)} s, saved as ${saved.name}`; +} + +if (params.get('autorun') === '1') { + runCompare({ + filter: params.get('filter') ?? '', + ops: (params.get('ops') ?? 'stringify,parse,serialize,deserialize').split(',').map(s => s.trim()) as Op[], + timeMs: Number(params.get('time') ?? 6000), + throttle: Number(params.get('throttle') ?? 1), + series: params.get('series') ?? undefined, + run: params.get('run') ?? undefined, + }); +} diff --git a/benchmarks/compare.html b/benchmarks/compare.html new file mode 100644 index 0000000..cc292ae --- /dev/null +++ b/benchmarks/compare.html @@ -0,0 +1,16 @@ + + + + + superjson vs devalue + + + +

superjson (base and PR) against devalue, interleaved

+ 
loading…
+ + + diff --git a/benchmarks/devalue-features.ts b/benchmarks/devalue-features.ts new file mode 100644 index 0000000..67758b5 --- /dev/null +++ b/benchmarks/devalue-features.ts @@ -0,0 +1,83 @@ +/** + * Round-trips one value per feature through superjson (current src) and devalue, and prints ok, lossy or throws. + * Run: bun benchmarks/devalue-features.ts + */ +import { isDeepStrictEqual } from 'node:util'; +import { currentEntry, writeBundle } from './build.js'; +import { parse, stringify } from 'devalue'; + +const SuperJSON = (await import(await writeBundle('probe-features', currentEntry, 'bun'))).default; + +class Point { + constructor(public x: number, public y: number) {} +} + +const cyclic: Record = { name: 'loop' }; +cyclic.self = cyclic; +const shared = { id: 1 }; + +const cases: [string, unknown][] = [ + ['Date', new Date('2026-10-22T10:00:00.000Z')], + ['undefined in object and array', { a: undefined, b: [undefined, 1] }], + ['NaN, Infinity, -0', { n: NaN, i: Infinity, z: -0 }], + ['BigInt', { big: 12345678901234567890n }], + ['Map and Set', { m: new Map([[1, 'a']]), s: new Set([1, 2]) }], + ['RegExp', /ab+c/gi], + ['URL', new URL('https://example.com/a?b=1')], + ['URLSearchParams', new URLSearchParams('a=1&b=2')], + ['ArrayBuffer', new ArrayBuffer(4)], + ['Uint8Array and Float64Array', { u: new Uint8Array([1, 2]), f: new Float64Array([1.5]) }], + ['BigInt64Array', new BigInt64Array([1n, -1n])], + ['Error with message', new Error('boom')], + ['Error with cause', new Error('outer', { cause: new Error('inner') })], + ['class instance (not registered)', new Point(1, 2)], + ['symbol (unregistered)', { s: Symbol.for('app.key') }], + ['cycle', cyclic], + ['shared references', { a: shared, b: shared }], + ['own __proto__ key', JSON.parse('{"__proto__": {"polluted": true}, "x": 1}')], + ['sparse array', [1, , 3]], + ['Promise', Promise.resolve(1)], + ['Temporal.Instant', globalThis.Temporal ? globalThis.Temporal.Instant.from('2026-10-22T10:00:00Z') : 'no Temporal in this runtime'], +]; + +const rows: string[] = []; +for (const [name, value] of cases) { + let sj: string; + try { + const sjs = new SuperJSON({ dedupe: true }); + const back = sjs.parse(sjs.stringify(value)); + sj = isDeepStrictEqual(back, value) ? 'ok' : 'lossy'; + } catch (error) { + sj = `throws: ${(error as Error).message.split('\n')[0].slice(0, 70)}`; + } + let dv: string; + try { + const back = parse(stringify(value)); + dv = isDeepStrictEqual(back, value) ? 'ok' : 'lossy'; + } catch (error) { + dv = `throws: ${(error as Error).message.split('\n')[0].slice(0, 70)}`; + } + rows.push(`${name}\t${sj}\t${dv}`); +} + +const custom = { p: new Point(3, 4) }; +let dvCustom: string; +try { + const text = stringify(custom, { Point: (v: unknown) => v instanceof Point && [v.x, v.y] }); + const back = parse(text, { Point: ([x, y]: number[]) => new Point(x, y) }); + dvCustom = isDeepStrictEqual(back, custom) ? 'ok (reducer and reviver)' : 'lossy'; +} catch (error) { + dvCustom = `throws: ${(error as Error).message.split('\n')[0].slice(0, 70)}`; +} +let sjCustom: string; +try { + const sjs = new SuperJSON(); + sjs.registerClass(Point); + const back = sjs.parse(sjs.stringify(custom)); + sjCustom = isDeepStrictEqual(back, custom) ? 'ok (registerClass)' : 'lossy'; +} catch (error) { + sjCustom = `throws: ${(error as Error).message.split('\n')[0].slice(0, 70)}`; +} +rows.push(`custom class with registration (superjson) or reducer and reviver (devalue)\t${sjCustom}\t${dvCustom}`); + +console.log(rows.join('\n')); diff --git a/benchmarks/devalue-lib.ts b/benchmarks/devalue-lib.ts new file mode 100644 index 0000000..9993f30 --- /dev/null +++ b/benchmarks/devalue-lib.ts @@ -0,0 +1,27 @@ +import { parse, stringify } from 'devalue'; + +/** + * devalue behind the SuperJSON call shape, so the harness can time it next to superjson. + * devalue has no class, symbol or custom-type registry, so the harness skips fixture setup for it. + */ +export default class DevalueCodec { + static readonly format = 'devalue'; + + constructor(_options?: { dedupe?: boolean }) {} + + serialize(value: unknown) { + return { json: stringify(value) }; + } + + deserialize(payload: { json: string }) { + return parse(payload.json); + } + + stringify(value: unknown) { + return stringify(value); + } + + parse(text: string) { + return parse(text); + } +} diff --git a/benchmarks/ledger-run.ts b/benchmarks/ledger-run.ts index 310f9ff..7f32643 100644 --- a/benchmarks/ledger-run.ts +++ b/benchmarks/ledger-run.ts @@ -18,6 +18,7 @@ const { values } = parseArgs({ time: { type: 'string', default: '6000' }, headed: { type: 'boolean', default: false }, repeat: { type: 'string', default: '1' }, + page: { type: 'string', default: '/steps' }, }, }); @@ -103,7 +104,7 @@ async function runOnce(run: number): Promise { `# ${values.browser}: fixed loop ${unthrottled.toFixed(0)} ms unthrottled, ${throttled.toFixed(0)} ms at rate ${rate} (${(throttled / unthrottled).toFixed(2)}× measured)` ); - const url = new URL('/steps', server.url); + const url = new URL(values.page!, server.url); url.search = new URLSearchParams({ autorun: '1', throttle: String(rate), @@ -138,8 +139,9 @@ for (const c of cases) { console.error(`\n${c.fixture} · ${c.op}`); steps.forEach((s: any, i: number) => { const r = c.results[i]; + const shown = r.ms === null ? `${'refused'.padStart(8)} ` : `${r.ms.toFixed(2).padStart(8)} ms ${(c.results[base].ms / r.ms).toFixed(2)}×`; console.error( - ` ${String(i).padStart(2)} ${s.kind.padEnd(9)} ${s.sha} ${r.ms.toFixed(2).padStart(8)} ms ${(c.results[base].ms / r.ms).toFixed(2)}×${r.identical ? '' : ' ✗ OUTPUT DIFFERS'} ${s.subject.slice(0, 70)}` + ` ${String(i).padStart(2)} ${s.kind.padEnd(9)} ${s.sha} ${shown}${r.identical ? '' : ' ✗ OUTPUT DIFFERS'} ${s.subject.slice(0, 70)}` ); }); } diff --git a/benchmarks/server.ts b/benchmarks/server.ts index 98e33cb..fb5ddc9 100644 --- a/benchmarks/server.ts +++ b/benchmarks/server.ts @@ -26,6 +26,7 @@ export async function startServer({ port, baselineRef, onResults, onLedger }: Se const baseline = await ensureBaseline(baselineRef); const resultsDir = join(BENCH, '.results'); const ledgersDir = join(resultsDir, 'ledgers'); + const compareDir = join(resultsDir, 'compare'); mkdirSync(ledgersDir, { recursive: true }); const savedFile = async (dir: string, encoded: string) => { const name = basename(decodeURIComponent(encoded)); @@ -61,6 +62,35 @@ export async function startServer({ port, baselineRef, onResults, onLedger }: Se return new Response(Bun.file(join(BENCH, 'steps.html')), { headers }); case '/steps.js': return js(await bundle(join(BENCH, 'steps-app.ts'), 'browser')); + case '/compare': + return new Response(Bun.file(join(BENCH, 'compare.html')), { headers }); + case '/compare.js': + return js(await bundle(join(BENCH, 'compare-app.ts'), 'browser')); + case '/lib/devalue.js': + return js(await bundle(join(BENCH, 'devalue-lib.ts'), 'browser')); + case '/api/compare/libs': { + const ours = (await $`git -C ${ROOT} rev-parse perf/faster-serialize`.text()).trim(); + const devalueVersion = JSON.parse(await Bun.file(join(ROOT, 'node_modules/devalue/package.json')).text()).version; + + return Response.json( + [ + { id: 'base', label: 'superjson origin/main', url: `/lib/ref/${baseline.sha}.js`, sha: baseline.sha, ref: baseline.ref, kind: 'baseline', subject: 'superjson origin/main (base)' }, + { id: 'ours', label: 'superjson PR #364', url: `/lib/ref/${ours}.js`, sha: ours, ref: 'perf/faster-serialize', kind: 'step', subject: 'superjson perf/faster-serialize (PR #364)' }, + { id: 'devalue', label: `devalue ${devalueVersion}`, url: '/lib/devalue.js', sha: `devalue@${devalueVersion}`, ref: `npm devalue ${devalueVersion}`, kind: 'lib', subject: `devalue ${devalueVersion} through a SuperJSON-shaped adapter` }, + ], + { headers } + ); + } + case '/api/compare': { + const body = await req.json(); + const stamp = new Date().toISOString().replace(/[:.]/g, '-'); + const name = `${stamp}-x${body.throttle ?? 1}.json`; + mkdirSync(compareDir, { recursive: true }); + await Bun.write(join(compareDir, name), JSON.stringify(body)); + onLedger?.(body); + + return Response.json({ ok: true, name }, { headers }); + } case '/api/steps': return Response.json(await resolveSteps(), { headers }); case '/api/ledgers': { diff --git a/benchmarks/suite.ts b/benchmarks/suite.ts index 6615faf..8662ebe 100644 --- a/benchmarks/suite.ts +++ b/benchmarks/suite.ts @@ -45,13 +45,31 @@ const dataOf = (fixture: Fixture) => { return dataCache.get(fixture); }; +/** devalue exposes `format = 'devalue'` and has no registries, so it skips fixture setup and fidelity is checked instead of bytes. */ +export const isDevalue = (Lib: SuperJSONCtor): boolean => 'format' in Lib && Lib.format === 'devalue'; + const instance = (Lib: SuperJSONCtor, fixture: Fixture) => { const sj = new Lib({ dedupe: fixture.dedupe }); - fixture.setup?.(sj); + if (!isDevalue(Lib)) { + fixture.setup?.(sj); + } return sj; }; +/** devalue's text differs from superjson's, so it counts as identical when its round trip keeps superjson's canonical form. */ +export function fidelityMatches(reference: SuperJSONCtor, lib: SuperJSONCtor, c: Case): boolean { + const ref = instance(reference, c.fixture); + const sj = instance(lib, c.fixture); + const data = dataOf(c.fixture) as any; + const canon = (v: any) => JSON.stringify(ref.serialize(v), stableBigint); + try { + return canon(sj.parse(sj.stringify(data))) === canon(data); + } catch { + return false; + } +} + /** Returns the function to time. All inputs are prepared up front so only the op itself is measured. */ export function prepare(Lib: SuperJSONCtor, c: Case): () => unknown { const sj = instance(Lib, c.fixture); @@ -215,7 +233,9 @@ export async function runSteps( ): Promise<{ bytes: number; results: StepResult[] }> { const reference = versions[referenceIndex]; const expected = outputSignature(reference, reference, c); - const identical = versions.map(v => outputSignature(reference, v, c) === expected); + const identical = versions.map(v => + isDevalue(v) ? fidelityMatches(reference, v, c) : outputSignature(reference, v, c) === expected + ); const fns = versions.map(v => prepare(v, c)); fns.forEach(fn => timeIt(fn, 2)); const counts = fns.map(fn => calibrate(fn, opts.minSampleMs)); diff --git a/package-lock.json b/package-lock.json index de4ee07..17fb514 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "benchmark": "^2.1.4", "copy-anything": "4.0.5", "decimal.js": "^10.3.1", + "devalue": "^6.0.3", "mongodb": "^3.6.6", "tsdx": "^0.14.1", "typescript": "^5.9.3", @@ -5452,6 +5453,16 @@ "node": ">=8" } }, + "node_modules/devalue": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/devalue/-/devalue-6.0.3.tgz", + "integrity": "sha512-3woMEITftG1wKeO/4d/QXH1DVKctTSZg3jNGPpjp9hNr5Hos8MCFF5McTML5ZtgKDObm2R591B0spRp2IVernA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/diff-sequences": { "version": "25.2.6", "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-25.2.6.tgz", diff --git a/package.json b/package.json index eb34491..58f2864 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "benchmark": "^2.1.4", "copy-anything": "4.0.5", "decimal.js": "^10.3.1", + "devalue": "^6.0.3", "mongodb": "^3.6.6", "tsdx": "^0.14.1", "typescript": "^5.9.3", From 6bf7a7a6991ac2a42be3206b58312f0b7bd86a1d Mon Sep 17 00:00:00 2001 From: Martin Date: Thu, 22 Oct 2026 19:08:24 +0200 Subject: [PATCH 2/2] bench: add superjson 2.2.3, the version col-web ships, as the base column --- benchmarks/server.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/server.ts b/benchmarks/server.ts index fb5ddc9..42c241c 100644 --- a/benchmarks/server.ts +++ b/benchmarks/server.ts @@ -70,11 +70,13 @@ export async function startServer({ port, baselineRef, onResults, onLedger }: Se return js(await bundle(join(BENCH, 'devalue-lib.ts'), 'browser')); case '/api/compare/libs': { const ours = (await $`git -C ${ROOT} rev-parse perf/faster-serialize`.text()).trim(); + const raw = (await $`git -C ${ROOT} rev-list -n 1 v2.2.3`.text()).trim(); const devalueVersion = JSON.parse(await Bun.file(join(ROOT, 'node_modules/devalue/package.json')).text()).version; return Response.json( [ - { id: 'base', label: 'superjson origin/main', url: `/lib/ref/${baseline.sha}.js`, sha: baseline.sha, ref: baseline.ref, kind: 'baseline', subject: 'superjson origin/main (base)' }, + { id: 'raw', label: 'superjson 2.2.3 (col-web)', url: `/lib/ref/${raw}.js`, sha: raw, ref: 'v2.2.3', kind: 'baseline', subject: 'superjson 2.2.3, the version col-web ships' }, + { id: 'base', label: 'superjson origin/main', url: `/lib/ref/${baseline.sha}.js`, sha: baseline.sha, ref: baseline.ref, kind: 'reference', subject: 'superjson origin/main (2.2.5)' }, { id: 'ours', label: 'superjson PR #364', url: `/lib/ref/${ours}.js`, sha: ours, ref: 'perf/faster-serialize', kind: 'step', subject: 'superjson perf/faster-serialize (PR #364)' }, { id: 'devalue', label: `devalue ${devalueVersion}`, url: '/lib/devalue.js', sha: `devalue@${devalueVersion}`, ref: `npm devalue ${devalueVersion}`, kind: 'lib', subject: `devalue ${devalueVersion} through a SuperJSON-shaped adapter` }, ],