diff --git a/app/README.md b/app/README.md index 50e1603736..eb377a28ef 100644 --- a/app/README.md +++ b/app/README.md @@ -4,8 +4,8 @@ An agentic AI coding app for the Mac, with the TensorFold engine built in. It al any OpenAI‑compatible LLM endpoint, local or cloud. The look comes from the TensorFold brand: graphite and navy surfaces, an electric-blue primary and the fold gradient. -The current desktop release is [beta 9](https://github.com/ashhart/TensorFold/releases/tag/v1.0.0-beta.9). -It launches the bundled native Zig engine directly. +The current desktop release is [beta 10](https://github.com/ashhart/TensorFold/releases/tag/v1.0.0-beta.10). +It launches the bundled TensorFold 1.1.4 native Zig engine directly. It reads, edits, and runs your code through a tool‑use loop, with streaming responses, live diffs, and per‑action approval. diff --git a/app/build/engine.json b/app/build/engine.json index 9aba2c16c0..47d0129a41 100644 --- a/app/build/engine.json +++ b/app/build/engine.json @@ -1,10 +1,10 @@ { - "version": "1.1.0", + "version": "1.1.4", "platform": "macos-arm64", - "archive": "tensorfold-1.1.0-macos-arm64.tar.gz", - "root": "tensorfold-1.1.0-macos-arm64", - "url": "https://github.com/ashhart/TensorFold/releases/download/v1.1.0/tensorfold-1.1.0-macos-arm64.tar.gz", - "sha256": "99c66f56fcde07a7dd8df6ea555bcbd406d0537b7d92a453112a54bef5a72707", + "archive": "tensorfold-1.1.4-macos-arm64.tar.gz", + "root": "tensorfold-1.1.4-macos-arm64", + "url": "https://github.com/ashhart/TensorFold/releases/download/v1.1.4/tensorfold-1.1.4-macos-arm64.tar.gz", + "sha256": "e60096c8af989f26eb622bde3099ff6ed3ecc37cdd46ade0397b3afd16145c6f", "required": [ "bin/tensorfold-native", "lib/libtfgrammar.dylib", diff --git a/app/docs/beta/1.0.0-beta.10.md b/app/docs/beta/1.0.0-beta.10.md new file mode 100644 index 0000000000..cec98b064b --- /dev/null +++ b/app/docs/beta/1.0.0-beta.10.md @@ -0,0 +1,22 @@ +# TensorFold app 1.0.0-beta.10 + +This macOS app beta bundles the released TensorFold 1.1.4 native engine for Apple silicon on macOS 13.3 or later. +The engine archive is pinned to its published SHA-256 and its internal file checksums are verified before packaging. + +## In this beta + +- Update the bundled native engine from 1.1.0 to 1.1.4. See the [engine release notes](https://github.com/ashhart/TensorFold/releases/tag/v1.1.4) for its platform-specific changes and measurements. +- Benchmarks use the engine's community-v2 protocol, recording request routes, cache format and natural sampled stops. Saved community-v1 results remain readable with their original protocol label. +- Google sign-in accepts Google's issuer callback parameter and shows a useful explanation when sign-in fails. +- Update checks and downloads work through fake-IP proxies while retaining the destination checks used for Hub downloads. +- The model library uses the engine's geometry admission checks before offering a checkpoint for loading. +- Update the shipped HTTP, email and schema-validation dependencies to patched versions. + +## Limits and qualification + +The beta.9 limitations still apply, including disabled Cortheon web research, experimental Living Weights and incomplete distributed-inference qualification. +Google and messaging connections require user-owned accounts and permissions; synthetic service checks do not establish every live-account workflow. +Window-close flashing reported in Full mode remains under investigation. +No new mobile or watch installer is included. + +Download the signed Mac installer from https://tensorfold.dev/download/mac, or use the in-app update prompt after the signed feed detects this version. diff --git a/app/electron/agent/engine/benchmark/controller.ts b/app/electron/agent/engine/benchmark/controller.ts index 9990e22005..cb163e1240 100644 --- a/app/electron/agent/engine/benchmark/controller.ts +++ b/app/electron/agent/engine/benchmark/controller.ts @@ -125,7 +125,7 @@ export class ModelBenchmarks { job.receipt = receipt ? parseBenchmarkReceipt(JSON.stringify(receipt), { ...input, engineVersion: checkpoint?.engineVersion ?? BENCHMARK_ENGINE_VERSION }, this.now()) : null if (job.receipt) { if (checkpoint && (job.receipt.model.revision !== checkpoint.revision || job.receipt.model.config_sha256 !== checkpoint.configSha256)) { job.receipt = null; throw new Error('The cached checkpoint changed during the benchmark.') } - job.snapshot.metadata = structuredClone({ runtime: job.receipt.runtime, model: job.receipt.model, hardware: job.receipt.hardware, settings: job.receipt.settings }) + job.snapshot.metadata = structuredClone({ suite: job.receipt.suite, runtime: job.receipt.runtime, model: job.receipt.model, hardware: job.receipt.hardware, settings: job.receipt.settings }) job.snapshot.cells = benchmarkCells(job.receipt) job.snapshot.receiptAvailable = true job.snapshot.protocolComplete = benchmarkProtocolComplete(job.receipt) diff --git a/app/electron/agent/engine/benchmark/fake-benchmark.mjs b/app/electron/agent/engine/benchmark/fake-benchmark.mjs index 5776fbeef8..8f9d48ad68 100644 --- a/app/electron/agent/engine/benchmark/fake-benchmark.mjs +++ b/app/electron/agent/engine/benchmark/fake-benchmark.mjs @@ -8,7 +8,7 @@ import { fileURLToPath } from 'node:url' let args = process.argv.slice(2) let scenario = 'complete' if (args[0]?.startsWith('scenario:')) scenario = args.shift().slice(9) -const engineVersion = scenario === 'old-version' ? '1.0.12' : scenario === 'unknown-version' ? '9.9.9' : '1.1.0' +const engineVersion = scenario === 'old-version' ? '1.0.12' : scenario === 'unknown-version' ? '9.9.9' : JSON.parse(readFileSync(new URL('../../../../build/engine.json', import.meta.url), 'utf8')).version if (args[0] === '--version') { console.log(`tensorfold-native ${engineVersion}`); process.exit(0) } if (args[0] !== 'benchmark') { const fixture = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../test-fixtures/fake-engine.mjs') @@ -19,8 +19,8 @@ const serial = args.includes('--serial') const root = path.join(process.env.HF_HUB_CACHE ?? '', `models--${model.replace('/', '--')}`) let revision = 'a'.repeat(40), config = Buffer.from('{}') try { revision = readFileSync(path.join(root, 'refs', 'main'), 'utf8').trim(); config = readFileSync(path.join(root, 'snapshots', revision, 'config.json')) } catch {} -const receipt = { schema_version: 1, suite: { id: 'community-v1', sha256: '9bb717da00666b90b4006174abd9ed46108affc37456b43c64a224a9c938d5be' }, run_id: '00000000-0000-4000-8000-000000000001', created_at: '2026-01-01T00:00:00Z', runtime: { tensorfold_version: scenario === 'mismatched-receipt' ? '1.0.12' : engineVersion, backend: 'mlx', python_version: 'none', platform: 'macos', dependencies: {} }, model: { repo_id: model, revision, family: 'qwen3_5', config_sha256: createHash('sha256').update(config).digest('hex'), tokenizer_sha256: null, quantization: 'mlx-4bit-g64', drafter_repo_id: null, local: false }, hardware: { cpu_model: 'Synthetic CPU', system_memory_bytes: 16 * 1024 ** 3, memory_type: 'unified', gpus: [{ name: 'Synthetic GPU', memory_bytes: null, cores: 8 }], gpu_count: 1, source: 'detected' }, settings: { tokens: 256, repetitions: 5, temperatures: [1, 0], serial, context_tokens: 4096, managed: true, rank_count: 1 }, samples: [] } -const sample = (fixture_id, temperature, repeat) => ({ fixture_id, temperature, repeat, seed: 1234 + Math.max(0, repeat), status: 'ok', prompt_tokens: 24, completion_tokens: 256, cached_tokens: 0, delivery_seconds: 2.55, delivery_tps: 100, ttft_seconds: 0.1, end_to_end_seconds: 2.8, server_decode_tps: 110, server_decode_seconds: null, prefill_seconds: null, token_sha: 'd'.repeat(12), error_code: null }) +const receipt = { schema_version: 1, suite: { id: 'community-v2', sha256: '51136f5d6775b2ff427834c6b1038b584de27e0a267cfc088ff2694ea069dcdc' }, run_id: '00000000-0000-4000-8000-000000000001', created_at: '2026-01-01T00:00:00Z', runtime: { tensorfold_version: scenario === 'mismatched-receipt' ? '1.0.12' : engineVersion, backend: 'mlx', python_version: 'none', platform: 'macos', dependencies: {} }, model: { repo_id: model, revision, family: 'qwen3_5', config_sha256: createHash('sha256').update(config).digest('hex'), tokenizer_sha256: null, quantization: 'mlx-4bit-g64', drafter_repo_id: null, local: false }, hardware: { cpu_model: 'Synthetic CPU', system_memory_bytes: 16 * 1024 ** 3, memory_type: 'unified', gpus: [{ name: 'Synthetic GPU', memory_bytes: null, cores: 8 }], gpu_count: 1, source: 'detected' }, settings: { tokens: 256, repetitions: 5, temperatures: [1, 0], serial, context_tokens: 4096, managed: true, rank_count: 1, kv_cache: 'bf16' }, samples: [] } +const sample = (fixture_id, temperature, repeat) => ({ fixture_id, temperature, repeat, seed: 1234 + Math.max(0, repeat), status: 'ok', prompt_tokens: 24, completion_tokens: 256, cached_tokens: 0, delivery_seconds: 2.55, delivery_tps: 100, ttft_seconds: 0.1, end_to_end_seconds: 2.8, server_decode_tps: 110, server_decode_seconds: null, prefill_seconds: null, token_sha: 'd'.repeat(12), error_code: null, finish_reason: 'length', route: fixture_id === 'chat' ? '/v1/chat/completions' : '/v1/completions' }) const save = () => writeFileSync(output, JSON.stringify(receipt), { mode: 0o600, flag: 'wx' }) const say = text => scenario === 'stderr' ? console.error(text) : console.log(text) if (scenario === 'missing') process.exit(1) diff --git a/app/electron/agent/engine/benchmark/fixtures.ts b/app/electron/agent/engine/benchmark/fixtures.ts index 29ec40354d..93dbf5bb67 100644 --- a/app/electron/agent/engine/benchmark/fixtures.ts +++ b/app/electron/agent/engine/benchmark/fixtures.ts @@ -4,5 +4,5 @@ import { BENCHMARK_ENGINE_VERSION } from './version' /** Synthetic receipt factory for regression tests, with no user or model output. */ export function syntheticBenchmarkReceipt(model = 'example-org/synthetic-model'): ModelBenchmarkReceipt { - return { schema_version: 1, suite: { id: 'community-v1', sha256: BENCHMARK_SUITE_SHA }, run_id: '00000000-0000-4000-8000-000000000001', created_at: '2026-01-01T00:00:00Z', runtime: { tensorfold_version: BENCHMARK_ENGINE_VERSION, backend: 'mlx', python_version: 'none', platform: 'macos', dependencies: {} }, model: { repo_id: model, revision: 'a'.repeat(40), family: 'qwen3_5', config_sha256: 'b'.repeat(64), tokenizer_sha256: 'c'.repeat(64), quantization: 'mlx-affine-4bit-g64', drafter_repo_id: null, local: false }, hardware: { cpu_model: 'Synthetic CPU', system_memory_bytes: 16 * 1024 ** 3, memory_type: 'unified', gpus: [{ name: 'Synthetic GPU', memory_bytes: null, cores: 8 }], gpu_count: 1, source: 'detected' }, settings: { tokens: 256, repetitions: 5, temperatures: [1, 0], serial: false, context_tokens: 4096, managed: true, rank_count: 1 }, samples: [1, 0].flatMap(temperature => (['code', 'chat'] as const).flatMap(fixture_id => Array.from({ length: 5 }, (_, repeat) => ({ fixture_id, temperature, repeat, seed: 1234 + repeat, status: 'ok' as const, prompt_tokens: 24, completion_tokens: 256, cached_tokens: 0, delivery_seconds: 2.55, delivery_tps: 100, ttft_seconds: 0.1 + repeat * 0.1, end_to_end_seconds: 2.8, server_decode_tps: 110, server_decode_seconds: null, prefill_seconds: null, token_sha: 'd'.repeat(12), error_code: null })))) } + return { schema_version: 1, suite: { id: 'community-v2', sha256: BENCHMARK_SUITE_SHA }, run_id: '00000000-0000-4000-8000-000000000001', created_at: '2026-01-01T00:00:00Z', runtime: { tensorfold_version: BENCHMARK_ENGINE_VERSION, backend: 'mlx', python_version: 'none', platform: 'macos', dependencies: {} }, model: { repo_id: model, revision: 'a'.repeat(40), family: 'qwen3_5', config_sha256: 'b'.repeat(64), tokenizer_sha256: 'c'.repeat(64), quantization: 'mlx-affine-4bit-g64', drafter_repo_id: null, local: false }, hardware: { cpu_model: 'Synthetic CPU', system_memory_bytes: 16 * 1024 ** 3, memory_type: 'unified', gpus: [{ name: 'Synthetic GPU', memory_bytes: null, cores: 8 }], gpu_count: 1, source: 'detected' }, settings: { tokens: 256, repetitions: 5, temperatures: [1, 0], serial: false, context_tokens: 4096, managed: true, rank_count: 1, kv_cache: 'bf16' }, samples: [1, 0].flatMap(temperature => (['code', 'chat'] as const).flatMap(fixture_id => Array.from({ length: 5 }, (_, repeat) => ({ fixture_id, temperature, repeat, seed: 1234 + repeat, status: 'ok' as const, prompt_tokens: 24, completion_tokens: 256, cached_tokens: 0, delivery_seconds: 2.55, delivery_tps: 100, ttft_seconds: 0.1 + repeat * 0.1, end_to_end_seconds: 2.8, server_decode_tps: 110, server_decode_seconds: null, prefill_seconds: null, token_sha: 'd'.repeat(12), error_code: null, finish_reason: 'length' as const, route: fixture_id === 'chat' ? '/v1/chat/completions' as const : '/v1/completions' as const })))) } } diff --git a/app/electron/agent/engine/benchmark/receipt.test.ts b/app/electron/agent/engine/benchmark/receipt.test.ts index 1e4a545d3a..237960e20b 100644 --- a/app/electron/agent/engine/benchmark/receipt.test.ts +++ b/app/electron/agent/engine/benchmark/receipt.test.ts @@ -1,6 +1,6 @@ import assert from 'node:assert/strict' import { syntheticBenchmarkReceipt } from './fixtures' -import { parseBenchmarkReceipt, benchmarkCells, benchmarkProtocolComplete } from './receipt' +import { parseBenchmarkReceipt, benchmarkCells, benchmarkProtocolComplete, LEGACY_BENCHMARK_SUITE_SHA } from './receipt' import { assertBundledBenchmarkVersion, BENCHMARK_ENGINE_VERSION } from './version' const receipt = syntheticBenchmarkReceipt() @@ -12,11 +12,28 @@ assert.equal(benchmarkCells(parsed)[0].successful, 5) assert.equal(benchmarkCells(parsed)[0].deliveryTokensPerSecond?.median, 100) assert.equal(benchmarkCells(parsed)[0].firstTextSeconds?.median, 0.30000000000000004) assert.equal(parsed.samples[0].server_decode_seconds, null, 'missing engine duration stays unknown') -const current = structuredClone(receipt); current.runtime.tensorfold_version = '1.1.0' -assert.equal(parseBenchmarkReceipt(JSON.stringify(current), { model: current.model.repo_id, serial: false, engineVersion: '1.1.0' }).runtime.tensorfold_version, '1.1.0') -const historical = structuredClone(receipt); historical.runtime.tensorfold_version = '1.0.12' -assert.equal(parseBenchmarkReceipt(JSON.stringify(historical)).runtime.tensorfold_version, '1.0.12', 'reviewed historical receipts remain readable after the engine update') -assert.throws(() => parseBenchmarkReceipt(JSON.stringify(historical), { model: historical.model.repo_id, serial: false, engineVersion: '1.1.0' }), /invalid|unsupported/, 'a historical receipt cannot substitute for a live run on the new engine') +assert.equal(parseBenchmarkReceipt(JSON.stringify(receipt), { model: receipt.model.repo_id, serial: false, engineVersion: BENCHMARK_ENGINE_VERSION }).suite.id, 'community-v2') +for (const version of ['1.0.12', '1.1.0']) { + const historical = structuredClone(receipt) + historical.runtime.tensorfold_version = version + historical.suite = { id: 'community-v1', sha256: LEGACY_BENCHMARK_SUITE_SHA } + delete historical.settings.kv_cache + for (const sample of historical.samples) { delete sample.finish_reason; delete sample.route } + assert.equal(benchmarkProtocolComplete(parseBenchmarkReceipt(JSON.stringify(historical))), true, 'reviewed historical receipts remain readable after the engine update') + assert.throws(() => parseBenchmarkReceipt(JSON.stringify(historical), { model: historical.model.repo_id, serial: false, engineVersion: BENCHMARK_ENGINE_VERSION }), /invalid|unsupported/, 'a historical receipt cannot substitute for a live run on the new engine') + historical.runtime.tensorfold_version = BENCHMARK_ENGINE_VERSION + assert.throws(() => parseBenchmarkReceipt(JSON.stringify(historical)), /invalid|unsupported/, 'the current engine must emit its reviewed v2 schema') +} +for (const count of [128, 200, 255]) { + const natural = structuredClone(receipt) + Object.assign(natural.samples[0], { finish_reason: 'stop', completion_tokens: count, delivery_seconds: (count - 1) / 100 }) + assert.equal(benchmarkProtocolComplete(parseBenchmarkReceipt(JSON.stringify(natural))), true, 'v2 permits sampled natural stops from 128 tokens') +} +const chatOnly = structuredClone(receipt) +for (const sample of chatOnly.samples) sample.route = '/v1/chat/completions' +assert.equal(benchmarkProtocolComplete(parseBenchmarkReceipt(JSON.stringify(chatOnly))), true, 'chat-only models may consistently use the chat route for code') +const fp8 = structuredClone(receipt); fp8.settings.kv_cache = 'fp8' +assert.equal(parseBenchmarkReceipt(JSON.stringify(fp8)).settings.kv_cache, 'fp8') for (const mutate of [ (value: any) => { value.prompt = 'SYNTHETIC_PRIVATE_PROMPT' }, (value: any) => { value.model.repo_id = '/private/operator/model' }, @@ -32,6 +49,20 @@ for (const mutate of [ (value: any) => { value.runtime.dependencies.private_path = '/private/operator' }, (value: any) => { value.runtime.tensorfold_version = '9.9.9' }, (value: any) => { value.runtime.tensorfold_version = '1.1.0-preview' }, + (value: any) => { value.runtime.tensorfold_version = '1.1.0' }, + (value: any) => { delete value.settings.kv_cache }, + (value: any) => { value.settings.kv_cache = null }, + (value: any) => { value.settings.kv_cache = 'fp16' }, + (value: any) => { delete value.samples[0].route }, + (value: any) => { value.samples[0].route = '/private/path' }, + (value: any) => { value.samples[5].route = '/v1/completions' }, + (value: any) => { value.samples[1].route = '/v1/chat/completions' }, + (value: any) => { delete value.samples[0].finish_reason }, + (value: any) => { value.samples[0].finish_reason = null }, + (value: any) => { value.samples[0].finish_reason = 'unknown' }, + (value: any) => { Object.assign(value.samples[0], { finish_reason: 'stop', completion_tokens: 127, delivery_seconds: 1.26 }) }, + (value: any) => { Object.assign(value.samples[0], { finish_reason: 'length', completion_tokens: 128, delivery_seconds: 1.27 }) }, + (value: any) => { Object.assign(value.samples[10], { finish_reason: 'stop', completion_tokens: 128, delivery_seconds: 1.27 }) }, ]) { const invalid = structuredClone(receipt); mutate(invalid) assert.throws(() => parseBenchmarkReceipt(JSON.stringify(invalid)), /invalid|unsupported/) diff --git a/app/electron/agent/engine/benchmark/receipt.ts b/app/electron/agent/engine/benchmark/receipt.ts index fdaa19fc49..1861ab8629 100644 --- a/app/electron/agent/engine/benchmark/receipt.ts +++ b/app/electron/agent/engine/benchmark/receipt.ts @@ -3,7 +3,8 @@ import { isRepoId } from '../paths' import { strictBenchmarkJson } from './strict-json' import { supportedBenchmarkReceiptVersion } from './version' -export const BENCHMARK_SUITE_SHA = '9bb717da00666b90b4006174abd9ed46108affc37456b43c64a224a9c938d5be' +export const LEGACY_BENCHMARK_SUITE_SHA = '9bb717da00666b90b4006174abd9ed46108affc37456b43c64a224a9c938d5be' +export const BENCHMARK_SUITE_SHA = '51136f5d6775b2ff427834c6b1038b584de27e0a267cfc088ff2694ea069dcdc' export const BENCHMARK_RECEIPT_MAX_BYTES = 1024 * 1024 const SAMPLE_KEYS = ['fixture_id', 'temperature', 'repeat', 'seed', 'status', 'prompt_tokens', 'completion_tokens', 'cached_tokens', 'delivery_seconds', 'delivery_tps', 'ttft_seconds', 'end_to_end_seconds', 'server_decode_tps', 'server_decode_seconds', 'prefill_seconds', 'token_sha', 'error_code'] const ERROR_CODES = new Set(['http_error', 'connection_error', 'timeout', 'cancelled', 'server_error', 'invalid_stream', 'truncated_stream', 'missing_usage', 'invalid_usage', 'empty_output', 'unexpected_finish', 'token_count_mismatch', 'unmeasured_delivery']) @@ -28,11 +29,13 @@ export function parseBenchmarkReceipt(raw: string, expected?: { model: string; s if (value.schema_version !== 1 || typeof value.run_id !== 'string' || !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(value.run_id)) invalid() if (typeof value.created_at !== 'string' || !/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d{1,6})?Z$/.test(value.created_at) || !Number.isFinite(Date.parse(value.created_at)) || Date.parse(value.created_at) > now + 900_000 || new Date(value.created_at).toISOString().slice(0, 19) !== value.created_at.slice(0, 19)) invalid() object(value.suite, ['id', 'sha256']) - if (value.suite.id !== 'community-v1' || value.suite.sha256 !== BENCHMARK_SUITE_SHA) invalid() + const v2 = value.suite.id === 'community-v2' + if (v2 ? value.suite.sha256 !== BENCHMARK_SUITE_SHA : value.suite.id !== 'community-v1' || value.suite.sha256 !== LEGACY_BENCHMARK_SUITE_SHA) invalid() const runtime = value.runtime object(runtime, ['tensorfold_version', 'backend', 'python_version', 'platform', 'dependencies']) text(runtime.tensorfold_version); text(runtime.python_version) if (!supportedBenchmarkReceiptVersion(runtime.tensorfold_version) || expected?.engineVersion !== undefined && runtime.tensorfold_version !== expected.engineVersion || runtime.python_version !== 'none' || !['mlx', 'cuda', 'rocm'].includes(runtime.backend) || runtime.platform !== (runtime.backend === 'mlx' ? 'macos' : 'linux')) invalid() + if (v2 !== (runtime.tensorfold_version === '1.1.4')) invalid() if (!runtime.dependencies || typeof runtime.dependencies !== 'object' || Array.isArray(runtime.dependencies) || Object.keys(runtime.dependencies).some(key => !['mlx', 'mlx_lm', 'torch', 'triton', 'cuda', 'driver'].includes(key))) invalid() Object.values(runtime.dependencies).forEach(item => text(item, true)) const model = value.model @@ -47,13 +50,20 @@ export function parseBenchmarkReceipt(raw: string, expected?: { model: string; s for (const gpu of hardware.gpus) { object(gpu, ['name', 'memory_bytes', 'cores']); text(gpu.name); number(gpu.memory_bytes, true, true); number(gpu.cores, true, true, 100_000) } if (hardware.gpu_count !== null && hardware.gpu_count !== hardware.gpus.length) invalid() const settings = value.settings - object(settings, ['tokens', 'repetitions', 'temperatures', 'serial', 'context_tokens', 'managed', 'rank_count']) + object(settings, ['tokens', 'repetitions', 'temperatures', 'serial', 'context_tokens', 'managed', 'rank_count', ...(v2 ? ['kv_cache'] : [])]) if (settings.tokens !== 256 || settings.repetitions !== 5 || settings.context_tokens !== 4096 || settings.managed !== true || settings.rank_count !== 1 || typeof settings.serial !== 'boolean' || !Array.isArray(settings.temperatures) || settings.temperatures.length !== 2 || settings.temperatures[0] !== 1 || settings.temperatures[1] !== 0) invalid() + if (v2 && !['bf16', 'fp8'].includes(settings.kv_cache)) invalid() if (expected && (model.repo_id !== expected.model || settings.serial !== expected.serial)) invalid() if (!Array.isArray(value.samples) || value.samples.length < 1 || value.samples.length > 24) invalid() const seen = new Set() + const routes = new Map() for (const sample of value.samples) { - object(sample, SAMPLE_KEYS) + object(sample, v2 ? [...SAMPLE_KEYS, 'finish_reason', 'route'] : SAMPLE_KEYS) + if (v2) { + if (![null, 'stop', 'length'].includes(sample.finish_reason) || !['/v1/completions', '/v1/chat/completions'].includes(sample.route) || sample.fixture_id === 'chat' && sample.route !== '/v1/chat/completions') invalid() + if (routes.has(sample.fixture_id) && routes.get(sample.fixture_id) !== sample.route) invalid() + routes.set(sample.fixture_id, sample.route) + } if (!['code', 'chat'].includes(sample.fixture_id) || ![0, 1].includes(sample.temperature) || !Number.isInteger(sample.repeat) || sample.repeat < -1 || sample.repeat > 4 || sample.seed !== 1234 + Math.max(0, sample.repeat) || !['ok', 'early_eos', 'error', 'unmeasured'].includes(sample.status) || sample.repeat === -1 && sample.status !== 'error') invalid() const key = `${sample.fixture_id}:${sample.temperature}:${sample.repeat}` if (seen.has(key)) invalid(); seen.add(key) @@ -67,7 +77,8 @@ export function parseBenchmarkReceipt(raw: string, expected?: { model: string; s const rate = (sample.completion_tokens - 1) / sample.delivery_seconds if (Math.abs(sample.delivery_tps - rate) > Math.max(1e-6, rate * 1e-6)) invalid() } - if (sample.status === 'ok' && (sample.completion_tokens !== 256 || sample.delivery_tps === null || sample.error_code !== null)) invalid() + const naturalStop = v2 && sample.temperature > 0 && sample.finish_reason === 'stop' && sample.completion_tokens >= 128 && sample.completion_tokens < 256 + if (sample.status === 'ok' && (sample.completion_tokens !== 256 && !naturalStop || sample.delivery_tps === null || sample.error_code !== null || v2 && sample.finish_reason === null)) invalid() if (sample.status === 'early_eos' && (sample.completion_tokens === null || sample.completion_tokens >= 256)) invalid() } return value as ModelBenchmarkReceipt diff --git a/app/electron/agent/engine/benchmark/state.ts b/app/electron/agent/engine/benchmark/state.ts index b1e01a5040..69339ca486 100644 --- a/app/electron/agent/engine/benchmark/state.ts +++ b/app/electron/agent/engine/benchmark/state.ts @@ -17,5 +17,5 @@ export function savedBenchmarkSnapshot(raw: any, receipt: ModelBenchmarkReceipt const progress = raw.progress if (!progress || typeof progress !== 'object' || Object.keys(progress).sort().join(',') !== 'completedRequests,fixture,phase,repeat,temperature,totalRequests' || progress.totalRequests !== 24 || progress.completedRequests !== null && (!Number.isInteger(progress.completedRequests) || progress.completedRequests < 0 || progress.completedRequests > 24) || ![null, 'code', 'chat'].includes(progress.fixture) || ![null, 0, 1].includes(progress.temperature) || ![null, 'loading', 'warmup', 'measured'].includes(progress.phase) || progress.repeat !== null && (!Number.isInteger(progress.repeat) || progress.repeat < -1 || progress.repeat > 4)) invalid() if (!Array.isArray(raw.warnings) || raw.warnings.length > 10) invalid() - return { id: raw.id, model: raw.model, serial: raw.serial, state: raw.state, revision: raw.revision, startedAt: raw.startedAt, finishedAt: raw.finishedAt, progress: structuredClone(progress), cells: benchmarkCells(receipt ?? undefined), receiptAvailable: receipt !== null, protocolComplete: receipt ? benchmarkProtocolComplete(receipt) : false, previousModel: raw.previousModel, restored: raw.restored, restoreError: message(raw.restoreError), error: message(raw.error), storageError: message(raw.storageError), warnings: raw.warnings.map((value: unknown) => message(value) ?? invalid()), saved: raw.saved, metadata: receipt ? structuredClone({ runtime: receipt.runtime, model: receipt.model, hardware: receipt.hardware, settings: receipt.settings }) : null } + return { id: raw.id, model: raw.model, serial: raw.serial, state: raw.state, revision: raw.revision, startedAt: raw.startedAt, finishedAt: raw.finishedAt, progress: structuredClone(progress), cells: benchmarkCells(receipt ?? undefined), receiptAvailable: receipt !== null, protocolComplete: receipt ? benchmarkProtocolComplete(receipt) : false, previousModel: raw.previousModel, restored: raw.restored, restoreError: message(raw.restoreError), error: message(raw.error), storageError: message(raw.storageError), warnings: raw.warnings.map((value: unknown) => message(value) ?? invalid()), saved: raw.saved, metadata: receipt ? structuredClone({ suite: receipt.suite, runtime: receipt.runtime, model: receipt.model, hardware: receipt.hardware, settings: receipt.settings }) : null } } diff --git a/app/electron/agent/engine/benchmark/storage.test.ts b/app/electron/agent/engine/benchmark/storage.test.ts index 83ba35e4f1..30d5ccab91 100644 --- a/app/electron/agent/engine/benchmark/storage.test.ts +++ b/app/electron/agent/engine/benchmark/storage.test.ts @@ -3,7 +3,7 @@ import { mkdtemp, readFile, stat, writeFile, chmod, symlink, link, rm } from 'no import os from 'node:os' import path from 'node:path' import { syntheticBenchmarkReceipt } from './fixtures' -import { benchmarkCells } from './receipt' +import { benchmarkCells, LEGACY_BENCHMARK_SUITE_SHA } from './receipt' import type { ModelBenchmarkSnapshot } from '../../../../shared/model-benchmark' const home = await mkdtemp(path.join(os.tmpdir(), 'tensorfold-benchmark-storage-')) @@ -24,7 +24,11 @@ try { assert.equal(reloaded[0].snapshot.metadata?.hardware.cpu_model, 'Synthetic CPU') assert.equal(reloaded[0].snapshot.cells[0].deliveryTokensPerSecond?.median, 100) assert.equal(reloaded[0].receipt?.samples.length, 20) + assert.equal(reloaded[0].snapshot.metadata?.suite?.id, 'community-v2') const historical = structuredClone(receipt); historical.runtime.tensorfold_version = '1.0.12' + historical.suite = { id: 'community-v1', sha256: LEGACY_BENCHMARK_SUITE_SHA } + delete historical.settings.kv_cache + for (const sample of historical.samples) { delete sample.finish_reason; delete sample.route } const historicalSnapshot = { ...snapshot, id: 'benchmark_historical_stored' } await store.save({ snapshot: historicalSnapshot, receipt: historical }) const afterUpgrade = await createModelBenchmarkStore(home).list() diff --git a/app/electron/agent/engine/benchmark/version.ts b/app/electron/agent/engine/benchmark/version.ts index 775a9b7b23..87ab03b88a 100644 --- a/app/electron/agent/engine/benchmark/version.ts +++ b/app/electron/agent/engine/benchmark/version.ts @@ -2,7 +2,7 @@ import engineManifest from '../../../../build/engine.json' export const BENCHMARK_ENGINE_VERSION = engineManifest.version // Schema compatibility is reviewed explicitly; a newer binary cannot validate its own receipts by version alone. -const REVIEWED_VERSIONS = new Set(['1.0.12', '1.1.0']) +const REVIEWED_VERSIONS = new Set(['1.0.12', '1.1.0', '1.1.4']) export function supportedBenchmarkReceiptVersion(value: unknown): value is string { return typeof value === 'string' && REVIEWED_VERSIONS.has(value) } export function assertBundledBenchmarkVersion(value: unknown): string { if (value !== BENCHMARK_ENGINE_VERSION || !supportedBenchmarkReceiptVersion(value)) throw new Error(`This benchmark requires the bundled TensorFold ${BENCHMARK_ENGINE_VERSION} engine.`) diff --git a/app/package-lock.json b/app/package-lock.json index c9b1c84cd3..404274a39f 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -1,12 +1,12 @@ { "name": "tensorfold-studio", - "version": "1.0.0-beta.9", + "version": "1.0.0-beta.10", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "tensorfold-studio", - "version": "1.0.0-beta.9", + "version": "1.0.0-beta.10", "hasInstallScript": true, "dependencies": { "@agentclientprotocol/sdk": "^1.1.0", @@ -20,7 +20,7 @@ "imapflow": "^1.7.8", "katex": "0.19.0", "kokoro-js": "1.2.1", - "nodemailer": "^10.1.0", + "nodemailer": "10.1.4", "phonemizer": "file:mobile/vendor/tensorfold-kokoro-phonemizer", "qrcode": "^1.5.4", "react": "^18.3.1", @@ -29,7 +29,7 @@ "remark-gfm": "^4.0.0", "sprintf-js": "file:mobile/vendor/sprintf-js", "ssh2": "^1.17.0", - "undici": "7.30.0", + "undici": "7.30.1", "zustand": "^5.0.2" }, "devDependencies": { @@ -7051,9 +7051,9 @@ } }, "node_modules/nodemailer": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-10.1.0.tgz", - "integrity": "sha512-fXW8AulB25aOTuwfRYV6lqHyfz2VM4rr4SGXH03mE7x5znEUpzL5mu6anARUpZN7744Ziwkmr4AxSLPJsBQspg==", + "version": "10.1.4", + "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-10.1.4.tgz", + "integrity": "sha512-EsMnOR2oRv4LGnWvHgcVYkkQ4O2j8KO4WTw6l+PHR4sOXo8SAgIraxtEnGDmotjbhv0hDsu3IuDIFZm8iXwO4A==", "license": "MIT-0", "engines": { "node": ">=20.0.0" @@ -8666,9 +8666,9 @@ } }, "node_modules/undici": { - "version": "7.30.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.30.0.tgz", - "integrity": "sha512-dkrQXeHSaoamnItlYbmzG0wFYrM0ZwDxCIg0A7aKjTyyhh9svRzCNFEzV+Vm05/yehjCzjDZ31KXfGEjYSztDQ==", + "version": "7.30.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.30.1.tgz", + "integrity": "sha512-03KrMbjIFDWQR5RNPrSw3bd1r12JpyFQl9/yxjUbJtqkjmllqC5lFjZh6ujeuJCKr8EOQvRW872Oy/az+nLYxg==", "license": "MIT", "engines": { "node": ">=20.18.1" @@ -9140,9 +9140,9 @@ } }, "node_modules/zod": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", - "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "version": "4.6.7", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.7.tgz", + "integrity": "sha512-g8W2D2kU6a8untc053fX1uVBKzM7yqAxW/3zwk83+6RyF2+SKOuV+c0wBDhUwzV1PaGapmZH+rmcUzPf+3CGwg==", "license": "MIT", "peer": true, "funding": { diff --git a/app/package.json b/app/package.json index c44cae359c..029e26de6b 100644 --- a/app/package.json +++ b/app/package.json @@ -1,6 +1,6 @@ { "name": "tensorfold-studio", - "version": "1.0.0-beta.9", + "version": "1.0.0-beta.10", "private": true, "packageManager": "npm@10.9.8", "engines": { @@ -187,7 +187,7 @@ "imapflow": "^1.7.8", "katex": "0.19.0", "kokoro-js": "1.2.1", - "nodemailer": "^10.1.0", + "nodemailer": "10.1.4", "phonemizer": "file:mobile/vendor/tensorfold-kokoro-phonemizer", "qrcode": "^1.5.4", "react": "^18.3.1", @@ -196,7 +196,7 @@ "remark-gfm": "^4.0.0", "sprintf-js": "file:mobile/vendor/sprintf-js", "ssh2": "^1.17.0", - "undici": "7.30.0", + "undici": "7.30.1", "zustand": "^5.0.2" }, "devDependencies": { diff --git a/app/scripts/package-checks.test.mjs b/app/scripts/package-checks.test.mjs index 760fd44ce7..c44eee9370 100644 --- a/app/scripts/package-checks.test.mjs +++ b/app/scripts/package-checks.test.mjs @@ -40,9 +40,9 @@ for (const script of ['pack:mac:dev', 'pack:mac', 'dist:mac']) { assert.match(pkg.scripts[script], /node scripts\/fetch-engine\.mjs && node scripts\/build-computer-helper\.mjs && node scripts\/package-mac\.mjs/, `${script} stages the pinned engine and builds the computer-use helper before packaging`) } const engineManifest = JSON.parse(await readFile(path.join(root, 'build', 'engine.json'), 'utf8')) -assert.equal(engineManifest.version, '1.1.0') -assert.equal(engineManifest.sha256, '99c66f56fcde07a7dd8df6ea555bcbd406d0537b7d92a453112a54bef5a72707') -assert.equal(engineManifest.url, 'https://github.com/ashhart/TensorFold/releases/download/v1.1.0/tensorfold-1.1.0-macos-arm64.tar.gz') +assert.equal(engineManifest.version, '1.1.4') +assert.equal(engineManifest.sha256, 'e60096c8af989f26eb622bde3099ff6ed3ecc37cdd46ade0397b3afd16145c6f') +assert.equal(engineManifest.url, 'https://github.com/ashhart/TensorFold/releases/download/v1.1.4/tensorfold-1.1.4-macos-arm64.tar.gz') const ignore = await readFile(path.join(root, '.gitignore'), 'utf8') assert.match(ignore, /^build\/engine\/$/m, 'the staged engine is never committed') diff --git a/app/scripts/package-config.test.mjs b/app/scripts/package-config.test.mjs index 76aece7c6d..48db381883 100644 --- a/app/scripts/package-config.test.mjs +++ b/app/scripts/package-config.test.mjs @@ -13,7 +13,7 @@ async function main() { assert.equal(pkg.main, 'dist-electron/main.js') assert.equal(pkg.packageManager, 'npm@10.9.8') assert.deepEqual(pkg.engines, { node: '>=22 <23', npm: '10.9.8' }) - assert.equal(pkg.dependencies?.undici, '7.30.0', 'DNS pinning must ship as a direct production dependency') + assert.equal(pkg.dependencies?.undici, '7.30.1', 'DNS pinning must ship as a direct production dependency') assert.deepEqual(pkg.devEngines?.packageManager, { name: 'npm', version: '10.9.8', onFail: 'error' }) assert.equal(pkg.build?.appId, 'dev.tensorfold.studio') assert.equal(pkg.build?.productName, 'TensorFold') diff --git a/app/shared/model-benchmark.ts b/app/shared/model-benchmark.ts index 68f0946cd9..acc0aa5945 100644 --- a/app/shared/model-benchmark.ts +++ b/app/shared/model-benchmark.ts @@ -48,7 +48,7 @@ export interface ModelBenchmarkSnapshot { storageError: string | null warnings: string[] saved: boolean - metadata?: Pick | null + metadata?: (Pick & Partial>) | null } export interface ModelBenchmarkSample { fixture_id: ModelBenchmarkFixture @@ -68,16 +68,19 @@ export interface ModelBenchmarkSample { prefill_seconds: number | null token_sha: string | null error_code: string | null + /** Required by community-v2; absent in historical community-v1 receipts. */ + finish_reason?: 'stop' | 'length' | null + route?: '/v1/completions' | '/v1/chat/completions' } -/** Exact community-v1 export schema. Supplemental GUI state stays outside this object. */ +/** Reviewed community-v1/v2 export schemas. The parser enforces each protocol's exact fields. */ export interface ModelBenchmarkReceipt { schema_version: 1 - suite: { id: 'community-v1'; sha256: string } + suite: { id: 'community-v1' | 'community-v2'; sha256: string } run_id: string created_at: string runtime: { tensorfold_version: string; backend: 'mlx' | 'cuda' | 'rocm'; python_version: string; platform: 'macos' | 'linux'; dependencies: Record } model: { repo_id: string; revision: string | null; family: string | null; config_sha256: string | null; tokenizer_sha256: string | null; quantization: string | null; drafter_repo_id: string | null; local: boolean } hardware: { cpu_model: string | null; system_memory_bytes: number | null; memory_type: 'unified' | 'dedicated' | 'unknown'; gpus: { name: string; memory_bytes: number | null; cores: number | null }[]; gpu_count: number | null; source: 'detected' | 'partial' | 'unavailable' } - settings: { tokens: number; repetitions: number; temperatures: number[]; serial: boolean; context_tokens: number | null; managed: boolean; rank_count: number | null } + settings: { tokens: number; repetitions: number; temperatures: number[]; serial: boolean; context_tokens: number | null; managed: boolean; rank_count: number | null; kv_cache?: 'bf16' | 'fp8' | null } samples: ModelBenchmarkSample[] } diff --git a/app/src/components/models/ModelDetailsDrawer.tsx b/app/src/components/models/ModelDetailsDrawer.tsx index 8cb9d0a5e1..6dbca992c0 100644 --- a/app/src/components/models/ModelDetailsDrawer.tsx +++ b/app/src/components/models/ModelDetailsDrawer.tsx @@ -73,7 +73,7 @@ export function ModelDetailsDrawer({ model, status, blocked, onClose, onBenchmar
{tx("Source and technical details")}
{model.localPath && }

{tx("Quantization is not reported separately by this engine; check the model identifier and its source details.")}

- + } function Fact({ label, value }: { label: string; value: string }) { return
{label}
{value}
} diff --git a/app/src/components/models/benchmark/BenchmarkResults.tsx b/app/src/components/models/benchmark/BenchmarkResults.tsx index 4b63be5367..b9b8554677 100644 --- a/app/src/components/models/benchmark/BenchmarkResults.tsx +++ b/app/src/components/models/benchmark/BenchmarkResults.tsx @@ -2,7 +2,7 @@ import { currentIntlLocale } from '../../../lib/preferences/current-locale' import { OriginalDiagnostic } from '../../OriginalDiagnostic' import { useAuthoredText } from '../../../lib/preferences/useAuthoredText' import type { ModelBenchmarkSnapshot } from '../../../../shared/model-benchmark' -import { benchmarkActive, benchmarkCellLabel, benchmarkMetric, benchmarkRange } from './benchmark-display' +import { benchmarkActive, benchmarkCellLabel, benchmarkMetric, benchmarkRange, benchmarkSuite } from './benchmark-display' export function BenchmarkResults({ run }: { run: ModelBenchmarkSnapshot }) { const tx = useAuthoredText() @@ -13,7 +13,7 @@ export function BenchmarkResults({ run }: { run: ModelBenchmarkSnapshot }) { return

{run.model}

{run.serial ? tx("Serial baseline") : tx("Standard run")} · {new Date(run.startedAt).toLocaleString(currentIntlLocale())}

{stateLabel}

{active && <>

{reported ? tx("{value0} of {value1} requests reported", { value0: count, value1: run.progress.totalRequests }) : tx("Request counts have not been reported yet; progress is indeterminate.")}{run.progress.phase ? run.progress.phase === 'warmup' ? tx(' · Warm-up') : run.progress.phase === 'measured' ? tx(' · Measured request') : tx(' · Model loading') : ''}{run.progress.fixture ? ` · ${run.progress.fixture}` : ''}{run.progress.temperature !== null ? tx(" · Temperature {value0}", { value0: run.progress.temperature }) : ''}

} - {!active &&

{run.protocolComplete ? tx("All community-v1 cells meet the complete protocol.") : tx("The complete community-v1 protocol was not satisfied; incomplete cells are not performance scores.")}

} + {!active &&

{run.protocolComplete ? tx("All community-v1 cells meet the complete protocol.").replace('community-v1', benchmarkSuite(run)) : tx("The complete community-v1 protocol was not satisfied; incomplete cells are not performance scores.").replace('community-v1', benchmarkSuite(run))}

}
{run.error &&

{{run.error}}

}
{run.cells.map((cell) =>

{benchmarkCellLabel(cell)}

{cell.complete ? tx("Complete") : active && cell.attempted === 0 ? tx("Not reported yet") : tx("Incomplete")}

{cell.successful} {tx("of ")}{cell.expected} {tx("valid measured samples")}{cell.failedWarmups ? tx(" · {value0} failed warm-up", { value0: cell.failedWarmups }) : ''}

{tx("Median delivery speed")}
{benchmarkMetric(cell.deliveryTokensPerSecond, cell.complete, 'tokens/s')}
{benchmarkRange(cell.deliveryTokensPerSecond, cell.complete, 'tokens/s') &&

{tx("Observed range ")}{benchmarkRange(cell.deliveryTokensPerSecond, cell.complete, 'tokens/s')}

}
{tx("Median first-text latency")}
{benchmarkMetric(cell.firstTextSeconds, cell.complete, 's')}
{!cell.complete &&

{cell.earlyEos ? tx("Early endings: {count}. ", { count: cell.earlyEos }) : ''}{cell.errors ? tx("Errors: {count}. ", { count: cell.errors }) : ''}{cell.unmeasured ? tx("{value0} unmeasured. ", { value0: cell.unmeasured }) : ''}{tx("Medians require all five valid samples and a successful warm-up.")}

}
{tx("Additional timings")}
)}
diff --git a/app/src/components/models/benchmark/ModelBenchmarkDrawer.test.tsx b/app/src/components/models/benchmark/ModelBenchmarkDrawer.test.tsx index 8ef61390b8..7e74493bb2 100644 --- a/app/src/components/models/benchmark/ModelBenchmarkDrawer.test.tsx +++ b/app/src/components/models/benchmark/ModelBenchmarkDrawer.test.tsx @@ -81,4 +81,9 @@ assert.match(renderToStaticMarkup(React.createElement(MetadataView, { run: fixtu const recorded = fixtureRun({ metadata: { runtime: { tensorfold_version: '1.0.12', backend: 'mlx', python_version: '3.11', platform: 'macos', dependencies: { mlx: 'fixture-version' } }, model: { repo_id: 'fixture/recorded-model', revision: 'a'.repeat(40), family: 'fixture', config_sha256: 'b'.repeat(64), tokenizer_sha256: null, quantization: '4-bit', drafter_repo_id: null, local: false }, hardware: { cpu_model: 'Synthetic recorded CPU', system_memory_bytes: 8 * 1024 ** 3, memory_type: 'unified', gpus: [{ name: 'Synthetic recorded GPU', memory_bytes: null, cores: null }], gpu_count: 1, source: 'partial' }, settings: { tokens: 256, repetitions: 5, temperatures: [1, 0], serial: false, context_tokens: 4096, managed: true, rank_count: 1 } } }) const recordedMarkup = renderToStaticMarkup(React.createElement(MetadataView, { run: recorded })) assert.match(recordedMarkup, /Synthetic recorded CPU/); assert.match(recordedMarkup, /Synthetic recorded GPU/); assert.match(recordedMarkup, /TensorFold 1.0.12/); assert.match(recordedMarkup, /belong to this recorded run/); assert.doesNotMatch(recordedMarkup, /peak|power|watts/i) +assert.match(display.benchmarkBasicSummary(recorded), /community-v1/, 'old snapshots retain their original protocol label') +const v2 = fixtureRun({ metadata: { ...recorded.metadata!, suite: { id: 'community-v2', sha256: 'fixture' } } }) +assert.match(display.benchmarkBasicSummary(v2), /community-v2/) +const v2Markup = renderToStaticMarkup(React.createElement(ResultsView, { run: v2 })) +assert.match(v2Markup, /All community-v2 cells/); assert.doesNotMatch(v2Markup, /community-v1/) console.log('Benchmark drawer actual handlers: cached defaults, blocked/duplicate Run, serial selection, close/cancel, history, guarded export/copy/community link and truthful incomplete/missing metrics passed') diff --git a/app/src/components/models/benchmark/ModelBenchmarkDrawer.tsx b/app/src/components/models/benchmark/ModelBenchmarkDrawer.tsx index 3e272b3603..2bd7549559 100644 --- a/app/src/components/models/benchmark/ModelBenchmarkDrawer.tsx +++ b/app/src/components/models/benchmark/ModelBenchmarkDrawer.tsx @@ -29,7 +29,7 @@ export function ModelBenchmarkDrawer({ models, activeModel, preferredModel, bloc if (blockedReason || state.loading || state.error || running || acting || !valid) return void controller.start({ id: `benchmark_${crypto.randomUUID().replaceAll('-', '')}`, model, serial }) } - return {running ? : }}> + return {running ? : }}>

{tx("Uses fixed code and chat fixtures, a 4,096-token context and 256 output tokens; 24 requests include four warm-ups and 20 measured samples at temperatures 0 and 1.")}

{tx("This temporarily reserves an idle app-managed engine, pauses its server and restores the previous model and profile afterward; finish chat/API work first, and no models are downloaded automatically.")}

{blockedReason && !running &&

{blockedReason}

} diff --git a/app/src/components/models/benchmark/benchmark-display.ts b/app/src/components/models/benchmark/benchmark-display.ts index e12ae1fdef..6e1781d381 100644 --- a/app/src/components/models/benchmark/benchmark-display.ts +++ b/app/src/components/models/benchmark/benchmark-display.ts @@ -14,8 +14,9 @@ export function benchmarkRange(value: ModelBenchmarkSpread | null, complete: boo return complete && value && [value.min, value.max].every((number) => Number.isFinite(number) && number >= 0) ? `${Number(value.min.toFixed(2))}–${Number(value.max.toFixed(2))} ${unit}` : null } export function benchmarkCellLabel(cell: ModelBenchmarkCell): string { return `${cell.fixture === 'code' ? 'Code' : 'Chat'} · Temperature ${cell.temperature}` } +export const benchmarkSuite = (run: ModelBenchmarkSnapshot) => run.metadata?.suite?.id ?? 'community-v1' export function benchmarkBasicSummary(run: ModelBenchmarkSnapshot): string { - return [`TensorFold · community-v1`, `Model: ${run.model}`, `Mode: ${run.serial ? 'Serial baseline' : 'Standard'}`, `State: ${run.state}; complete protocol: ${run.protocolComplete ? 'yes' : 'no'}`, + return [`TensorFold · ${benchmarkSuite(run)}`, `Model: ${run.model}`, `Mode: ${run.serial ? 'Serial baseline' : 'Standard'}`, `State: ${run.state}; complete protocol: ${run.protocolComplete ? 'yes' : 'no'}`, ...run.cells.map((cell) => `${benchmarkCellLabel(cell)}: ${cell.complete ? `median delivery ${benchmarkMetric(cell.deliveryTokensPerSecond, true, 'tokens/s')}; first text ${benchmarkMetric(cell.firstTextSeconds, true, 's')}` : `incomplete, ${cell.successful}/${cell.expected} valid samples`}`), 'Measured speed is not a quality ranking.'].join('\n') }