diff --git a/build.zig b/build.zig index edebcb739c..a35d791f28 100644 --- a/build.zig +++ b/build.zig @@ -76,6 +76,20 @@ pub fn build(b: *std.Build) void { const slide_test_hooks = b.option(bool, "slide-test-hooks", "Living Weights test hooks, such as TF_SLIDE_FORGET_FAIL (never in a release)") orelse false; if (slide_test_hooks and release_version != null) @panic("-Dslide-test-hooks is for test builds: a -Dversion release never carries it"); build_options.addOption(bool, "slide_test_hooks", slide_test_hooks); + const lfm3_risk_fast = b.option(bool, "lfm3-risk-fast", "Developer LFM3 fused range predicate candidate") orelse false; + const lfm3_defer_head = b.option(bool, "lfm3-defer-prompt-head", "Developer LFM3 unobserved intermediate prompt head omission") orelse false; + const nax_prompt = b.option(bool, "lfm3-nax-prompt", "Development-only tensor-unit LFM3 prompt projections") orelse (release_version == null); + const tiled_attention = b.option(bool, "lfm3-tiled-attention", "Development-only tiled LFM3 prompt attention") orelse (release_version == null); + const prompt_rows = b.option(u32, "lfm3-prompt-rows", "Development-only LFM3 prompt rows: 128, 256 or 2048") orelse + (if (release_version == null and nax_prompt and tiled_attention) @as(u32, 2048) else 128); + switch (prompt_rows) { + 128, 256, 2048 => {}, + else => @panic("Invalid LFM3 prompt row count"), + } + if (prompt_rows > 128 and (!nax_prompt or !tiled_attention)) + @panic("Wide LFM3 prompts require both development prompt options"); + if ((nax_prompt or tiled_attention or prompt_rows > 128) and release_version != null) @panic("Tiled LFM3 candidates are development-only options"); + if (lfm3_defer_head and release_version != null) @panic("Deferred prompt head is a development-only option"); const optimize = b.option(std.builtin.OptimizeMode, "optimize", "debug, safe, fast or small (default fast)") orelse .fast; // Nemotron's draft vocabulary (zig/src/families/nemotron/draft_ids.txt), for every backend const ids_files = b.addWriteFiles(); @@ -87,7 +101,7 @@ pub fn build(b: *std.Build) void { @import("zig/build/kernel_symbols.zig").tests(b, test_step); @import("zig/build/checkpoint_security.zig").tests(b, test_step); if (b.graph.host.result.os.tag == .macos) { - const step5_mods = metalEngineModules(b, b.graph.host, .Debug, draft_ids); + const step5_mods = metalEngineModules(b, b.graph.host, .Debug, draft_ids, false, false, false, false, 128); const empty = b.addWriteFiles().add("step5-compile-only.metallib", ""); step5_mods.sources.addImport("nemotron_packed_metallib", embedded(b, empty, "nemotron_packed.metallib")); @import("zig/build/step5_host.zig").steps(b, step5_mods.metal, step5_mods.engine, step5_mods.lanes); @@ -96,8 +110,9 @@ pub fn build(b: *std.Build) void { @import("zig/build/digest.zig").steps(b, test_step, draft_ids, build_options); @import("zig/build/flashnext_budget.zig").steps(b, test_step); @import("zig/build/flashnext_batch_host.zig").tests(b, test_step); + @import("zig/build/lfm3_host.zig").steps(b, test_step); if (b.graph.host.result.os.tag == .macos) { - const budget_mods = metalEngineModules(b, b.graph.host, .Debug, draft_ids); + const budget_mods = metalEngineModules(b, b.graph.host, .Debug, draft_ids, false, false, false, false, 128); const budget_empty = b.addWriteFiles().add("budget-compile-only.metallib", ""); budget_mods.sources.addImport("nemotron_packed_metallib", embedded(b, budget_empty, "nemotron_packed.metallib")); @import("zig/build/fsr.zig").deviceSteps(b, budget_mods.metal, budget_mods.engine, budget_mods.sources); @@ -139,7 +154,7 @@ pub fn build(b: *std.Build) void { const cuda_options = @import("zig/build/options.zig").cuda(b); const tp_options = @import("zig/build/options.zig").tp(b); switch (target.result.os.tag) { - .macos => metalTargets(b, target, optimize, draft_ids, build_options, test_step), + .macos => metalTargets(b, target, optimize, draft_ids, build_options, test_step, lfm3_risk_fast, lfm3_defer_head, nax_prompt, tiled_attention, prompt_rows), .linux, .windows => cuda_build.targets(b, target, optimize, draft_ids, build_options, cuda_options), else => {}, } @@ -148,7 +163,7 @@ pub fn build(b: *std.Build) void { const gemma4_contract_host = @import("zig/build/gemma4_host.zig").tests(b, gemma4_host_sources); gemma4_contract_host.dependOn(@import("zig/build/glm_rows_contract.zig").host(b, gemma4_host_sources)); if (b.graph.host.result.os.tag == .macos) { - const gemma4_host_mods = metalEngineModules(b, b.graph.host, .Debug, draft_ids); + const gemma4_host_mods = metalEngineModules(b, b.graph.host, .Debug, draft_ids, false, false, false, false, 128); gemma4_host_mods.sources.addImport("nemotron_packed_metallib", embedded(b, b.addWriteFiles().add("gemma4-compile-empty.metallib", ""), "nemotron_packed.metallib")); @import("zig/build/gemma4_host.zig").compileHost(b, gemma4_host_mods.metal, gemma4_host_mods.engine, gemma4_host_mods.lanes, gemma4_host_mods.sources); @import("zig/build/qwen27_host.zig").compileHost(b, gemma4_host_mods.metal, gemma4_host_mods.engine, gemma4_host_mods.lanes); diff --git a/docs/receipts/lfm3/README.md b/docs/receipts/lfm3/README.md new file mode 100644 index 0000000000..69b27f3315 --- /dev/null +++ b/docs/receipts/lfm3/README.md @@ -0,0 +1,206 @@ +# LFM3 Metal qualification receipts + +## Current M5 long-context checks + +[`current-m5-long-context.json`](current-m5-long-context.json) records all 26 completed requests: +VL6/8 text prompts at 32k/64k, text4/6/8 at 131072/262128 prompt tokens, and VL4/6/8 at 127984 prompt tokens. +Each cell has two fresh-process legs. All requests return 16 tokens with zero cached input; +all 13 repeated output-token hashes match. All 16 servers exit zero, and all three packets release their leases. +The receipt includes artifact/checkpoint pins, exact input and output hashes, timings and process memory measurements. +These are native text-context checks, including VL checkpoints; they do not establish image-context, FP64 or comparative speed acceptance. +The earlier 20-request snapshot remains in `current-m5-long-context-partial.json`. + +## Tensor-unit route gates (M5 Ultra) + +- [`m5-nax-exactness.json`](m5-nax-exactness.json): prompt cuts, resume and verification windows against their one-pass references, byte for byte, at all six widths. +- [`m5-nax-serving-lifecycle.json`](m5-nax-serving-lifecycle.json): served drafted/plain, concurrent/solo, cache, image and cancellation checks at all six widths, plus cancellation and concurrency with 31k-token prompts. +- [`m5-nax-real-image-rows.json`](m5-nax-real-image-rows.json): native projected rows of five real images against an independent FP64 graph and official MLX BF16. +- [`m5-nax-32k-fidelity.json`](m5-nax-32k-fidelity.json): 32768-token prompt boundary and 16 teacher-forced rows against independent FP32 and official MLX BF16, for the tensor-unit and the earlier prompt route. +- [`m5-nax-launch-fusion-rejected.json`](m5-nax-launch-fusion-rejected.json): fused decode dispatches measured and not adopted. + +These receipts cover native LFM3-1.7B and LFM3-VL-2B from the official MLX exports. +They distinguish implementation checks from the still-required maintainer model/performance gate. +Checkpoint revisions and run instructions are in the [family recipe](../../recipes/lfm3-native.md). + +## Builds and hardware + +- Initial corrected native: `a8df9730ee637237545311a6753dd4bdcb13e860700d65a01a033ae818ee0d18`. +- Optimized benchmark native: `5c8a04d4af599879797f4f5bed8e00ab7d9d70732e698bd2a1ec2a01ec762e09`. +- Guarded row-pair build: `251079309919aa9b21ca8cc6a89c58fc3fc9a9baf800fa5ce6a6af313a5b5f65`. + +The guarded row-pair build additionally closes a diagnostic encoder when counter setup fails. +That change does not alter the production projection or attention arithmetic. Its serving checks were rerun separately. + +Correctness runs used an M4 Pro with 48 GB RAM, macOS 26.5.1 and Zig 0.17.0. +Controlled performance runs used an M5 Ultra with 256 GiB RAM and macOS 27.0 (26A428). +A measured footprint on the M5 is not a smaller-RAM qualification. + +## Served and operator checks + +The optimized M4 binary passed all six greedy/seeded-sampling suites: 303 passed, zero failed or skipped. +These cover drafted/plain, concurrent/solo, real cache reuse, edited prefixes, image order/content, +corrupt input and cancellation/recovery. The guarded row-pair binary also passed 303/303 on the M5. + +Text4 and VL4 grammar runs on both hosts each passed 32 configurations, concurrent comparisons and five refusals. +On each host, full-token capture verified 64/64 configuration groups, including hidden reasoning, across drafted/plain/concurrent requests. +An additional 80 checks covered real required/named tools, typed arguments, streaming, tool-result history and image-conditioned calls. +Grammar requires the matching optional library; these tests used an isolated bundle with library SHA-256 +`04133accdcecd01cd7dd4e8730855905850967ed8c6bd6d8a637258264ae7687`. + +The 4-bit text long suite passed 73/73, including chunk and 8,193-token seams. +The row-pair LFM3-only contract run on the M4 reported: + +```text +SUMMARY 50 kernels: 50 pass gates 1-4, 0 fail, 0 known findings; 13 controls: 13 red as planted, 0 missed; 0 limit problems; gates 19521 ms, limits 133 ms +``` + +The full earlier M5 runner reported 272/273 kernels passing, two known findings, all 298 controls red and zero limit problems. +Its Qwen4 HC activation failure reproduces exactly in a separate build of pinned base `b30d5b9d3`. +That is not a global pass or a final-source global-suite receipt. + +## Shared-stream attention follow-up + +`shared-attention.json` records the subsequent shared-dispatch source and binary identity. +Full and sliding attention reuse the serial partial and merge arithmetic, with stream-local cache addresses and row maps. +All 52 LFM3 kernels passed gates 1–4, all 14 controls were red as planted, and launch limits had zero problems. +The real text4 default exactness suite passed 68/68, including unequal two/four-stream state and logits. +All six sampled/cancellation serving suites on the M5 passed 303/303 with zero failures or skips. +This follow-up is byte-preserving shared scheduling; it does not qualify tensor-unit prompt attention or a speed gate. + +## Numerical evidence + +`projection-fp64-bounds.json` covers 66 projection/layout cells and 60,954,624 outputs. +The actual shared FP32 helper was captured before BF16 storage and compared with independently unpacked affine FP64 products. +No raw FP32 or BF16-store error exceeded the existing derived bounds. Worst-column errors were retained. + +The row-pair optimization invokes the same shared helper and preserves its accumulation order. +All 132 checkpoint comparisons at widths 1 and 64 produced 123,814,080 bytes identical to the old selector. +A separate 8,193-token before/after check compared all 2,097,408,000 BF16 logit bytes exactly. +Thus this optimization does not worsen any output-based precision metric on those comparisons. + +`fidelity-investigation.json` records the first actual primitive difference and its causal propagation. +Official MLX 0.32.4 uses different effective affine reconstruction in its vector and matrix paths. +The native engine consistently follows the FP32 affine equation rather than changing arithmetic with batch width. +The first captured FP32 error was about 6.81e-9, inside the existing derived 5.52e-5 pre-store bound; +it crossed a BF16 midpoint and propagated through recurrent history. + +`model-fp64-comparison.json` retains all 18 originally disjoint native/MLX positions, including repeated histories. +With direct binary64-to-BF16 rounding, native belongs to the ideal model maximum set in nine cases, MLX in 16, +both in eight and neither in one. This ideal FP64 model is a diagnostic; it does not execute the native FP32 reductions. +No new tie tolerance or end-to-end acceptance threshold was applied. The primitive evidence does not replace that model gate. + +The oracle is reproducible with `tools/lfm3/fp64_reference.py`; its tests include all finite BF16 encodings, +adjacent midpoints, double rounding, subnormals, overflow and six-bit cross-word unpacking. + +## Binned projection candidate + +`binned-projection.json` records the explicit candidate at `75478ed17`; ordinary model loading leaves it disabled. +It preserves each input value while partitioning exponent ranges and exact code nibbles. M5 uses the shared +`nax.h` tensor-unit fragments; M4 uses its fallback. Group affine sums use compensated FP32 pairs. +The integer argument is limited to finite normal aligned products without overflow or underflow. +It does not establish universal FP64 equivalence for extreme BF16 subnormals or arbitrary inputs. + +The actual native M5 tool produced **61,907,040/61,907,040** direct FP64-to-BF16 reference values across 132 captures: +11 projection classes, all six checkpoints, and prompt/decode widths 64 and 1. All 66 first-row comparisons matched. +The six small attention-gate cells use the unchanged row fallback and are included explicitly. +No column had more FP64 error than either the retained original-native or MLX result. +Those two comparator captures are from the M4, not a same-device M5 performance comparison. +The file retains worst-column metrics, input/output hashes, artifact identities and the complete contract summaries. +Both M4 and M5 passed all six operator variants, both planted controls and launch limits. +These primitive results do not settle whole-model fidelity or served performance. + +To reconstruct the fixtures, save `fixture_tokens` as a little-endian uint32 NPY array and run the default +`tf-lfm3-forward MODEL TOKENS.npy baseline.bf16 trace TRACE_DIR` for each named checkpoint. +Concatenate the trace files ending in `_.bf16` in filename order to recreate each `input_file`; +the first `2*k` bytes recreate its `one_row_file`. Verify the recorded input hash before comparing outputs. +Run `tf-lfm3-project MODEL MODULE INPUT.bf16 ROWS OUTPUT.bf16 --binned` at widths 64 and 1. +Save the receipt’s `cases` array as `manifest.json` beside the inputs for `tools/lfm3/projection_compare.py`. +Use the frozen candidate commit for these artifact identities. + +`binned-row-twin.json` records the later opt-in row twin at `5a2bdea76`: all 132 captures retain +123,814,080 bytes exactly against the earlier FP64-exact tiled captures. Both M4 and M5 pass six +variants and two planted controls. The twin remains disabled for ordinary model loading because +its serving performance has not passed; these operator results do not qualify a whole model. + +## Controlled performance + +`cold-prefill-ab.json` contains all 16 matched pairs / 32 fresh-server cold requests. +It includes full output-token hashes, body hashes, client TTFT, engine prefill time and peak physical process footprint. +Both sides used the same body, exact prompt counts, 16 output tokens, context 70,000, four configured lanes and zero cached tokens. +The 2k and 8k cells have three alternating pairs each; 32k and 64k have one pair each. +Only 4-bit performance was measured. + +Engine prefill seconds, before to after: + +- Text: 2k 1.239530 to 0.812959; 8k 5.307524 to 3.608895; 32k 27.314691 to 20.488389; 64k 73.402432 to 61.632933. +- VL decoder, text requests: 2k 1.261845 to 0.815432; 8k 5.402433 to 3.617449; 32k 27.686643 to 20.536702; 64k 73.920498 to 59.646930. + +These are engine prefill times, not client TTFT. All 16 paired full output-token hashes matched. +At 64k, physical footprint was 2,926,167,768 to 2,925,315,776 bytes for text and +3,586,115,720 to 3,584,559,264 bytes for VL. Device/pinned allocator peaks were not collected. + +`decode-concurrency.json` records 256-token, three-repetition code/prose runs. +All 60 concurrent-to-solo comparisons and all eight serial checks matched. +Small decode-rate differences are descriptive; they are not a separately established speedup. + +Earlier standard-server comparisons still showed MLX-VLM faster, especially for long cold prefill. +The faster arithmetic-changing matrix candidate failed the strict no-worse precision comparison and was excluded. +The slower uniform-exponential candidate was also excluded. No unvalidated GQA experiment is enabled. + +## Paired FFN and CPU greedy follow-up + +`greedy-dual.json` records the subsequent immutable `b804c6ed` native build. +The FFN gate/up projections share one dispatch while retaining each projection's shared helper and BF16 stores. +A SIMD CPU greedy reduction preserves the first maximum, signed-zero ties and nonfinite rejection; +masked and positive-temperature sampling keep the original path. +All 58 LFM3 kernel checks passed, all 16 controls were red, and launch limits had zero problems. +The real text4 exactness suite passed 68/68; six M5 sampled/cancellation serving suites passed 303/303. +Actual projection pairs retained 247,628,160 bytes, and six complete teacher-forced logit arrays retained 394,752,000 bytes. + +Matched M5 old/new/new/old runs used three repetitions and 256 output tokens per request. +All 240 concurrent full-token digests matched across the corresponding prompts and legs; +240/240 concurrent-to-solo and 32/32 serial checks passed. +Text code/prose single-stream rates changed from 309.27/322.59 to 317.39/332.83 tok/s; +four-stream aggregate rates changed from 682.50/717.25 to 836.65/856.70 tok/s. +VL decoder code/prose rates changed from 292.69/290.90 to 301.68/299.90 tok/s; +four-stream rates changed from 656.00/658.15 to 813.40/810.10 tok/s. +These are medians of the two run medians, with all raw three-repetition cells retained. +This ABBA experiment measures only 4-bit performance and does not assert standard-server parity. + +## Reproduce + +Use the pinned Zig version and a complete local checkpoint. Developer references need NumPy; the MLX reference also needs MLX. + +```sh +zig build native tf-lfm3-forward tf-lfm3-project tf-lfm3-profile tf-lfm3-exact tf-kernel-contract -Dcpu=apple_m1 +zig build test test-lfm3-host test-kernel-contract test-golden fuzz-cortheon -Dcpu=apple_m1 +python3 -m unittest discover -s tools/lfm3 -p 'test_*.py' +./zig-out/bin/tf-kernel-contract --only LFM3 +./zig-out/bin/tf-lfm3-exact MODEL_DIR --long +python3 tools/lfm3/serve_check.py --binary ./zig-out/native/bin/tensorfold-native --model-dir MODEL_DIR --sampled --cancel --output served.json +python3 tools/lfm3/reference.py --model-dir MODEL_DIR --tokens TOKENS.npy --output mlx.bf16 +python3 tools/lfm3/fp64_reference.py --model-dir MODEL_DIR --tokens TOKENS.npy --output fp64.bf16 --weight-mode exact +./zig-out/bin/tf-lfm3-forward MODEL_DIR TOKENS.npy native.bf16 +./zig-out/bin/tf-lfm3-profile MODEL_DIR TOKENS.npy profile.json +``` + +Repeat the served command for all six checkpoints. Run one model process per GPU, using the native lock and admission guards. +The synthetic cold-prompt request bodies and rendered token arrays are regenerated by `tools/lfm3/cold_prompts.py` (`--write DIR`); it checks all 16 files against the SHA-256 hashes in `cold-prompt-sha256.json`. +The native server did not return its rendered arrays, so matching counts are not described as a captured native-token identity proof. +To repeat cold HTTP measurements, start a fresh owned server with `--context 70000 --parallel 4 --prompt-cache-gib 0`, +POST a saved body to `/v1/chat/completions`, and retain usage, full token hash and timings from the stream. +The supplied bodies omit the optional model alias. Keep the exact bodies unchanged between compared builds. + +## All-width matched baseline + +`all-width-performance-baseline.json` records the shared-attention binary before the SIMD greedy and paired-FFN follow-ups. +Both native and the standard server completed 72 retained single-stream requests, 360 retained concurrency requests and 48 cold requests in total. +Every measured decode/concurrency reply has 256 tokens; each cold request has 16 output tokens and zero cached prompt tokens. +All 180 native concurrent-to-solo and 24 serial comparisons matched. Exact public prompts, client hashes, token hashes and timings are retained. +There is one cold sample per cell and three decode/concurrency repetitions. These baseline rates remain below the standard server; they are not a performance-gate pass. +The native binary SHA is `d1656249bdb581b147faacbd3b3c557c00b6c45b710eeddce517a67178b6ba2b`. + +The public long-prompt wrapper is `tools/lfm3/bench_decode.py`; its prompts match the recorded JSON exactly. +Use `python3 tools/lfm3/bench_decode.py decode URL MODEL --tokens 256 --reps 3 --temperatures 0 --output decode.json`. +For concurrency use the `concurrent` mode with `--levels 1,4 --alone --serial` on native, or `--levels 1,4 --no-seed` on standard. +Verify every reply length in the evidence; a shorter standard reply does not establish an equal-length rate. diff --git a/docs/receipts/lfm3/all-fp64-top-differences.json b/docs/receipts/lfm3/all-fp64-top-differences.json new file mode 100644 index 0000000000..5584687ea4 --- /dev/null +++ b/docs/receipts/lfm3/all-fp64-top-differences.json @@ -0,0 +1,293 @@ +{ + "scope": "Existing post-RoPE baseline prompt captures against direct FP64/BF16 oracle, selected 17 cells previously containing native/MLX disjoint maxima; not final prompt-tile qualification; nominal fixture names can exceed actual encoded row count, which is reported per cell", + "accepted_threshold": null, + "fidelity_pass": null, + "cells": { + "_trimmed_list": { + "count": 17, + "sha256": "d7c68c084ca9b8616e0e0d7b4b2428a8291cd6bd2d849b45c730793884627e07", + "first": [ + { + "rows": 257, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.0259931724353837, + "maximum_absolute_error": 2.3125, + "differences": [ + { + "row": 58, + "native_top": 3599, + "reference_top": 2292, + "native_maximum_set": [ + 3599 + ], + "reference_maximum_set": [ + 2292, + 3599 + ], + "maximum_sets_overlap": true, + "native_margin": 0.375, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.375, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": { + "_trimmed_list": { + "count": 20, + "sha256": "3ccf7ff882d65195a0d7b7f65f1e0073641db0b9d054f9c39bd93f4055e5f299", + "first": [ + { + "row": 106, + "rms_error": 0.3824222834630468, + "maximum_absolute_error": 2.3125 + }, + { + "row": 82, + "rms_error": 0.30074669025232487, + "maximum_absolute_error": 1.609375 + } + ], + "last": [ + { + "row": 77, + "rms_error": 0.09738425343068764, + "maximum_absolute_error": 0.34375 + } + ] + } + }, + "worst_columns": { + "_trimmed_list": { + "count": 20, + "sha256": "30caf217b30f5c02c6179ddecf1a69bcb564f48d3e4e27d31d2f8585afa0a2fd", + "first": [ + { + "column": 2789, + "rms_error": 0.19791373342423282, + "maximum_absolute_error": 2.3125 + }, + { + "column": 17913, + "rms_error": 0.18852943222159388, + "maximum_absolute_error": 2.0625 + } + ], + "last": [ + { + "column": 33006, + "rms_error": 0.15496943800268753, + "maximum_absolute_error": 1.66015625 + } + ] + } + }, + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "1.7B-6-prose-257", + "phase": "prompt", + "native_logits_sha256": "2fb12160d203b843337c07b5a3e5b7a1f35de658857c3e06d3827c77dac1d001", + "reference_logits_sha256": "3f4c9736cbcbbb893519fa7d1e70e3953df7337e09c4d4764a642cc6e116c89f", + "token_fixture_sha256": "522ec0f7a7ef8481d2f5533e3962ddad1f418d4337b095282e40c096e2befb1d" + }, + { + "rows": 481, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.024320309575124863, + "maximum_absolute_error": 2.3125, + "differences": [ + { + "row": 58, + "native_top": 3599, + "reference_top": 2292, + "native_maximum_set": [ + 3599 + ], + "reference_maximum_set": [ + 2292, + 3599 + ], + "maximum_sets_overlap": true, + "native_margin": 0.375, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.375, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": { + "_trimmed_list": { + "count": 20, + "sha256": "d9c1eb0ea949368664537761d9a1e96d0818ec8cf3e638ed44a0cdeb6fa4a56e", + "first": [ + { + "row": 106, + "rms_error": 0.3824222834630468, + "maximum_absolute_error": 2.3125 + }, + { + "row": 82, + "rms_error": 0.30074669025232487, + "maximum_absolute_error": 1.609375 + } + ], + "last": [ + { + "row": 214, + "rms_error": 0.10219409674230935, + "maximum_absolute_error": 0.4375 + } + ] + } + }, + "worst_columns": { + "_trimmed_list": { + "count": 20, + "sha256": "3c0cc6aea45e742f24c20a41fa20eecbb3003ba61c370e68b19165ce8d34b054", + "first": [ + { + "column": 2789, + "rms_error": 0.1578289865837652, + "maximum_absolute_error": 2.3125 + }, + { + "column": 17913, + "rms_error": 0.15538353652730102, + "maximum_absolute_error": 2.0625 + } + ], + "last": [ + { + "column": 8639, + "rms_error": 0.13764913746301133, + "maximum_absolute_error": 1.15625 + } + ] + } + }, + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "1.7B-6-prose-513", + "phase": "prompt", + "native_logits_sha256": "b6c07fed3bdccf311a1a46b1b2dc3a5f3de069d0b0a433d679a25b41cfa15968", + "reference_logits_sha256": "3906de3fe32531a42940b5a11bb42bde5bfa5f7e574e65b1220e561aebe88f42", + "token_fixture_sha256": "4b9e533cc1948ee72c4231178ec3af21808d7542c97f35997bb9538161d9a4cb" + } + ], + "last": [ + { + "rows": 513, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "relative_rms": 0.00937199471206947, + "maximum_absolute_error": 1.03125, + "differences": [ + { + "row": 5, + "native_top": 440, + "reference_top": 334, + "native_maximum_set": [ + 440 + ], + "reference_maximum_set": [ + 334, + 440 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + }, + { + "row": 33, + "native_top": 1579, + "reference_top": 2925, + "native_maximum_set": [ + 1579, + 2925 + ], + "reference_maximum_set": [ + 2925 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + } + ], + "worst_rows": { + "_trimmed_list": { + "count": 20, + "sha256": "c7eb19b07754ef4dd2cc6f9befd30c455d576d9ac2f73ce6ff9c0d19111f8e3f", + "first": [ + { + "row": 477, + "rms_error": 0.11058203618233109, + "maximum_absolute_error": 0.25 + }, + { + "row": 461, + "rms_error": 0.10228833277998309, + "maximum_absolute_error": 0.6875 + } + ], + "last": [ + { + "row": 23, + "rms_error": 0.07232874153553556, + "maximum_absolute_error": 0.1875 + } + ] + } + }, + "worst_columns": { + "_trimmed_list": { + "count": 20, + "sha256": "c611ad7791b56c188fa597ba82b87f7e78e0338d06c6acc828ed72d54d904ca2", + "first": [ + { + "column": 334, + "rms_error": 0.09123395835991276, + "maximum_absolute_error": 0.59375 + }, + { + "column": 10625, + "rms_error": 0.0874009612140456, + "maximum_absolute_error": 0.40625 + } + ], + "last": [ + { + "column": 16420, + "rms_error": 0.07075007746790489, + "maximum_absolute_error": 0.359375 + } + ] + } + }, + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-8-reasoning-513", + "phase": "prompt", + "native_logits_sha256": "0fbe7f9ff161bf9b0961bd4d64b081f775dc51bb8eb72e7c1bcef7ac9bcbd582", + "reference_logits_sha256": "3abcad4537454a6995fc3a7a75fc74d47a0e148eb0f2e914ee9c46bc1fd9595a", + "token_fixture_sha256": "c8ce58a8d512375ea4635bc85d0ea8a235506aa56026087bda216ea61357a863" + } + ] + } + }, + "rows": 5581, + "top1_differences": 37, + "disjoint_maximum_sets": 12, + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "_trim": { + "note": "Raw capture replaced by counts, first/last items and SHA-256 of the canonical JSON of each trimmed list or string.", + "original_sha256": "7abcce70668e3d60b27ac1d7a60e199f84b5ca214b6a73ec0a65f75fbb057477", + "original_bytes": 119943 + } +} diff --git a/docs/receipts/lfm3/all-width-performance-baseline.json b/docs/receipts/lfm3/all-width-performance-baseline.json new file mode 100644 index 0000000000..ec8ada986e --- /dev/null +++ b/docs/receipts/lfm3/all-width-performance-baseline.json @@ -0,0 +1,583 @@ +{ + "native_sha256": "d1656249bdb581b147faacbd3b3c557c00b6c45b710eeddce517a67178b6ba2b", + "scope": "shared-attention baseline before greedy SIMD and paired FFN dispatch", + "hardware": "Apple M5 Ultra, 256 GiB, macOS 27.0 (26A428)", + "standard": { + "mlx": "0.32.4", + "mlx_vlm_source": "230fc308efb577d87446306bfb8a3e9f1a129558" + }, + "settings": { + "context": 70000, + "parallel": 4, + "native_prompt_cache_gib": 0, + "standard_APC_ENABLED": 0, + "standard_APC_DISK_ENABLED": 0, + "temperature": 0, + "top_p": 1, + "top_k": 0, + "repetition_penalty": 1, + "thinking": false, + "decode_tokens": 256, + "decode_reps": 3, + "cold_tokens": 16, + "cold_reps": 1 + }, + "cells": { + "_trimmed_list": { + "count": 12, + "sha256": "13d4c5749cdc89b9e5110f976f276166cc0ab1eff66a30ce35e393559af8e9af", + "first": [ + { + "family": "text", + "bits": 4, + "engine": "native", + "decode": [ + { + "label": "text-4-native", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 308.4647098580153, + "decode_tps_all": [ + 308.52, + 308.46, + 307.92 + ], + "ttft_s_median": 0.02111462503671646, + "prefill_tps_median": 2510.109, + "itl_mean_ms_median": 3.242, + "itl_p50_ms_median": 3.305, + "itl_p95_ms_median": 3.76, + "round_gap_p50_ms_median": 3.318, + "round_gap_p95_ms_median": 3.76, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ] + }, + { + "label": "text-4-native", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 321.89276348274177, + "decode_tps_all": [ + 321.89, + 322.39, + 321.74 + ], + "ttft_s_median": 0.0213663331232965, + "prefill_tps_median": 2527.341, + "itl_mean_ms_median": 3.107, + "itl_p50_ms_median": 3.315, + "itl_p95_ms_median": 3.768, + "round_gap_p50_ms_median": 3.367, + "round_gap_p95_ms_median": 3.77, + "tokens_per_chunk_median": 1.103, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ] + } + ], + "concurrent": { + "label": "text-4-native", + "tokens": 256, + "mixed": false, + "cells": { + "_trimmed_value": { + "bytes": 32731, + "sha256": "80c9b2c6f95ef9fc726a41fd1f6b56aa8c96b03b905c56fd13fb7d4d9c3f4102", + "keys": null, + "count": 4 + } + } + }, + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.825507041066885, + "elapsed_s": 0.9079782911576331, + "engine_prefill_s": 0.813756542, + "full_token_sha256": "c244b715e827fd1ce59aa4e249cb47900812abeee1482b0a79a1b251fe0f3262", + "output_text_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "normalized_body_sha256": "b9bc5e2feab4faa89e3cff58510809a260dfe738c75102107e2ccb2a018c3f3e", + "peak_physical_process_footprint_bytes": 1491076776 + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 3.614595875144005, + "elapsed_s": 3.695393417030573, + "engine_prefill_s": 3.601386791, + "full_token_sha256": "3f913af21b98f7168ba1fcd27bd9f16b62982dcf6168b991091bbd6e4d9e7fed", + "output_text_sha256": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349", + "normalized_body_sha256": "e4d6edf5df4aa5f7d6633ccf51897404f88b486f5cc919ed353e40acb5aa8ce1", + "peak_physical_process_footprint_bytes": 1629882048 + }, + { + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 20.516632792074233, + "elapsed_s": 20.598936625290662, + "engine_prefill_s": 20.498580167, + "full_token_sha256": "e15e7490e640d938a0a846dfc9de1dd7a51e50ae97ca4fd42a213cbb633f66f5", + "output_text_sha256": "382055acebd5e91b010ffa41b6497ebdb6d22c9bbdd7a2ab220c18df2e11f6b3", + "normalized_body_sha256": "667df3a5602ea177f4c8014d7c12c748d3df65fa026d4891a8dd87977c8c8fcf", + "peak_physical_process_footprint_bytes": 2185103040 + }, + { + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 59.7936379997991, + "elapsed_s": 59.87897541699931, + "engine_prefill_s": 59.772125333, + "full_token_sha256": "cf6c4062b60dc49e0ca503b7be2aaf3d247d73be037d269cf73e127a5e35aa32", + "output_text_sha256": "49b9640f88ffd97f99e1c6a354211b68d7b226874d3d8c1afbc261003f02cb39", + "normalized_body_sha256": "f82e80f89b846734ed458943f26d45bac00bfdae2fe2c20e0680ef1060e490ca", + "peak_physical_process_footprint_bytes": 2925315776 + } + ] + }, + { + "family": "text", + "bits": 4, + "engine": "standard", + "decode": [ + { + "label": "text-4-standard", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 345.11249650803427, + "decode_tps_all": [ + 344.67, + 345.18, + 345.11 + ], + "ttft_s_median": 0.017844750080257654, + "prefill_tps_median": 2970.061, + "itl_mean_ms_median": 2.898, + "itl_p50_ms_median": 2.877, + "itl_p95_ms_median": 2.953, + "round_gap_p50_ms_median": 2.877, + "round_gap_p95_ms_median": 2.953, + "tokens_per_chunk_median": 1.0, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ] + }, + { + "label": "text-4-standard", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 344.8503001737025, + "decode_tps_all": [ + 344.96, + 344.78, + 344.85 + ], + "ttft_s_median": 0.020470124669373035, + "prefill_tps_median": 2637.991, + "itl_mean_ms_median": 2.9, + "itl_p50_ms_median": 2.889, + "itl_p95_ms_median": 2.989, + "round_gap_p50_ms_median": 2.889, + "round_gap_p95_ms_median": 2.989, + "tokens_per_chunk_median": 1.004, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "fc6c1799b10d5ece11495fb9eb854a5562db013147b5382e1e92d4dc95bf5ae7" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "fc6c1799b10d5ece11495fb9eb854a5562db013147b5382e1e92d4dc95bf5ae7" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "fc6c1799b10d5ece11495fb9eb854a5562db013147b5382e1e92d4dc95bf5ae7" + } + ] + } + ], + "concurrent": { + "label": "text-4-standard", + "tokens": 256, + "mixed": false, + "cells": { + "_trimmed_value": { + "bytes": 9931, + "sha256": "5d1cb1db49abd07604a8ce4269762134d06ae52faa19b0531cdda1d7ce2a6581", + "keys": null, + "count": 4 + } + } + }, + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.1617831252515316, + "elapsed_s": 0.20617104228585958, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "normalized_body_sha256": "b9bc5e2feab4faa89e3cff58510809a260dfe738c75102107e2ccb2a018c3f3e", + "peak_physical_process_footprint_bytes": 2587510944 + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.738223166204989, + "elapsed_s": 0.7843746249563992, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349", + "normalized_body_sha256": "e4d6edf5df4aa5f7d6633ccf51897404f88b486f5cc919ed353e40acb5aa8ce1", + "peak_physical_process_footprint_bytes": 2904115336 + }, + { + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 1.6866859588772058, + "elapsed_s": 1.7399688339792192, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "5bb9e62f6a8c22891e1f0f3814891df33bf136e0e41a2f3e531f4cbf2af171a2", + "normalized_body_sha256": "667df3a5602ea177f4c8014d7c12c748d3df65fa026d4891a8dd87977c8c8fcf", + "peak_physical_process_footprint_bytes": 3534162152 + }, + { + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 4.055460750125349, + "elapsed_s": 4.124586749821901, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "49b9640f88ffd97f99e1c6a354211b68d7b226874d3d8c1afbc261003f02cb39", + "normalized_body_sha256": "f82e80f89b846734ed458943f26d45bac00bfdae2fe2c20e0680ef1060e490ca", + "peak_physical_process_footprint_bytes": 4395944072 + } + ] + } + ], + "last": [ + { + "family": "vl", + "bits": 8, + "engine": "standard", + "decode": [ + { + "label": "vl-8-standard", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 273.7432577926277, + "decode_tps_all": [ + 273.69, + 273.9, + 273.74 + ], + "ttft_s_median": 0.018855708185583353, + "prefill_tps_median": 2810.82, + "itl_mean_ms_median": 3.653, + "itl_p50_ms_median": 3.628, + "itl_p95_ms_median": 3.668, + "round_gap_p50_ms_median": 3.628, + "round_gap_p95_ms_median": 3.668, + "tokens_per_chunk_median": 1.0, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + } + ] + }, + { + "label": "vl-8-standard", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 273.8349864799858, + "decode_tps_all": [ + 273.83, + 273.88, + 273.2 + ], + "ttft_s_median": 0.018880541902035475, + "prefill_tps_median": 2860.087, + "itl_mean_ms_median": 3.652, + "itl_p50_ms_median": 3.627, + "itl_p95_ms_median": 3.676, + "round_gap_p50_ms_median": 3.627, + "round_gap_p95_ms_median": 3.676, + "tokens_per_chunk_median": 1.0, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "aa1580092ce8b46dc61bd425d5697c7031f2e7e503428df48248be6c514babdf" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "aa1580092ce8b46dc61bd425d5697c7031f2e7e503428df48248be6c514babdf" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "aa1580092ce8b46dc61bd425d5697c7031f2e7e503428df48248be6c514babdf" + } + ] + } + ], + "concurrent": { + "label": "vl-8-standard", + "tokens": 256, + "mixed": false, + "cells": { + "_trimmed_value": { + "bytes": 9912, + "sha256": "ddf9c11d08218a375365d7a62f1ef7175fd9bf24509c0b8f84b07ab07b0fa350", + "keys": null, + "count": 4 + } + } + }, + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.1605673748999834, + "elapsed_s": 0.2167126671411097, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "normalized_body_sha256": "b9bc5e2feab4faa89e3cff58510809a260dfe738c75102107e2ccb2a018c3f3e", + "peak_physical_process_footprint_bytes": 3761376352 + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.44067829102277756, + "elapsed_s": 0.5023705409839749, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "b7b834e8f232d6ff122c84f11c241563a5b5bd6ae97dfaabd0e38994edaff19e", + "normalized_body_sha256": "e4d6edf5df4aa5f7d6633ccf51897404f88b486f5cc919ed353e40acb5aa8ce1", + "peak_physical_process_footprint_bytes": 4019571664 + }, + { + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 1.7595829591155052, + "elapsed_s": 1.8293205839581788, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "cd68ec38f174ad2fc03030a610282a0678fb8c6662ab51a33bf72ca2bc7f3192", + "normalized_body_sha256": "667df3a5602ea177f4c8014d7c12c748d3df65fa026d4891a8dd87977c8c8fcf", + "peak_physical_process_footprint_bytes": 4726967512 + }, + { + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 4.19275641720742, + "elapsed_s": 4.273371207993478, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "ac0fb26c45837a298e9195e00cbefd1205ae81470b3b3e54b3555dd440946ea7", + "normalized_body_sha256": "f82e80f89b846734ed458943f26d45bac00bfdae2fe2c20e0680ef1060e490ca", + "peak_physical_process_footprint_bytes": 5640195192 + } + ] + } + ] + } + }, + "counts": { + "measured_decode_requests": 72, + "measured_concurrent_requests": 360, + "cold_requests": 48, + "native_alone_equal": 180, + "native_serial_equal": 24 + }, + "decode_prompts": [ + { + "name": "long-code-chat", + "kind": "chat", + "prompt": "Write a complete Python implementation of a red-black tree with insertion, deletion, searching, rotations, and extensive unit tests. Include detailed explanations after each method. Continue until the implementation and every test are complete." + }, + { + "name": "long-prose-chat", + "kind": "chat", + "prompt": "Write a detailed 2000-word tutorial explaining how matrix multiplication runs on a GPU, including memory layout, tiling, shared memory, synchronization, numerical precision, and several worked numerical examples. Explain every step thoroughly." + } + ], + "zig": "0.17.0", + "benchmark_sources_sha256": { + "bench_openai.py": "d5af4fbb756cdfb573546c0e0017652a4bb8f509d1251af683be7a3cbdd426f6", + "bench_concurrent.py": "0e72e26769f082459329fc068c25f6f72a9155e422962ea54cb5d1226227752d", + "bench_evidence.py": "54c28474f0804dbd120e3fb5859be0d1667ef21ca502880d0bed098ec9bb42ff", + "bench_metrics.py": "26f4c3ecbb03df4f12e24d18840fea989d9ef8c63117ca11eb944ef670458a27" + }, + "long_prompt_wrapper_sha256": "48c0709f4c40734a60f0d22fa73adee69f68e40ecc18f43a574447af82fb252e", + "_trim": { + "note": "Raw capture replaced by counts, first/last items and SHA-256 of the canonical JSON of each trimmed list or string.", + "original_sha256": "c871bd05735bf3b98b9d7596b4f3f2c03941813071c918652b25786f231fd76c", + "original_bytes": 556016 + } +} diff --git a/docs/receipts/lfm3/attention-range-text.json b/docs/receipts/lfm3/attention-range-text.json new file mode 100644 index 0000000000..47e6437b25 --- /dev/null +++ b/docs/receipts/lfm3/attention-range-text.json @@ -0,0 +1,41 @@ +{ + "scope": "Text-slice applied artifact CPU compilation and source review; no text-binary device rerun or final performance acceptance claimed", + "commit": "4199067fa66ce858ba060b9c7e4e7fae8c65c7f7", + "build": { + "command": "zig build native tf-kernel-contract test-lfm3-host -Dcpu=apple_m1 --summary all", + "actual_exit": 0, + "log": "$TMPDIR/lfm3-range-text-build.log" + }, + "format_check": "passed", + "lean": { + "baseline_problems": 428, + "current_problems": 428, + "new_problems": 0 + }, + "text_registry_has_new_lfm3_vision_imports": false, + "source_sha256": { + "zig/kernels/metal/lfm3/attention.metal": "6dc51acbb45e2e65028e72fed72d318629eb7b382e773c5837db9baa8de5d3b9", + "zig/src/families/lfm3/forward.zig": "618e9ecf0e96f4709c0ad543585ec58d06f14501cabdc2b9caeba9ab1d73e7b6", + "zig/src/families/lfm3/state.zig": "1161aea9174fd6a6dbecc09bfd26565459c9d7b8d1674ca06c7c78c1b0265517", + "zig/tests/contract/metal_lfm3.zig": "60abd8dd905a669c13c565241a5537ce9164bf3f9ab2400c70e666af8786e18d", + "zig/tests/contract/metal_lfm3_segments.zig": "434e8215ec6d4459f7b377100bf099f81ccee85a768bb3b6981c6c16ab9b4d29", + "zig/tests/contract/metal_lfm3_attention_sums.zig": "146e77b310349ddee5dc87bfce3d6b05950ce1412c96e1b3cf46e2b247116665", + "zig/tests/contract/metal_lfm3_alpha.zig": "f9df1bdc9d49a8a1847579f79898694ee5b083ec4ff9ac5d1460ca8b2f33d287", + "zig/tests/contract/metal_main.zig": "0c3a9bacccebcd14d6a80227c225c45afe8241ad641dcd6e459dc7d26dbe3a53", + "zig/tests/contract/metal_limits.zig": "5c094102286e9936f96c822277e35e8b065a18f967302d6808420c9cb08d271c", + "zig/src/core/contract/lfm3_records.zig": "b26c6ebf598b95927ccb4fb09d4ad23e6416cc21de5c3c2af9f56b8b793f803f" + }, + "artifact_sha256": { + "zig-out/native/bin/tensorfold-native": "623acac2dd42b51af61bcd9da242a6ec15af55deafdafd46cf4e54054329544f", + "zig-out/bin/tf-kernel-contract": "7216e3e315b252ff6f473d47203257da121997044dfbf3ba0036b55a2153d746" + }, + "guarded_combined_main_source_qualification": { + "state_passed": 408, + "serving_passed": 303, + "public_exact_logit_bytes": 394752000, + "registered_kernels_passed": 70, + "limit_problems": 0, + "receipt": "attention-range.json" + }, + "performance_accepted": false +} diff --git a/docs/receipts/lfm3/attention-range.json b/docs/receipts/lfm3/attention-range.json new file mode 100644 index 0000000000..524dd307d1 --- /dev/null +++ b/docs/receipts/lfm3/attention-range.json @@ -0,0 +1,136 @@ +{ + "scope": "Applied public source matches all ten guarded private candidate source hashes; final applied artifact build pending. No final performance acceptance.", + "complete": { + "actual_exit": 0, + "models": 6, + "state_passed": 408, + "serving_passed": 303, + "all_current_public_logits_exact": true, + "logit_bytes": 394752000 + }, + "models": [ + { + "model": "LFM3-1.7B-MLX-4bit", + "state_passed": 68, + "state_failed": 0, + "serving_counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "logit_bytes": 65792000, + "logits_current_public_exact": true, + "logits_sha256": "f1dbd42c2556fcb05031a00b209c61fadee45e74ee1ff94628d4802479fcb518" + }, + { + "model": "LFM3-1.7B-MLX-6bit", + "state_passed": 68, + "state_failed": 0, + "serving_counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "logit_bytes": 65792000, + "logits_current_public_exact": true, + "logits_sha256": "08bf833a94fa0403bc65c70677d20372b5ab0616fb0a58d26119c486069ef59c" + }, + { + "model": "LFM3-1.7B-MLX-8bit", + "state_passed": 68, + "state_failed": 0, + "serving_counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "logit_bytes": 65792000, + "logits_current_public_exact": true, + "logits_sha256": "1345bd80acaa16fe08a00a14d839e3e37cf9d0d8cd7e98c01b68cd1b8d2f51d4" + }, + { + "model": "LFM3-VL-2B-MLX-4bit", + "state_passed": 68, + "state_failed": 0, + "serving_counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "logit_bytes": 65792000, + "logits_current_public_exact": true, + "logits_sha256": "a4dde0a00ee9acb294c2b5a791e7e73cd05c5d6904ffe171e439ad1b01e25c5d" + }, + { + "model": "LFM3-VL-2B-MLX-6bit", + "state_passed": 68, + "state_failed": 0, + "serving_counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "logit_bytes": 65792000, + "logits_current_public_exact": true, + "logits_sha256": "93ea83c90c5623f4e17502a526e2358f9e1ba3f2d03999295745257dc95bbca6" + }, + { + "model": "LFM3-VL-2B-MLX-8bit", + "state_passed": 68, + "state_failed": 0, + "serving_counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "logit_bytes": 65792000, + "logits_current_public_exact": true, + "logits_sha256": "5a013f3eed97f143429f09a81a829b07dc32360734cd54855e18e514160e0a48" + } + ], + "registered_contracts": { + "passed": 70, + "failed": 0, + "known_findings": 0, + "controls_red": 2, + "controls_missed": 0, + "limit_problems": 0 + }, + "postnorm_control_stage_exit": 0, + "source_and_binary_sha256": { + "$TMPDIR/lfm3-combined-qualify.py": "9d9a5f942609de71f0494ce5b23ef0f5bd8f80b65aa9732ff0d5ea601a183a76", + "$TMPDIR/lfm3-combined-manifest.json": "2c86619e73c86e6d21a99f1276102294c6c538005af393af5ea4bd6178b61ddc", + "$TMPDIR/lfm3-combined-promotion.patch": "58df0a67e783c6d3af89fe18fb878083c9f1055ed09a692340b13612eaf0d9f2", + "$TMPDIR/lfm3-postnorm-branchless-combined/zig-out/native/bin/tensorfold-native": "ad72d7ec17b681ede54676ba75f7d4d1b39282aa0dae7cf27d8b201660fce489" + }, + "event_log": "events.json", + "manifest": "manifest.json", + "public_changes_applied": true, + "performance_accepted": false, + "applied_public_source_matches_qualified_private": true, + "committed_or_pushed": false, + "format_check": "passed", + "lean": { + "baseline_problems": 428, + "current_problems": 428, + "new_problems": 0 + }, + "build": { + "status": "passed", + "log": "$TMPDIR/lfm3-applied-build.log", + "command": "zig build native tf-kernel-contract tf-lfm3-forward tf-lfm3-exact test-lfm3-host test-lfm3-image-host -Dcpu=apple_m1 --summary all", + "actual_exit": 0 + }, + "performance_scope": { + "acceptance": false, + "isolated_branchless_local_M4_vs_pre_postnorm_baseline_throughput_percent": -5.3498, + "combined_vs_current_public_and_standard": "pending guarded matched local M4 benchmark", + "M5_performance_claim": false + }, + "applied_public_artifact_sha256": { + "zig-out/native/bin/tensorfold-native": "3d4a3daeda697420bfbba6d81972896c2bf9e31efaf6f12321ee8487a299354b", + "zig-out/bin/tf-kernel-contract": "2786719bfe0314754349463721b8e1faa7cd06f6f2efa142db19acacd83d54e6", + "zig-out/bin/tf-lfm3-forward": "243da3eb79553e7937c36959db13df84baa9d7fcc3e4724d164a708f2fd1eacc", + "zig-out/bin/tf-lfm3-exact": "e078bbbe23f5a36ac7813b9cef1dab6b9ecfd404d39a99ec21969e3ee5cd55ad" + } +} diff --git a/docs/receipts/lfm3/binned-nonfinite.json b/docs/receipts/lfm3/binned-nonfinite.json new file mode 100644 index 0000000000..0a19f12dae --- /dev/null +++ b/docs/receipts/lfm3/binned-nonfinite.json @@ -0,0 +1,145 @@ +{ + "status": "verified; binned candidate remains disabled by default", + "regression": { + "inputs": "BF16 X=+1 [1,64], codes=1, scale=+1, bias=+Inf, group64, bits4", + "before": "NaN (0x7fc0), where the independent factored FP64 expression is +Inf", + "cause": "FMA product residual inf-inf contaminated the low component of an infinite product.", + "fix": "Keep the original computed compensated product when both components are finite. On nonfinite/overflow results only, normalize the input pair and recompute, returning nonfinite results with a zero low component.", + "before_public_shader_sha256": "8119aa792197ebf3c9b12f767fe0dfe12482189340be8e75ae93f7910ac944ca", + "probe_binary_sha256": "18a8d11c077ae817f560d7ed31ea0045c8a1b2c058612d793feb60b627025588", + "before_probe_shader_hash": "not retained; original arithmetic is re-executed by the registered nonfinite negative control", + "finite_domain_scope": "The operation order is unchanged for original compensated products whose two components are finite. Finite operands that overflow can intentionally change; this is not a universal all-finite-input equivalence claim." + }, + "native_nonfinite_probe": { + "shader_sha256": "3fd70fb76ff2ea9ec12b55f3cccb7a2c8ddeeb1e8010f244a7910b101e1d463c", + "cases_per_host": 30, + "raw_candidate_corrected_canonical_pairs_per_host": 960, + "finite_outputs_per_host": 96, + "nonfinite_outputs_per_host": 864, + "axes": [ + "input", + "scale", + "bias" + ], + "planted_values": [ + "NaN", + "+Inf", + "-Inf" + ], + "bits": [ + 4, + 6, + 8 + ], + "group": 64, + "hosts": { + "M4 Pro": { + "raw_bytes_equal": true, + "signed_inf_nan_classes_match": true, + "nonfinite_outputs_certified": 0 + }, + "M5 Ultra": { + "raw_bytes_equal": true, + "signed_inf_nan_classes_match": true, + "nonfinite_outputs_certified": 0 + } + } + }, + "kernel_contract": { + "fixture": "Existing nonfinite input rows plus a positive finite last row, NaN and both infinities in scales/biases, and q6/q8 nibble/bin cancellation leaving (high=0,low=2).", + "new_control": "TF_BP_BAD_NONFINITE restores the original product-residual bug and must fail gate1.", + "M4 Pro": { + "kernels_pass": 6, + "kernels_fail": 0, + "controls_red_as_planted": 3, + "controls_missed": 0, + "limit_problems": 0 + }, + "M5 Ultra": { + "kernels_pass": 6, + "kernels_fail": 0, + "controls_red_as_planted": 3, + "controls_missed": 0, + "limit_problems": 0 + } + }, + "finite_checkpoint_proof": { + "host": "M5 Ultra", + "cases": 132, + "bytes": 123814080, + "all_canonical_bytes_equal": true, + "prompt_decode_pairs": 66, + "all_prompt_decode_first_row_bytes_equal": true, + "comparison": "All132 captures match the frozen FP64-exact canonical captures in binned-projection.json; includes six unchanged n16 fallbacks.", + "binary_sha256": "18d7dbf6148dc0541dc69422b0a4026fe2aa325482a1941095ffcda5da2eb465", + "capture_manifest_sha256": "60c0243fd00107f3e16c73efb3022d02ad19d50022d3fb52414597687f525bdf" + }, + "artifacts": { + "project": "18d7dbf6148dc0541dc69422b0a4026fe2aa325482a1941095ffcda5da2eb465", + "contract": "09506e643820fecac7be980dff611bb96f010e81b7e5239238fbc76527665b87" + }, + "source_sha256": { + "zig/kernels/metal/core/binned_projection.metal": "3029d1cee80ad6897d0a5dd3e788291ad9bb5f74d1fe1fb689c04368569d618f", + "zig/tests/contract/metal_binned_projection.zig": "100d93e632d73d5ce73221646e4ce2d1be42f06ac3aa9b0862a667c3262e31b6", + "zig/tests/contract/metal_main.zig": "b6e42170977de56d6c12c68c994b7b7fe1fd0f2af7f5865a94d3274116e548db", + "zig/tests/contract/metal_limits.zig": "a898d53688271ca6da39daac167e582d7237c9e24d8ba849b211e1b3a9b3a864", + "zig/src/core/contract/lfm3_records.zig": "9ba50b5b5e4d0ffc9834e2c83c6b4c11872e57758b6372402ee293a0d41fd34f" + }, + "scope": "No production candidate activation or performance acceptance is asserted by this receipt.", + "reproduce_kernel_contract": [ + "zig build tf-kernel-contract tf-lfm3-project", + "zig-out/bin/tf-kernel-contract --only 'LFM3 binned'" + ], + "tested_source_commit": "43e5f1a90", + "comment_only_successor_commit": "8955faee0", + "comment_only_successor_scope": "Comments only; tested mathematical operations and line positions are unchanged. The source hashes below identify the actual pre-comment binaries, not the successor.", + "low_component_regression": { + "before_shader_sha256": "c839dbf5d8945f907b516de9f8ca0cb2e7d47b8acc4d63c349dc22ec863e1a3c", + "operator_fixture": { + "bits": [ + 6, + 8 + ], + "input_first_group": [ + "2^26", + "-2^22", + "2", + "remaining entries zero" + ], + "codes_first_group": [ + 1, + 16, + 1 + ], + "remaining_codes": "zero", + "compensated_qx": [ + "high=0", + "low=2" + ], + "scales": [ + "+Inf", + "-Inf", + "NaN" + ], + "bias_inf_columns": "Retained, with a positive input sum; original nonfinite input rows also remain." + }, + "before_native_helper_cases": 4, + "before_native_helper_result": "(0,+/-2)*+Inf produced NaN; finite BF16-maximum products produced signed infinity with a NaN low component.", + "after_native_helper_cases_per_host": 7, + "after_checks": [ + "signed low-only products with +/-Inf", + "low-component overflow with finite BF16 maximum", + "zero represented sum (1,-1) times Inf gives NaN", + "low-component overflow with full finite FP32 maximum" + ], + "after_result": "Six infinite products match raw pair/BF16 bytes with zero low components; the zero-sum case matches the NaN class and a zero low component.", + "M4 Pro": "pass", + "M5 Ultra": "pass", + "probe_program_sha256": "763320e00066edcd336a17a103c666d71ad84db90c2a7c349410a4f58b90e9c1", + "before_probe_shader_sha256": "a474c1dec0889494a33b1bca611d02947fd170caaa49248e321890fd81601e9f", + "after_probe_shader_sha256": "c8e8a0dbf3cf65fd3d64de428690e21a7e8ffa16eb823ce6f9df715a225d421e", + "inputs_sha256": "a92318e04f453f7278b534d246aa528935351e4c4165622a936b4c5fba106057", + "expected_sha256": "d5be7739c64d768ca08422016ae6977a0499aafd7794bc054aa54b6729285d09", + "before_native_helper_host": "M4 Pro" + } +} diff --git a/docs/receipts/lfm3/binned-projection.json b/docs/receipts/lfm3/binned-projection.json new file mode 100644 index 0000000000..5c82dcd2f2 --- /dev/null +++ b/docs/receipts/lfm3/binned-projection.json @@ -0,0 +1,334 @@ +{ + "status": "explicit qualification candidate; disabled during ordinary model loading", + "source_commit": "75478ed17", + "shader_sha256": "605188026ccfe9f2e60bd01ee425a954367272c21a0cf6d2e43b7af5ea99261d", + "native_project_tool_sha256": "6c1ed257a51327e0f7d3ae8806fcff488ba07198376432508f4e8d38a5ed1609", + "contract_runner_sha256": "e35dfdacaeba5b35811e60f5c4fe11f951d3025752f746beaae2cb46c761ae14", + "original_row_project_tool_sha256": "244137da5e7afccad8fc6fb97598d53aec96cf2d097d8bfbdf296310ba013428", + "native_platform": { + "gpu": "M5 Ultra", + "ram_gib": 256, + "os": "macOS 27.0 (26A428)", + "zig": "0.17.0", + "path": "nax.h tensor-unit fragments selected by Device.tensorUnits()" + }, + "comparison_platform": { + "gpu": "M4 Pro", + "ram_gb": 48, + "os": "macOS 26.5.1", + "mlx": "0.32.4", + "scope": "Retained original-native and MLX projection captures use identical input rows on M4; this is not a same-device M5 timing comparison." + }, + "reference": { + "arithmetic": "Independent CPU FP64 dequantized checkpoint products", + "rounding": "Direct binary64-to-BF16 round-to-nearest-even", + "scope": "Projection fixtures only; not a whole-model fidelity acceptance policy." + }, + "integer_bin_argument": { + "group64_max_exact_integer_sum": 15667200, + "group32_max_exact_integer_sum": 15667200, + "fp32_exact_integer_limit": 16777216, + "group64_exponent_span": 6, + "group32_exponent_span": 7, + "code_parts": "q = low_nibble + 16 * high_nibble; four-bit codes need only the low part", + "scope": "Finite normal aligned products without overflow/underflow; no universal exactness claim for extreme BF16 subnormals or arbitrary inputs." + }, + "summary": { + "cells": 66, + "prompt_decode_equal_cells": 66, + "outputs": 61907040, + "correctly_rounded": 61907040, + "columns_worse_than_original": 0, + "columns_worse_than_mlx": 0, + "native_hardware": "M5 Ultra", + "comparison_hardware": "M4 Pro", + "binary_sha256": "6c1ed257a51327e0f7d3ae8806fcff488ba07198376432508f4e8d38a5ed1609", + "model_acceptance_threshold": null + }, + "contracts": { + "m4": { + "hardware": "M4 Pro", + "exit": 0, + "summary": "SUMMARY 6 kernels: 6 pass gates 1-4, 0 fail, 0 known findings; 2 controls: 2 red as planted, 0 missed; 0 limit problems; gates 9716 ms, limits 5147 ms" + }, + "m5": { + "hardware": "M5 Ultra", + "exit": 0, + "summary": "SUMMARY 6 kernels: 6 pass gates 1-4, 0 fail, 0 known findings; 2 controls: 2 red as planted, 0 missed; 0 limit problems; gates 8967 ms, limits 918 ms" + } + }, + "fixture_tokens": { + "_trimmed_list": { + "count": 64, + "sha256": "3c782c154742a2226fdb1585116768d20b09661031e0264f4efa0e91d6b59cef", + "first": [ + 89509, + 3420 + ], + "last": [ + 22 + ] + } + }, + "cases": { + "_trimmed_list": { + "count": 66, + "sha256": "3bc8ae852836fceee16a92faa6eda417799034976358998a14fe2e02bc9a3374", + "first": [ + { + "name": "1.7B-4-conv_in", + "class": "conv_in", + "shape": [ + 64, + 2048, + 6144 + ], + "input_sha256": "c3c833599efae0b6f0abeeab1c674d0f88fd3828c89f7e3453505feb1abfa34e", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.00043982049783110983, + "max_abs": 0.007671507657505572, + "worst_column_rms": 0.002848056337488529, + "worst_column": 3716, + "correctly_rounded": 393216, + "outputs": 393216 + }, + "original_native": { + "rms": 0.00043982049813563945, + "max_abs": 0.007671507657505572, + "worst_column_rms": 0.002848056337488529, + "worst_column": 3716, + "correctly_rounded": 393170, + "outputs": 393216 + }, + "mlx": { + "rms": 0.0005148634877763631, + "max_abs": 0.007671507657505572, + "worst_column_rms": 0.002895652676547294, + "worst_column": 3716, + "correctly_rounded": 239774, + "outputs": 393216 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "764c618b648c35902ec16e7f8578687e9b514a4c06ab5e7191c122ce8c704ffe" + }, + { + "mode": "decode", + "native": { + "rms": 0.00048554828079777217, + "max_abs": 0.0037449413212016225, + "worst_column_rms": 0.0037449413212016225, + "worst_column": 1969, + "correctly_rounded": 6144, + "outputs": 6144 + }, + "original_native": { + "rms": 0.00048554828079777217, + "max_abs": 0.0037449413212016225, + "worst_column_rms": 0.0037449413212016225, + "worst_column": 1969, + "correctly_rounded": 6144, + "outputs": 6144 + }, + "mlx": { + "rms": 0.00048554828079777217, + "max_abs": 0.0037449413212016225, + "worst_column_rms": 0.0037449413212016225, + "worst_column": 1969, + "correctly_rounded": 6144, + "outputs": 6144 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "fa0a8412931265b2da7c5083d58a2bb9f9a776a65f8476ee623a7293dc07a1d2" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-4bit", + "module": "language_model.model.layers.0.conv.in_proj", + "input_stage": "layer_00_operator_norm", + "input_file": "1.7B-4-conv_in.bf16", + "one_row_file": "1.7B-4-conv_in-one.bf16", + "model_directory": "LFM3-1.7B-MLX-4bit", + "rows": 64, + "k": 2048, + "n": 6144 + }, + { + "name": "1.7B-4-conv_out", + "class": "conv_out", + "shape": [ + 64, + 2048, + 2048 + ], + "input_sha256": "a0044e29906efde3689ad19b91868257d8487c11655ac8e83d1e5e73bd3e163e", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 1.602854875446973e-05, + "max_abs": 0.0009396018046068377, + "worst_column_rms": 0.00032164384276088477, + "worst_column": 1688, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 1.602854875697783e-05, + "max_abs": 0.0009396018046068377, + "worst_column_rms": 0.00032164384276088477, + "worst_column": 1688, + "correctly_rounded": 131057, + "outputs": 131072 + }, + "mlx": { + "rms": 1.7543640518602656e-05, + "max_abs": 0.0012221553097768734, + "worst_column_rms": 0.00032858266117196354, + "worst_column": 1688, + "correctly_rounded": 94899, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "72ccfcb68f3a6e49a45a3c61501b886efdf56617095f4ea39c6f8fa6dacf812c" + }, + { + "mode": "decode", + "native": { + "rms": 2.696892560157749e-06, + "max_abs": 5.5160145456811804e-05, + "worst_column_rms": 5.5160145456811804e-05, + "worst_column": 1700, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 2.696892560157749e-06, + "max_abs": 5.5160145456811804e-05, + "worst_column_rms": 5.5160145456811804e-05, + "worst_column": 1700, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "mlx": { + "rms": 2.6968925601881646e-06, + "max_abs": 5.5160145456811804e-05, + "worst_column_rms": 5.5160145456811804e-05, + "worst_column": 1700, + "correctly_rounded": 2047, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "571e0e60f83da426e1f36973ce381c6c07dc0250805f58875cb4c6ce3b16f635" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-4bit", + "module": "language_model.model.layers.0.conv.out_proj", + "input_stage": "layer_00_conv_gated", + "input_file": "1.7B-4-conv_out.bf16", + "one_row_file": "1.7B-4-conv_out-one.bf16", + "model_directory": "LFM3-1.7B-MLX-4bit", + "rows": 64, + "k": 2048, + "n": 2048 + } + ], + "last": [ + { + "name": "VL-2B-8-vocabulary_head", + "class": "vocabulary_head", + "shape": [ + 64, + 2048, + 128000 + ], + "input_sha256": "69e9450f2984d88628637b19bfd27885d6e09ebe0841a6c99a60340fc2a6c237", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.00637062298687024, + "max_abs": 0.06121470243670046, + "worst_column_rms": 0.01631051233544983, + "worst_column": 278, + "correctly_rounded": 8192000, + "outputs": 8192000 + }, + "original_native": { + "rms": 0.006370622995905622, + "max_abs": 0.06121470243670046, + "worst_column_rms": 0.01631051233544983, + "worst_column": 278, + "correctly_rounded": 8191553, + "outputs": 8192000 + }, + "mlx": { + "rms": 0.007425419414880146, + "max_abs": 0.06121470243670046, + "worst_column_rms": 0.016382519090016284, + "worst_column": 278, + "correctly_rounded": 6260448, + "outputs": 8192000 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "61f3c553b0a0daf33f2dd29f46677112dae2639fd338014b7ebdd22558c6b1cb" + }, + { + "mode": "decode", + "native": { + "rms": 0.00450617515076936, + "max_abs": 0.02677314502216177, + "worst_column_rms": 0.02677314502216177, + "worst_column": 1022, + "correctly_rounded": 128000, + "outputs": 128000 + }, + "original_native": { + "rms": 0.0045061751651349746, + "max_abs": 0.02677314502216177, + "worst_column_rms": 0.02677314502216177, + "worst_column": 1022, + "correctly_rounded": 127992, + "outputs": 128000 + }, + "mlx": { + "rms": 0.004506175151630346, + "max_abs": 0.02677314502216177, + "worst_column_rms": 0.02677314502216177, + "worst_column": 1022, + "correctly_rounded": 127994, + "outputs": 128000 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "e250665cdfd5e271e66e762ae85ec1bae526ef802cc62ec9f736be9d52622f13" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-8bit", + "module": "language_model.model.embed_tokens", + "input_stage": "final_norm", + "input_file": "VL-2B-8-vocabulary_head.bf16", + "one_row_file": "VL-2B-8-vocabulary_head-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-8bit", + "rows": 64, + "k": 2048, + "n": 128000 + } + ] + } + }, + "_trim": { + "note": "Raw capture replaced by counts, first/last items and SHA-256 of the canonical JSON of each trimmed list or string.", + "original_sha256": "f69ea8b1f4b260a8e50c566854b6325dd9edcfec57b6c47eaf39a29cf7ba8ba4", + "original_bytes": 189089 + } +} diff --git a/docs/receipts/lfm3/binned-row-twin.json b/docs/receipts/lfm3/binned-row-twin.json new file mode 100644 index 0000000000..a1e8140f73 --- /dev/null +++ b/docs/receipts/lfm3/binned-row-twin.json @@ -0,0 +1,941 @@ +{ + "scope": "Explicitly opt-in canonical row twin; ordinary model loading remains disabled. No served performance pass.", + "commit": "5a2bdea76", + "comparisons": 132, + "bytes": 123814080, + "cases": [ + { + "name": "1.7B-4-conv_in", + "rows": 64, + "bytes": 786432, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "764c618b648c35902ec16e7f8578687e9b514a4c06ab5e7191c122ce8c704ffe" + }, + { + "name": "1.7B-4-conv_in", + "rows": 1, + "bytes": 12288, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "fa0a8412931265b2da7c5083d58a2bb9f9a776a65f8476ee623a7293dc07a1d2" + }, + { + "name": "1.7B-4-conv_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "72ccfcb68f3a6e49a45a3c61501b886efdf56617095f4ea39c6f8fa6dacf812c" + }, + { + "name": "1.7B-4-conv_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "571e0e60f83da426e1f36973ce381c6c07dc0250805f58875cb4c6ce3b16f635" + }, + { + "name": "1.7B-4-ffn_gate", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "c2f30477ae8dc9605d0624cd0e488921a3d628f5e22756d0d699ceea360166a3" + }, + { + "name": "1.7B-4-ffn_gate", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "88fe235008e7a04f2aa0a365a599700df32f4e4b511487ae47f84cbd9042147d" + }, + { + "name": "1.7B-4-ffn_up", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "bfa00c6ce34772edc11f01c7de19f0b3c9c3b2cf28288500c70cc1a8fe823787" + }, + { + "name": "1.7B-4-ffn_up", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "7936b781b393ee17409bd70f5bc5592c64963a8d08141633b37e072a3c13362d" + }, + { + "name": "1.7B-4-ffn_down", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "81e564394395e6f5ae3e1ba6f1c524ea76ec070846e90020ae4db3e9a1d8f122" + }, + { + "name": "1.7B-4-ffn_down", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "c33b3ac7d6ecc10fc84d4c45adb28d52fd65037a2f34a2293123484fe9747e76" + }, + { + "name": "1.7B-4-attention_q", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "35d010f3010570b13756e02ffad15c2e3d88a20c8fa17d848fc9644e7e46f41f" + }, + { + "name": "1.7B-4-attention_q", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "a2a0743f91f9e9b823701b82fd82860e202ad3a82b6c94cd8b8a200f5a48f86c" + }, + { + "name": "1.7B-4-attention_k", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "b6119e6fdfa2e6b2e57e63a5efbdde3777ea84cebadd42f70b8eb2d24a0b38c6" + }, + { + "name": "1.7B-4-attention_k", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "49cd1a700b8e55f3ae9081f9dfec38426a2c42cfd091122200705bc882dde26e" + }, + { + "name": "1.7B-4-attention_v", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "c8e358a2139f014b68ef723b2ad0b600422c1600c93dd497e42818487a4cf8b1" + }, + { + "name": "1.7B-4-attention_v", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "f807cedaf1556dc08feeb85ab4e26c8e9adb2597a9a27f7cce06259b40e95643" + }, + { + "name": "1.7B-4-attention_gate", + "rows": 64, + "bytes": 2048, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "acaf96d0437193e28abe706219a35cd47abb6783f846a6c3edcf376c73ebbbd5" + }, + { + "name": "1.7B-4-attention_gate", + "rows": 1, + "bytes": 32, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "04c0a5e4019680d000290cddf3e2a236028ab4f341059eab523bab6f452402d3" + }, + { + "name": "1.7B-4-attention_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "fcc3661b25c7a9b58e7cf37790af16a9f9775061dc504e8880574cf5b3064509" + }, + { + "name": "1.7B-4-attention_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "9bc91b6132035b4c9c3e26eb2b0e1ff0471e8b1675f0de175c4555e4e3e4a6d3" + }, + { + "name": "1.7B-4-vocabulary_head", + "rows": 64, + "bytes": 16384000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "6786268290e834bbc1d6f7367f17f3d8240fede1a1c66608edf843bdd3ddd3b9" + }, + { + "name": "1.7B-4-vocabulary_head", + "rows": 1, + "bytes": 256000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "ab9a7a2a9c4ce1da97742a4d0d5bfd497ee5f97be6e7f40ddbb35074779490f1" + }, + { + "name": "1.7B-6-conv_in", + "rows": 64, + "bytes": 786432, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "e690ade98b46e3c1ed023f33a6c729070b280b6b367b6356d046bad571f9e7d5" + }, + { + "name": "1.7B-6-conv_in", + "rows": 1, + "bytes": 12288, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "8812effabc5786e48e33c14f58aa0aed55ba1a2041ff8c68aa7c0e57c1d5db14" + }, + { + "name": "1.7B-6-conv_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "7b42d71bdf4c84a41f239b9928d62a42d591e1a19a32595add3b17bb0402ec08" + }, + { + "name": "1.7B-6-conv_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "6b6acdf7bfcca8b64f078b9bc8ce48a03490595f2870672482e1509380bcd107" + }, + { + "name": "1.7B-6-ffn_gate", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "6ff356e3c153ebc2efebc895be224be29f28512a97f76eaee2e29b20c56aab32" + }, + { + "name": "1.7B-6-ffn_gate", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "dc997a64e86ef2c76b962890db232dddd6bdd55819092a846abd0b0f5b82e6f3" + }, + { + "name": "1.7B-6-ffn_up", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "ad79c7c7bfd66a7545a9d76490566a10a3fd3dd0ee2fd74f97ca0e144a52f2f4" + }, + { + "name": "1.7B-6-ffn_up", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "546cd059b084c49f08ca8244d16b9ee8a5c2c4c797492adf45d359cb77ac141a" + }, + { + "name": "1.7B-6-ffn_down", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "3a6b7f54b9e6e33077d0bc94526b59f8bd460b4ef08470658b17df388abf6789" + }, + { + "name": "1.7B-6-ffn_down", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "d58d233e1dd4097bce3dda862fee878e15ffa51016c11d235f7544a7cc6e3db3" + }, + { + "name": "1.7B-6-attention_q", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "20779bc07a475a9ca2e2e43f9dfcc20e5e335854cb233f1633a45d8fa425140e" + }, + { + "name": "1.7B-6-attention_q", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "0bacb201fc20be38fbe4bc2589867d82c517ee19f3c261766f80d0f1f80c1cbf" + }, + { + "name": "1.7B-6-attention_k", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "92fef97597874698bad91388b36fcb1579fd9176643637397e43bca03a61d4e0" + }, + { + "name": "1.7B-6-attention_k", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "b59457203b990344119a76037c9ad8e4dbeaaa005239ef3d6c12d1dad72d229f" + }, + { + "name": "1.7B-6-attention_v", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "087f15e21a21ff48959028377610106d96c0bfd2f78e37ce6f039eaea7f95fb2" + }, + { + "name": "1.7B-6-attention_v", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "3b40c61d73782c5b2193028ea246f7964508017bf74fbf63e69750c99edce57a" + }, + { + "name": "1.7B-6-attention_gate", + "rows": 64, + "bytes": 2048, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "a71bd4049b62100ef10b401077aa7f1ee7b92cf39165bf06f217847b417f08c4" + }, + { + "name": "1.7B-6-attention_gate", + "rows": 1, + "bytes": 32, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "bd5cc9257542ff115afb330579f0fc9ca115c7b01bd446b1059f8e4ecbf2bba0" + }, + { + "name": "1.7B-6-attention_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "924ec41f566032e6928c145693aef559153b4cc4e0e3582002ca4cb3f1bb5574" + }, + { + "name": "1.7B-6-attention_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "53b6cdfc5c6ae2b1989b3250cb5fe665338d00065a9f340ee99b7878d3c796f9" + }, + { + "name": "1.7B-6-vocabulary_head", + "rows": 64, + "bytes": 16384000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "4aa176ed5fda5e148f8ae28a9aa7ab2476b91e447a1eb84d3c543a20f9fc043b" + }, + { + "name": "1.7B-6-vocabulary_head", + "rows": 1, + "bytes": 256000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "a7d12df1673fad09167e8ff16292935fc0eec2044e57119bdd14c9cf7ad914c1" + }, + { + "name": "1.7B-8-conv_in", + "rows": 64, + "bytes": 786432, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "d6b35c4877de2f0a8d67cd1f068b9826d6ee1529c653dbf90fa542ef8f2636ae" + }, + { + "name": "1.7B-8-conv_in", + "rows": 1, + "bytes": 12288, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "7e9ceb9220c05a6b5069aee118ad7bff5449da2d9e91c9ff8c5eea17cf7263e8" + }, + { + "name": "1.7B-8-conv_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "f1469364d1355c14c380655d42e4aef357c54ee58a03f767e65073c69bcc77db" + }, + { + "name": "1.7B-8-conv_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "347e2cee0ad33b3cf957d1487e3fd2200eed2b6f4de206da8f2b0758ab471c86" + }, + { + "name": "1.7B-8-ffn_gate", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "3359e636ffbda8f5c9a3267435991cca16cc88e19b4919f566a4c3d3a96a90cd" + }, + { + "name": "1.7B-8-ffn_gate", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "5d7265f9038c45a2a7414e9bf751a2cdee8ead2da6d0144ef8bc73500083f6f1" + }, + { + "name": "1.7B-8-ffn_up", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "6c9c205deb589766242f9176c5d5cb06b5f114ae4c0144c4b168c16fed380ace" + }, + { + "name": "1.7B-8-ffn_up", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "44147335dcb02a2216a2d116959141df0a9aad2bec658fae007ae864ea0ec157" + }, + { + "name": "1.7B-8-ffn_down", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "c9535b39ba4ef83911282db1f28fca22373b318d7749ac18c90ca0a91a236bde" + }, + { + "name": "1.7B-8-ffn_down", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "8555da8c544587c8042e0bf321c6288974d6f2e58eabf4079966ed53504e8510" + }, + { + "name": "1.7B-8-attention_q", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "cfeac51b5f537a6e2bcc4c466a9912a06d2b4c0d4d0e5418f04678bc99febb76" + }, + { + "name": "1.7B-8-attention_q", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "c12320e5ac155f07a393be2542fff7995eccd912097c79f8ac63aa3dadf750d0" + }, + { + "name": "1.7B-8-attention_k", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "0d64a33538106896db23021a7251d26e7622f90e950561d15f0d041b70ce4802" + }, + { + "name": "1.7B-8-attention_k", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "bd2d6913ec4438c993dcebe577870b403caae745a38a7af3c9fe7ea2c849230e" + }, + { + "name": "1.7B-8-attention_v", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "8bbf123e32dd083bd8515984bd3f5d24e8520e68366cf5cbff99f6c42f9e9bbd" + }, + { + "name": "1.7B-8-attention_v", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "5672c89b1209a47f4db84bbbed85ad5677542e28f1f4bcdbc95098d2d1027aa0" + }, + { + "name": "1.7B-8-attention_gate", + "rows": 64, + "bytes": 2048, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "9d5e0d6dc77e56482a4652be9b53cdecfd809c91c6daf9cd86d598cf6ea5515d" + }, + { + "name": "1.7B-8-attention_gate", + "rows": 1, + "bytes": 32, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "490faa9e36e8cd958d4d64e8f85a708191d50e1c8b402c5686e8bdeefa97d7ae" + }, + { + "name": "1.7B-8-attention_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "9309a2af2e0a4d428c0c5dc0cea2732f468c93aaff21a7de62689c7c949b445f" + }, + { + "name": "1.7B-8-attention_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "9aee14bdf4e46e44048a96d2ecce55556f03067b17f970bb6b693c6d3c6cb9fd" + }, + { + "name": "1.7B-8-vocabulary_head", + "rows": 64, + "bytes": 16384000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "60e65041ceddf857ed5d3f834ee2e3e9031aa35657546522c649c229cc69cadd" + }, + { + "name": "1.7B-8-vocabulary_head", + "rows": 1, + "bytes": 256000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "0eb8a7d5cff089e99330ff2b8de752bcbde52172e7a80d47f97553714027587c" + }, + { + "name": "VL-2B-4-conv_in", + "rows": 64, + "bytes": 786432, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "d14b70a308f20d5218b334611a93b842b44e8608c806f4e7e7fffc0ad14a9e9f" + }, + { + "name": "VL-2B-4-conv_in", + "rows": 1, + "bytes": 12288, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "719399211617079e28556f60b5da3216dc841a0c5f40f51223f210e081c14003" + }, + { + "name": "VL-2B-4-conv_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "a781c7e21bd0ed2ed093e11d769fa51868182e8a96a8ccaf8006527cf768a1cb" + }, + { + "name": "VL-2B-4-conv_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "8bc576734bd022af17e8c1d5f6e0e38088c42165aa06e297822024a3762dd806" + }, + { + "name": "VL-2B-4-ffn_gate", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "4f23d79f8e70a5da00d1b6c7369456881ce93f202bd7677c731a42b8bf13afa7" + }, + { + "name": "VL-2B-4-ffn_gate", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "fc3260f84dd8a6cd21baa4e4690c37e26e2a09b75f27bdccfc0b48e1a4e155a2" + }, + { + "name": "VL-2B-4-ffn_up", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "50a86965edc35a4281b42ad4574c34de38cf90ae70309a694c54f6fc9033b813" + }, + { + "name": "VL-2B-4-ffn_up", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "14890b0147213d78a0d31c91dbdb82efbef19a820cc80d281b37d0441d2963b5" + }, + { + "name": "VL-2B-4-ffn_down", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "10283fdeee7e16d060a6e045bb7f91c25dadc1cdc0251176a658a9b4c2126733" + }, + { + "name": "VL-2B-4-ffn_down", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "98c725a270344e983bedaeaba0de68165a5ff08404ca24012c943835d7a41b4d" + }, + { + "name": "VL-2B-4-attention_q", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "ccdfc7269632f8e52d30bbc34c55535b7fce5f67ad555f75a4b40f4ed77ebf03" + }, + { + "name": "VL-2B-4-attention_q", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "80c2bb98cc9414ddeb8708afe894f12a1c78380124bcf65b261af28d5afba456" + }, + { + "name": "VL-2B-4-attention_k", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "6c46ee9e09a66308ad24f7fcc48c806f5066f11b77cad9612cd356ffe4634728" + }, + { + "name": "VL-2B-4-attention_k", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "1b51d5ed6ef29f58baa2788eafaa855f89bb12320babdc361f2cd9f58816ac5b" + }, + { + "name": "VL-2B-4-attention_v", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "b36ddb06295061232d83d792cf1d992f88aff3c9394bb745575bfa07e66ffdb4" + }, + { + "name": "VL-2B-4-attention_v", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "e46f37723d8adf961c8e8e62cb1d09d74a33fdd36562fb52bb7f804aeccb129c" + }, + { + "name": "VL-2B-4-attention_gate", + "rows": 64, + "bytes": 2048, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "a18b3d969a2b0b1ded99fc9bdf947a780f5c8b711ea1c57a893bb2d9f2dcb810" + }, + { + "name": "VL-2B-4-attention_gate", + "rows": 1, + "bytes": 32, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "3137099219d5c4ecf39915438ad8f89e506b5b84b2898fa3dcd4b0306f33f77d" + }, + { + "name": "VL-2B-4-attention_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "193973f3a4fc797a29993f8a8c52b3659f13a2a044740cc8061011a9e120bcf8" + }, + { + "name": "VL-2B-4-attention_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "2f9d4462c35d055eb24a15602a150770bd2387229cdd4ae7a028b26a4dc86927" + }, + { + "name": "VL-2B-4-vocabulary_head", + "rows": 64, + "bytes": 16384000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "64bd27b14f896401645b37e539c37ae3d35d28c1df24bc48f6a4c18fabf78a4c" + }, + { + "name": "VL-2B-4-vocabulary_head", + "rows": 1, + "bytes": 256000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "95b91cd0681074c14320eaa82430d3ddd9d21a5015e726c98c797b96206ffd0e" + }, + { + "name": "VL-2B-6-conv_in", + "rows": 64, + "bytes": 786432, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "767f316b194381f15ad6ecff5a88bb2764cf709d4e1ef2d841e32a52cc960ad3" + }, + { + "name": "VL-2B-6-conv_in", + "rows": 1, + "bytes": 12288, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "f747092d8186ca673cd682c56a429eaf2c27631c02b9903ba803532865043db5" + }, + { + "name": "VL-2B-6-conv_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "e133c7b157ae2341d7c533a153789ed4c8ff5fe93df6146d57cf855ee81f5d93" + }, + { + "name": "VL-2B-6-conv_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "23087af72979d3a2dc227423c1b8c31d4ff5954224f07798b82dd730c494570a" + }, + { + "name": "VL-2B-6-ffn_gate", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "5e22fb9cad2fbf1de6f75a4bd152691c72fec6c5daffab2da265ed56e0d31fcd" + }, + { + "name": "VL-2B-6-ffn_gate", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "2d240b08a3628383264bb75eb0b522693059025db8cadb554d77b493703d0e9f" + }, + { + "name": "VL-2B-6-ffn_up", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "db0fa182e03793b1f02d4b297035039cf6af4001d0bb3f70cef2674b15eb4270" + }, + { + "name": "VL-2B-6-ffn_up", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "4ff805d85a0816552f0774749cd18aa1daac3e67e74345b907331358fca800a8" + }, + { + "name": "VL-2B-6-ffn_down", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "2daecfa2bf09af4963cbc61d1fc1cd8c187d12b7f94ad5e3faa7bbdfcf164c52" + }, + { + "name": "VL-2B-6-ffn_down", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "531988206cfc6f52e7e6b3c227132392bf3204b87a84f422bcd5bb80c162d68a" + }, + { + "name": "VL-2B-6-attention_q", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "7dd67e902a2eef702d22c1f1ea921c09e4ee7d054d66e5c7d9e2cbb7ecaa31fb" + }, + { + "name": "VL-2B-6-attention_q", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "731b98f325784057f48d09e9eb0e5b87f1c2e4e1089729873689fea506272ca7" + }, + { + "name": "VL-2B-6-attention_k", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "ec9d623b94b96e7f16efd8b78fbeb6594daef7712d34101e8171efb9d9a07e76" + }, + { + "name": "VL-2B-6-attention_k", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "897083fbf229dca38a8de1d30fac9a6098d55ec30874d2ba14a692d34bddb436" + }, + { + "name": "VL-2B-6-attention_v", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "c9767157863db3b698250112aedfbf14601174a6e1bde137f72881df5f50c1a3" + }, + { + "name": "VL-2B-6-attention_v", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "a8d94974c7616852ac893dd29ba55aa7bf8e82c9760dd7793157eb96df59358f" + }, + { + "name": "VL-2B-6-attention_gate", + "rows": 64, + "bytes": 2048, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "ae7f4042b17c5c7b77562de99ec66e581e5815e9f2e61d8cec0c461d7d0eb2cb" + }, + { + "name": "VL-2B-6-attention_gate", + "rows": 1, + "bytes": 32, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "0e7e3e65904ebd568f7b50e3856311cd08f4ce5bd2b5c25e7cccaeb04bf1d2ad" + }, + { + "name": "VL-2B-6-attention_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "df60fd6d1502456327eda9c18e9a1c8f083f57f168347c50ed13cbf327386c92" + }, + { + "name": "VL-2B-6-attention_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "12f97abed04460ce87b0fedebcae80babd945ccb9cde3de78e704f17cb224e9e" + }, + { + "name": "VL-2B-6-vocabulary_head", + "rows": 64, + "bytes": 16384000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "a2cdac531945e51aa61719171460f3e40cf8b658f6f4532dd46fa944b10483ad" + }, + { + "name": "VL-2B-6-vocabulary_head", + "rows": 1, + "bytes": 256000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "3af09e6edb5b63ee9acb212a1fe4f23d82d7dfdf39ae63617c6fb60ebc523362" + }, + { + "name": "VL-2B-8-conv_in", + "rows": 64, + "bytes": 786432, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "71bf9b014f1cc1b8edc7167067e790c3a5e536a88109751b5ed4f09528098597" + }, + { + "name": "VL-2B-8-conv_in", + "rows": 1, + "bytes": 12288, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "abe2d50106ca7025d34167f9b593f034e3cccfd56c2aa5473efa2505b6469c9a" + }, + { + "name": "VL-2B-8-conv_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "3a88ff8d8fb1b7d99092f109dedf5f81ca6d76c32433c9b6aeb3596b7f6ab8b2" + }, + { + "name": "VL-2B-8-conv_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "d32cc9511e429d97a0c492a42d9b7b6184ad6165ac1ed9e3b355ed6765af85fc" + }, + { + "name": "VL-2B-8-ffn_gate", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "e0b4f14b9e9b91516bd391a06838292f5ff79ae9cbc429fe404bde9f5a130e08" + }, + { + "name": "VL-2B-8-ffn_gate", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "f5caab73d84c4262377769654faaba8c285b694455e329264919a3f9641db7ef" + }, + { + "name": "VL-2B-8-ffn_up", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "fcce4ac31aa0f9ef38b3cd0cb7c25398dcf6ed68a309362418b5814af62eca3e" + }, + { + "name": "VL-2B-8-ffn_up", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "112d5b19e6a549b3ea21198dc4f933dbaeeb6f3579cc895c076c6b7f406cd105" + }, + { + "name": "VL-2B-8-ffn_down", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "6896f97343dfd72dfcdb74d4f1f5141bd4283249844922b8def07210c5abf213" + }, + { + "name": "VL-2B-8-ffn_down", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "6fa80d2c836c95a713b9a496e3d75954b69f48c775fe975f6011f3a9583f0870" + }, + { + "name": "VL-2B-8-attention_q", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "2406496d7aac10e40363859bcbe52cdcbbd424d233cedd915f90ea89bbc351f4" + }, + { + "name": "VL-2B-8-attention_q", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "7ebc981eadeeb63851ef1ac62366369946d4cd85699291ef54ea48ae45b9ed9c" + }, + { + "name": "VL-2B-8-attention_k", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "e04f2457b18a1378260dcc1ab3f6372c8f82a54c65b95ca3b95575fc36798874" + }, + { + "name": "VL-2B-8-attention_k", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "ca775c01620b1478372d4823c64a976b5e0ef3ad0d3f1c4d90e731cc4f8671a9" + }, + { + "name": "VL-2B-8-attention_v", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "3624b048e8946768c2f095071dd36e53475847d188b87da9146ae6d9fe064040" + }, + { + "name": "VL-2B-8-attention_v", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "861913b23c3fcbafe53476e6537fe0c03da18bb338a6d50842dd13f7e84376e7" + }, + { + "name": "VL-2B-8-attention_gate", + "rows": 64, + "bytes": 2048, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "5ac650fc043aaa905783be602dfbddbc6a8f0c0a94d78d7188743bc70801508e" + }, + { + "name": "VL-2B-8-attention_gate", + "rows": 1, + "bytes": 32, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "95cbe3c3772e06917f9b2aa90cafd9ffd5cbeacf97a131ed3a77862627308818" + }, + { + "name": "VL-2B-8-attention_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "0c026f593adfcb6b3ed08c37bebbfc5d0cfc265eb1e6bf4f2b3dd3af8d3de347" + }, + { + "name": "VL-2B-8-attention_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "e5632e43d6d113a76edddfecc4f162eacc1c6c407b8dcf58b35487528da6e1e1" + }, + { + "name": "VL-2B-8-vocabulary_head", + "rows": 64, + "bytes": 16384000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "61f3c553b0a0daf33f2dd29f46677112dae2639fd338014b7ebdd22558c6b1cb" + }, + { + "name": "VL-2B-8-vocabulary_head", + "rows": 1, + "bytes": 256000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "e250665cdfd5e271e66e762ae85ec1bae526ef802cc62ec9f736be9d52622f13" + } + ], + "operator_contract": { + "m4": "SUMMARY 6 kernels: 6 pass gates 1-4, 0 fail, 0 known findings; 2 controls: 2 red as planted, 0 missed; 0 limit problems; gates 10718 ms, limits 147 ms", + "m5": "SUMMARY 6 kernels: 6 pass gates 1-4, 0 fail, 0 known findings; 2 controls: 2 red as planted, 0 missed; 0 limit problems; gates 8872 ms, limits 88 ms" + }, + "native_project_sha256": "184f7fd1f7e1794f211f6ca0c5c6c6dcd8a2930a883565a16812fe33ddf7c233", + "sources_sha256": { + "zig/kernels/metal/core/binned_projection.metal": "8119aa792197ebf3c9b12f767fe0dfe12482189340be8e75ae93f7910ac944ca", + "zig/src/core/binned_projection.zig": "14d6a42a263d639435eae208a6e808830dfb43e86a6bbfad3c67d4499af85d4f" + } +} diff --git a/docs/receipts/lfm3/cold-prefill-ab.json b/docs/receipts/lfm3/cold-prefill-ab.json new file mode 100644 index 0000000000..2b636885db --- /dev/null +++ b/docs/receipts/lfm3/cold-prefill-ab.json @@ -0,0 +1,383 @@ +{ + "baseline_native_sha256": "a8df9730ee637237545311a6753dd4bdcb13e860700d65a01a033ae818ee0d18", + "rowaxis_native_sha256": "5c8a04d4af599879797f4f5bed8e00ab7d9d70732e698bd2a1ec2a01ec762e09", + "completed_matching_pairs": 16, + "cells": [ + { + "family": "text", + "prompt_tokens": 2048, + "requested_pairs": 3, + "completed_pairs": 3, + "before_prefill_median_s": 1.239529791, + "after_prefill_median_s": 0.812958542, + "speed_ratio": 1.524714640368463, + "phases": { + "before": [ + { + "rep": 0, + "prefill_s": 1.239792167, + "ttft_s": 1.2504969998262823, + "token_sha256": "c244b715e827fd1ce59aa4e249cb47900812abeee1482b0a79a1b251fe0f3262", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 1491863256 + }, + { + "rep": 1, + "prefill_s": 1.238933042, + "ttft_s": 1.2502966253086925, + "token_sha256": "c244b715e827fd1ce59aa4e249cb47900812abeee1482b0a79a1b251fe0f3262", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 1491846848 + }, + { + "rep": 2, + "prefill_s": 1.239529791, + "ttft_s": 1.2511707078665495, + "token_sha256": "c244b715e827fd1ce59aa4e249cb47900812abeee1482b0a79a1b251fe0f3262", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 1491830440 + } + ], + "after": [ + { + "rep": 0, + "prefill_s": 0.81294775, + "ttft_s": 0.8246144163422287, + "token_sha256": "c244b715e827fd1ce59aa4e249cb47900812abeee1482b0a79a1b251fe0f3262", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 1490699896 + }, + { + "rep": 1, + "prefill_s": 0.812958542, + "ttft_s": 0.8243804168887436, + "token_sha256": "c244b715e827fd1ce59aa4e249cb47900812abeee1482b0a79a1b251fe0f3262", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 1490667128 + }, + { + "rep": 2, + "prefill_s": 0.813141375, + "ttft_s": 0.8247690829448402, + "token_sha256": "c244b715e827fd1ce59aa4e249cb47900812abeee1482b0a79a1b251fe0f3262", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 1490699896 + } + ] + } + }, + { + "family": "text", + "prompt_tokens": 8192, + "requested_pairs": 3, + "completed_pairs": 3, + "before_prefill_median_s": 5.307524, + "after_prefill_median_s": 3.608894709, + "speed_ratio": 1.4706785395439477, + "phases": { + "before": [ + { + "rep": 0, + "prefill_s": 5.306101708, + "ttft_s": 5.318894917145371, + "token_sha256": "3f913af21b98f7168ba1fcd27bd9f16b62982dcf6168b991091bbd6e4d9e7fed", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 1630586536 + }, + { + "rep": 1, + "prefill_s": 5.307524, + "ttft_s": 5.319628749974072, + "token_sha256": "3f913af21b98f7168ba1fcd27bd9f16b62982dcf6168b991091bbd6e4d9e7fed", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 1630570152 + }, + { + "rep": 2, + "prefill_s": 5.31343825, + "ttft_s": 5.325319999828935, + "token_sha256": "3f913af21b98f7168ba1fcd27bd9f16b62982dcf6168b991091bbd6e4d9e7fed", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 1630652120 + } + ], + "after": [ + { + "rep": 0, + "prefill_s": 3.608894709, + "ttft_s": 3.621469959151, + "token_sha256": "3f913af21b98f7168ba1fcd27bd9f16b62982dcf6168b991091bbd6e4d9e7fed", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 1629488736 + }, + { + "rep": 1, + "prefill_s": 3.610029625, + "ttft_s": 3.6229068338871, + "token_sha256": "3f913af21b98f7168ba1fcd27bd9f16b62982dcf6168b991091bbd6e4d9e7fed", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 1629570752 + }, + { + "rep": 2, + "prefill_s": 3.607758625, + "ttft_s": 3.620500334072858, + "token_sha256": "3f913af21b98f7168ba1fcd27bd9f16b62982dcf6168b991091bbd6e4d9e7fed", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 1629488784 + } + ] + } + }, + { + "family": "text", + "prompt_tokens": 32768, + "requested_pairs": 1, + "completed_pairs": 1, + "before_prefill_median_s": 27.314690833, + "after_prefill_median_s": 20.488389292, + "speed_ratio": 1.333179023675884, + "phases": { + "before": [ + { + "rep": 0, + "prefill_s": 27.314690833, + "ttft_s": 27.331630500033498, + "token_sha256": "e15e7490e640d938a0a846dfc9de1dd7a51e50ae97ca4fd42a213cbb633f66f5", + "body_sha256": "b11cb39bb728d97c62594c8a4f2004846f06aeddebda9e4c825136a31448bcb6", + "peak_physical_footprint_bytes": 2185791144 + } + ], + "after": [ + { + "rep": 0, + "prefill_s": 20.488389292, + "ttft_s": 20.504653749987483, + "token_sha256": "e15e7490e640d938a0a846dfc9de1dd7a51e50ae97ca4fd42a213cbb633f66f5", + "body_sha256": "b11cb39bb728d97c62594c8a4f2004846f06aeddebda9e4c825136a31448bcb6", + "peak_physical_footprint_bytes": 2184644240 + } + ] + } + }, + { + "family": "text", + "prompt_tokens": 65536, + "requested_pairs": 1, + "completed_pairs": 1, + "before_prefill_median_s": 73.402432042, + "after_prefill_median_s": 61.632932625, + "speed_ratio": 1.1909612104394003, + "phases": { + "before": [ + { + "rep": 0, + "prefill_s": 73.402432042, + "ttft_s": 73.4243381661363, + "token_sha256": "cf6c4062b60dc49e0ca503b7be2aaf3d247d73be037d269cf73e127a5e35aa32", + "body_sha256": "3aa9a0a57dfaccac1354b37af6015a319cb24fce9c0fb30f4b6ddb3f12a48639", + "peak_physical_footprint_bytes": 2926167768 + } + ], + "after": [ + { + "rep": 0, + "prefill_s": 61.632932625, + "ttft_s": 61.653554582968354, + "token_sha256": "cf6c4062b60dc49e0ca503b7be2aaf3d247d73be037d269cf73e127a5e35aa32", + "body_sha256": "3aa9a0a57dfaccac1354b37af6015a319cb24fce9c0fb30f4b6ddb3f12a48639", + "peak_physical_footprint_bytes": 2925315776 + } + ] + } + }, + { + "family": "vl", + "prompt_tokens": 2048, + "requested_pairs": 3, + "completed_pairs": 3, + "before_prefill_median_s": 1.2618445, + "after_prefill_median_s": 0.815432416, + "speed_ratio": 1.5474544244755655, + "phases": { + "before": [ + { + "rep": 0, + "prefill_s": 1.261758042, + "ttft_s": 1.273614791687578, + "token_sha256": "21c3879cda05ccf4cde74c1f54d326d2c271f673656d166241e6643c21f49d89", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 2151958712 + }, + { + "rep": 1, + "prefill_s": 1.2618445, + "ttft_s": 1.2731465827673674, + "token_sha256": "21c3879cda05ccf4cde74c1f54d326d2c271f673656d166241e6643c21f49d89", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 2151925920 + }, + { + "rep": 2, + "prefill_s": 1.26217825, + "ttft_s": 1.2739000832661986, + "token_sha256": "21c3879cda05ccf4cde74c1f54d326d2c271f673656d166241e6643c21f49d89", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 2151893128 + } + ], + "after": [ + { + "rep": 0, + "prefill_s": 0.815139, + "ttft_s": 0.8269048747606575, + "token_sha256": "21c3879cda05ccf4cde74c1f54d326d2c271f673656d166241e6643c21f49d89", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 2149976200 + }, + { + "rep": 1, + "prefill_s": 0.815432416, + "ttft_s": 0.8269431251101196, + "token_sha256": "21c3879cda05ccf4cde74c1f54d326d2c271f673656d166241e6643c21f49d89", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 2149976200 + }, + { + "rep": 2, + "prefill_s": 0.816996041, + "ttft_s": 0.8286756249144673, + "token_sha256": "21c3879cda05ccf4cde74c1f54d326d2c271f673656d166241e6643c21f49d89", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 2150041736 + } + ] + } + }, + { + "family": "vl", + "prompt_tokens": 8192, + "requested_pairs": 3, + "completed_pairs": 3, + "before_prefill_median_s": 5.402433084, + "after_prefill_median_s": 3.61744925, + "speed_ratio": 1.4934371460774467, + "phases": { + "before": [ + { + "rep": 0, + "prefill_s": 5.399245833, + "ttft_s": 5.410978707950562, + "token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 2290616432 + }, + { + "rep": 1, + "prefill_s": 5.403565458, + "ttft_s": 5.416440249886364, + "token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 2290583664 + }, + { + "rep": 2, + "prefill_s": 5.402433084, + "ttft_s": 5.415588291827589, + "token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 2290583664 + } + ], + "after": [ + { + "rep": 0, + "prefill_s": 3.61744925, + "ttft_s": 3.630715959239751, + "token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 2288797856 + }, + { + "rep": 1, + "prefill_s": 3.624671708, + "ttft_s": 3.6372522921301425, + "token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 2288797856 + }, + { + "rep": 2, + "prefill_s": 3.608385375, + "ttft_s": 3.6207989170216024, + "token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 2288748680 + } + ] + } + }, + { + "family": "vl", + "prompt_tokens": 32768, + "requested_pairs": 1, + "completed_pairs": 1, + "before_prefill_median_s": 27.686642834, + "after_prefill_median_s": 20.536701917, + "speed_ratio": 1.3481542920521907, + "phases": { + "before": [ + { + "rep": 0, + "prefill_s": 27.686642834, + "ttft_s": 27.70385383302346, + "token_sha256": "f35dfbfd86653608bd3aa0fe6672229ecfe9ec17bf45ebcdb91dc7c7e894af4a", + "body_sha256": "b11cb39bb728d97c62594c8a4f2004846f06aeddebda9e4c825136a31448bcb6", + "peak_physical_footprint_bytes": 2845788224 + } + ], + "after": [ + { + "rep": 0, + "prefill_s": 20.536701917, + "ttft_s": 20.55388933280483, + "token_sha256": "f35dfbfd86653608bd3aa0fe6672229ecfe9ec17bf45ebcdb91dc7c7e894af4a", + "body_sha256": "b11cb39bb728d97c62594c8a4f2004846f06aeddebda9e4c825136a31448bcb6", + "peak_physical_footprint_bytes": 2843887776 + } + ] + } + }, + { + "family": "vl", + "prompt_tokens": 65536, + "requested_pairs": 1, + "completed_pairs": 1, + "before_prefill_median_s": 73.920498208, + "after_prefill_median_s": 59.64693, + "speed_ratio": 1.2393009700247775, + "phases": { + "before": [ + { + "rep": 0, + "prefill_s": 73.920498208, + "ttft_s": 73.94245912507176, + "token_sha256": "6dc17e16c1bdd265cbfecc43f466db1fb29c9240e7818d912ecc773b568b4856", + "body_sha256": "3aa9a0a57dfaccac1354b37af6015a319cb24fce9c0fb30f4b6ddb3f12a48639", + "peak_physical_footprint_bytes": 3586115720 + } + ], + "after": [ + { + "rep": 0, + "prefill_s": 59.64693, + "ttft_s": 59.66807400016114, + "token_sha256": "6dc17e16c1bdd265cbfecc43f466db1fb29c9240e7818d912ecc773b568b4856", + "body_sha256": "3aa9a0a57dfaccac1354b37af6015a319cb24fce9c0fb30f4b6ddb3f12a48639", + "peak_physical_footprint_bytes": 3584559264 + } + ] + } + } + ] +} diff --git a/docs/receipts/lfm3/cold-prompt-sha256.json b/docs/receipts/lfm3/cold-prompt-sha256.json new file mode 100644 index 0000000000..9398e0ac47 --- /dev/null +++ b/docs/receipts/lfm3/cold-prompt-sha256.json @@ -0,0 +1,18 @@ +{ + "text-2048.body.json": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "text-2048.ids.json": "f5edac3447405086d7ec361d6c899410ea7cbbb2ef697c27fd7486ef5ff877fe", + "text-32768.body.json": "6e4c2ecd84a08e202dddce7f9b50bada5c006f9d529a74177f38dc11c08e0bb4", + "text-32768.ids.json": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f", + "text-65536.body.json": "d6dea630f39ee2ddd13e8a88a90050d615d518e21f5a9219e1dc3d2297617b63", + "text-65536.ids.json": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed", + "text-8192.body.json": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "text-8192.ids.json": "a9b3cdf070b9ca5ea523565bcb82c80b6af5aae71f640a8fba0460c9069d22cd", + "vl-2048.body.json": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "vl-2048.ids.json": "f5edac3447405086d7ec361d6c899410ea7cbbb2ef697c27fd7486ef5ff877fe", + "vl-32768.body.json": "6e4c2ecd84a08e202dddce7f9b50bada5c006f9d529a74177f38dc11c08e0bb4", + "vl-32768.ids.json": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f", + "vl-65536.body.json": "d6dea630f39ee2ddd13e8a88a90050d615d518e21f5a9219e1dc3d2297617b63", + "vl-65536.ids.json": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed", + "vl-8192.body.json": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "vl-8192.ids.json": "a9b3cdf070b9ca5ea523565bcb82c80b6af5aae71f640a8fba0460c9069d22cd" +} diff --git a/docs/receipts/lfm3/corrected-pixel-decoder.json b/docs/receipts/lfm3/corrected-pixel-decoder.json new file mode 100644 index 0000000000..1c2a811963 --- /dev/null +++ b/docs/receipts/lfm3/corrected-pixel-decoder.json @@ -0,0 +1,1783 @@ +{ + "scope": "Corrected-pixel real-image teacher-forced decoder, all 90 rows and all 128000 vocabulary columns, independent ideal-FP64 and prescribed FP32/BF16 graphs.", + "host": "Apple M4", + "source_build": { + "source_head_at_build": "830eace481170a233ada1dc34313b8b8c5ccc5cd", + "zig_source_tree": "62366349b2dad53884fd74a80f50f904ed36da7d", + "working_source_clean_before_after": true, + "build": "tf-lfm3-forward -p PRIVATE_PREFIX -Dcpu=apple_m1 --summary all", + "build_steps": [ + 55, + 55 + ], + "GPU_execution_at_build": false, + "build_log_sha256": "e5638918bb7c15b7f43f545f41bd655cad66583be77b08e0d0ca53e439e4fc1d", + "compiler": "Zig0.17.0", + "Xcode": "26.6 build17F113" + }, + "plan_manifest_sha256": "3f279383959c71ced8a0ea9056425e04e04da647450f5ae1ae0c0bcd7a5ac707", + "pins": { + "artifact/lfm3-corrected-pixel-decoder-plan/reference.py": "afc808ea000fcb7b88a8bce4baf6f4f3336b161315a668637559667404d6ae57", + "artifact/lfm3-corrected-pixel-decoder-plan/run_guarded.py": "bcc88d4392dad6ea0e8d6c37ceef4bf7268149422a23dda2be337640b7195a38", + "artifact/lfm3-corrected-pixel-forward-build/bin/tf-lfm3-forward": "f7311f7bba02d7a2e618e0bac55178847946db40d31b5f0705c911177b78b2de", + "artifact/lfm3-local-standard-packet/guard.py": "0caf67448d401704bd849744584248729decf5fd5b994f669a518cb61891df37", + "artifact/lfm3-production-elementary-provider-chunked/provider.py": "97aec6e37977bbef32c60e7f86253c29c7b8161159dd6c9aded657e4bd69ddd0", + "artifact/lfm3-production-elementary-provider/tf-lfm3-row-tune": "ce433fcac7b724cb14b22d8f5392373e950bf3e3347633c96bdc89cc06934048", + "artifact/lfm3-vl6-row22-cpu-replay/row.so": "11ad0f66632bfcba548dbede9027ffdb112cd418ff2be80610af56a957424753", + "tools/lfm3/fp64_reference.py": "0fb1128ade095db87b6126792ab14930792ab1f7b41b998570356bc586f16b2e", + "tools/lfm3/row_projection_reference.py": "d03f224a037a920ba66b9828059d1489e48729cd168077e76789d0ec5c199c0c", + "tools/lfm3/row_projection_replay.c": "ff336f75fcf0a4f6c0ac92ed627bc1a6abb0b36b8f4cc95d876f2a8bb2f5cc42", + "tools/lfm3/elementary_reference.py": "3e4db276278867b7683536044ed7bcbe4c7643867d722fdda5b8ad895b3963e7", + "tools/lfm3/text_rounding_reference.py": "52665eef48570d45a2554861fe8f22b31874f8f464366adf882f40c3d311db27", + "tools/lfm3/text_attention_rounding.py": "ae67f2b2c26f89f75b1f36822a4b30a9f710df2b42f9f7ec42679373028b89e8", + "tools/lfm3/compare_logits.py": "b1bf273c8f3ce89591bccb8f098c82763cddff11373dbe850302450fcee1502c", + "artifact/lfm3-corrected-pixel-decoder-plan/vl4.json": "e6520745f9ac0cf1ea10cfa03cb489c3cb7d0c2e1863b8a5cd907db62a2ecfda", + "artifact/lfm3-real-cat-tokens.npy": "54c970db8c6ba82895cb2f18a48d397ea568ae885241129e9c5ee7dbbc91a24d", + "artifact/lfm3-corrected-pixel-capture/vl4.bf16": "ffa0401f452f39a07833d6692ac9eff0d5485cc93a14fb3433bdad74c42ab05e", + "artifact/lfm3-corrected-pixel-reference-results/vision4/ideal-FP64-image70.bf16": "a67172521a00ffc5c50b848769a92f732f12456276cf484e76509f91c703d0c8", + "artifact/lfm3-corrected-pixel-reference-results/vision4/named-image70.bf16": "ffa0401f452f39a07833d6692ac9eff0d5485cc93a14fb3433bdad74c42ab05e", + "artifact/lfm3-corrected-pixel-reference-results/vision4/receipt.json": "98556f6663ca894b3f185e7873f145fb199f76d32a24bf62b6a5db6023b35bd3", + "artifact/lfm3-corrected-pixel-reference-results/vision68-identity/receipt.json": "6e28b34d17c1b5ff57811e1375bf8292fdbe18b2ddeeebe78d12191c64ebab98", + "artifact/lfm3-corrected-pixel-decoder-plan/vl6.json": "78a6a862e467c18549636cac4543dd852497c75250b64a711ed0295c6d922b6f", + "artifact/lfm3-corrected-pixel-capture/vl6.bf16": "0bfbdb4d6e8f3485609f0b6e8cf4b84bd8daba4c2ae62f56a4861aad3688b4df", + "artifact/lfm3-corrected-pixel-reference-results/vision6/ideal-FP64-image70.bf16": "221133d871ce2db48dfcb28af88f8e3facb80f604d9fa8b4e37afef07e04fde4", + "artifact/lfm3-corrected-pixel-reference-results/vision6/named-image70.bf16": "0bfbdb4d6e8f3485609f0b6e8cf4b84bd8daba4c2ae62f56a4861aad3688b4df", + "artifact/lfm3-corrected-pixel-reference-results/vision6/receipt.json": "cd65b824e33a1e20bd635c4f7500819b36e4c6702135d3a75e409943cdb3fc13", + "artifact/lfm3-corrected-pixel-decoder-plan/vl8.json": "a5d36e820ee88c34ef38cadbda95170c192324bbb9a20d062d7cf2d62c316860", + "artifact/lfm3-corrected-pixel-capture/vl8.bf16": "0bfbdb4d6e8f3485609f0b6e8cf4b84bd8daba4c2ae62f56a4861aad3688b4df" + }, + "input_lineage": { + "corrected_pixel_F32_patches_sha256": "9d39c11551f105b8982e3a9acceffcdf687d360c17a7931b7fff21757a6e5473", + "image_rows": 70, + "image_start": 4, + "tokens": [ + 124899, + 5922, + 207, + 125009, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125010, + 90869, + 532, + 4145, + 22, + 124900, + 207, + 124899, + 63514, + 207, + 597, + 4145, + 5533, + 267, + 5140, + 22 + ], + "suffix_rows": [ + 74, + 89 + ], + "ideal_decoder_image_input": "Independently computed ideal-FP64 vision70 rows.", + "named_decoder_image_input": "Independently computed prescribed FP32/BF16 vision70 rows.", + "native_activation_injection": false, + "vision6_to8_transfer": "Only vision tensor/input identity proven separately; each decoder here independently evaluated." + }, + "cases": [ + { + "bits": 4, + "rows": 90, + "vocabulary": 128000, + "values": 11520000, + "named_native_equal_words": 11520000, + "native_sha256": "2417b3ab43226a995aea39f6dafd01d100cda63ea2c08be26c97b42b26d7486c", + "named_sha256": "2417b3ab43226a995aea39f6dafd01d100cda63ea2c08be26c97b42b26d7486c", + "ideal_sha256": "81edcc2f6e32e3434dfce66ee6d6e1a770be267a719f075573163d0c13ae608f", + "native_vs_ideal_FP64_all90": { + "rows": 90, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 1, + "relative_rms": 0.022357823032940326, + "maximum_absolute_error": 1.7265625, + "differences": [ + { + "row": 11, + "native_top": 267, + "reference_top": 7179, + "native_maximum_set": [ + 267, + 7179 + ], + "reference_maximum_set": [ + 7179 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + }, + { + "row": 34, + "native_top": 2911, + "reference_top": 5594, + "native_maximum_set": [ + 2911 + ], + "reference_maximum_set": [ + 5594 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.21875, + "native_reference_contender_gap": 0.15625, + "reference_native_contender_gap": 0.21875 + } + ], + "worst_rows": [ + { + "row": 31, + "rms_error": 0.3863085412632454, + "maximum_absolute_error": 1.7265625 + }, + { + "row": 33, + "rms_error": 0.3330400018062991, + "maximum_absolute_error": 0.81640625 + }, + { + "row": 24, + "rms_error": 0.20343373091929323, + "maximum_absolute_error": 0.65625 + }, + { + "row": 0, + "rms_error": 0.14167088715067103, + "maximum_absolute_error": 0.59228515625 + }, + { + "row": 14, + "rms_error": 0.14100924992911187, + "maximum_absolute_error": 0.5 + }, + { + "row": 79, + "rms_error": 0.13411523814017293, + "maximum_absolute_error": 0.3125 + }, + { + "row": 41, + "rms_error": 0.1336002662514358, + "maximum_absolute_error": 0.421875 + }, + { + "row": 34, + "rms_error": 0.11779374364374101, + "maximum_absolute_error": 0.703125 + }, + { + "row": 39, + "rms_error": 0.11620973283786575, + "maximum_absolute_error": 0.6171875 + }, + { + "row": 64, + "rms_error": 0.11234823782165897, + "maximum_absolute_error": 0.375 + }, + { + "row": 53, + "rms_error": 0.11058961649192356, + "maximum_absolute_error": 0.70458984375 + }, + { + "row": 76, + "rms_error": 0.09546533245241906, + "maximum_absolute_error": 1.0 + }, + { + "row": 10, + "rms_error": 0.09450923916929485, + "maximum_absolute_error": 0.359375 + }, + { + "row": 66, + "rms_error": 0.09330024186941319, + "maximum_absolute_error": 0.46875 + }, + { + "row": 44, + "rms_error": 0.09231120455527364, + "maximum_absolute_error": 0.4921875 + }, + { + "row": 61, + "rms_error": 0.09134544713165614, + "maximum_absolute_error": 0.25 + }, + { + "row": 40, + "rms_error": 0.08934515185917324, + "maximum_absolute_error": 0.4921875 + }, + { + "row": 54, + "rms_error": 0.08633833376502414, + "maximum_absolute_error": 0.40625 + }, + { + "row": 19, + "rms_error": 0.08557473946899023, + "maximum_absolute_error": 0.4375 + }, + { + "row": 5, + "rms_error": 0.08211735116187491, + "maximum_absolute_error": 0.25 + } + ], + "worst_columns": [ + { + "column": 1427, + "rms_error": 0.20979932612579705, + "maximum_absolute_error": 1.70263671875 + }, + { + "column": 92335, + "rms_error": 0.20560901851574673, + "maximum_absolute_error": 1.7265625 + }, + { + "column": 59969, + "rms_error": 0.20221007672904207, + "maximum_absolute_error": 1.578125 + }, + { + "column": 42385, + "rms_error": 0.19531838810840238, + "maximum_absolute_error": 1.625 + }, + { + "column": 39091, + "rms_error": 0.19417556458524513, + "maximum_absolute_error": 1.640625 + }, + { + "column": 23910, + "rms_error": 0.19398074819307062, + "maximum_absolute_error": 1.62060546875 + }, + { + "column": 9053, + "rms_error": 0.19159144753879584, + "maximum_absolute_error": 1.40625 + }, + { + "column": 518, + "rms_error": 0.19064029708659155, + "maximum_absolute_error": 1.484375 + }, + { + "column": 24012, + "rms_error": 0.19017803551103526, + "maximum_absolute_error": 1.6484375 + }, + { + "column": 417, + "rms_error": 0.18482742476815525, + "maximum_absolute_error": 1.5078125 + }, + { + "column": 57782, + "rms_error": 0.18402265945450397, + "maximum_absolute_error": 1.5625 + }, + { + "column": 12733, + "rms_error": 0.18203132567542835, + "maximum_absolute_error": 1.46875 + }, + { + "column": 79410, + "rms_error": 0.1797184719354099, + "maximum_absolute_error": 1.390625 + }, + { + "column": 98817, + "rms_error": 0.17967523357889909, + "maximum_absolute_error": 1.421875 + }, + { + "column": 20473, + "rms_error": 0.17934775414341161, + "maximum_absolute_error": 1.46875 + }, + { + "column": 21170, + "rms_error": 0.1786359049567866, + "maximum_absolute_error": 1.375 + }, + { + "column": 27366, + "rms_error": 0.17833913679492197, + "maximum_absolute_error": 1.4375 + }, + { + "column": 13326, + "rms_error": 0.17800397267879936, + "maximum_absolute_error": 1.46875 + }, + { + "column": 23506, + "rms_error": 0.17673391856257692, + "maximum_absolute_error": 1.296875 + }, + { + "column": 57003, + "rms_error": 0.17535357406073135, + "maximum_absolute_error": 1.34375 + } + ], + "accepted_threshold": null, + "fidelity_pass": null + }, + "native_vs_ideal_FP64_suffix16": { + "rows": 16, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.012747055368292898, + "maximum_absolute_error": 1.0, + "differences": [], + "worst_rows": [ + { + "row": 79, + "rms_error": 0.13411523814017293, + "maximum_absolute_error": 0.3125 + }, + { + "row": 76, + "rms_error": 0.09546533245241906, + "maximum_absolute_error": 1.0 + }, + { + "row": 82, + "rms_error": 0.07769397965510104, + "maximum_absolute_error": 0.28125 + }, + { + "row": 81, + "rms_error": 0.07562315079595386, + "maximum_absolute_error": 0.341796875 + }, + { + "row": 78, + "rms_error": 0.07257791645652735, + "maximum_absolute_error": 0.2421875 + }, + { + "row": 86, + "rms_error": 0.06240101252752554, + "maximum_absolute_error": 0.38330078125 + }, + { + "row": 84, + "rms_error": 0.059990395069031074, + "maximum_absolute_error": 0.328125 + }, + { + "row": 89, + "rms_error": 0.054601865044352646, + "maximum_absolute_error": 0.328125 + }, + { + "row": 83, + "rms_error": 0.05300709190864203, + "maximum_absolute_error": 0.28125 + }, + { + "row": 88, + "rms_error": 0.04824179547491646, + "maximum_absolute_error": 0.25 + }, + { + "row": 85, + "rms_error": 0.0463168691723059, + "maximum_absolute_error": 0.2265625 + }, + { + "row": 87, + "rms_error": 0.04248637870893665, + "maximum_absolute_error": 0.248046875 + }, + { + "row": 74, + "rms_error": 0.04157631149902542, + "maximum_absolute_error": 0.21875 + }, + { + "row": 77, + "rms_error": 0.03715444406743506, + "maximum_absolute_error": 0.1875 + }, + { + "row": 75, + "rms_error": 0.033932348752290474, + "maximum_absolute_error": 0.15625 + }, + { + "row": 80, + "rms_error": 0.03249433165839135, + "maximum_absolute_error": 0.17578125 + } + ], + "worst_columns": [ + { + "column": 8818, + "rms_error": 0.2573197091158039, + "maximum_absolute_error": 1.0 + }, + { + "column": 7986, + "rms_error": 0.2271379303572591, + "maximum_absolute_error": 0.8828125 + }, + { + "column": 4524, + "rms_error": 0.22408632297523942, + "maximum_absolute_error": 0.859375 + }, + { + "column": 17606, + "rms_error": 0.21593737987288628, + "maximum_absolute_error": 0.8125 + }, + { + "column": 47479, + "rms_error": 0.20979053768496908, + "maximum_absolute_error": 0.8125 + }, + { + "column": 89785, + "rms_error": 0.19938865596916125, + "maximum_absolute_error": 0.75 + }, + { + "column": 36123, + "rms_error": 0.19902837296076206, + "maximum_absolute_error": 0.765625 + }, + { + "column": 27387, + "rms_error": 0.19763466336429056, + "maximum_absolute_error": 0.751953125 + }, + { + "column": 25491, + "rms_error": 0.19657019983684634, + "maximum_absolute_error": 0.7578125 + }, + { + "column": 19235, + "rms_error": 0.19650529935839858, + "maximum_absolute_error": 0.7587890625 + }, + { + "column": 11353, + "rms_error": 0.1907850812797534, + "maximum_absolute_error": 0.7265625 + }, + { + "column": 27120, + "rms_error": 0.1794277885065123, + "maximum_absolute_error": 0.6220703125 + }, + { + "column": 13579, + "rms_error": 0.17493898145369335, + "maximum_absolute_error": 0.640625 + }, + { + "column": 11479, + "rms_error": 0.17425826783453177, + "maximum_absolute_error": 0.640625 + }, + { + "column": 21817, + "rms_error": 0.17298392707381488, + "maximum_absolute_error": 0.65625 + }, + { + "column": 1009, + "rms_error": 0.1700831995755906, + "maximum_absolute_error": 0.625 + }, + { + "column": 278, + "rms_error": 0.16506254717527535, + "maximum_absolute_error": 0.5625 + }, + { + "column": 26484, + "rms_error": 0.16420925479518414, + "maximum_absolute_error": 0.609375 + }, + { + "column": 2697, + "rms_error": 0.16407412533216695, + "maximum_absolute_error": 0.578125 + }, + { + "column": 58530, + "rms_error": 0.16312978920172735, + "maximum_absolute_error": 0.578125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null + }, + "compared_native_stages": 0, + "limits": [ + "IdealFP64 differences preserved, no tolerance or top-set policy changed.", + "Namedgraph ordinary attention omits selected integer-exponent replay; observed equality validates only this input.", + "FP64 multiply-add thenFP32 fma helper is not universally correct fmaf.", + "6\u21928 transfer applies only independently proven identical vision tensors/input; each decoder graph is independently computed on its own quantizedcheckpoint.", + "No native intermediate stage comparisons." + ], + "fidelity_pass": null, + "accepted_threshold": null, + "performance_claim": false, + "independent_decoder_run": true, + "decoder_transfer_from_other_width": false, + "checkpoint_sha256": "58268dfdf441ae4020d2a52fc97a814645e8a9624946e2d490403d4dcd71a255", + "config_sha256": "0efb7708e0771e3e104ac628954cdc71070dd436e0b7974615bd1b9eb836f9d9", + "input_hashes": { + "tokens": "54c970db8c6ba82895cb2f18a48d397ea568ae885241129e9c5ee7dbbc91a24d", + "native_image_rows": "ffa0401f452f39a07833d6692ac9eff0d5485cc93a14fb3433bdad74c42ab05e", + "independent_ideal_image": "a67172521a00ffc5c50b848769a92f732f12456276cf484e76509f91c703d0c8", + "independent_named_image": "ffa0401f452f39a07833d6692ac9eff0d5485cc93a14fb3433bdad74c42ab05e", + "vision_reference_receipt": "98556f6663ca894b3f185e7873f145fb199f76d32a24bf62b6a5db6023b35bd3", + "vision_identity_receipt": "6e28b34d17c1b5ff57811e1375bf8292fdbe18b2ddeeebe78d12191c64ebab98" + }, + "raw_receipt_sha256": "c10a070d72c5ff36a2ef62a5735697817a422b0752956325892021a19d3754b0", + "loaded_reference_modules": { + "tools/lfm3/fp64_reference.py": "0fb1128ade095db87b6126792ab14930792ab1f7b41b998570356bc586f16b2e", + "tools/lfm3/row_projection_reference.py": "d03f224a037a920ba66b9828059d1489e48729cd168077e76789d0ec5c199c0c", + "tools/lfm3/compare_logits.py": "b1bf273c8f3ce89591bccb8f098c82763cddff11373dbe850302450fcee1502c", + "tools/lfm3/elementary_reference.py": "3e4db276278867b7683536044ed7bcbe4c7643867d722fdda5b8ad895b3963e7", + "tools/lfm3/text_rounding_reference.py": "52665eef48570d45a2554861fe8f22b31874f8f464366adf882f40c3d311db27", + "tools/lfm3/text_attention_rounding.py": "ae67f2b2c26f89f75b1f36822a4b30a9f710df2b42f9f7ec42679373028b89e8" + }, + "elementary_calls_receipt_sha256": "e7f5ef9842fed904bd7e9a500a27e94180bde9f7418fd70adb48411867937f32" + }, + { + "bits": 6, + "rows": 90, + "vocabulary": 128000, + "values": 11520000, + "named_native_equal_words": 11520000, + "native_sha256": "090dfcc2d4a86039b651544489ba64232cda63352a7e2acfa8395896e6681abd", + "named_sha256": "090dfcc2d4a86039b651544489ba64232cda63352a7e2acfa8395896e6681abd", + "ideal_sha256": "01b638b3ea22b179379904d1643bd6b64a806fbd0cb8bd94229aa73cf6fe3fb9", + "native_vs_ideal_FP64_all90": { + "rows": 90, + "vocabulary": 128000, + "top1_differences": 4, + "disjoint_maximum_sets": 4, + "relative_rms": 0.03184506055566759, + "maximum_absolute_error": 3.9765625, + "differences": [ + { + "row": 14, + "native_top": 73, + "reference_top": 1319, + "native_maximum_set": [ + 73 + ], + "reference_maximum_set": [ + 1319 + ], + "maximum_sets_overlap": false, + "native_margin": 0.15625, + "reference_margin": 0.125, + "native_reference_contender_gap": 0.15625, + "reference_native_contender_gap": 0.125 + }, + { + "row": 43, + "native_top": 76, + "reference_top": 10, + "native_maximum_set": [ + 76 + ], + "reference_maximum_set": [ + 10 + ], + "maximum_sets_overlap": false, + "native_margin": 0.03125, + "reference_margin": 0.28125, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.375 + }, + { + "row": 52, + "native_top": 1628, + "reference_top": 933, + "native_maximum_set": [ + 1628 + ], + "reference_maximum_set": [ + 933 + ], + "maximum_sets_overlap": false, + "native_margin": 0.09375, + "reference_margin": 0.34375, + "native_reference_contender_gap": 0.09375, + "reference_native_contender_gap": 0.75 + }, + { + "row": 89, + "native_top": 6066, + "reference_top": 440, + "native_maximum_set": [ + 6066 + ], + "reference_maximum_set": [ + 440 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.125, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.125 + } + ], + "worst_rows": [ + { + "row": 31, + "rms_error": 0.6817091762210523, + "maximum_absolute_error": 3.9765625 + }, + { + "row": 17, + "rms_error": 0.3855200220352792, + "maximum_absolute_error": 2.328125 + }, + { + "row": 24, + "rms_error": 0.29071305191446567, + "maximum_absolute_error": 0.8984375 + }, + { + "row": 14, + "rms_error": 0.21156460556604134, + "maximum_absolute_error": 0.78125 + }, + { + "row": 41, + "rms_error": 0.20567940097410894, + "maximum_absolute_error": 0.68359375 + }, + { + "row": 54, + "rms_error": 0.18406604129150933, + "maximum_absolute_error": 0.501953125 + }, + { + "row": 19, + "rms_error": 0.17525203858353874, + "maximum_absolute_error": 0.796875 + }, + { + "row": 64, + "rms_error": 0.1525908824928283, + "maximum_absolute_error": 0.5029296875 + }, + { + "row": 34, + "rms_error": 0.14846789993952306, + "maximum_absolute_error": 0.6640625 + }, + { + "row": 53, + "rms_error": 0.14799893407626782, + "maximum_absolute_error": 0.84130859375 + }, + { + "row": 18, + "rms_error": 0.14656350294627024, + "maximum_absolute_error": 0.578125 + }, + { + "row": 52, + "rms_error": 0.14612344925795098, + "maximum_absolute_error": 0.828125 + }, + { + "row": 29, + "rms_error": 0.1374869722707254, + "maximum_absolute_error": 0.48046875 + }, + { + "row": 60, + "rms_error": 0.13163154624720513, + "maximum_absolute_error": 0.34375 + }, + { + "row": 40, + "rms_error": 0.12995604117646795, + "maximum_absolute_error": 0.9453125 + }, + { + "row": 3, + "rms_error": 0.12302304532129478, + "maximum_absolute_error": 0.4375 + }, + { + "row": 73, + "rms_error": 0.11714780877150698, + "maximum_absolute_error": 0.296875 + }, + { + "row": 33, + "rms_error": 0.1094471985680796, + "maximum_absolute_error": 0.6640625 + }, + { + "row": 23, + "rms_error": 0.10710137370027979, + "maximum_absolute_error": 0.73681640625 + }, + { + "row": 21, + "rms_error": 0.10207552512692736, + "maximum_absolute_error": 0.296875 + } + ], + "worst_columns": [ + { + "column": 461, + "rms_error": 0.4361162010438823, + "maximum_absolute_error": 3.92578125 + }, + { + "column": 82, + "rms_error": 0.4346224634745526, + "maximum_absolute_error": 3.9765625 + }, + { + "column": 4834, + "rms_error": 0.406660088977096, + "maximum_absolute_error": 3.7109375 + }, + { + "column": 104982, + "rms_error": 0.38757688313373473, + "maximum_absolute_error": 3.13671875 + }, + { + "column": 3341, + "rms_error": 0.3856819899918962, + "maximum_absolute_error": 3.5390625 + }, + { + "column": 14998, + "rms_error": 0.3850086423666284, + "maximum_absolute_error": 3.484375 + }, + { + "column": 18728, + "rms_error": 0.3718763316355032, + "maximum_absolute_error": 3.3828125 + }, + { + "column": 1420, + "rms_error": 0.3643339288368358, + "maximum_absolute_error": 3.21484375 + }, + { + "column": 55707, + "rms_error": 0.3608243967954695, + "maximum_absolute_error": 3.26953125 + }, + { + "column": 5190, + "rms_error": 0.36045134058082956, + "maximum_absolute_error": 3.306640625 + }, + { + "column": 417, + "rms_error": 0.3555122365336174, + "maximum_absolute_error": 2.8515625 + }, + { + "column": 51619, + "rms_error": 0.3529028474113053, + "maximum_absolute_error": 3.23046875 + }, + { + "column": 518, + "rms_error": 0.35177576832445556, + "maximum_absolute_error": 2.66015625 + }, + { + "column": 59610, + "rms_error": 0.35022396384839966, + "maximum_absolute_error": 3.1806640625 + }, + { + "column": 12443, + "rms_error": 0.35005619171946445, + "maximum_absolute_error": 3.0859375 + }, + { + "column": 15131, + "rms_error": 0.34886656071571204, + "maximum_absolute_error": 2.91796875 + }, + { + "column": 32240, + "rms_error": 0.3483694925355373, + "maximum_absolute_error": 3.1005859375 + }, + { + "column": 24457, + "rms_error": 0.3448792107302648, + "maximum_absolute_error": 2.5 + }, + { + "column": 15446, + "rms_error": 0.3442066444426652, + "maximum_absolute_error": 3.03125 + }, + { + "column": 4425, + "rms_error": 0.3438285237030011, + "maximum_absolute_error": 3.11865234375 + } + ], + "accepted_threshold": null, + "fidelity_pass": null + }, + "native_vs_ideal_FP64_suffix16": { + "rows": 16, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 1, + "relative_rms": 0.01177118128146598, + "maximum_absolute_error": 0.609375, + "differences": [ + { + "row": 89, + "native_top": 6066, + "reference_top": 440, + "native_maximum_set": [ + 6066 + ], + "reference_maximum_set": [ + 440 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.125, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.125 + } + ], + "worst_rows": [ + { + "row": 76, + "rms_error": 0.10091868484554894, + "maximum_absolute_error": 0.609375 + }, + { + "row": 86, + "rms_error": 0.09955829832699667, + "maximum_absolute_error": 0.40625 + }, + { + "row": 82, + "rms_error": 0.07869171853281849, + "maximum_absolute_error": 0.34375 + }, + { + "row": 89, + "rms_error": 0.06554237531313618, + "maximum_absolute_error": 0.390625 + }, + { + "row": 87, + "rms_error": 0.058100667327155374, + "maximum_absolute_error": 0.375 + }, + { + "row": 84, + "rms_error": 0.054905126073209604, + "maximum_absolute_error": 0.28125 + }, + { + "row": 81, + "rms_error": 0.05457037704459559, + "maximum_absolute_error": 0.3828125 + }, + { + "row": 88, + "rms_error": 0.051123148813793685, + "maximum_absolute_error": 0.265625 + }, + { + "row": 74, + "rms_error": 0.05050178504830527, + "maximum_absolute_error": 0.28125 + }, + { + "row": 83, + "rms_error": 0.048607154181645885, + "maximum_absolute_error": 0.25 + }, + { + "row": 85, + "rms_error": 0.047260047417249845, + "maximum_absolute_error": 0.21875 + }, + { + "row": 79, + "rms_error": 0.04204592860587671, + "maximum_absolute_error": 0.140625 + }, + { + "row": 80, + "rms_error": 0.039622123603917404, + "maximum_absolute_error": 0.21875 + }, + { + "row": 78, + "rms_error": 0.03959908222293029, + "maximum_absolute_error": 0.25 + }, + { + "row": 77, + "rms_error": 0.030233986960441547, + "maximum_absolute_error": 0.15625 + }, + { + "row": 75, + "rms_error": 0.021145048036500656, + "maximum_absolute_error": 0.109375 + } + ], + "worst_columns": [ + { + "column": 1368, + "rms_error": 0.16893825626521555, + "maximum_absolute_error": 0.56640625 + }, + { + "column": 915, + "rms_error": 0.16818223737622257, + "maximum_absolute_error": 0.609375 + }, + { + "column": 4507, + "rms_error": 0.1617790441482419, + "maximum_absolute_error": 0.6015625 + }, + { + "column": 1064, + "rms_error": 0.15638498197877243, + "maximum_absolute_error": 0.5625 + }, + { + "column": 267, + "rms_error": 0.15595675607560258, + "maximum_absolute_error": 0.53125 + }, + { + "column": 1037, + "rms_error": 0.15158489936795955, + "maximum_absolute_error": 0.4248046875 + }, + { + "column": 11510, + "rms_error": 0.14912895976306764, + "maximum_absolute_error": 0.541015625 + }, + { + "column": 39914, + "rms_error": 0.1488481162754504, + "maximum_absolute_error": 0.390625 + }, + { + "column": 106378, + "rms_error": 0.14854026048305893, + "maximum_absolute_error": 0.5 + }, + { + "column": 56852, + "rms_error": 0.14758053407575789, + "maximum_absolute_error": 0.51171875 + }, + { + "column": 57005, + "rms_error": 0.14631500625256283, + "maximum_absolute_error": 0.515625 + }, + { + "column": 27715, + "rms_error": 0.14607037851993482, + "maximum_absolute_error": 0.484375 + }, + { + "column": 59750, + "rms_error": 0.1454422566251192, + "maximum_absolute_error": 0.53125 + }, + { + "column": 2146, + "rms_error": 0.1452887154877859, + "maximum_absolute_error": 0.46875 + }, + { + "column": 5711, + "rms_error": 0.1450148045447081, + "maximum_absolute_error": 0.49609375 + }, + { + "column": 576, + "rms_error": 0.14464090672276295, + "maximum_absolute_error": 0.48828125 + }, + { + "column": 2898, + "rms_error": 0.1446248859513871, + "maximum_absolute_error": 0.40625 + }, + { + "column": 2972, + "rms_error": 0.14213597967703323, + "maximum_absolute_error": 0.390625 + }, + { + "column": 19751, + "rms_error": 0.1413118753533306, + "maximum_absolute_error": 0.4765625 + }, + { + "column": 1187, + "rms_error": 0.14130154099754627, + "maximum_absolute_error": 0.3203125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null + }, + "compared_native_stages": 0, + "limits": [ + "IdealFP64 differences preserved, no tolerance or top-set policy changed.", + "Namedgraph ordinary attention omits selected integer-exponent replay; observed equality validates only this input.", + "FP64 multiply-add thenFP32 fma helper is not universally correct fmaf.", + "6\u21928 transfer applies only independently proven identical vision tensors/input; each decoder graph is independently computed on its own quantizedcheckpoint.", + "No native intermediate stage comparisons." + ], + "fidelity_pass": null, + "accepted_threshold": null, + "performance_claim": false, + "independent_decoder_run": true, + "decoder_transfer_from_other_width": false, + "checkpoint_sha256": "838c39147541b12e86a4f1b43d6a7814d7fb4efe9d5ff8a151c85ac06f321eef", + "config_sha256": "4c03aa298372cbcb478eb08be0a4a59be917b4949756066ce4eac06033d200bc", + "input_hashes": { + "tokens": "54c970db8c6ba82895cb2f18a48d397ea568ae885241129e9c5ee7dbbc91a24d", + "native_image_rows": "0bfbdb4d6e8f3485609f0b6e8cf4b84bd8daba4c2ae62f56a4861aad3688b4df", + "independent_ideal_image": "221133d871ce2db48dfcb28af88f8e3facb80f604d9fa8b4e37afef07e04fde4", + "independent_named_image": "0bfbdb4d6e8f3485609f0b6e8cf4b84bd8daba4c2ae62f56a4861aad3688b4df", + "vision_reference_receipt": "cd65b824e33a1e20bd635c4f7500819b36e4c6702135d3a75e409943cdb3fc13", + "vision_identity_receipt": "6e28b34d17c1b5ff57811e1375bf8292fdbe18b2ddeeebe78d12191c64ebab98" + }, + "raw_receipt_sha256": "70ba18ca52b1178b8b8aa25c9d29aeac98e8c096ff79234fa8e2b3ce33e600be", + "loaded_reference_modules": { + "tools/lfm3/fp64_reference.py": "0fb1128ade095db87b6126792ab14930792ab1f7b41b998570356bc586f16b2e", + "tools/lfm3/row_projection_reference.py": "d03f224a037a920ba66b9828059d1489e48729cd168077e76789d0ec5c199c0c", + "tools/lfm3/compare_logits.py": "b1bf273c8f3ce89591bccb8f098c82763cddff11373dbe850302450fcee1502c", + "tools/lfm3/elementary_reference.py": "3e4db276278867b7683536044ed7bcbe4c7643867d722fdda5b8ad895b3963e7", + "tools/lfm3/text_rounding_reference.py": "52665eef48570d45a2554861fe8f22b31874f8f464366adf882f40c3d311db27", + "tools/lfm3/text_attention_rounding.py": "ae67f2b2c26f89f75b1f36822a4b30a9f710df2b42f9f7ec42679373028b89e8" + }, + "elementary_calls_receipt_sha256": "9878a3ae6fa97070f6d22234ac56447096536cb26d9abb1330f2cf393522a2ef" + }, + { + "bits": 8, + "rows": 90, + "vocabulary": 128000, + "values": 11520000, + "named_native_equal_words": 11520000, + "native_sha256": "c4ada8a06059f4f663adb463d564a48ed5fb48f59008f5d8f24b5cb9ec26848e", + "named_sha256": "c4ada8a06059f4f663adb463d564a48ed5fb48f59008f5d8f24b5cb9ec26848e", + "ideal_sha256": "c4f0978b4b43990c56acf7759f1c4874a54b4c97e3259fb64cb54853b7d15ea3", + "native_vs_ideal_FP64_all90": { + "rows": 90, + "vocabulary": 128000, + "top1_differences": 7, + "disjoint_maximum_sets": 2, + "relative_rms": 0.030672305970061556, + "maximum_absolute_error": 3.9375, + "differences": [ + { + "row": 14, + "native_top": 594, + "reference_top": 73, + "native_maximum_set": [ + 594 + ], + "reference_maximum_set": [ + 73, + 594 + ], + "maximum_sets_overlap": true, + "native_margin": 0.25, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.25, + "reference_native_contender_gap": 0.0 + }, + { + "row": 18, + "native_top": 309, + "reference_top": 40662, + "native_maximum_set": [ + 309, + 40662 + ], + "reference_maximum_set": [ + 40662 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.375, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.6875 + }, + { + "row": 43, + "native_top": 76, + "reference_top": 10, + "native_maximum_set": [ + 76 + ], + "reference_maximum_set": [ + 10 + ], + "maximum_sets_overlap": false, + "native_margin": 0.1875, + "reference_margin": 0.1875, + "native_reference_contender_gap": 0.1875, + "reference_native_contender_gap": 0.1875 + }, + { + "row": 46, + "native_top": 25496, + "reference_top": 23, + "native_maximum_set": [ + 25496 + ], + "reference_maximum_set": [ + 23, + 25496 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + }, + { + "row": 52, + "native_top": 1628, + "reference_top": 933, + "native_maximum_set": [ + 1628 + ], + "reference_maximum_set": [ + 933 + ], + "maximum_sets_overlap": false, + "native_margin": 0.34375, + "reference_margin": 0.5, + "native_reference_contender_gap": 0.53125, + "reference_native_contender_gap": 0.5 + }, + { + "row": 56, + "native_top": 32048, + "reference_top": 4752, + "native_maximum_set": [ + 32048 + ], + "reference_maximum_set": [ + 4752, + 32048 + ], + "maximum_sets_overlap": true, + "native_margin": 0.09375, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.09375, + "reference_native_contender_gap": 0.0 + }, + { + "row": 88, + "native_top": 309, + "reference_top": 22, + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22, + 309 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 31, + "rms_error": 0.6916239688724547, + "maximum_absolute_error": 3.9375 + }, + { + "row": 17, + "rms_error": 0.3294247851607478, + "maximum_absolute_error": 2.15234375 + }, + { + "row": 64, + "rms_error": 0.30245018327941037, + "maximum_absolute_error": 0.625 + }, + { + "row": 41, + "rms_error": 0.17945556417538017, + "maximum_absolute_error": 0.75341796875 + }, + { + "row": 24, + "rms_error": 0.17406751550162675, + "maximum_absolute_error": 0.73046875 + }, + { + "row": 34, + "rms_error": 0.1445714960268575, + "maximum_absolute_error": 0.734375 + }, + { + "row": 29, + "rms_error": 0.14388185427874117, + "maximum_absolute_error": 0.5625 + }, + { + "row": 54, + "rms_error": 0.14268417084418752, + "maximum_absolute_error": 0.515625 + }, + { + "row": 53, + "rms_error": 0.14037637833532524, + "maximum_absolute_error": 0.876953125 + }, + { + "row": 19, + "rms_error": 0.13927104568019405, + "maximum_absolute_error": 0.6875 + }, + { + "row": 52, + "rms_error": 0.13363253175177817, + "maximum_absolute_error": 0.9609375 + }, + { + "row": 33, + "rms_error": 0.12750569912333592, + "maximum_absolute_error": 0.671875 + }, + { + "row": 40, + "rms_error": 0.127051574905554, + "maximum_absolute_error": 0.9921875 + }, + { + "row": 82, + "rms_error": 0.1198935001460607, + "maximum_absolute_error": 0.390625 + }, + { + "row": 8, + "rms_error": 0.11902762357850875, + "maximum_absolute_error": 0.359375 + }, + { + "row": 63, + "rms_error": 0.11584276190501128, + "maximum_absolute_error": 0.34375 + }, + { + "row": 14, + "rms_error": 0.11043077089726261, + "maximum_absolute_error": 0.5 + }, + { + "row": 23, + "rms_error": 0.10686595149668394, + "maximum_absolute_error": 0.818359375 + }, + { + "row": 42, + "rms_error": 0.10412159121042476, + "maximum_absolute_error": 0.375 + }, + { + "row": 78, + "rms_error": 0.10188237019612166, + "maximum_absolute_error": 0.40625 + } + ], + "worst_columns": [ + { + "column": 4834, + "rms_error": 0.4272392562076876, + "maximum_absolute_error": 3.9375 + }, + { + "column": 1420, + "rms_error": 0.4072810095851561, + "maximum_absolute_error": 3.6611328125 + }, + { + "column": 5190, + "rms_error": 0.38279238818875627, + "maximum_absolute_error": 3.53125 + }, + { + "column": 1427, + "rms_error": 0.3774396091789617, + "maximum_absolute_error": 3.3828125 + }, + { + "column": 104982, + "rms_error": 0.3730385842103811, + "maximum_absolute_error": 3.18359375 + }, + { + "column": 3341, + "rms_error": 0.3704571798908096, + "maximum_absolute_error": 3.38671875 + }, + { + "column": 15131, + "rms_error": 0.3649059705027634, + "maximum_absolute_error": 3.08984375 + }, + { + "column": 14998, + "rms_error": 0.3604175868968939, + "maximum_absolute_error": 3.25 + }, + { + "column": 18728, + "rms_error": 0.35932770408629344, + "maximum_absolute_error": 3.265625 + }, + { + "column": 42385, + "rms_error": 0.3587140625998213, + "maximum_absolute_error": 3.2744140625 + }, + { + "column": 55707, + "rms_error": 0.3554542137965256, + "maximum_absolute_error": 3.173828125 + }, + { + "column": 518, + "rms_error": 0.3554054231937565, + "maximum_absolute_error": 2.802734375 + }, + { + "column": 7427, + "rms_error": 0.35458752926631115, + "maximum_absolute_error": 3.14453125 + }, + { + "column": 93784, + "rms_error": 0.3526706298591616, + "maximum_absolute_error": 3.234375 + }, + { + "column": 20057, + "rms_error": 0.3482760435567775, + "maximum_absolute_error": 3.08203125 + }, + { + "column": 8323, + "rms_error": 0.3476132598471024, + "maximum_absolute_error": 2.671875 + }, + { + "column": 51619, + "rms_error": 0.34553521790806563, + "maximum_absolute_error": 3.1171875 + }, + { + "column": 11576, + "rms_error": 0.3387996991758052, + "maximum_absolute_error": 3.0673828125 + }, + { + "column": 12443, + "rms_error": 0.3364088908401562, + "maximum_absolute_error": 2.984375 + }, + { + "column": 27942, + "rms_error": 0.3358226430725191, + "maximum_absolute_error": 3.046875 + } + ], + "accepted_threshold": null, + "fidelity_pass": null + }, + "native_vs_ideal_FP64_suffix16": { + "rows": 16, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.014368072050965845, + "maximum_absolute_error": 0.609375, + "differences": [ + { + "row": 88, + "native_top": 309, + "reference_top": 22, + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22, + 309 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 82, + "rms_error": 0.1198935001460607, + "maximum_absolute_error": 0.390625 + }, + { + "row": 78, + "rms_error": 0.10188237019612166, + "maximum_absolute_error": 0.40625 + }, + { + "row": 86, + "rms_error": 0.0998781455178678, + "maximum_absolute_error": 0.609375 + }, + { + "row": 81, + "rms_error": 0.08580416761508934, + "maximum_absolute_error": 0.5361328125 + }, + { + "row": 89, + "rms_error": 0.08208782593190767, + "maximum_absolute_error": 0.46875 + }, + { + "row": 80, + "rms_error": 0.07270305780828373, + "maximum_absolute_error": 0.265625 + }, + { + "row": 83, + "rms_error": 0.07086549662700121, + "maximum_absolute_error": 0.328125 + }, + { + "row": 74, + "rms_error": 0.06462174052271798, + "maximum_absolute_error": 0.296875 + }, + { + "row": 76, + "rms_error": 0.05771098738739263, + "maximum_absolute_error": 0.5 + }, + { + "row": 84, + "rms_error": 0.056249772951973746, + "maximum_absolute_error": 0.34375 + }, + { + "row": 87, + "rms_error": 0.049322321678761265, + "maximum_absolute_error": 0.296875 + }, + { + "row": 85, + "rms_error": 0.047184947727720104, + "maximum_absolute_error": 0.25 + }, + { + "row": 75, + "rms_error": 0.0465256409042671, + "maximum_absolute_error": 0.140625 + }, + { + "row": 79, + "rms_error": 0.045989686188802954, + "maximum_absolute_error": 0.1875 + }, + { + "row": 88, + "rms_error": 0.04416602790051088, + "maximum_absolute_error": 0.265625 + }, + { + "row": 77, + "rms_error": 0.043429916921016054, + "maximum_absolute_error": 0.1875 + } + ], + "worst_columns": [ + { + "column": 267, + "rms_error": 0.1834066533370075, + "maximum_absolute_error": 0.5 + }, + { + "column": 5140, + "rms_error": 0.17878846534927656, + "maximum_absolute_error": 0.609375 + }, + { + "column": 563, + "rms_error": 0.1635794225975237, + "maximum_absolute_error": 0.390625 + }, + { + "column": 933, + "rms_error": 0.15843880135567934, + "maximum_absolute_error": 0.4375 + }, + { + "column": 417, + "rms_error": 0.15751290162544399, + "maximum_absolute_error": 0.421875 + }, + { + "column": 293, + "rms_error": 0.15707162366508126, + "maximum_absolute_error": 0.5361328125 + }, + { + "column": 9952, + "rms_error": 0.15069554121474835, + "maximum_absolute_error": 0.4375 + }, + { + "column": 378, + "rms_error": 0.14906119771904355, + "maximum_absolute_error": 0.50390625 + }, + { + "column": 125025, + "rms_error": 0.14899700525333687, + "maximum_absolute_error": 0.40625 + }, + { + "column": 90738, + "rms_error": 0.14806360286126388, + "maximum_absolute_error": 0.41796875 + }, + { + "column": 120715, + "rms_error": 0.14643355441669773, + "maximum_absolute_error": 0.375 + }, + { + "column": 8905, + "rms_error": 0.14537257111636775, + "maximum_absolute_error": 0.478515625 + }, + { + "column": 16123, + "rms_error": 0.1427775125404699, + "maximum_absolute_error": 0.5 + }, + { + "column": 116965, + "rms_error": 0.14275714661300895, + "maximum_absolute_error": 0.40625 + }, + { + "column": 339, + "rms_error": 0.14146680009735554, + "maximum_absolute_error": 0.40625 + }, + { + "column": 335, + "rms_error": 0.14112278467404366, + "maximum_absolute_error": 0.4052734375 + }, + { + "column": 7504, + "rms_error": 0.1408181453543369, + "maximum_absolute_error": 0.296875 + }, + { + "column": 32442, + "rms_error": 0.14054836728298803, + "maximum_absolute_error": 0.46875 + }, + { + "column": 42913, + "rms_error": 0.1402591609036901, + "maximum_absolute_error": 0.28125 + }, + { + "column": 64012, + "rms_error": 0.1399944269199803, + "maximum_absolute_error": 0.421875 + } + ], + "accepted_threshold": null, + "fidelity_pass": null + }, + "compared_native_stages": 0, + "limits": [ + "IdealFP64 differences preserved, no tolerance or top-set policy changed.", + "Namedgraph ordinary attention omits selected integer-exponent replay; observed equality validates only this input.", + "FP64 multiply-add thenFP32 fma helper is not universally correct fmaf.", + "6\u21928 transfer applies only independently proven identical vision tensors/input; each decoder graph is independently computed on its own quantizedcheckpoint.", + "No native intermediate stage comparisons." + ], + "fidelity_pass": null, + "accepted_threshold": null, + "performance_claim": false, + "independent_decoder_run": true, + "decoder_transfer_from_other_width": false, + "checkpoint_sha256": "50c9476a7f00f5349e81b42ab88b47635b9eba3c7ab5851c598d270cda4c6857", + "config_sha256": "9f7dc06d04d0610c7e27fb6a110c42a3a09b686a6ec53b0eb383f30a8be5a457", + "input_hashes": { + "tokens": "54c970db8c6ba82895cb2f18a48d397ea568ae885241129e9c5ee7dbbc91a24d", + "native_image_rows": "0bfbdb4d6e8f3485609f0b6e8cf4b84bd8daba4c2ae62f56a4861aad3688b4df", + "independent_ideal_image": "221133d871ce2db48dfcb28af88f8e3facb80f604d9fa8b4e37afef07e04fde4", + "independent_named_image": "0bfbdb4d6e8f3485609f0b6e8cf4b84bd8daba4c2ae62f56a4861aad3688b4df", + "vision_reference_receipt": "cd65b824e33a1e20bd635c4f7500819b36e4c6702135d3a75e409943cdb3fc13", + "vision_identity_receipt": "6e28b34d17c1b5ff57811e1375bf8292fdbe18b2ddeeebe78d12191c64ebab98" + }, + "raw_receipt_sha256": "8fb7b190e05b0fa5ae9536bc284b2b88b8e6754c84a8a1f5791a39a57d8b6245", + "loaded_reference_modules": { + "tools/lfm3/fp64_reference.py": "0fb1128ade095db87b6126792ab14930792ab1f7b41b998570356bc586f16b2e", + "tools/lfm3/row_projection_reference.py": "d03f224a037a920ba66b9828059d1489e48729cd168077e76789d0ec5c199c0c", + "tools/lfm3/compare_logits.py": "b1bf273c8f3ce89591bccb8f098c82763cddff11373dbe850302450fcee1502c", + "tools/lfm3/elementary_reference.py": "3e4db276278867b7683536044ed7bcbe4c7643867d722fdda5b8ad895b3963e7", + "tools/lfm3/text_rounding_reference.py": "52665eef48570d45a2554861fe8f22b31874f8f464366adf882f40c3d311db27", + "tools/lfm3/text_attention_rounding.py": "ae67f2b2c26f89f75b1f36822a4b30a9f710df2b42f9f7ec42679373028b89e8" + }, + "elementary_calls_receipt_sha256": "85bcb3771b1b429ca97219a39172418aef999116182bac9ecba1feee0801547f" + } + ], + "execution": { + "actual_exit": 0, + "owned_children_exit_zero": 6, + "lock_released": true, + "complete_receipt_sha256": "b4dbe3ba5dd85e57c13d7cf546a43ccf3e3c30ded9666baba5bc4dab7adff01f", + "events_sha256": "9d034a1dffda05a54f35d4a51dbcdf5fc221dc7959cefed591d154238b1fc837" + }, + "total_named_native_equal_words": 34560000, + "ideal_FP64_fidelity_pass": null, + "cancellation_test": false, + "performance_claim": false, + "limitations": [ + "No native intermediate stage comparisons; stage hashes are not stage equality proof.", + "Observed named-graph equality does not waive any ideal-FP64 discrepancy or establish universal arithmetic equivalence.", + "Old-pixel outputs are not used as reference or transferred." + ] +} diff --git a/docs/receipts/lfm3/corrected-pixel-serving.json b/docs/receipts/lfm3/corrected-pixel-serving.json new file mode 100644 index 0000000000..5057ba9dd7 --- /dev/null +++ b/docs/receipts/lfm3/corrected-pixel-serving.json @@ -0,0 +1,3449 @@ +{ + "schema_version": 1, + "source_commit": "50b4e0da3", + "scope": "Corrected CPU normalization; real native VL4/6/8 serving self-consistency using generated solid red/blue PNGs and ordered-image requests", + "binary_sha256": "b8404ae0c6edec9968f59dbe699bfb61fa8638c564519b94ce7a2863dfcb3377", + "client_sha256": "87327328ef1a1fcc35536efa0c18829779ee278a4d325f2af69356bc68a71f5c", + "counts": { + "passed": 183, + "failed": 0, + "skipped": 0, + "cancellation_checks": 27 + }, + "cases": [ + { + "model": "LFM3-VL-2B-MLX-4bit", + "counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 9, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c", + "text": " \nThe story is a summary of the events that occur in the events that occur in the events that occur in the events" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370", + "text": " A Python function that returns the larger of two integers can be written as: A can be returned as: A can be" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + }, + { + "name": "red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead", + "text": "Red<|im_end|>\n<|im_end|>\n \"Red\" is a broad category; the specific shade appears to be a bright, saturated red" + } + }, + { + "name": "blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef", + "text": "Blue<|im_end|>\n<|im_end|>\n \"Blue\" is a broad category, but the image matches the specific shade \"Blue\" as" + } + }, + { + "name": "red_blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40", + "text": "Red, Blue<|im_end|>\n<|im_end|>\n<|im_end|>\n Name the solid color of the first image, then the second image." + } + }, + { + "name": "blue_red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf", + "text": "Blue, Red<|im_end|>\n<|im_end|>\n<|im_end|>\n you have to think about the visual characteristics of the images and the visual characteristics" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c", + "text": " \nThe story is a summary of the events that occur in the events that occur in the events that occur in the events" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c", + "text": " \nThe story is a summary of the events that occur in the events that occur in the events that occur in the events" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c", + "text": " \nThe story is a summary of the events that occur in the events that occur in the events that occur in the events" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370", + "text": " A Python function that returns the larger of two integers can be written as: A can be returned as: A can be" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370", + "text": " A Python function that returns the larger of two integers can be written as: A can be returned as: A can be" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + }, + { + "name": "red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead", + "text": "Red<|im_end|>\n<|im_end|>\n \"Red\" is a broad category; the specific shade appears to be a bright, saturated red" + } + }, + { + "name": "red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead", + "text": "Red<|im_end|>\n<|im_end|>\n \"Red\" is a broad category; the specific shade appears to be a bright, saturated red" + } + }, + { + "name": "blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef", + "text": "Blue<|im_end|>\n<|im_end|>\n \"Blue\" is a broad category, but the image matches the specific shade \"Blue\" as" + } + }, + { + "name": "blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef", + "text": "Blue<|im_end|>\n<|im_end|>\n \"Blue\" is a broad category, but the image matches the specific shade \"Blue\" as" + } + }, + { + "name": "red_blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40", + "text": "Red, Blue<|im_end|>\n<|im_end|>\n<|im_end|>\n Name the solid color of the first image, then the second image." + } + }, + { + "name": "red_blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40", + "text": "Red, Blue<|im_end|>\n<|im_end|>\n<|im_end|>\n Name the solid color of the first image, then the second image." + } + }, + { + "name": "blue_red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf", + "text": "Blue, Red<|im_end|>\n<|im_end|>\n<|im_end|>\n you have to think about the visual characteristics of the images and the visual characteristics" + } + }, + { + "name": "blue_red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf", + "text": "Blue, Red<|im_end|>\n<|im_end|>\n<|im_end|>\n you have to think about the visual characteristics of the images and the visual characteristics" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370", + "text": " A Python function that returns the larger of two integers can be written as: A can be returned as: A can be" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370", + "text": " A Python function that returns the larger of two integers can be written as: A can be returned as: A can be" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370", + "text": " A Python function that returns the larger of two integers can be written as: A can be returned as: A can be" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370", + "text": " A Python function that returns the larger of two integers can be written as: A can be returned as: A can be" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370", + "text": " A Python function that returns the larger of two integers can be written as: A can be returned as: A can be" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370", + "text": " A Python function that returns the larger of two integers can be written as: A can be returned as: A can be" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370", + "text": " A Python function that returns the larger of two integers can be written as: A can be returned as: A can be" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + ] + }, + { + "name": "red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red" + ], + "observed": [ + "red", + "red", + "red" + ], + "text": "Red<|im_end|>\n<|im_end|>\n \"Red\" is a broad category; the specific shade appears to be a bright, saturated red" + } + }, + { + "name": "blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue" + ], + "observed": [ + "blue", + "blue", + "blue" + ], + "text": "Blue<|im_end|>\n<|im_end|>\n \"Blue\" is a broad category, but the image matches the specific shade \"Blue\" as" + } + }, + { + "name": "red_blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red", + "blue" + ], + "observed": [ + "red", + "blue" + ], + "text": "Red, Blue<|im_end|>\n<|im_end|>\n<|im_end|>\n Name the solid color of the first image, then the second image." + } + }, + { + "name": "blue_red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue", + "red" + ], + "observed": [ + "blue", + "red" + ], + "text": "Blue, Red<|im_end|>\n<|im_end|>\n<|im_end|>\n you have to think about the visual characteristics of the images and the visual characteristics" + } + }, + { + "name": "images/concurrent_equals_solos", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead", + "text": "Red<|im_end|>\n<|im_end|>\n \"Red\" is a broad category; the specific shade appears to be a bright, saturated red" + }, + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef", + "text": "Blue<|im_end|>\n<|im_end|>\n \"Blue\" is a broad category, but the image matches the specific shade \"Blue\" as" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40", + "text": "Red, Blue<|im_end|>\n<|im_end|>\n<|im_end|>\n Name the solid color of the first image, then the second image." + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf", + "text": "Blue, Red<|im_end|>\n<|im_end|>\n<|im_end|>\n you have to think about the visual characteristics of the images and the visual characteristics" + } + ] + }, + { + "name": "images/corrupt_png_rejected", + "status": "passed", + "evidence": { + "error": "{\"error\": {\"message\": \"image 1 could not be decoded; send PNG, JPEG or WebP\", \"type\": \"invalid_request_error\"}}", + "status": 400, + "complete": false + } + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "cmpl-e3b621ee6c97577bdeda474bdf75e6c1", + "first_content": " The", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "chatcmpl-eb11c533a1224b98f9286b1d894e1f07", + "first_content": "The", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + } + }, + { + "name": "red/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "chatcmpl-7b2d6bc31c4c2ef8fee8254e582683eb", + "first_content": "Red", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 2, + "server_disconnections_after": 3, + "server_confirmed_cancel": true + } + }, + { + "name": "red/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead", + "text": "Red<|im_end|>\n<|im_end|>\n \"Red\" is a broad category; the specific shade appears to be a bright, saturated red" + } + }, + { + "name": "red/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead", + "text": "Red<|im_end|>\n<|im_end|>\n \"Red\" is a broad category; the specific shade appears to be a bright, saturated red" + } + } + ], + "raw_receipt_sha256": "194a953509382b29dfcebfb30371cd1329f2bbfb13f7aec69d284c4a1a3a7c12" + }, + { + "model": "LFM3-VL-2B-MLX-6bit", + "counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 9, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942", + "text": " The story is about a child who watches birds near a certain place. The story is about a child who watches birds near" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8", + "text": "The sky appears blue because molecules in the atmosphere scatter short blue wavelengths of sunlight more efficiently than longer red wavelengths. This scattering" + } + }, + { + "name": "red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is Red.<|im_end|>\n, ., and" + } + }, + { + "name": "blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0", + "text": "Blue<|im_end|>\n<|im_end|>\nfollowing the instruction to not mention the reasoning. Here is the final answer:\n\nBlue<|im_end|>\n" + } + }, + { + "name": "red_blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5", + "text": "Red, blue<|im_end|>\n<|im_end|>\n<|im_end|>\n named after the color of the sky on a clear day, but with a" + } + }, + { + "name": "blue_red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603", + "text": "Blue, red<|im_end|>\n<|im_end|>\n 2. The first image has a blue color, and the second image has" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942", + "text": " The story is about a child who watches birds near a certain place. The story is about a child who watches birds near" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942", + "text": " The story is about a child who watches birds near a certain place. The story is about a child who watches birds near" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942", + "text": " The story is about a child who watches birds near a certain place. The story is about a child who watches birds near" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8", + "text": "The sky appears blue because molecules in the atmosphere scatter short blue wavelengths of sunlight more efficiently than longer red wavelengths. This scattering" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8", + "text": "The sky appears blue because molecules in the atmosphere scatter short blue wavelengths of sunlight more efficiently than longer red wavelengths. This scattering" + } + }, + { + "name": "red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is Red.<|im_end|>\n, ., and" + } + }, + { + "name": "red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is Red.<|im_end|>\n, ., and" + } + }, + { + "name": "blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0", + "text": "Blue<|im_end|>\n<|im_end|>\nfollowing the instruction to not mention the reasoning. Here is the final answer:\n\nBlue<|im_end|>\n" + } + }, + { + "name": "blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0", + "text": "Blue<|im_end|>\n<|im_end|>\nfollowing the instruction to not mention the reasoning. Here is the final answer:\n\nBlue<|im_end|>\n" + } + }, + { + "name": "red_blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5", + "text": "Red, blue<|im_end|>\n<|im_end|>\n<|im_end|>\n named after the color of the sky on a clear day, but with a" + } + }, + { + "name": "red_blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5", + "text": "Red, blue<|im_end|>\n<|im_end|>\n<|im_end|>\n named after the color of the sky on a clear day, but with a" + } + }, + { + "name": "blue_red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603", + "text": "Blue, red<|im_end|>\n<|im_end|>\n 2. The first image has a blue color, and the second image has" + } + }, + { + "name": "blue_red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603", + "text": "Blue, red<|im_end|>\n<|im_end|>\n 2. The first image has a blue color, and the second image has" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8", + "text": "The sky appears blue because molecules in the atmosphere scatter short blue wavelengths of sunlight more efficiently than longer red wavelengths. This scattering" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8", + "text": "The sky appears blue because molecules in the atmosphere scatter short blue wavelengths of sunlight more efficiently than longer red wavelengths. This scattering" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8", + "text": "The sky appears blue because molecules in the atmosphere scatter short blue wavelengths of sunlight more efficiently than longer red wavelengths. This scattering" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8", + "text": "The sky appears blue because molecules in the atmosphere scatter short blue wavelengths of sunlight more efficiently than longer red wavelengths. This scattering" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8", + "text": "The sky appears blue because molecules in the atmosphere scatter short blue wavelengths of sunlight more efficiently than longer red wavelengths. This scattering" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8", + "text": "The sky appears blue because molecules in the atmosphere scatter short blue wavelengths of sunlight more efficiently than longer red wavelengths. This scattering" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8", + "text": "The sky appears blue because molecules in the atmosphere scatter short blue wavelengths of sunlight more efficiently than longer red wavelengths. This scattering" + } + ] + }, + { + "name": "red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red" + ], + "observed": [ + "red", + "red" + ], + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is Red.<|im_end|>\n, ., and" + } + }, + { + "name": "blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue" + ], + "observed": [ + "blue", + "blue" + ], + "text": "Blue<|im_end|>\n<|im_end|>\nfollowing the instruction to not mention the reasoning. Here is the final answer:\n\nBlue<|im_end|>\n" + } + }, + { + "name": "red_blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red", + "blue" + ], + "observed": [ + "red", + "blue" + ], + "text": "Red, blue<|im_end|>\n<|im_end|>\n<|im_end|>\n named after the color of the sky on a clear day, but with a" + } + }, + { + "name": "blue_red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue", + "red" + ], + "observed": [ + "blue", + "red", + "blue" + ], + "text": "Blue, red<|im_end|>\n<|im_end|>\n 2. The first image has a blue color, and the second image has" + } + }, + { + "name": "images/concurrent_equals_solos", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is Red.<|im_end|>\n, ., and" + }, + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0", + "text": "Blue<|im_end|>\n<|im_end|>\nfollowing the instruction to not mention the reasoning. Here is the final answer:\n\nBlue<|im_end|>\n" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5", + "text": "Red, blue<|im_end|>\n<|im_end|>\n<|im_end|>\n named after the color of the sky on a clear day, but with a" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603", + "text": "Blue, red<|im_end|>\n<|im_end|>\n 2. The first image has a blue color, and the second image has" + } + ] + }, + { + "name": "images/corrupt_png_rejected", + "status": "passed", + "evidence": { + "error": "{\"error\": {\"message\": \"image 1 could not be decoded; send PNG, JPEG or WebP\", \"type\": \"invalid_request_error\"}}", + "status": 400, + "complete": false + } + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "cmpl-6a67bcb048b34befb20a0bbef9010b66", + "first_content": " The", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "chatcmpl-b2dc19d2fd409ca9371fcda393f9a48a", + "first_content": "The", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + }, + { + "name": "red/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "chatcmpl-a8a2656e3aa816b19358432a6f397cbc", + "first_content": "Red", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 2, + "server_disconnections_after": 3, + "server_confirmed_cancel": true + } + }, + { + "name": "red/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is Red.<|im_end|>\n, ., and" + } + }, + { + "name": "red/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is Red.<|im_end|>\n, ., and" + } + } + ], + "raw_receipt_sha256": "12be5ff9e214b398748cebb682c4f9e99d9d2ce274b57259a4a61f0395d2d2d5" + }, + { + "model": "LFM3-VL-2B-MLX-8bit", + "counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 9, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb", + "text": " The story is about a person who is always in a state of being in a state of a state of a state of" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + }, + { + "name": "red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is: Red<|im_end|>\nThe image contains a" + } + }, + { + "name": "blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6", + "text": "Blue<|im_end|>\nThe image contains a single solid color: Blue.<|im_end|>\n, I see a uniform blue hue throughout the" + } + }, + { + "name": "red_blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f", + "text": "Red, blue<|im_end|>\n<|im_end|>\n<|im_end|>\n colored with colored pencils pencil? No, solid color.<|im_end|>\n correction:" + } + }, + { + "name": "blue_red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005", + "text": "Blue, red<|im_end|>\n<|im_end|>\n<|im_end|>\n you have to rely on the visual information from the images provided. Rely" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb", + "text": " The story is about a person who is always in a state of being in a state of a state of a state of" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb", + "text": " The story is about a person who is always in a state of being in a state of a state of a state of" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb", + "text": " The story is about a person who is always in a state of being in a state of a state of a state of" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + }, + { + "name": "red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is: Red<|im_end|>\nThe image contains a" + } + }, + { + "name": "red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is: Red<|im_end|>\nThe image contains a" + } + }, + { + "name": "blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6", + "text": "Blue<|im_end|>\nThe image contains a single solid color: Blue.<|im_end|>\n, I see a uniform blue hue throughout the" + } + }, + { + "name": "blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6", + "text": "Blue<|im_end|>\nThe image contains a single solid color: Blue.<|im_end|>\n, I see a uniform blue hue throughout the" + } + }, + { + "name": "red_blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f", + "text": "Red, blue<|im_end|>\n<|im_end|>\n<|im_end|>\n colored with colored pencils pencil? No, solid color.<|im_end|>\n correction:" + } + }, + { + "name": "red_blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f", + "text": "Red, blue<|im_end|>\n<|im_end|>\n<|im_end|>\n colored with colored pencils pencil? No, solid color.<|im_end|>\n correction:" + } + }, + { + "name": "blue_red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005", + "text": "Blue, red<|im_end|>\n<|im_end|>\n<|im_end|>\n you have to rely on the visual information from the images provided. Rely" + } + }, + { + "name": "blue_red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005", + "text": "Blue, red<|im_end|>\n<|im_end|>\n<|im_end|>\n you have to rely on the visual information from the images provided. Rely" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + ] + }, + { + "name": "red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red" + ], + "observed": [ + "red", + "red" + ], + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is: Red<|im_end|>\nThe image contains a" + } + }, + { + "name": "blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue" + ], + "observed": [ + "blue", + "blue", + "blue" + ], + "text": "Blue<|im_end|>\nThe image contains a single solid color: Blue.<|im_end|>\n, I see a uniform blue hue throughout the" + } + }, + { + "name": "red_blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red", + "blue" + ], + "observed": [ + "red", + "blue" + ], + "text": "Red, blue<|im_end|>\n<|im_end|>\n<|im_end|>\n colored with colored pencils pencil? No, solid color.<|im_end|>\n correction:" + } + }, + { + "name": "blue_red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue", + "red" + ], + "observed": [ + "blue", + "red" + ], + "text": "Blue, red<|im_end|>\n<|im_end|>\n<|im_end|>\n you have to rely on the visual information from the images provided. Rely" + } + }, + { + "name": "images/concurrent_equals_solos", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is: Red<|im_end|>\nThe image contains a" + }, + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6", + "text": "Blue<|im_end|>\nThe image contains a single solid color: Blue.<|im_end|>\n, I see a uniform blue hue throughout the" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f", + "text": "Red, blue<|im_end|>\n<|im_end|>\n<|im_end|>\n colored with colored pencils pencil? No, solid color.<|im_end|>\n correction:" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005", + "text": "Blue, red<|im_end|>\n<|im_end|>\n<|im_end|>\n you have to rely on the visual information from the images provided. Rely" + } + ] + }, + { + "name": "images/corrupt_png_rejected", + "status": "passed", + "evidence": { + "error": "{\"error\": {\"message\": \"image 1 could not be decoded; send PNG, JPEG or WebP\", \"type\": \"invalid_request_error\"}}", + "status": 400, + "complete": false + } + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "cmpl-7b8e23ef851393842939a081fd748526", + "first_content": " The", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "chatcmpl-775efa25b95bed4a0876c3a6e48e827d", + "first_content": "The", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + }, + { + "name": "red/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "chatcmpl-392f668ef84cf1be4e8c21e9bb356aa7", + "first_content": "Red", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 2, + "server_disconnections_after": 3, + "server_confirmed_cancel": true + } + }, + { + "name": "red/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is: Red<|im_end|>\nThe image contains a" + } + }, + { + "name": "red/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is: Red<|im_end|>\nThe image contains a" + } + } + ], + "raw_receipt_sha256": "51de00cf2a1c61b880632d87a7209736397945b8190675c7e1fcfe6fb2ad9855" + } + ], + "cleanup": { + "client_children_exit_codes": [ + 0, + 0, + 0 + ], + "six_server_logs_end_drained_stopped": true, + "server_exit_codes": "not separately recorded", + "lease_released": true + }, + "limits": [ + "Same-engine serving checks, not independent FP64 acceptance.", + "No real-cat input consumed by this suite; unused controller PNG pin omitted.", + "Corrected-input teacher-forced decoder reference remains separate.", + "No speed qualification." + ] +} diff --git a/docs/receipts/lfm3/corrected-pixel-vision.json b/docs/receipts/lfm3/corrected-pixel-vision.json new file mode 100644 index 0000000000..fde239e6ca --- /dev/null +++ b/docs/receipts/lfm3/corrected-pixel-vision.json @@ -0,0 +1,171 @@ +{ + "schema_version": 1, + "scope": "Corrected CPU pixel normalization and NEW fixed-input vision-only requalification; decoder/serving requalification is separate", + "public_pixel_source": "50b4e0da3", + "PNG_sha256": "005500792ee8ea6f35ac369d900fc788cd77667f145e7100c2cff57fae5355d9", + "grid": [ + 14, + 20 + ], + "corrected_input": { + "type": "CPU processor FP32patch matrix; not GPU model activations", + "shape": [ + 280, + 1536 + ], + "sha256": "9d39c11551f105b8982e3a9acceffcdf687d360c17a7931b7fff21757a6e5473" + }, + "host_processor_regression": { + "change": { + "before": "(f32(pixel)*f32(1/255)-0.5)/0.5", + "after": "(f32(pixel)-127.5)/127.5", + "changed_lut_entries": 207, + "lut_entries": 256, + "max_absolute_difference": 1.1920928955078125e-07 + }, + "reference_fixture": { + "source": "https://github.com/jschmied/TensorFold/blob/1325da3bfab19d2ada499e613f1f0c90b7d95dd3/zig/src/core/picture/patches_test.zig", + "processor": "transformers 5.19.0 / torchvision 0.29.0 as recorded by fixture author", + "input": "RGB217x300; (7*x+13*y+50*c+(x*y)%31)%256", + "grid": [ + 14, + 20 + ], + "pixel_values_sha256": "53153c22eb4e065031bbdf4de0f038db37ef736a880e3438ac8e0d73f82dcea8" + }, + "regression": { + "before_exit": 1, + "before_pixel_values_sha256": "6e7d91ab24b4b18bc383dc5fcf441e2a973904f5a9b6a2af0272f34c48cf0229", + "after_exit": 0, + "after_pixel_values_sha256": "53153c22eb4e065031bbdf4de0f038db37ef736a880e3438ac8e0d73f82dcea8", + "command": "zig build test-lfm3-host -Dcpu=apple_m1 --summary all", + "after_build_steps": [ + 12, + 12 + ], + "image_tests": [ + 26, + 26 + ], + "optional_JPEG_cases": "three unavailable JPEG fixture cases; unrelated to RGB preparation regression", + "logs_sha256": { + "lfm3-pixel-parity-before.log": "ac5bdb487acc143649eb9db314387148d2deeb4c0d2f49efd17580e67ef70941", + "lfm3-pixel-parity-after.log": "028c13fb4b3378574ce6f140297c4352ecf7b72f2c8053916e82c37be349a57d" + } + }, + "source_receipt": "pixel-normalization.json" + }, + "actual_new_native_vision": [ + { + "bits": 4, + "projection_bytes": 286720, + "projection_sha256": "ffa0401f452f39a07833d6692ac9eff0d5485cc93a14fb3433bdad74c42ab05e", + "patches_sha256": "9d39c11551f105b8982e3a9acceffcdf687d360c17a7931b7fff21757a6e5473" + }, + { + "bits": 6, + "projection_bytes": 286720, + "projection_sha256": "0bfbdb4d6e8f3485609f0b6e8cf4b84bd8daba4c2ae62f56a4861aad3688b4df", + "patches_sha256": "9d39c11551f105b8982e3a9acceffcdf687d360c17a7931b7fff21757a6e5473" + }, + { + "bits": 8, + "projection_bytes": 286720, + "projection_sha256": "0bfbdb4d6e8f3485609f0b6e8cf4b84bd8daba4c2ae62f56a4861aad3688b4df", + "patches_sha256": "9d39c11551f105b8982e3a9acceffcdf687d360c17a7931b7fff21757a6e5473" + } + ], + "independent_graphs": { + "4": { + "scope": "NEW corrected fixed CPU-processor patches into independent idealFP64 and prescribedFP32/SFU vision graphs; no projected/native activation injection", + "values": 143360, + "named_native_equal_words": 143360, + "native_sha256": "ffa0401f452f39a07833d6692ac9eff0d5485cc93a14fb3433bdad74c42ab05e", + "named_sha256": "ffa0401f452f39a07833d6692ac9eff0d5485cc93a14fb3433bdad74c42ab05e", + "ideal_sha256": "a67172521a00ffc5c50b848769a92f732f12456276cf484e76509f91c703d0c8", + "native_vs_ideal_FP64": { + "RMS": 0.0005607575567752688, + "relative_RMS": 0.015839316732953166, + "maximum_absolute_error": 0.005889892578125, + "BF16_equal_words": 17164 + }, + "compared_native_stages": 0, + "receipt_sha256": "98556f6663ca894b3f185e7873f145fb199f76d32a24bf62b6a5db6023b35bd3", + "runner_sha256": "64a55e7515edd2e0bd0f92a169487a7e6d2727449e67173ba9d0a2619e0824ee", + "reference_source_sha256": { + "tools/lfm3/fp64_reference.py": "0fb1128ade095db87b6126792ab14930792ab1f7b41b998570356bc586f16b2e", + "tools/lfm3/vision_fp64_reference.py": "742b8b4f44dde824c3a5c2d72a632401a0f9df7b7f23a8d3729d15f4f8c6a4d2", + "tools/lfm3/elementary_reference.py": "3e4db276278867b7683536044ed7bcbe4c7643867d722fdda5b8ad895b3963e7", + "tools/lfm3/row_projection_reference.py": "d03f224a037a920ba66b9828059d1489e48729cd168077e76789d0ec5c199c0c", + "tools/lfm3/text_rounding_reference.py": "52665eef48570d45a2554861fe8f22b31874f8f464366adf882f40c3d311db27", + "tools/lfm3/vision_rounding_reference.py": "8b2f5679bbd3f05b3e86a5748c3803f744502cffd0159bee2a218da38bd6f1c4" + }, + "provider_sha256": "97aec6e37977bbef32c60e7f86253c29c7b8161159dd6c9aded657e4bd69ddd0", + "optional_accelerator_sha256": "11ad0f66632bfcba548dbede9027ffdb112cd418ff2be80610af56a957424753", + "production_primitive_executable_sha256": "ce433fcac7b724cb14b22d8f5392373e950bf3e3347633c96bdc89cc06934048" + }, + "6": { + "scope": "NEW corrected fixed CPU-processor patches into independent idealFP64 and prescribedFP32/SFU vision graphs; no projected/native activation injection", + "values": 143360, + "named_native_equal_words": 143360, + "native_sha256": "0bfbdb4d6e8f3485609f0b6e8cf4b84bd8daba4c2ae62f56a4861aad3688b4df", + "named_sha256": "0bfbdb4d6e8f3485609f0b6e8cf4b84bd8daba4c2ae62f56a4861aad3688b4df", + "ideal_sha256": "221133d871ce2db48dfcb28af88f8e3facb80f604d9fa8b4e37afef07e04fde4", + "native_vs_ideal_FP64": { + "RMS": 0.0006593466304518665, + "relative_RMS": 0.018721776792383023, + "maximum_absolute_error": 0.015380859375, + "BF16_equal_words": 16385 + }, + "compared_native_stages": 0, + "receipt_sha256": "cd65b824e33a1e20bd635c4f7500819b36e4c6702135d3a75e409943cdb3fc13", + "runner_sha256": "64a55e7515edd2e0bd0f92a169487a7e6d2727449e67173ba9d0a2619e0824ee", + "reference_source_sha256": { + "tools/lfm3/fp64_reference.py": "0fb1128ade095db87b6126792ab14930792ab1f7b41b998570356bc586f16b2e", + "tools/lfm3/vision_fp64_reference.py": "742b8b4f44dde824c3a5c2d72a632401a0f9df7b7f23a8d3729d15f4f8c6a4d2", + "tools/lfm3/elementary_reference.py": "3e4db276278867b7683536044ed7bcbe4c7643867d722fdda5b8ad895b3963e7", + "tools/lfm3/row_projection_reference.py": "d03f224a037a920ba66b9828059d1489e48729cd168077e76789d0ec5c199c0c", + "tools/lfm3/text_rounding_reference.py": "52665eef48570d45a2554861fe8f22b31874f8f464366adf882f40c3d311db27", + "tools/lfm3/vision_rounding_reference.py": "8b2f5679bbd3f05b3e86a5748c3803f744502cffd0159bee2a218da38bd6f1c4" + }, + "provider_sha256": "97aec6e37977bbef32c60e7f86253c29c7b8161159dd6c9aded657e4bd69ddd0", + "optional_accelerator_sha256": "11ad0f66632bfcba548dbede9027ffdb112cd418ff2be80610af56a957424753", + "production_primitive_executable_sha256": "ce433fcac7b724cb14b22d8f5392373e950bf3e3347633c96bdc89cc06934048" + } + }, + "six_to_eight_vision_identity_transfer": { + "scope": "6\u21928 corrected-input vision only; no decoder or old-image-output transfer", + "tensor_count": 138, + "tensor_bytes": 378760704, + "named_metadata_content_sha256": "be7e3c3f041f78b3f01218a0b9900f0750f26b5ed5c3b496c07abe9e0efb6ae1", + "corrected_CPU_input_sha256": "9d39c11551f105b8982e3a9acceffcdf687d360c17a7931b7fff21757a6e5473", + "actual_native70_sha256": "0bfbdb4d6e8f3485609f0b6e8cf4b84bd8daba4c2ae62f56a4861aad3688b4df", + "identity_receipt_sha256": "6e28b34d17c1b5ff57811e1375bf8292fdbe18b2ddeeebe78d12191c64ebab98", + "same_reference_graph_as": "6", + "reference_source_sha256": { + "tools/lfm3/fp64_reference.py": "0fb1128ade095db87b6126792ab14930792ab1f7b41b998570356bc586f16b2e", + "tools/lfm3/vision_fp64_reference.py": "742b8b4f44dde824c3a5c2d72a632401a0f9df7b7f23a8d3729d15f4f8c6a4d2", + "tools/lfm3/elementary_reference.py": "3e4db276278867b7683536044ed7bcbe4c7643867d722fdda5b8ad895b3963e7", + "tools/lfm3/row_projection_reference.py": "d03f224a037a920ba66b9828059d1489e48729cd168077e76789d0ec5c199c0c", + "tools/lfm3/text_rounding_reference.py": "52665eef48570d45a2554861fe8f22b31874f8f464366adf882f40c3d311db27", + "tools/lfm3/vision_rounding_reference.py": "8b2f5679bbd3f05b3e86a5748c3803f744502cffd0159bee2a218da38bd6f1c4" + }, + "native_vs_ideal_FP64": { + "RMS": 0.0006593466304518665, + "relative_RMS": 0.018721776792383023, + "maximum_absolute_error": 0.015380859375, + "BF16_equal_words": 16385 + }, + "named_native_equal_words": 143360 + }, + "all_requested_vision_scopes_complete": true, + "limits": [ + "Ideal FP64 RMS/maxabs/BF16 differences retained; no tolerance or fidelity acceptance gate changed.", + "These are new corrected-input image outputs; historical old-pixel image receipts are not byte-transferred.", + "Fixed CPU-processor matrix is the vision operator input; independent Pillow preprocessing differed from Torch uint8resizing and was not used.", + "No native intermediate stage comparisons or decoder/image-conditioned serving assertion from these vision-only graphs.", + "The 6\u21928 transfer, when present, requires actual identical current vision tensor bytes, relevant parameters, new patch input and current native output." + ], + "fidelity_pass": null, + "performance_claim": false +} diff --git a/docs/receipts/lfm3/current-m5-long-context-partial.json b/docs/receipts/lfm3/current-m5-long-context-partial.json new file mode 100644 index 0000000000..77c7425304 --- /dev/null +++ b/docs/receipts/lfm3/current-m5-long-context-partial.json @@ -0,0 +1,1268 @@ +{ + "schema_version": 1, + "experiment": "Current public M5 exact-token context qualification: first 20 completed requests", + "status": "partial", + "planned_requests": 26, + "published_completed_requests": 20, + "remaining_requests": 6, + "remaining_scope": [ + "VL4/6/8:127984 prompt tokens, two fresh legs each" + ], + "remaining_status": "running or not yet attempted at this publication snapshot; no qualification assertion", + "source_commit": "20947578c08ead20e8e0a18790b1fd5f5fb3b6e3", + "runtime_base": "2e97b1fbbe80f2446e49893fc297da9be914fc71", + "source_archive_sha256": "6001dd8298380ade2764eb75875a7c8a7732f191384b729e711f77e7614d2814", + "native_sha256": "437218121714a1761187e009e8075d8751e5327ce2fd9b65ac21f3aeead214c4", + "approved_payload_archive_sha256": "579394ca68c41325fd8dc917e1b5dc6a172f2b0e7128873612c646b649581215", + "native_build": { + "steps_passed": 57, + "steps_total": 57, + "exit": 0 + }, + "scope": { + "engine": "current native only", + "request_kind": "text prompts, including VL checkpoints", + "actual_simultaneous_requests": 1, + "configured_parallel": 4, + "output_tokens_per_request": 16, + "temperature": 0, + "thinking": false, + "prefix_cache_gib": 0, + "drafts": "production default; actual response acknowledgement retained", + "live_tokenize": "full rendered ID array byte/value equality against pinned fixture before every request", + "performance_comparison": false, + "image_context": false, + "fp64_fidelity": false, + "whole_matrix_complete": false + }, + "packets": [ + { + "name": "vl68-32k64k", + "executed_manifest_sha256": "7bb9931400304960faa834b97fb732341a625052dd9033e38fc1022ed121221f", + "events_snapshot_sha256": "14a59913647c6980861f755cc7acf7b4ed6dfb101194825229cd6a2866a29f00", + "events_snapshot_scope": "complete packet", + "models": { + "LFM3-VL-2B-MLX-6bit": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-6bit", + "revision": "6d71bfae121abedc75f6d531fc7f6ceb4f5e3501", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 4138, + "sha256": "4c03aa298372cbcb478eb08be0a4a59be917b4949756066ce4eac06033d200bc", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 323, + "sha256": "30b1115bcdc4e5671567a5b1255a2f3d2f5a912a79292051c6f1e9db81b5e8c4", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 1783964482, + "sha256": "838c39147541b12e86a4f1b43d6a7814d7fb4efe9d5ff8a151c85ac06f321eef", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 61255, + "sha256": "abc29f6e9809c65c5ce62e6b1754b3f103d82266adf74d6d661dd3f8cd443cb5", + "verified_against_pinned_revision": true + }, + "processor_config.json": { + "bytes": 1254, + "sha256": "f183f5a5e5b3a32e666fce05033e6cb7ca496d26e709a2cee611a13eb7d6b610", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908258, + "sha256": "a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1759, + "sha256": "d63d1ce2f332047e398764b4170352b96629a5194643105bfd6d70c90b750396", + "verified_against_pinned_revision": true + } + } + }, + "LFM3-VL-2B-MLX-8bit": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-8bit", + "revision": "83336d5be3874a2bdd3297bde77b2114dbd3a816", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 4138, + "sha256": "9f7dc06d04d0610c7e27fb6a110c42a3a09b686a6ec53b0eb383f30a8be5a457", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 323, + "sha256": "30b1115bcdc4e5671567a5b1255a2f3d2f5a912a79292051c6f1e9db81b5e8c4", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 2216068156, + "sha256": "50c9476a7f00f5349e81b42ab88b47635b9eba3c7ab5851c598d270cda4c6857", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 61255, + "sha256": "31fe8faaf68bfc2fbce07e9d97ff47c1abad234d783ed615f2d938392b7cf6f0", + "verified_against_pinned_revision": true + }, + "processor_config.json": { + "bytes": 1254, + "sha256": "f183f5a5e5b3a32e666fce05033e6cb7ca496d26e709a2cee611a13eb7d6b610", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908258, + "sha256": "a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1759, + "sha256": "d63d1ce2f332047e398764b4170352b96629a5194643105bfd6d70c90b750396", + "verified_against_pinned_revision": true + } + } + } + }, + "context_capacity": 70000, + "client_sha256": { + "bench_auth.py": "1896120b70d0df05646870a252697e20ce84c63e09dcba88b2a4e5a55ccb3c5b", + "bench_cache.py": "24364dd13c033450f45710fc8685e2c64d6a72d6dd6e223d0d7b19aabbdc1d98", + "bench_concurrent.py": "0e72e26769f082459329fc068c25f6f72a9155e422962ea54cb5d1226227752d", + "bench_evidence.py": "54c28474f0804dbd120e3fb5859be0d1667ef21ca502880d0bed098ec9bb42ff", + "bench_metrics.py": "26f4c3ecbb03df4f12e24d18840fea989d9ef8c63117ca11eb944ef670458a27", + "bench_openai.py": "d5af4fbb756cdfb573546c0e0017652a4bb8f509d1251af683be7a3cbdd426f6", + "bench_timing.py": "158f636ef0b3bed460ce19ef4872b6d8607ff7a8590562d06bae8bd55e6f44ed", + "bench_decode.py": "c011106f4f182db19371e3f91f70bcc374160786dd105f2d713a7fea07ab8c4c", + "cold_client.py": "7b8ea55aa88834814f95a6f679654e16074b7a8364e34b39a276f009fe0231f2" + }, + "fixture_sha256": { + "vl-32768.body.json": "6e4c2ecd84a08e202dddce7f9b50bada5c006f9d529a74177f38dc11c08e0bb4", + "vl-32768.ids.json": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f", + "vl-65536.body.json": "d6dea630f39ee2ddd13e8a88a90050d615d518e21f5a9219e1dc3d2297617b63", + "vl-65536.ids.json": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed" + }, + "guard_sha256": "3c4118b62c3797e1871032fcbbb1bf7d390f143c2e93ddf564b309e7f49a4664", + "host": "ProductName:\t\tmacOS\nProductVersion:\t\t27.0\nBuildVersion:\t\t26A428\n", + "physical_memory_bytes": 274877906944, + "gpu_device": "Apple M5 Ultra", + "requirements_sha256": "275b369ea72d5757b1e3f91e68ed5b746d83b5f7667adc8f5d9d28bb8933ddd6", + "memory_admission": [ + { + "event": "memory_admission", + "time_ns": 1793249253131489000, + "physical_memory_bytes": 274877906944, + "working_set": 239143780352, + "ceiling": 192414534830, + "budgets": [ + { + "family": "VL-2B", + "bits": 6, + "capacity": 128000, + "checkpoint_bytes": 1783964482, + "cache_per_stream_bytes": 2884159488, + "streams": 4, + "measured_scratch128_bytes": 62104192, + "existing_margin_bytes": 2147483648, + "conservative_total_bytes": 15530190274 + }, + { + "family": "VL-2B", + "bits": 8, + "capacity": 128000, + "checkpoint_bytes": 2216068156, + "cache_per_stream_bytes": 2884159488, + "streams": 4, + "measured_scratch128_bytes": 62104192, + "existing_margin_bytes": 2147483648, + "conservative_total_bytes": 15962293948 + } + ], + "limits_raised": false + } + ], + "lease": [ + { + "event": "lock_acquired", + "time_ns": 1793249253063149000, + "pid": 39375, + "device": 16777233, + "inode": 2622134 + }, + { + "event": "lock_release", + "time_ns": 1793249873155825000, + "pid": 39375 + } + ], + "server_cleanups": [ + { + "event": "owned_cleanup_complete", + "time_ns": 1793249413560157000, + "tag": "vl-6-0-public", + "pid": 39388, + "exit": 0, + "memory": { + "max_rss_bytes": 2041888768, + "peak_physical_footprint_bytes": 3778742552 + } + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793249571314589000, + "tag": "vl-6-1-public", + "pid": 39434, + "exit": 0, + "memory": { + "max_rss_bytes": 2041282560, + "peak_physical_footprint_bytes": 3777857840 + } + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793249722355473000, + "tag": "vl-8-0-public", + "pid": 39466, + "exit": 0, + "memory": { + "max_rss_bytes": 2473476096, + "peak_physical_footprint_bytes": 4210215576 + } + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793249873088871000, + "tag": "vl-8-1-public", + "pid": 39502, + "exit": 0, + "memory": { + "max_rss_bytes": 2473590784, + "peak_physical_footprint_bytes": 4210543280 + } + } + ], + "requests": [ + { + "tag": "vl-6-0-public", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 16, + "total_tokens": 32784, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 36.00836220802739, + "elapsed_seconds": 36.10532324993983, + "output_text_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "reply_token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0", + "drafts_acknowledged": true, + "raw_response_sha256": "03327af93e2f925b11c0cddcfd62eb7bfe7fce5ab22f983399afd44cf9e3a5a9", + "actual_tokenize_sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450", + "rendered_ids_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f", + "fixture_body_sha256": "6e4c2ecd84a08e202dddce7f9b50bada5c006f9d529a74177f38dc11c08e0bb4", + "exact_tokenize_event": { + "time_ns": 1793249257343034000, + "tag": "vl-6-0-public", + "prompt_tokens": 32768, + "ids_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f" + } + }, + { + "tag": "vl-6-0-public", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 16, + "total_tokens": 65552, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 119.1769048338756, + "elapsed_seconds": 119.28063683398068, + "output_text_sha256": "4026f32b09f16475aea9189543a259259e5dea4a9c58e40d969af677ffb1251b", + "reply_token_sha256": "01496c951534bedc261fc706cd6de05a865b636542badf970ce10254c9b17dfd", + "drafts_acknowledged": true, + "raw_response_sha256": "c4aa30182579e0c70976ab0d3dcbdbe36ef81bb8b3bc3c02e6c7563d06d34afa", + "actual_tokenize_sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082", + "rendered_ids_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed", + "fixture_body_sha256": "d6dea630f39ee2ddd13e8a88a90050d615d518e21f5a9219e1dc3d2297617b63", + "exact_tokenize_event": { + "time_ns": 1793249293617106000, + "tag": "vl-6-0-public", + "prompt_tokens": 65536, + "ids_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed" + } + }, + { + "tag": "vl-6-1-public", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 16, + "total_tokens": 32784, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 36.01983079081401, + "elapsed_seconds": 36.11669062497094, + "output_text_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "reply_token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0", + "drafts_acknowledged": true, + "raw_response_sha256": "76afa7b3cd0d50c864ece3938443f3aed02430b974540b923bdc2a750b1709bb", + "actual_tokenize_sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450", + "rendered_ids_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f", + "fixture_body_sha256": "6e4c2ecd84a08e202dddce7f9b50bada5c006f9d529a74177f38dc11c08e0bb4", + "exact_tokenize_event": { + "time_ns": 1793249414813199000, + "tag": "vl-6-1-public", + "prompt_tokens": 32768, + "ids_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f" + } + }, + { + "tag": "vl-6-1-public", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 16, + "total_tokens": 65552, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 119.46985933324322, + "elapsed_seconds": 119.5737877083011, + "output_text_sha256": "4026f32b09f16475aea9189543a259259e5dea4a9c58e40d969af677ffb1251b", + "reply_token_sha256": "01496c951534bedc261fc706cd6de05a865b636542badf970ce10254c9b17dfd", + "drafts_acknowledged": true, + "raw_response_sha256": "92314ef7cbb0b8040f37c120e80466fec5529508f36c975939ad0c6f29968f29", + "actual_tokenize_sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082", + "rendered_ids_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed", + "fixture_body_sha256": "d6dea630f39ee2ddd13e8a88a90050d615d518e21f5a9219e1dc3d2297617b63", + "exact_tokenize_event": { + "time_ns": 1793249451116889000, + "tag": "vl-6-1-public", + "prompt_tokens": 65536, + "ids_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed" + } + }, + { + "tag": "vl-8-0-public", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 16, + "total_tokens": 32784, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 34.09193491702899, + "elapsed_seconds": 34.190794500056654, + "output_text_sha256": "363f5de76e594bc939dc4dfea2b5c71c087224382a91db7f7aef7ef650b86791", + "reply_token_sha256": "6b68989060076239ed31666b235e266a2e83536962209d719c5e58346bfe6251", + "drafts_acknowledged": true, + "raw_response_sha256": "1d8ff6db01586184ab2be9d55cb91ca5405416c84b56b1022519850e028237ca", + "actual_tokenize_sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450", + "rendered_ids_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f", + "fixture_body_sha256": "6e4c2ecd84a08e202dddce7f9b50bada5c006f9d529a74177f38dc11c08e0bb4", + "exact_tokenize_event": { + "time_ns": 1793249572575420000, + "tag": "vl-8-0-public", + "prompt_tokens": 32768, + "ids_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f" + } + }, + { + "tag": "vl-8-0-public", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 16, + "total_tokens": 65552, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 114.42618841677904, + "elapsed_seconds": 114.5319452919066, + "output_text_sha256": "b3fc28d243e9bb51d0db0f4729560c1a1382e069e5cee04f555a3eb4233fc207", + "reply_token_sha256": "7240d3de4d84db461fc2e1f43a7b9f8a79a876eb933afb924e661ad8d8268c3b", + "drafts_acknowledged": true, + "raw_response_sha256": "bf09569bb27a6352a43f30a51b87ed51118020d11f97f23caefd384586bfb11c", + "actual_tokenize_sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082", + "rendered_ids_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed", + "fixture_body_sha256": "d6dea630f39ee2ddd13e8a88a90050d615d518e21f5a9219e1dc3d2297617b63", + "exact_tokenize_event": { + "time_ns": 1793249606961361000, + "tag": "vl-8-0-public", + "prompt_tokens": 65536, + "ids_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed" + } + }, + { + "tag": "vl-8-1-public", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 16, + "total_tokens": 32784, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 34.15882908413187, + "elapsed_seconds": 34.25784600013867, + "output_text_sha256": "363f5de76e594bc939dc4dfea2b5c71c087224382a91db7f7aef7ef650b86791", + "reply_token_sha256": "6b68989060076239ed31666b235e266a2e83536962209d719c5e58346bfe6251", + "drafts_acknowledged": true, + "raw_response_sha256": "1e4eba9e0bfd5c47765c43d7a3983732b8382ff68118f8e036f34fbdd53e8739", + "actual_tokenize_sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450", + "rendered_ids_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f", + "fixture_body_sha256": "6e4c2ecd84a08e202dddce7f9b50bada5c006f9d529a74177f38dc11c08e0bb4", + "exact_tokenize_event": { + "time_ns": 1793249723607015000, + "tag": "vl-8-1-public", + "prompt_tokens": 32768, + "ids_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f" + } + }, + { + "tag": "vl-8-1-public", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 16, + "total_tokens": 65552, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 114.3223563330248, + "elapsed_seconds": 114.4282983331941, + "output_text_sha256": "b3fc28d243e9bb51d0db0f4729560c1a1382e069e5cee04f555a3eb4233fc207", + "reply_token_sha256": "7240d3de4d84db461fc2e1f43a7b9f8a79a876eb933afb924e661ad8d8268c3b", + "drafts_acknowledged": true, + "raw_response_sha256": "c3e6c4a433193ff2bde8ca6fb8e72a3760eabf5af7e13bc3613ebf2aa08f38c2", + "actual_tokenize_sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082", + "rendered_ids_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed", + "fixture_body_sha256": "d6dea630f39ee2ddd13e8a88a90050d615d518e21f5a9219e1dc3d2297617b63", + "exact_tokenize_event": { + "time_ns": 1793249758105970000, + "tag": "vl-8-1-public", + "prompt_tokens": 65536, + "ids_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed" + } + } + ], + "repeat_equality": [ + { + "bits": 6, + "prompt_tokens": 32768, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0" + }, + { + "bits": 6, + "prompt_tokens": 65536, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "01496c951534bedc261fc706cd6de05a865b636542badf970ce10254c9b17dfd" + }, + { + "bits": 8, + "prompt_tokens": 32768, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "6b68989060076239ed31666b235e266a2e83536962209d719c5e58346bfe6251" + }, + { + "bits": 8, + "prompt_tokens": 65536, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "7240d3de4d84db461fc2e1f43a7b9f8a79a876eb933afb924e661ad8d8268c3b" + } + ] + }, + { + "name": "text-all-native-window", + "executed_manifest_sha256": "351960c08c0d953374c2ede787a3a84b6321cd77da51485300abb2cfa1a9aa5b", + "events_snapshot_sha256": "de23611dbf5eeaa452b8d5eb5dd0bb7aab2cf9d37beceda4af3476c9a03e2d33", + "events_snapshot_scope": "complete packet", + "models": { + "LFM3-1.7B-MLX-4bit": { + "repository": "LiquidAI/LFM3-1.7B-MLX-4bit", + "revision": "d0ca73dcfa70ce18733754a492cdc1fb0123262d", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 2423, + "sha256": "7df2ea819f9a31b770192a0a9dc1b7953a7dd8c41b816cc3905bbec99ef332bd", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 328, + "sha256": "0095ba7158e938b1e2d3082d412f00d8be85e42cbbd7608b63866f7a081cf270", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 973083386, + "sha256": "1be5bf672013cd91aa62148f77c5eea8d2784a95845d5fe826e08c7b84823815", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 50700, + "sha256": "2c8b88e25f6a1055164dd051436b170872200047bce4b3ec5f2a58f6ef929a75", + "verified_against_pinned_revision": true + }, + "special_tokens_map.json": { + "bytes": 92, + "sha256": "7946f864cbfb62a1b041bda937244210a1a87f4a6cab3a48b791131b2cd57f88", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908634, + "sha256": "41927bd314d17d7d7fadb44e10cd0919f7b71528d7741596d8cbd15c6945ec44", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1859, + "sha256": "b77214b9f12be74b9e5e1dab35d7dace077a31be969314bf38866fd6c8163853", + "verified_against_pinned_revision": true + } + } + }, + "LFM3-1.7B-MLX-6bit": { + "repository": "LiquidAI/LFM3-1.7B-MLX-6bit", + "revision": "26be898c36b545d625d84ac0761c6c3eb664aef3", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 2423, + "sha256": "2b5879bf6d1d99b43e17726c7ee3fb4868e4c2e29c4a49784803df90f494fc2e", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 328, + "sha256": "0095ba7158e938b1e2d3082d412f00d8be85e42cbbd7608b63866f7a081cf270", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 1405187179, + "sha256": "503d68675793fba64907842e176d066c8c3adca22e14c8b3538f9ffb76bd5e0e", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 50701, + "sha256": "940f7e7b4133162a48fbca232e663c32240364aea9d60f1715b62bed41a81e2c", + "verified_against_pinned_revision": true + }, + "special_tokens_map.json": { + "bytes": 92, + "sha256": "7946f864cbfb62a1b041bda937244210a1a87f4a6cab3a48b791131b2cd57f88", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908634, + "sha256": "41927bd314d17d7d7fadb44e10cd0919f7b71528d7741596d8cbd15c6945ec44", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1859, + "sha256": "b77214b9f12be74b9e5e1dab35d7dace077a31be969314bf38866fd6c8163853", + "verified_against_pinned_revision": true + } + } + }, + "LFM3-1.7B-MLX-8bit": { + "repository": "LiquidAI/LFM3-1.7B-MLX-8bit", + "revision": "ee7667f3cfccb2a41725f91820531823fcabc6b7", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 2423, + "sha256": "4ef95e289e7105a09bf010b3ceaa015d3f01163104c039a4e7948739d5b9035b", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 328, + "sha256": "0095ba7158e938b1e2d3082d412f00d8be85e42cbbd7608b63866f7a081cf270", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 1837290805, + "sha256": "2b06cea98d75527c248a0b04383f7cf0a1c72fb4ed7b45096618fc4e59d015ad", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 50701, + "sha256": "0f4f85710fd22ffb8a1977520a734cb7dc7e1f7858a0f91d9c8b56e6d36b78cc", + "verified_against_pinned_revision": true + }, + "special_tokens_map.json": { + "bytes": 92, + "sha256": "7946f864cbfb62a1b041bda937244210a1a87f4a6cab3a48b791131b2cd57f88", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908634, + "sha256": "41927bd314d17d7d7fadb44e10cd0919f7b71528d7741596d8cbd15c6945ec44", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1859, + "sha256": "b77214b9f12be74b9e5e1dab35d7dace077a31be969314bf38866fd6c8163853", + "verified_against_pinned_revision": true + } + } + } + }, + "context_capacity": 262144, + "client_sha256": { + "bench_auth.py": "1896120b70d0df05646870a252697e20ce84c63e09dcba88b2a4e5a55ccb3c5b", + "bench_cache.py": "24364dd13c033450f45710fc8685e2c64d6a72d6dd6e223d0d7b19aabbdc1d98", + "bench_concurrent.py": "0e72e26769f082459329fc068c25f6f72a9155e422962ea54cb5d1226227752d", + "bench_evidence.py": "54c28474f0804dbd120e3fb5859be0d1667ef21ca502880d0bed098ec9bb42ff", + "bench_metrics.py": "26f4c3ecbb03df4f12e24d18840fea989d9ef8c63117ca11eb944ef670458a27", + "bench_openai.py": "d5af4fbb756cdfb573546c0e0017652a4bb8f509d1251af683be7a3cbdd426f6", + "bench_timing.py": "158f636ef0b3bed460ce19ef4872b6d8607ff7a8590562d06bae8bd55e6f44ed", + "bench_decode.py": "c011106f4f182db19371e3f91f70bcc374160786dd105f2d713a7fea07ab8c4c", + "cold_client.py": "7b8ea55aa88834814f95a6f679654e16074b7a8364e34b39a276f009fe0231f2" + }, + "fixture_sha256": { + "text-131072.body.json": "d8c3cfe0099049d8264273d510f2b300cb2917b4eecee248b8f9bfc9ffdbbaae", + "text-131072.ids.json": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff", + "text-262128.body.json": "247e79c1ac3e88a314e8e9dff18b9b2fac208ef03a8ab0c4a9574361447304f0", + "text-262128.ids.json": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358" + }, + "guard_sha256": "3c4118b62c3797e1871032fcbbb1bf7d390f143c2e93ddf564b309e7f49a4664", + "host": "ProductName:\t\tmacOS\nProductVersion:\t\t27.0\nBuildVersion:\t\t26A428\n", + "physical_memory_bytes": 274877906944, + "gpu_device": "Apple M5 Ultra", + "requirements_sha256": "275b369ea72d5757b1e3f91e68ed5b746d83b5f7667adc8f5d9d28bb8933ddd6", + "memory_admission": [ + { + "event": "memory_admission", + "time_ns": 1793249873317272000, + "physical_memory_bytes": 274877906944, + "working_set": 239143780352, + "ceiling": 192414534830, + "budgets": [ + { + "family": "1.7B", + "bits": 4, + "capacity": 262144, + "checkpoint_bytes": 973083386, + "cache_per_stream_bytes": 5906155520, + "streams": 4, + "measured_scratch128_bytes": 62104192, + "existing_margin_bytes": 2147483648, + "conservative_total_bytes": 26807293306 + }, + { + "family": "1.7B", + "bits": 6, + "capacity": 262144, + "checkpoint_bytes": 1405187179, + "cache_per_stream_bytes": 5906155520, + "streams": 4, + "measured_scratch128_bytes": 62104192, + "existing_margin_bytes": 2147483648, + "conservative_total_bytes": 27239397099 + }, + { + "family": "1.7B", + "bits": 8, + "capacity": 262144, + "checkpoint_bytes": 1837290805, + "cache_per_stream_bytes": 5906155520, + "streams": 4, + "measured_scratch128_bytes": 62104192, + "existing_margin_bytes": 2147483648, + "conservative_total_bytes": 27671500725 + } + ], + "limits_raised": false + } + ], + "lease": [ + { + "event": "lock_acquired", + "time_ns": 1793249873251776000, + "pid": 39546, + "device": 16777233, + "inode": 2622134 + }, + { + "event": "lock_release", + "time_ns": 1793264748356781000, + "pid": 39546 + } + ], + "server_cleanups": [ + { + "event": "owned_cleanup_complete", + "time_ns": 1793252343951170000, + "tag": "text-4-0-public", + "pid": 39567, + "exit": 0, + "memory": { + "max_rss_bytes": 1306099712, + "peak_physical_footprint_bytes": 7431800536 + } + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793254812861949000, + "tag": "text-4-1-public", + "pid": 39668, + "exit": 0, + "memory": { + "max_rss_bytes": 1304674304, + "peak_physical_footprint_bytes": 7431816920 + } + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793257300236461000, + "tag": "text-6-0-public", + "pid": 39775, + "exit": 0, + "memory": { + "max_rss_bytes": 1736687616, + "peak_physical_footprint_bytes": 7880951896 + } + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793259789078224000, + "tag": "text-6-1-public", + "pid": 39925, + "exit": 0, + "memory": { + "max_rss_bytes": 1737752576, + "peak_physical_footprint_bytes": 7864256576 + } + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793262268490697000, + "tag": "text-8-0-public", + "pid": 40037, + "exit": 0, + "memory": { + "max_rss_bytes": 2168832000, + "peak_physical_footprint_bytes": 8286505336 + } + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793264748283480000, + "tag": "text-8-1-public", + "pid": 40166, + "exit": 0, + "memory": { + "max_rss_bytes": 2168946688, + "peak_physical_footprint_bytes": 8296384888 + } + } + ], + "requests": [ + { + "tag": "text-4-0-public", + "prompt_tokens": 131072, + "usage": { + "prompt_tokens": 131072, + "completion_tokens": 16, + "total_tokens": 131088, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 463.8665871671401, + "elapsed_seconds": 464.01015008287504, + "output_text_sha256": "93527a09d1b7d0901b65ecdd1b49611d78a3f2a09b25547d1c69d390dc32d43e", + "reply_token_sha256": "eab2cbad6c3db4db52f33d47d1fc19e1af989c9ea6647e61ea843a8d0dfc9faf", + "drafts_acknowledged": true, + "raw_response_sha256": "d0edf9fff4c58b08c6f2554b371397c9a0ec2ba8dcfd01d9621b674b86875f20", + "actual_tokenize_sha256": "1ad37fede50ddc2307adacc2f216ec67ae3d4f466b9834d3acbeb86b8c34b452", + "rendered_ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff", + "fixture_body_sha256": "d8c3cfe0099049d8264273d510f2b300cb2917b4eecee248b8f9bfc9ffdbbaae", + "exact_tokenize_event": { + "time_ns": 1793249876210952000, + "tag": "text-4-0-public", + "prompt_tokens": 131072, + "ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff" + } + }, + { + "tag": "text-4-0-public", + "prompt_tokens": 262128, + "usage": { + "prompt_tokens": 262128, + "completion_tokens": 16, + "total_tokens": 262144, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 2002.808624625206, + "elapsed_seconds": 2003.032133874949, + "output_text_sha256": "db6d16f0b8ab4c4e65f82a9bfbfed8dc5cb5649337698c701f5b5c28801b453f", + "reply_token_sha256": "da7391ebf4662f42f3730deab9c314430745a31a5bec92f01bc440c7c3d18cdd", + "drafts_acknowledged": true, + "raw_response_sha256": "7971ded691ccc17e5f8ffda24039c9fa8ce75080e997ece989967c098a0632c5", + "actual_tokenize_sha256": "803d527a17209dc622cc8fd8fba1b7db6a79f98532062e57fdca36ad4052e422", + "rendered_ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358", + "fixture_body_sha256": "247e79c1ac3e88a314e8e9dff18b9b2fac208ef03a8ab0c4a9574361447304f0", + "exact_tokenize_event": { + "time_ns": 1793250340471807000, + "tag": "text-4-0-public", + "prompt_tokens": 262128, + "ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358" + } + }, + { + "tag": "text-4-1-public", + "prompt_tokens": 131072, + "usage": { + "prompt_tokens": 131072, + "completion_tokens": 16, + "total_tokens": 131088, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 463.9125559167005, + "elapsed_seconds": 464.0565038328059, + "output_text_sha256": "93527a09d1b7d0901b65ecdd1b49611d78a3f2a09b25547d1c69d390dc32d43e", + "reply_token_sha256": "eab2cbad6c3db4db52f33d47d1fc19e1af989c9ea6647e61ea843a8d0dfc9faf", + "drafts_acknowledged": true, + "raw_response_sha256": "8a6b4cc23c3ce16499fda0a0427c733300231d7749aeb45642c0ed1e7a8b3b2d", + "actual_tokenize_sha256": "1ad37fede50ddc2307adacc2f216ec67ae3d4f466b9834d3acbeb86b8c34b452", + "rendered_ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff", + "fixture_body_sha256": "d8c3cfe0099049d8264273d510f2b300cb2917b4eecee248b8f9bfc9ffdbbaae", + "exact_tokenize_event": { + "time_ns": 1793252345245842000, + "tag": "text-4-1-public", + "prompt_tokens": 131072, + "ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff" + } + }, + { + "tag": "text-4-1-public", + "prompt_tokens": 262128, + "usage": { + "prompt_tokens": 262128, + "completion_tokens": 16, + "total_tokens": 262144, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 2002.407886958681, + "elapsed_seconds": 2002.630378708709, + "output_text_sha256": "db6d16f0b8ab4c4e65f82a9bfbfed8dc5cb5649337698c701f5b5c28801b453f", + "reply_token_sha256": "da7391ebf4662f42f3730deab9c314430745a31a5bec92f01bc440c7c3d18cdd", + "drafts_acknowledged": true, + "raw_response_sha256": "a8d97410eaa3972af83f16fd2654b9d87f669319c3bb573cce13b9068e9f8aa9", + "actual_tokenize_sha256": "803d527a17209dc622cc8fd8fba1b7db6a79f98532062e57fdca36ad4052e422", + "rendered_ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358", + "fixture_body_sha256": "247e79c1ac3e88a314e8e9dff18b9b2fac208ef03a8ab0c4a9574361447304f0", + "exact_tokenize_event": { + "time_ns": 1793252809603842000, + "tag": "text-4-1-public", + "prompt_tokens": 262128, + "ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358" + } + }, + { + "tag": "text-6-0-public", + "prompt_tokens": 131072, + "usage": { + "prompt_tokens": 131072, + "completion_tokens": 16, + "total_tokens": 131088, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 468.3503332915716, + "elapsed_seconds": 468.5017660828307, + "output_text_sha256": "53a2f067164e0bb60a42dfaa508697eb29c05ade85e604e23986531fe73d9ba9", + "reply_token_sha256": "27537795b880c9473753a2a60282a66b75c55c5c566f5122ca261a511424122c", + "drafts_acknowledged": true, + "raw_response_sha256": "11814b0a864e001c7cf3d4e30f1791878c05c2ecb9919f75e5524b03c6c7295c", + "actual_tokenize_sha256": "1ad37fede50ddc2307adacc2f216ec67ae3d4f466b9834d3acbeb86b8c34b452", + "rendered_ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff", + "fixture_body_sha256": "d8c3cfe0099049d8264273d510f2b300cb2917b4eecee248b8f9bfc9ffdbbaae", + "exact_tokenize_event": { + "time_ns": 1793254814148541000, + "tag": "text-6-0-public", + "prompt_tokens": 131072, + "ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff" + } + }, + { + "tag": "text-6-0-public", + "prompt_tokens": 262128, + "usage": { + "prompt_tokens": 262128, + "completion_tokens": 16, + "total_tokens": 262144, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 2016.505903666839, + "elapsed_seconds": 2016.7560635828413, + "output_text_sha256": "3d39e6371bfc165fc82c941cbe31c7bcc11398dc2722de4de297f2f80802e474", + "reply_token_sha256": "7b57da6fe8ea8351c13c86156951ce94941445e7f881eaded945d97676b5558b", + "drafts_acknowledged": true, + "raw_response_sha256": "5bed39d4af0840184334c4e60f15d3f89270ff49724a7f2b49520373a41a5a1a", + "actual_tokenize_sha256": "803d527a17209dc622cc8fd8fba1b7db6a79f98532062e57fdca36ad4052e422", + "rendered_ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358", + "fixture_body_sha256": "247e79c1ac3e88a314e8e9dff18b9b2fac208ef03a8ab0c4a9574361447304f0", + "exact_tokenize_event": { + "time_ns": 1793255282929303000, + "tag": "text-6-0-public", + "prompt_tokens": 262128, + "ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358" + } + }, + { + "tag": "text-6-1-public", + "prompt_tokens": 131072, + "usage": { + "prompt_tokens": 131072, + "completion_tokens": 16, + "total_tokens": 131088, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 468.44259300036356, + "elapsed_seconds": 468.59457087516785, + "output_text_sha256": "53a2f067164e0bb60a42dfaa508697eb29c05ade85e604e23986531fe73d9ba9", + "reply_token_sha256": "27537795b880c9473753a2a60282a66b75c55c5c566f5122ca261a511424122c", + "drafts_acknowledged": true, + "raw_response_sha256": "10fc71f0f074543a8db5d10989f70fc775ee0987939b294985e7d276051b095b", + "actual_tokenize_sha256": "1ad37fede50ddc2307adacc2f216ec67ae3d4f466b9834d3acbeb86b8c34b452", + "rendered_ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff", + "fixture_body_sha256": "d8c3cfe0099049d8264273d510f2b300cb2917b4eecee248b8f9bfc9ffdbbaae", + "exact_tokenize_event": { + "time_ns": 1793257301509910000, + "tag": "text-6-1-public", + "prompt_tokens": 131072, + "ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff" + } + }, + { + "tag": "text-6-1-public", + "prompt_tokens": 262128, + "usage": { + "prompt_tokens": 262128, + "completion_tokens": 16, + "total_tokens": 262144, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 2017.7195548326708, + "elapsed_seconds": 2017.9693661658093, + "output_text_sha256": "3d39e6371bfc165fc82c941cbe31c7bcc11398dc2722de4de297f2f80802e474", + "reply_token_sha256": "7b57da6fe8ea8351c13c86156951ce94941445e7f881eaded945d97676b5558b", + "drafts_acknowledged": true, + "raw_response_sha256": "df6073f58e68841c86245db5fdbcd637a33fd630dcf95f6a0a82160082c970db", + "actual_tokenize_sha256": "803d527a17209dc622cc8fd8fba1b7db6a79f98532062e57fdca36ad4052e422", + "rendered_ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358", + "fixture_body_sha256": "247e79c1ac3e88a314e8e9dff18b9b2fac208ef03a8ab0c4a9574361447304f0", + "exact_tokenize_event": { + "time_ns": 1793257770339886000, + "tag": "text-6-1-public", + "prompt_tokens": 262128, + "ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358" + } + }, + { + "tag": "text-8-0-public", + "prompt_tokens": 131072, + "usage": { + "prompt_tokens": 131072, + "completion_tokens": 16, + "total_tokens": 131088, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 465.25606929231435, + "elapsed_seconds": 465.41089862491935, + "output_text_sha256": "1a926710cb89d3ce12776b0668fc943e78b88dc03f9c725f3aad13b3e836a208", + "reply_token_sha256": "68a83382c9db4b63d1939f0e8410bfb8070c09cdad28eb7bc968c8ef217e922a", + "drafts_acknowledged": true, + "raw_response_sha256": "43a1adc00e00d9c22cb1b28f8bf912fae1c767ff3027a44a02eca899bec0fb6f", + "actual_tokenize_sha256": "1ad37fede50ddc2307adacc2f216ec67ae3d4f466b9834d3acbeb86b8c34b452", + "rendered_ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff", + "fixture_body_sha256": "d8c3cfe0099049d8264273d510f2b300cb2917b4eecee248b8f9bfc9ffdbbaae", + "exact_tokenize_event": { + "time_ns": 1793259790354592000, + "tag": "text-8-0-public", + "prompt_tokens": 131072, + "ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff" + } + }, + { + "tag": "text-8-0-public", + "prompt_tokens": 262128, + "usage": { + "prompt_tokens": 262128, + "completion_tokens": 16, + "total_tokens": 262144, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 2011.3112170421518, + "elapsed_seconds": 2011.5430155843496, + "output_text_sha256": "201bec012f54acfe4f6fb6899eda2cbd35c3d1df621f055e656d81ec9812bc2c", + "reply_token_sha256": "8fa7c55cb544af753f9420284bacfd3f6bdfe89e30b5740682222fa25427d23d", + "drafts_acknowledged": true, + "raw_response_sha256": "9edbe6f80cfc202d1ea0ae9ceaa75c09c516548a5e726639cf47de673f9784b8", + "actual_tokenize_sha256": "803d527a17209dc622cc8fd8fba1b7db6a79f98532062e57fdca36ad4052e422", + "rendered_ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358", + "fixture_body_sha256": "247e79c1ac3e88a314e8e9dff18b9b2fac208ef03a8ab0c4a9574361447304f0", + "exact_tokenize_event": { + "time_ns": 1793260256032140000, + "tag": "text-8-0-public", + "prompt_tokens": 262128, + "ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358" + } + }, + { + "tag": "text-8-1-public", + "prompt_tokens": 131072, + "usage": { + "prompt_tokens": 131072, + "completion_tokens": 16, + "total_tokens": 131088, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 465.5472393753007, + "elapsed_seconds": 465.70099454233423, + "output_text_sha256": "1a926710cb89d3ce12776b0668fc943e78b88dc03f9c725f3aad13b3e836a208", + "reply_token_sha256": "68a83382c9db4b63d1939f0e8410bfb8070c09cdad28eb7bc968c8ef217e922a", + "drafts_acknowledged": true, + "raw_response_sha256": "3140008b98854f6ec7c87d1775ac8afc161b25001d9b9b10a0ba8bd5721138b9", + "actual_tokenize_sha256": "1ad37fede50ddc2307adacc2f216ec67ae3d4f466b9834d3acbeb86b8c34b452", + "rendered_ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff", + "fixture_body_sha256": "d8c3cfe0099049d8264273d510f2b300cb2917b4eecee248b8f9bfc9ffdbbaae", + "exact_tokenize_event": { + "time_ns": 1793262269768495000, + "tag": "text-8-1-public", + "prompt_tokens": 131072, + "ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff" + } + }, + { + "tag": "text-8-1-public", + "prompt_tokens": 262128, + "usage": { + "prompt_tokens": 262128, + "completion_tokens": 16, + "total_tokens": 262144, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 2011.5570595832542, + "elapsed_seconds": 2011.7797447079793, + "output_text_sha256": "201bec012f54acfe4f6fb6899eda2cbd35c3d1df621f055e656d81ec9812bc2c", + "reply_token_sha256": "8fa7c55cb544af753f9420284bacfd3f6bdfe89e30b5740682222fa25427d23d", + "drafts_acknowledged": true, + "raw_response_sha256": "b027d39bc74b97f37c587e2d13fd1b0792f000a25acb409dc2330f249fc109d8", + "actual_tokenize_sha256": "803d527a17209dc622cc8fd8fba1b7db6a79f98532062e57fdca36ad4052e422", + "rendered_ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358", + "fixture_body_sha256": "247e79c1ac3e88a314e8e9dff18b9b2fac208ef03a8ab0c4a9574361447304f0", + "exact_tokenize_event": { + "time_ns": 1793262735773032000, + "tag": "text-8-1-public", + "prompt_tokens": 262128, + "ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358" + } + } + ], + "repeat_equality": [ + { + "bits": 4, + "prompt_tokens": 131072, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "eab2cbad6c3db4db52f33d47d1fc19e1af989c9ea6647e61ea843a8d0dfc9faf" + }, + { + "bits": 4, + "prompt_tokens": 262128, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "da7391ebf4662f42f3730deab9c314430745a31a5bec92f01bc440c7c3d18cdd" + }, + { + "bits": 6, + "prompt_tokens": 131072, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "27537795b880c9473753a2a60282a66b75c55c5c566f5122ca261a511424122c" + }, + { + "bits": 6, + "prompt_tokens": 262128, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "7b57da6fe8ea8351c13c86156951ce94941445e7f881eaded945d97676b5558b" + }, + { + "bits": 8, + "prompt_tokens": 131072, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "68a83382c9db4b63d1939f0e8410bfb8070c09cdad28eb7bc968c8ef217e922a" + }, + { + "bits": 8, + "prompt_tokens": 262128, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "8fa7c55cb544af753f9420284bacfd3f6bdfe89e30b5740682222fa25427d23d" + } + ] + } + ], + "counts": { + "completed_requests": 20, + "completion_tokens": 320, + "cached_tokens": 0, + "repeat_cells": 10, + "fresh_server_cleanups": 10, + "cleanup_exit_zero": 10 + }, + "limits": [ + "Remaining 6 planned requests are not asserted complete by this receipt.", + "Text4 total prompt+completion budget262144 is not a claim of every physical KV slot or simultaneous full-window workload.", + "VL checkpoints here receive text prompts; no image long-context acceptance is asserted.", + "Timings are native context diagnostics without a standard-server comparison or speed qualification.", + "Memory values in server_cleanups are per-process peak physical footprint and maximum RSS, not device allocator peaks.", + "Admission budgets are conservative source-derived estimates; no memory or wired limits were raised.", + "Both published packets completed and released their leases; the separate remaining VL native-window packet is active." + ] +} diff --git a/docs/receipts/lfm3/current-m5-long-context.json b/docs/receipts/lfm3/current-m5-long-context.json new file mode 100644 index 0000000000..fb58e6b4ab --- /dev/null +++ b/docs/receipts/lfm3/current-m5-long-context.json @@ -0,0 +1,1787 @@ +{ + "schema_version": 1, + "experiment": "Current public M5 exact-token context qualification: 26 completed requests", + "status": "complete", + "planned_requests": 26, + "published_completed_requests": 26, + "remaining_requests": 0, + "remaining_scope": [], + "remaining_status": "all planned context requests completed", + "source_commit": "20947578c08ead20e8e0a18790b1fd5f5fb3b6e3", + "runtime_base": "2e97b1fbbe80f2446e49893fc297da9be914fc71", + "source_archive_sha256": "6001dd8298380ade2764eb75875a7c8a7732f191384b729e711f77e7614d2814", + "native_sha256": "437218121714a1761187e009e8075d8751e5327ce2fd9b65ac21f3aeead214c4", + "approved_payload_archive_sha256": "579394ca68c41325fd8dc917e1b5dc6a172f2b0e7128873612c646b649581215", + "native_build": { + "steps_passed": 57, + "steps_total": 57, + "exit": 0 + }, + "scope": { + "engine": "current native only", + "request_kind": "text prompts, including VL checkpoints", + "actual_simultaneous_requests": 1, + "configured_parallel": 4, + "output_tokens_per_request": 16, + "temperature": 0, + "thinking": false, + "prefix_cache_gib": 0, + "drafts": "production default; actual response acknowledgement retained", + "live_tokenize": "full rendered ID array byte/value equality against pinned fixture before every request", + "performance_comparison": false, + "image_context": false, + "fp64_fidelity": false, + "whole_matrix_complete": true + }, + "packets": [ + { + "name": "vl68-32k64k", + "executed_manifest_sha256": "7bb9931400304960faa834b97fb732341a625052dd9033e38fc1022ed121221f", + "events_snapshot_sha256": "14a59913647c6980861f755cc7acf7b4ed6dfb101194825229cd6a2866a29f00", + "events_snapshot_scope": "complete packet", + "models": { + "LFM3-VL-2B-MLX-6bit": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-6bit", + "revision": "6d71bfae121abedc75f6d531fc7f6ceb4f5e3501", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 4138, + "sha256": "4c03aa298372cbcb478eb08be0a4a59be917b4949756066ce4eac06033d200bc", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 323, + "sha256": "30b1115bcdc4e5671567a5b1255a2f3d2f5a912a79292051c6f1e9db81b5e8c4", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 1783964482, + "sha256": "838c39147541b12e86a4f1b43d6a7814d7fb4efe9d5ff8a151c85ac06f321eef", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 61255, + "sha256": "abc29f6e9809c65c5ce62e6b1754b3f103d82266adf74d6d661dd3f8cd443cb5", + "verified_against_pinned_revision": true + }, + "processor_config.json": { + "bytes": 1254, + "sha256": "f183f5a5e5b3a32e666fce05033e6cb7ca496d26e709a2cee611a13eb7d6b610", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908258, + "sha256": "a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1759, + "sha256": "d63d1ce2f332047e398764b4170352b96629a5194643105bfd6d70c90b750396", + "verified_against_pinned_revision": true + } + } + }, + "LFM3-VL-2B-MLX-8bit": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-8bit", + "revision": "83336d5be3874a2bdd3297bde77b2114dbd3a816", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 4138, + "sha256": "9f7dc06d04d0610c7e27fb6a110c42a3a09b686a6ec53b0eb383f30a8be5a457", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 323, + "sha256": "30b1115bcdc4e5671567a5b1255a2f3d2f5a912a79292051c6f1e9db81b5e8c4", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 2216068156, + "sha256": "50c9476a7f00f5349e81b42ab88b47635b9eba3c7ab5851c598d270cda4c6857", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 61255, + "sha256": "31fe8faaf68bfc2fbce07e9d97ff47c1abad234d783ed615f2d938392b7cf6f0", + "verified_against_pinned_revision": true + }, + "processor_config.json": { + "bytes": 1254, + "sha256": "f183f5a5e5b3a32e666fce05033e6cb7ca496d26e709a2cee611a13eb7d6b610", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908258, + "sha256": "a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1759, + "sha256": "d63d1ce2f332047e398764b4170352b96629a5194643105bfd6d70c90b750396", + "verified_against_pinned_revision": true + } + } + } + }, + "context_capacity": 70000, + "client_sha256": { + "bench_auth.py": "1896120b70d0df05646870a252697e20ce84c63e09dcba88b2a4e5a55ccb3c5b", + "bench_cache.py": "24364dd13c033450f45710fc8685e2c64d6a72d6dd6e223d0d7b19aabbdc1d98", + "bench_concurrent.py": "0e72e26769f082459329fc068c25f6f72a9155e422962ea54cb5d1226227752d", + "bench_evidence.py": "54c28474f0804dbd120e3fb5859be0d1667ef21ca502880d0bed098ec9bb42ff", + "bench_metrics.py": "26f4c3ecbb03df4f12e24d18840fea989d9ef8c63117ca11eb944ef670458a27", + "bench_openai.py": "d5af4fbb756cdfb573546c0e0017652a4bb8f509d1251af683be7a3cbdd426f6", + "bench_timing.py": "158f636ef0b3bed460ce19ef4872b6d8607ff7a8590562d06bae8bd55e6f44ed", + "bench_decode.py": "c011106f4f182db19371e3f91f70bcc374160786dd105f2d713a7fea07ab8c4c", + "cold_client.py": "7b8ea55aa88834814f95a6f679654e16074b7a8364e34b39a276f009fe0231f2" + }, + "fixture_sha256": { + "vl-32768.body.json": "6e4c2ecd84a08e202dddce7f9b50bada5c006f9d529a74177f38dc11c08e0bb4", + "vl-32768.ids.json": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f", + "vl-65536.body.json": "d6dea630f39ee2ddd13e8a88a90050d615d518e21f5a9219e1dc3d2297617b63", + "vl-65536.ids.json": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed" + }, + "guard_sha256": "3c4118b62c3797e1871032fcbbb1bf7d390f143c2e93ddf564b309e7f49a4664", + "host": "ProductName:\t\tmacOS\nProductVersion:\t\t27.0\nBuildVersion:\t\t26A428\n", + "physical_memory_bytes": 274877906944, + "gpu_device": "Apple M5 Ultra", + "requirements_sha256": "275b369ea72d5757b1e3f91e68ed5b746d83b5f7667adc8f5d9d28bb8933ddd6", + "memory_admission": [ + { + "event": "memory_admission", + "time_ns": 1793249253131489000, + "physical_memory_bytes": 274877906944, + "working_set": 239143780352, + "ceiling": 192414534830, + "budgets": [ + { + "family": "VL-2B", + "bits": 6, + "capacity": 128000, + "checkpoint_bytes": 1783964482, + "cache_per_stream_bytes": 2884159488, + "streams": 4, + "measured_scratch128_bytes": 62104192, + "existing_margin_bytes": 2147483648, + "conservative_total_bytes": 15530190274 + }, + { + "family": "VL-2B", + "bits": 8, + "capacity": 128000, + "checkpoint_bytes": 2216068156, + "cache_per_stream_bytes": 2884159488, + "streams": 4, + "measured_scratch128_bytes": 62104192, + "existing_margin_bytes": 2147483648, + "conservative_total_bytes": 15962293948 + } + ], + "limits_raised": false + } + ], + "lease": [ + { + "event": "lock_acquired", + "time_ns": 1793249253063149000, + "pid": 39375, + "device": 16777233, + "inode": 2622134 + }, + { + "event": "lock_release", + "time_ns": 1793249873155825000, + "pid": 39375 + } + ], + "server_cleanups": [ + { + "event": "owned_cleanup_complete", + "time_ns": 1793249413560157000, + "tag": "vl-6-0-public", + "pid": 39388, + "exit": 0, + "memory": { + "max_rss_bytes": 2041888768, + "peak_physical_footprint_bytes": 3778742552 + } + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793249571314589000, + "tag": "vl-6-1-public", + "pid": 39434, + "exit": 0, + "memory": { + "max_rss_bytes": 2041282560, + "peak_physical_footprint_bytes": 3777857840 + } + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793249722355473000, + "tag": "vl-8-0-public", + "pid": 39466, + "exit": 0, + "memory": { + "max_rss_bytes": 2473476096, + "peak_physical_footprint_bytes": 4210215576 + } + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793249873088871000, + "tag": "vl-8-1-public", + "pid": 39502, + "exit": 0, + "memory": { + "max_rss_bytes": 2473590784, + "peak_physical_footprint_bytes": 4210543280 + } + } + ], + "requests": [ + { + "tag": "vl-6-0-public", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 16, + "total_tokens": 32784, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 36.00836220802739, + "elapsed_seconds": 36.10532324993983, + "output_text_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "reply_token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0", + "drafts_acknowledged": true, + "raw_response_sha256": "03327af93e2f925b11c0cddcfd62eb7bfe7fce5ab22f983399afd44cf9e3a5a9", + "actual_tokenize_sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450", + "rendered_ids_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f", + "fixture_body_sha256": "6e4c2ecd84a08e202dddce7f9b50bada5c006f9d529a74177f38dc11c08e0bb4", + "exact_tokenize_event": { + "time_ns": 1793249257343034000, + "tag": "vl-6-0-public", + "prompt_tokens": 32768, + "ids_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f" + } + }, + { + "tag": "vl-6-0-public", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 16, + "total_tokens": 65552, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 119.1769048338756, + "elapsed_seconds": 119.28063683398068, + "output_text_sha256": "4026f32b09f16475aea9189543a259259e5dea4a9c58e40d969af677ffb1251b", + "reply_token_sha256": "01496c951534bedc261fc706cd6de05a865b636542badf970ce10254c9b17dfd", + "drafts_acknowledged": true, + "raw_response_sha256": "c4aa30182579e0c70976ab0d3dcbdbe36ef81bb8b3bc3c02e6c7563d06d34afa", + "actual_tokenize_sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082", + "rendered_ids_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed", + "fixture_body_sha256": "d6dea630f39ee2ddd13e8a88a90050d615d518e21f5a9219e1dc3d2297617b63", + "exact_tokenize_event": { + "time_ns": 1793249293617106000, + "tag": "vl-6-0-public", + "prompt_tokens": 65536, + "ids_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed" + } + }, + { + "tag": "vl-6-1-public", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 16, + "total_tokens": 32784, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 36.01983079081401, + "elapsed_seconds": 36.11669062497094, + "output_text_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "reply_token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0", + "drafts_acknowledged": true, + "raw_response_sha256": "76afa7b3cd0d50c864ece3938443f3aed02430b974540b923bdc2a750b1709bb", + "actual_tokenize_sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450", + "rendered_ids_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f", + "fixture_body_sha256": "6e4c2ecd84a08e202dddce7f9b50bada5c006f9d529a74177f38dc11c08e0bb4", + "exact_tokenize_event": { + "time_ns": 1793249414813199000, + "tag": "vl-6-1-public", + "prompt_tokens": 32768, + "ids_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f" + } + }, + { + "tag": "vl-6-1-public", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 16, + "total_tokens": 65552, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 119.46985933324322, + "elapsed_seconds": 119.5737877083011, + "output_text_sha256": "4026f32b09f16475aea9189543a259259e5dea4a9c58e40d969af677ffb1251b", + "reply_token_sha256": "01496c951534bedc261fc706cd6de05a865b636542badf970ce10254c9b17dfd", + "drafts_acknowledged": true, + "raw_response_sha256": "92314ef7cbb0b8040f37c120e80466fec5529508f36c975939ad0c6f29968f29", + "actual_tokenize_sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082", + "rendered_ids_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed", + "fixture_body_sha256": "d6dea630f39ee2ddd13e8a88a90050d615d518e21f5a9219e1dc3d2297617b63", + "exact_tokenize_event": { + "time_ns": 1793249451116889000, + "tag": "vl-6-1-public", + "prompt_tokens": 65536, + "ids_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed" + } + }, + { + "tag": "vl-8-0-public", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 16, + "total_tokens": 32784, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 34.09193491702899, + "elapsed_seconds": 34.190794500056654, + "output_text_sha256": "363f5de76e594bc939dc4dfea2b5c71c087224382a91db7f7aef7ef650b86791", + "reply_token_sha256": "6b68989060076239ed31666b235e266a2e83536962209d719c5e58346bfe6251", + "drafts_acknowledged": true, + "raw_response_sha256": "1d8ff6db01586184ab2be9d55cb91ca5405416c84b56b1022519850e028237ca", + "actual_tokenize_sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450", + "rendered_ids_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f", + "fixture_body_sha256": "6e4c2ecd84a08e202dddce7f9b50bada5c006f9d529a74177f38dc11c08e0bb4", + "exact_tokenize_event": { + "time_ns": 1793249572575420000, + "tag": "vl-8-0-public", + "prompt_tokens": 32768, + "ids_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f" + } + }, + { + "tag": "vl-8-0-public", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 16, + "total_tokens": 65552, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 114.42618841677904, + "elapsed_seconds": 114.5319452919066, + "output_text_sha256": "b3fc28d243e9bb51d0db0f4729560c1a1382e069e5cee04f555a3eb4233fc207", + "reply_token_sha256": "7240d3de4d84db461fc2e1f43a7b9f8a79a876eb933afb924e661ad8d8268c3b", + "drafts_acknowledged": true, + "raw_response_sha256": "bf09569bb27a6352a43f30a51b87ed51118020d11f97f23caefd384586bfb11c", + "actual_tokenize_sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082", + "rendered_ids_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed", + "fixture_body_sha256": "d6dea630f39ee2ddd13e8a88a90050d615d518e21f5a9219e1dc3d2297617b63", + "exact_tokenize_event": { + "time_ns": 1793249606961361000, + "tag": "vl-8-0-public", + "prompt_tokens": 65536, + "ids_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed" + } + }, + { + "tag": "vl-8-1-public", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 16, + "total_tokens": 32784, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 34.15882908413187, + "elapsed_seconds": 34.25784600013867, + "output_text_sha256": "363f5de76e594bc939dc4dfea2b5c71c087224382a91db7f7aef7ef650b86791", + "reply_token_sha256": "6b68989060076239ed31666b235e266a2e83536962209d719c5e58346bfe6251", + "drafts_acknowledged": true, + "raw_response_sha256": "1e4eba9e0bfd5c47765c43d7a3983732b8382ff68118f8e036f34fbdd53e8739", + "actual_tokenize_sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450", + "rendered_ids_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f", + "fixture_body_sha256": "6e4c2ecd84a08e202dddce7f9b50bada5c006f9d529a74177f38dc11c08e0bb4", + "exact_tokenize_event": { + "time_ns": 1793249723607015000, + "tag": "vl-8-1-public", + "prompt_tokens": 32768, + "ids_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f" + } + }, + { + "tag": "vl-8-1-public", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 16, + "total_tokens": 65552, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 114.3223563330248, + "elapsed_seconds": 114.4282983331941, + "output_text_sha256": "b3fc28d243e9bb51d0db0f4729560c1a1382e069e5cee04f555a3eb4233fc207", + "reply_token_sha256": "7240d3de4d84db461fc2e1f43a7b9f8a79a876eb933afb924e661ad8d8268c3b", + "drafts_acknowledged": true, + "raw_response_sha256": "c3e6c4a433193ff2bde8ca6fb8e72a3760eabf5af7e13bc3613ebf2aa08f38c2", + "actual_tokenize_sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082", + "rendered_ids_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed", + "fixture_body_sha256": "d6dea630f39ee2ddd13e8a88a90050d615d518e21f5a9219e1dc3d2297617b63", + "exact_tokenize_event": { + "time_ns": 1793249758105970000, + "tag": "vl-8-1-public", + "prompt_tokens": 65536, + "ids_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed" + } + } + ], + "repeat_equality": [ + { + "bits": 6, + "prompt_tokens": 32768, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0" + }, + { + "bits": 6, + "prompt_tokens": 65536, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "01496c951534bedc261fc706cd6de05a865b636542badf970ce10254c9b17dfd" + }, + { + "bits": 8, + "prompt_tokens": 32768, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "6b68989060076239ed31666b235e266a2e83536962209d719c5e58346bfe6251" + }, + { + "bits": 8, + "prompt_tokens": 65536, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "7240d3de4d84db461fc2e1f43a7b9f8a79a876eb933afb924e661ad8d8268c3b" + } + ] + }, + { + "name": "text-all-native-window", + "executed_manifest_sha256": "351960c08c0d953374c2ede787a3a84b6321cd77da51485300abb2cfa1a9aa5b", + "events_snapshot_sha256": "de23611dbf5eeaa452b8d5eb5dd0bb7aab2cf9d37beceda4af3476c9a03e2d33", + "events_snapshot_scope": "complete packet", + "models": { + "LFM3-1.7B-MLX-4bit": { + "repository": "LiquidAI/LFM3-1.7B-MLX-4bit", + "revision": "d0ca73dcfa70ce18733754a492cdc1fb0123262d", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 2423, + "sha256": "7df2ea819f9a31b770192a0a9dc1b7953a7dd8c41b816cc3905bbec99ef332bd", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 328, + "sha256": "0095ba7158e938b1e2d3082d412f00d8be85e42cbbd7608b63866f7a081cf270", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 973083386, + "sha256": "1be5bf672013cd91aa62148f77c5eea8d2784a95845d5fe826e08c7b84823815", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 50700, + "sha256": "2c8b88e25f6a1055164dd051436b170872200047bce4b3ec5f2a58f6ef929a75", + "verified_against_pinned_revision": true + }, + "special_tokens_map.json": { + "bytes": 92, + "sha256": "7946f864cbfb62a1b041bda937244210a1a87f4a6cab3a48b791131b2cd57f88", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908634, + "sha256": "41927bd314d17d7d7fadb44e10cd0919f7b71528d7741596d8cbd15c6945ec44", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1859, + "sha256": "b77214b9f12be74b9e5e1dab35d7dace077a31be969314bf38866fd6c8163853", + "verified_against_pinned_revision": true + } + } + }, + "LFM3-1.7B-MLX-6bit": { + "repository": "LiquidAI/LFM3-1.7B-MLX-6bit", + "revision": "26be898c36b545d625d84ac0761c6c3eb664aef3", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 2423, + "sha256": "2b5879bf6d1d99b43e17726c7ee3fb4868e4c2e29c4a49784803df90f494fc2e", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 328, + "sha256": "0095ba7158e938b1e2d3082d412f00d8be85e42cbbd7608b63866f7a081cf270", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 1405187179, + "sha256": "503d68675793fba64907842e176d066c8c3adca22e14c8b3538f9ffb76bd5e0e", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 50701, + "sha256": "940f7e7b4133162a48fbca232e663c32240364aea9d60f1715b62bed41a81e2c", + "verified_against_pinned_revision": true + }, + "special_tokens_map.json": { + "bytes": 92, + "sha256": "7946f864cbfb62a1b041bda937244210a1a87f4a6cab3a48b791131b2cd57f88", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908634, + "sha256": "41927bd314d17d7d7fadb44e10cd0919f7b71528d7741596d8cbd15c6945ec44", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1859, + "sha256": "b77214b9f12be74b9e5e1dab35d7dace077a31be969314bf38866fd6c8163853", + "verified_against_pinned_revision": true + } + } + }, + "LFM3-1.7B-MLX-8bit": { + "repository": "LiquidAI/LFM3-1.7B-MLX-8bit", + "revision": "ee7667f3cfccb2a41725f91820531823fcabc6b7", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 2423, + "sha256": "4ef95e289e7105a09bf010b3ceaa015d3f01163104c039a4e7948739d5b9035b", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 328, + "sha256": "0095ba7158e938b1e2d3082d412f00d8be85e42cbbd7608b63866f7a081cf270", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 1837290805, + "sha256": "2b06cea98d75527c248a0b04383f7cf0a1c72fb4ed7b45096618fc4e59d015ad", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 50701, + "sha256": "0f4f85710fd22ffb8a1977520a734cb7dc7e1f7858a0f91d9c8b56e6d36b78cc", + "verified_against_pinned_revision": true + }, + "special_tokens_map.json": { + "bytes": 92, + "sha256": "7946f864cbfb62a1b041bda937244210a1a87f4a6cab3a48b791131b2cd57f88", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908634, + "sha256": "41927bd314d17d7d7fadb44e10cd0919f7b71528d7741596d8cbd15c6945ec44", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1859, + "sha256": "b77214b9f12be74b9e5e1dab35d7dace077a31be969314bf38866fd6c8163853", + "verified_against_pinned_revision": true + } + } + } + }, + "context_capacity": 262144, + "client_sha256": { + "bench_auth.py": "1896120b70d0df05646870a252697e20ce84c63e09dcba88b2a4e5a55ccb3c5b", + "bench_cache.py": "24364dd13c033450f45710fc8685e2c64d6a72d6dd6e223d0d7b19aabbdc1d98", + "bench_concurrent.py": "0e72e26769f082459329fc068c25f6f72a9155e422962ea54cb5d1226227752d", + "bench_evidence.py": "54c28474f0804dbd120e3fb5859be0d1667ef21ca502880d0bed098ec9bb42ff", + "bench_metrics.py": "26f4c3ecbb03df4f12e24d18840fea989d9ef8c63117ca11eb944ef670458a27", + "bench_openai.py": "d5af4fbb756cdfb573546c0e0017652a4bb8f509d1251af683be7a3cbdd426f6", + "bench_timing.py": "158f636ef0b3bed460ce19ef4872b6d8607ff7a8590562d06bae8bd55e6f44ed", + "bench_decode.py": "c011106f4f182db19371e3f91f70bcc374160786dd105f2d713a7fea07ab8c4c", + "cold_client.py": "7b8ea55aa88834814f95a6f679654e16074b7a8364e34b39a276f009fe0231f2" + }, + "fixture_sha256": { + "text-131072.body.json": "d8c3cfe0099049d8264273d510f2b300cb2917b4eecee248b8f9bfc9ffdbbaae", + "text-131072.ids.json": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff", + "text-262128.body.json": "247e79c1ac3e88a314e8e9dff18b9b2fac208ef03a8ab0c4a9574361447304f0", + "text-262128.ids.json": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358" + }, + "guard_sha256": "3c4118b62c3797e1871032fcbbb1bf7d390f143c2e93ddf564b309e7f49a4664", + "host": "ProductName:\t\tmacOS\nProductVersion:\t\t27.0\nBuildVersion:\t\t26A428\n", + "physical_memory_bytes": 274877906944, + "gpu_device": "Apple M5 Ultra", + "requirements_sha256": "275b369ea72d5757b1e3f91e68ed5b746d83b5f7667adc8f5d9d28bb8933ddd6", + "memory_admission": [ + { + "event": "memory_admission", + "time_ns": 1793249873317272000, + "physical_memory_bytes": 274877906944, + "working_set": 239143780352, + "ceiling": 192414534830, + "budgets": [ + { + "family": "1.7B", + "bits": 4, + "capacity": 262144, + "checkpoint_bytes": 973083386, + "cache_per_stream_bytes": 5906155520, + "streams": 4, + "measured_scratch128_bytes": 62104192, + "existing_margin_bytes": 2147483648, + "conservative_total_bytes": 26807293306 + }, + { + "family": "1.7B", + "bits": 6, + "capacity": 262144, + "checkpoint_bytes": 1405187179, + "cache_per_stream_bytes": 5906155520, + "streams": 4, + "measured_scratch128_bytes": 62104192, + "existing_margin_bytes": 2147483648, + "conservative_total_bytes": 27239397099 + }, + { + "family": "1.7B", + "bits": 8, + "capacity": 262144, + "checkpoint_bytes": 1837290805, + "cache_per_stream_bytes": 5906155520, + "streams": 4, + "measured_scratch128_bytes": 62104192, + "existing_margin_bytes": 2147483648, + "conservative_total_bytes": 27671500725 + } + ], + "limits_raised": false + } + ], + "lease": [ + { + "event": "lock_acquired", + "time_ns": 1793249873251776000, + "pid": 39546, + "device": 16777233, + "inode": 2622134 + }, + { + "event": "lock_release", + "time_ns": 1793264748356781000, + "pid": 39546 + } + ], + "server_cleanups": [ + { + "event": "owned_cleanup_complete", + "time_ns": 1793252343951170000, + "tag": "text-4-0-public", + "pid": 39567, + "exit": 0, + "memory": { + "max_rss_bytes": 1306099712, + "peak_physical_footprint_bytes": 7431800536 + } + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793254812861949000, + "tag": "text-4-1-public", + "pid": 39668, + "exit": 0, + "memory": { + "max_rss_bytes": 1304674304, + "peak_physical_footprint_bytes": 7431816920 + } + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793257300236461000, + "tag": "text-6-0-public", + "pid": 39775, + "exit": 0, + "memory": { + "max_rss_bytes": 1736687616, + "peak_physical_footprint_bytes": 7880951896 + } + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793259789078224000, + "tag": "text-6-1-public", + "pid": 39925, + "exit": 0, + "memory": { + "max_rss_bytes": 1737752576, + "peak_physical_footprint_bytes": 7864256576 + } + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793262268490697000, + "tag": "text-8-0-public", + "pid": 40037, + "exit": 0, + "memory": { + "max_rss_bytes": 2168832000, + "peak_physical_footprint_bytes": 8286505336 + } + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793264748283480000, + "tag": "text-8-1-public", + "pid": 40166, + "exit": 0, + "memory": { + "max_rss_bytes": 2168946688, + "peak_physical_footprint_bytes": 8296384888 + } + } + ], + "requests": [ + { + "tag": "text-4-0-public", + "prompt_tokens": 131072, + "usage": { + "prompt_tokens": 131072, + "completion_tokens": 16, + "total_tokens": 131088, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 463.8665871671401, + "elapsed_seconds": 464.01015008287504, + "output_text_sha256": "93527a09d1b7d0901b65ecdd1b49611d78a3f2a09b25547d1c69d390dc32d43e", + "reply_token_sha256": "eab2cbad6c3db4db52f33d47d1fc19e1af989c9ea6647e61ea843a8d0dfc9faf", + "drafts_acknowledged": true, + "raw_response_sha256": "d0edf9fff4c58b08c6f2554b371397c9a0ec2ba8dcfd01d9621b674b86875f20", + "actual_tokenize_sha256": "1ad37fede50ddc2307adacc2f216ec67ae3d4f466b9834d3acbeb86b8c34b452", + "rendered_ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff", + "fixture_body_sha256": "d8c3cfe0099049d8264273d510f2b300cb2917b4eecee248b8f9bfc9ffdbbaae", + "exact_tokenize_event": { + "time_ns": 1793249876210952000, + "tag": "text-4-0-public", + "prompt_tokens": 131072, + "ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff" + } + }, + { + "tag": "text-4-0-public", + "prompt_tokens": 262128, + "usage": { + "prompt_tokens": 262128, + "completion_tokens": 16, + "total_tokens": 262144, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 2002.808624625206, + "elapsed_seconds": 2003.032133874949, + "output_text_sha256": "db6d16f0b8ab4c4e65f82a9bfbfed8dc5cb5649337698c701f5b5c28801b453f", + "reply_token_sha256": "da7391ebf4662f42f3730deab9c314430745a31a5bec92f01bc440c7c3d18cdd", + "drafts_acknowledged": true, + "raw_response_sha256": "7971ded691ccc17e5f8ffda24039c9fa8ce75080e997ece989967c098a0632c5", + "actual_tokenize_sha256": "803d527a17209dc622cc8fd8fba1b7db6a79f98532062e57fdca36ad4052e422", + "rendered_ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358", + "fixture_body_sha256": "247e79c1ac3e88a314e8e9dff18b9b2fac208ef03a8ab0c4a9574361447304f0", + "exact_tokenize_event": { + "time_ns": 1793250340471807000, + "tag": "text-4-0-public", + "prompt_tokens": 262128, + "ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358" + } + }, + { + "tag": "text-4-1-public", + "prompt_tokens": 131072, + "usage": { + "prompt_tokens": 131072, + "completion_tokens": 16, + "total_tokens": 131088, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 463.9125559167005, + "elapsed_seconds": 464.0565038328059, + "output_text_sha256": "93527a09d1b7d0901b65ecdd1b49611d78a3f2a09b25547d1c69d390dc32d43e", + "reply_token_sha256": "eab2cbad6c3db4db52f33d47d1fc19e1af989c9ea6647e61ea843a8d0dfc9faf", + "drafts_acknowledged": true, + "raw_response_sha256": "8a6b4cc23c3ce16499fda0a0427c733300231d7749aeb45642c0ed1e7a8b3b2d", + "actual_tokenize_sha256": "1ad37fede50ddc2307adacc2f216ec67ae3d4f466b9834d3acbeb86b8c34b452", + "rendered_ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff", + "fixture_body_sha256": "d8c3cfe0099049d8264273d510f2b300cb2917b4eecee248b8f9bfc9ffdbbaae", + "exact_tokenize_event": { + "time_ns": 1793252345245842000, + "tag": "text-4-1-public", + "prompt_tokens": 131072, + "ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff" + } + }, + { + "tag": "text-4-1-public", + "prompt_tokens": 262128, + "usage": { + "prompt_tokens": 262128, + "completion_tokens": 16, + "total_tokens": 262144, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 2002.407886958681, + "elapsed_seconds": 2002.630378708709, + "output_text_sha256": "db6d16f0b8ab4c4e65f82a9bfbfed8dc5cb5649337698c701f5b5c28801b453f", + "reply_token_sha256": "da7391ebf4662f42f3730deab9c314430745a31a5bec92f01bc440c7c3d18cdd", + "drafts_acknowledged": true, + "raw_response_sha256": "a8d97410eaa3972af83f16fd2654b9d87f669319c3bb573cce13b9068e9f8aa9", + "actual_tokenize_sha256": "803d527a17209dc622cc8fd8fba1b7db6a79f98532062e57fdca36ad4052e422", + "rendered_ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358", + "fixture_body_sha256": "247e79c1ac3e88a314e8e9dff18b9b2fac208ef03a8ab0c4a9574361447304f0", + "exact_tokenize_event": { + "time_ns": 1793252809603842000, + "tag": "text-4-1-public", + "prompt_tokens": 262128, + "ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358" + } + }, + { + "tag": "text-6-0-public", + "prompt_tokens": 131072, + "usage": { + "prompt_tokens": 131072, + "completion_tokens": 16, + "total_tokens": 131088, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 468.3503332915716, + "elapsed_seconds": 468.5017660828307, + "output_text_sha256": "53a2f067164e0bb60a42dfaa508697eb29c05ade85e604e23986531fe73d9ba9", + "reply_token_sha256": "27537795b880c9473753a2a60282a66b75c55c5c566f5122ca261a511424122c", + "drafts_acknowledged": true, + "raw_response_sha256": "11814b0a864e001c7cf3d4e30f1791878c05c2ecb9919f75e5524b03c6c7295c", + "actual_tokenize_sha256": "1ad37fede50ddc2307adacc2f216ec67ae3d4f466b9834d3acbeb86b8c34b452", + "rendered_ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff", + "fixture_body_sha256": "d8c3cfe0099049d8264273d510f2b300cb2917b4eecee248b8f9bfc9ffdbbaae", + "exact_tokenize_event": { + "time_ns": 1793254814148541000, + "tag": "text-6-0-public", + "prompt_tokens": 131072, + "ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff" + } + }, + { + "tag": "text-6-0-public", + "prompt_tokens": 262128, + "usage": { + "prompt_tokens": 262128, + "completion_tokens": 16, + "total_tokens": 262144, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 2016.505903666839, + "elapsed_seconds": 2016.7560635828413, + "output_text_sha256": "3d39e6371bfc165fc82c941cbe31c7bcc11398dc2722de4de297f2f80802e474", + "reply_token_sha256": "7b57da6fe8ea8351c13c86156951ce94941445e7f881eaded945d97676b5558b", + "drafts_acknowledged": true, + "raw_response_sha256": "5bed39d4af0840184334c4e60f15d3f89270ff49724a7f2b49520373a41a5a1a", + "actual_tokenize_sha256": "803d527a17209dc622cc8fd8fba1b7db6a79f98532062e57fdca36ad4052e422", + "rendered_ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358", + "fixture_body_sha256": "247e79c1ac3e88a314e8e9dff18b9b2fac208ef03a8ab0c4a9574361447304f0", + "exact_tokenize_event": { + "time_ns": 1793255282929303000, + "tag": "text-6-0-public", + "prompt_tokens": 262128, + "ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358" + } + }, + { + "tag": "text-6-1-public", + "prompt_tokens": 131072, + "usage": { + "prompt_tokens": 131072, + "completion_tokens": 16, + "total_tokens": 131088, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 468.44259300036356, + "elapsed_seconds": 468.59457087516785, + "output_text_sha256": "53a2f067164e0bb60a42dfaa508697eb29c05ade85e604e23986531fe73d9ba9", + "reply_token_sha256": "27537795b880c9473753a2a60282a66b75c55c5c566f5122ca261a511424122c", + "drafts_acknowledged": true, + "raw_response_sha256": "10fc71f0f074543a8db5d10989f70fc775ee0987939b294985e7d276051b095b", + "actual_tokenize_sha256": "1ad37fede50ddc2307adacc2f216ec67ae3d4f466b9834d3acbeb86b8c34b452", + "rendered_ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff", + "fixture_body_sha256": "d8c3cfe0099049d8264273d510f2b300cb2917b4eecee248b8f9bfc9ffdbbaae", + "exact_tokenize_event": { + "time_ns": 1793257301509910000, + "tag": "text-6-1-public", + "prompt_tokens": 131072, + "ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff" + } + }, + { + "tag": "text-6-1-public", + "prompt_tokens": 262128, + "usage": { + "prompt_tokens": 262128, + "completion_tokens": 16, + "total_tokens": 262144, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 2017.7195548326708, + "elapsed_seconds": 2017.9693661658093, + "output_text_sha256": "3d39e6371bfc165fc82c941cbe31c7bcc11398dc2722de4de297f2f80802e474", + "reply_token_sha256": "7b57da6fe8ea8351c13c86156951ce94941445e7f881eaded945d97676b5558b", + "drafts_acknowledged": true, + "raw_response_sha256": "df6073f58e68841c86245db5fdbcd637a33fd630dcf95f6a0a82160082c970db", + "actual_tokenize_sha256": "803d527a17209dc622cc8fd8fba1b7db6a79f98532062e57fdca36ad4052e422", + "rendered_ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358", + "fixture_body_sha256": "247e79c1ac3e88a314e8e9dff18b9b2fac208ef03a8ab0c4a9574361447304f0", + "exact_tokenize_event": { + "time_ns": 1793257770339886000, + "tag": "text-6-1-public", + "prompt_tokens": 262128, + "ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358" + } + }, + { + "tag": "text-8-0-public", + "prompt_tokens": 131072, + "usage": { + "prompt_tokens": 131072, + "completion_tokens": 16, + "total_tokens": 131088, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 465.25606929231435, + "elapsed_seconds": 465.41089862491935, + "output_text_sha256": "1a926710cb89d3ce12776b0668fc943e78b88dc03f9c725f3aad13b3e836a208", + "reply_token_sha256": "68a83382c9db4b63d1939f0e8410bfb8070c09cdad28eb7bc968c8ef217e922a", + "drafts_acknowledged": true, + "raw_response_sha256": "43a1adc00e00d9c22cb1b28f8bf912fae1c767ff3027a44a02eca899bec0fb6f", + "actual_tokenize_sha256": "1ad37fede50ddc2307adacc2f216ec67ae3d4f466b9834d3acbeb86b8c34b452", + "rendered_ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff", + "fixture_body_sha256": "d8c3cfe0099049d8264273d510f2b300cb2917b4eecee248b8f9bfc9ffdbbaae", + "exact_tokenize_event": { + "time_ns": 1793259790354592000, + "tag": "text-8-0-public", + "prompt_tokens": 131072, + "ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff" + } + }, + { + "tag": "text-8-0-public", + "prompt_tokens": 262128, + "usage": { + "prompt_tokens": 262128, + "completion_tokens": 16, + "total_tokens": 262144, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 2011.3112170421518, + "elapsed_seconds": 2011.5430155843496, + "output_text_sha256": "201bec012f54acfe4f6fb6899eda2cbd35c3d1df621f055e656d81ec9812bc2c", + "reply_token_sha256": "8fa7c55cb544af753f9420284bacfd3f6bdfe89e30b5740682222fa25427d23d", + "drafts_acknowledged": true, + "raw_response_sha256": "9edbe6f80cfc202d1ea0ae9ceaa75c09c516548a5e726639cf47de673f9784b8", + "actual_tokenize_sha256": "803d527a17209dc622cc8fd8fba1b7db6a79f98532062e57fdca36ad4052e422", + "rendered_ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358", + "fixture_body_sha256": "247e79c1ac3e88a314e8e9dff18b9b2fac208ef03a8ab0c4a9574361447304f0", + "exact_tokenize_event": { + "time_ns": 1793260256032140000, + "tag": "text-8-0-public", + "prompt_tokens": 262128, + "ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358" + } + }, + { + "tag": "text-8-1-public", + "prompt_tokens": 131072, + "usage": { + "prompt_tokens": 131072, + "completion_tokens": 16, + "total_tokens": 131088, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 465.5472393753007, + "elapsed_seconds": 465.70099454233423, + "output_text_sha256": "1a926710cb89d3ce12776b0668fc943e78b88dc03f9c725f3aad13b3e836a208", + "reply_token_sha256": "68a83382c9db4b63d1939f0e8410bfb8070c09cdad28eb7bc968c8ef217e922a", + "drafts_acknowledged": true, + "raw_response_sha256": "3140008b98854f6ec7c87d1775ac8afc161b25001d9b9b10a0ba8bd5721138b9", + "actual_tokenize_sha256": "1ad37fede50ddc2307adacc2f216ec67ae3d4f466b9834d3acbeb86b8c34b452", + "rendered_ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff", + "fixture_body_sha256": "d8c3cfe0099049d8264273d510f2b300cb2917b4eecee248b8f9bfc9ffdbbaae", + "exact_tokenize_event": { + "time_ns": 1793262269768495000, + "tag": "text-8-1-public", + "prompt_tokens": 131072, + "ids_sha256": "aa206fe973ccb6e305f2d1a13c62efc3cbf61082fc1602a1f7b89071f67ea6ff" + } + }, + { + "tag": "text-8-1-public", + "prompt_tokens": 262128, + "usage": { + "prompt_tokens": 262128, + "completion_tokens": 16, + "total_tokens": 262144, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 2011.5570595832542, + "elapsed_seconds": 2011.7797447079793, + "output_text_sha256": "201bec012f54acfe4f6fb6899eda2cbd35c3d1df621f055e656d81ec9812bc2c", + "reply_token_sha256": "8fa7c55cb544af753f9420284bacfd3f6bdfe89e30b5740682222fa25427d23d", + "drafts_acknowledged": true, + "raw_response_sha256": "b027d39bc74b97f37c587e2d13fd1b0792f000a25acb409dc2330f249fc109d8", + "actual_tokenize_sha256": "803d527a17209dc622cc8fd8fba1b7db6a79f98532062e57fdca36ad4052e422", + "rendered_ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358", + "fixture_body_sha256": "247e79c1ac3e88a314e8e9dff18b9b2fac208ef03a8ab0c4a9574361447304f0", + "exact_tokenize_event": { + "time_ns": 1793262735773032000, + "tag": "text-8-1-public", + "prompt_tokens": 262128, + "ids_sha256": "87a99eaddef8771c9f66c157e63e97aeb99b3bc50d6cfc5c53e5c5c64a347358" + } + } + ], + "repeat_equality": [ + { + "bits": 4, + "prompt_tokens": 131072, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "eab2cbad6c3db4db52f33d47d1fc19e1af989c9ea6647e61ea843a8d0dfc9faf" + }, + { + "bits": 4, + "prompt_tokens": 262128, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "da7391ebf4662f42f3730deab9c314430745a31a5bec92f01bc440c7c3d18cdd" + }, + { + "bits": 6, + "prompt_tokens": 131072, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "27537795b880c9473753a2a60282a66b75c55c5c566f5122ca261a511424122c" + }, + { + "bits": 6, + "prompt_tokens": 262128, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "7b57da6fe8ea8351c13c86156951ce94941445e7f881eaded945d97676b5558b" + }, + { + "bits": 8, + "prompt_tokens": 131072, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "68a83382c9db4b63d1939f0e8410bfb8070c09cdad28eb7bc968c8ef217e922a" + }, + { + "bits": 8, + "prompt_tokens": 262128, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "8fa7c55cb544af753f9420284bacfd3f6bdfe89e30b5740682222fa25427d23d" + } + ] + }, + { + "name": "vl-all-native-window", + "executed_manifest_sha256": "46ba025c9d09a73008d4c796f91e1280b96a924f20ca6ee1871ab680c9aae2c1", + "events_snapshot_sha256": "729b66ac500af86cc6f3bea16c362df3faba27bd43dbc1aee243cef3eee755f8", + "events_snapshot_scope": "complete packet", + "models": { + "LFM3-VL-2B-MLX-4bit": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "revision": "c823fade98e35a3d6942ad4d38b1ea15f675cd12", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 4419, + "sha256": "0efb7708e0771e3e104ac628954cdc71070dd436e0b7974615bd1b9eb836f9d9", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 323, + "sha256": "30b1115bcdc4e5671567a5b1255a2f3d2f5a912a79292051c6f1e9db81b5e8c4", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 1590959088, + "sha256": "58268dfdf441ae4020d2a52fc97a814645e8a9624946e2d490403d4dcd71a255", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 61256, + "sha256": "38b756dbde167ab7c60756af7d5652414c31afdf4e77700b8d3c99395515d639", + "verified_against_pinned_revision": true + }, + "processor_config.json": { + "bytes": 1254, + "sha256": "f183f5a5e5b3a32e666fce05033e6cb7ca496d26e709a2cee611a13eb7d6b610", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908258, + "sha256": "a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1759, + "sha256": "d63d1ce2f332047e398764b4170352b96629a5194643105bfd6d70c90b750396", + "verified_against_pinned_revision": true + } + } + }, + "LFM3-VL-2B-MLX-6bit": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-6bit", + "revision": "6d71bfae121abedc75f6d531fc7f6ceb4f5e3501", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 4138, + "sha256": "4c03aa298372cbcb478eb08be0a4a59be917b4949756066ce4eac06033d200bc", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 323, + "sha256": "30b1115bcdc4e5671567a5b1255a2f3d2f5a912a79292051c6f1e9db81b5e8c4", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 1783964482, + "sha256": "838c39147541b12e86a4f1b43d6a7814d7fb4efe9d5ff8a151c85ac06f321eef", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 61255, + "sha256": "abc29f6e9809c65c5ce62e6b1754b3f103d82266adf74d6d661dd3f8cd443cb5", + "verified_against_pinned_revision": true + }, + "processor_config.json": { + "bytes": 1254, + "sha256": "f183f5a5e5b3a32e666fce05033e6cb7ca496d26e709a2cee611a13eb7d6b610", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908258, + "sha256": "a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1759, + "sha256": "d63d1ce2f332047e398764b4170352b96629a5194643105bfd6d70c90b750396", + "verified_against_pinned_revision": true + } + } + }, + "LFM3-VL-2B-MLX-8bit": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-8bit", + "revision": "83336d5be3874a2bdd3297bde77b2114dbd3a816", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 4138, + "sha256": "9f7dc06d04d0610c7e27fb6a110c42a3a09b686a6ec53b0eb383f30a8be5a457", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 323, + "sha256": "30b1115bcdc4e5671567a5b1255a2f3d2f5a912a79292051c6f1e9db81b5e8c4", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 2216068156, + "sha256": "50c9476a7f00f5349e81b42ab88b47635b9eba3c7ab5851c598d270cda4c6857", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 61255, + "sha256": "31fe8faaf68bfc2fbce07e9d97ff47c1abad234d783ed615f2d938392b7cf6f0", + "verified_against_pinned_revision": true + }, + "processor_config.json": { + "bytes": 1254, + "sha256": "f183f5a5e5b3a32e666fce05033e6cb7ca496d26e709a2cee611a13eb7d6b610", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908258, + "sha256": "a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1759, + "sha256": "d63d1ce2f332047e398764b4170352b96629a5194643105bfd6d70c90b750396", + "verified_against_pinned_revision": true + } + } + } + }, + "context_capacity": 128000, + "client_sha256": { + "bench_auth.py": "1896120b70d0df05646870a252697e20ce84c63e09dcba88b2a4e5a55ccb3c5b", + "bench_cache.py": "24364dd13c033450f45710fc8685e2c64d6a72d6dd6e223d0d7b19aabbdc1d98", + "bench_concurrent.py": "0e72e26769f082459329fc068c25f6f72a9155e422962ea54cb5d1226227752d", + "bench_evidence.py": "54c28474f0804dbd120e3fb5859be0d1667ef21ca502880d0bed098ec9bb42ff", + "bench_metrics.py": "26f4c3ecbb03df4f12e24d18840fea989d9ef8c63117ca11eb944ef670458a27", + "bench_openai.py": "d5af4fbb756cdfb573546c0e0017652a4bb8f509d1251af683be7a3cbdd426f6", + "bench_timing.py": "158f636ef0b3bed460ce19ef4872b6d8607ff7a8590562d06bae8bd55e6f44ed", + "bench_decode.py": "c011106f4f182db19371e3f91f70bcc374160786dd105f2d713a7fea07ab8c4c", + "cold_client.py": "7b8ea55aa88834814f95a6f679654e16074b7a8364e34b39a276f009fe0231f2" + }, + "fixture_sha256": { + "vl-127984.body.json": "858727a862148967fcfa6563399dd3fd38da3ea8d0038370b952967913991fa8", + "vl-127984.ids.json": "35662e48e60c43a39bb5cc46ffb09b9bb2e7fbbafda29b15d9a5d071f7d875ca" + }, + "guard_sha256": "3c4118b62c3797e1871032fcbbb1bf7d390f143c2e93ddf564b309e7f49a4664", + "host": "ProductName:\t\tmacOS\nProductVersion:\t\t27.0\nBuildVersion:\t\t26A428\n", + "physical_memory_bytes": 274877906944, + "gpu_device": "Apple M5 Ultra", + "requirements_sha256": "275b369ea72d5757b1e3f91e68ed5b746d83b5f7667adc8f5d9d28bb8933ddd6", + "memory_admission": [ + { + "event": "memory_admission", + "time_ns": 1793264748521570000, + "physical_memory_bytes": 274877906944, + "working_set": 239143780352, + "ceiling": 192414534830, + "budgets": [ + { + "family": "VL-2B", + "bits": 4, + "capacity": 128000, + "checkpoint_bytes": 1590959088, + "cache_per_stream_bytes": 2884159488, + "streams": 4, + "measured_scratch128_bytes": 62104192, + "existing_margin_bytes": 2147483648, + "conservative_total_bytes": 15337184880 + }, + { + "family": "VL-2B", + "bits": 6, + "capacity": 128000, + "checkpoint_bytes": 1783964482, + "cache_per_stream_bytes": 2884159488, + "streams": 4, + "measured_scratch128_bytes": 62104192, + "existing_margin_bytes": 2147483648, + "conservative_total_bytes": 15530190274 + }, + { + "family": "VL-2B", + "bits": 8, + "capacity": 128000, + "checkpoint_bytes": 2216068156, + "cache_per_stream_bytes": 2884159488, + "streams": 4, + "measured_scratch128_bytes": 62104192, + "existing_margin_bytes": 2147483648, + "conservative_total_bytes": 15962293948 + } + ], + "limits_raised": false + } + ], + "lease": [ + { + "event": "lock_acquired", + "time_ns": 1793264748456361000, + "pid": 40272, + "device": 16777233, + "inode": 2622134 + }, + { + "event": "lock_release", + "time_ns": 1793267288518497000, + "pid": 40272 + } + ], + "server_cleanups": [ + { + "event": "owned_cleanup_complete", + "time_ns": 1793265170362169000, + "tag": "vl-4-0-public", + "pid": 40297, + "exit": 0, + "memory": { + "max_rss_bytes": 1866711040, + "peak_physical_footprint_bytes": 5027203232 + } + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793265589467001000, + "tag": "vl-4-1-public", + "pid": 40335, + "exit": 0, + "memory": { + "max_rss_bytes": 1866612736, + "peak_physical_footprint_bytes": 5027170464 + } + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793266013289824000, + "tag": "vl-6-0-public", + "pid": 40372, + "exit": 0, + "memory": { + "max_rss_bytes": 2059649024, + "peak_physical_footprint_bytes": 5220256072 + } + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793266437559171000, + "tag": "vl-6-1-public", + "pid": 40412, + "exit": 0, + "memory": { + "max_rss_bytes": 2059485184, + "peak_physical_footprint_bytes": 5220321560 + } + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793266862870503000, + "tag": "vl-8-0-public", + "pid": 40450, + "exit": 0, + "memory": { + "max_rss_bytes": 2491842560, + "peak_physical_footprint_bytes": 5650401896 + } + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793267288451804000, + "tag": "vl-8-1-public", + "pid": 40489, + "exit": 0, + "memory": { + "max_rss_bytes": 2491924480, + "peak_physical_footprint_bytes": 5652597376 + } + } + ], + "requests": [ + { + "tag": "vl-4-0-public", + "prompt_tokens": 127984, + "usage": { + "prompt_tokens": 127984, + "completion_tokens": 16, + "total_tokens": 128000, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 417.68389100022614, + "elapsed_seconds": 417.8305515833199, + "output_text_sha256": "ccd73036537af583f6afa20104114c902919a3169a1308e36fa01f40270968b4", + "reply_token_sha256": "2cd3959a4daa4603c6708c8feb5c278ccefe64e1dd6a72ae94c175f08b17f5b9", + "drafts_acknowledged": true, + "raw_response_sha256": "1b71960a20d147b13934d56f7ad098f2c65db6bdad10fb437acc50c28f853fa4", + "actual_tokenize_sha256": "dd6ff738fbd506529512dd819b2fc2d965f0057776a97e7b6172b2c343f36d21", + "rendered_ids_sha256": "35662e48e60c43a39bb5cc46ffb09b9bb2e7fbbafda29b15d9a5d071f7d875ca", + "fixture_body_sha256": "858727a862148967fcfa6563399dd3fd38da3ea8d0038370b952967913991fa8", + "exact_tokenize_event": { + "time_ns": 1793264751925346000, + "tag": "vl-4-0-public", + "prompt_tokens": 127984, + "ids_sha256": "35662e48e60c43a39bb5cc46ffb09b9bb2e7fbbafda29b15d9a5d071f7d875ca" + } + }, + { + "tag": "vl-4-1-public", + "prompt_tokens": 127984, + "usage": { + "prompt_tokens": 127984, + "completion_tokens": 16, + "total_tokens": 128000, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 417.0292538749054, + "elapsed_seconds": 417.17672312492505, + "output_text_sha256": "ccd73036537af583f6afa20104114c902919a3169a1308e36fa01f40270968b4", + "reply_token_sha256": "2cd3959a4daa4603c6708c8feb5c278ccefe64e1dd6a72ae94c175f08b17f5b9", + "drafts_acknowledged": true, + "raw_response_sha256": "815b19cf16534428a47ac4a8116a6c4d9a5070d3d2227613f3f8026232081de0", + "actual_tokenize_sha256": "dd6ff738fbd506529512dd819b2fc2d965f0057776a97e7b6172b2c343f36d21", + "rendered_ids_sha256": "35662e48e60c43a39bb5cc46ffb09b9bb2e7fbbafda29b15d9a5d071f7d875ca", + "fixture_body_sha256": "858727a862148967fcfa6563399dd3fd38da3ea8d0038370b952967913991fa8", + "exact_tokenize_event": { + "time_ns": 1793265171656113000, + "tag": "vl-4-1-public", + "prompt_tokens": 127984, + "ids_sha256": "35662e48e60c43a39bb5cc46ffb09b9bb2e7fbbafda29b15d9a5d071f7d875ca" + } + }, + { + "tag": "vl-6-0-public", + "prompt_tokens": 127984, + "usage": { + "prompt_tokens": 127984, + "completion_tokens": 16, + "total_tokens": 128000, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 421.87937845801935, + "elapsed_seconds": 422.03323470801115, + "output_text_sha256": "ee3706db43062351c319f5dbdc8cb7ab3bdc5e160736c5b6bb557242d258e2c7", + "reply_token_sha256": "a09f77d7448e7a505d12252536706338fdde6187ee38914677ebddade9c64e4e", + "drafts_acknowledged": true, + "raw_response_sha256": "3bb7bf146c80e23531aadf67a553882be21d74e57a6b59ebe4361528d54077ea", + "actual_tokenize_sha256": "dd6ff738fbd506529512dd819b2fc2d965f0057776a97e7b6172b2c343f36d21", + "rendered_ids_sha256": "35662e48e60c43a39bb5cc46ffb09b9bb2e7fbbafda29b15d9a5d071f7d875ca", + "fixture_body_sha256": "858727a862148967fcfa6563399dd3fd38da3ea8d0038370b952967913991fa8", + "exact_tokenize_event": { + "time_ns": 1793265590762743000, + "tag": "vl-6-0-public", + "prompt_tokens": 127984, + "ids_sha256": "35662e48e60c43a39bb5cc46ffb09b9bb2e7fbbafda29b15d9a5d071f7d875ca" + } + }, + { + "tag": "vl-6-1-public", + "prompt_tokens": 127984, + "usage": { + "prompt_tokens": 127984, + "completion_tokens": 16, + "total_tokens": 128000, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 422.2483382499777, + "elapsed_seconds": 422.40192537521943, + "output_text_sha256": "ee3706db43062351c319f5dbdc8cb7ab3bdc5e160736c5b6bb557242d258e2c7", + "reply_token_sha256": "a09f77d7448e7a505d12252536706338fdde6187ee38914677ebddade9c64e4e", + "drafts_acknowledged": true, + "raw_response_sha256": "a3a5326ac5e91b2693ee95e3316cd8ed28a514a8dd5d2798bf52633dae25d1ee", + "actual_tokenize_sha256": "dd6ff738fbd506529512dd819b2fc2d965f0057776a97e7b6172b2c343f36d21", + "rendered_ids_sha256": "35662e48e60c43a39bb5cc46ffb09b9bb2e7fbbafda29b15d9a5d071f7d875ca", + "fixture_body_sha256": "858727a862148967fcfa6563399dd3fd38da3ea8d0038370b952967913991fa8", + "exact_tokenize_event": { + "time_ns": 1793266014544143000, + "tag": "vl-6-1-public", + "prompt_tokens": 127984, + "ids_sha256": "35662e48e60c43a39bb5cc46ffb09b9bb2e7fbbafda29b15d9a5d071f7d875ca" + } + }, + { + "tag": "vl-8-0-public", + "prompt_tokens": 127984, + "usage": { + "prompt_tokens": 127984, + "completion_tokens": 16, + "total_tokens": 128000, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 423.1031053336337, + "elapsed_seconds": 423.2528267917223, + "output_text_sha256": "4f75af35be12d31e8c60561af8cbb489c0fd0f3aad2cbac7b3407dd7f996feb8", + "reply_token_sha256": "53f08a02489cbc1f2693407c451ed9bfae09a86285e9aa1f21b303dbc3662895", + "drafts_acknowledged": true, + "raw_response_sha256": "95f341b9984ece250cb9a77c8573f1ca3964ea27b7b462517958bceee0b4df6a", + "actual_tokenize_sha256": "dd6ff738fbd506529512dd819b2fc2d965f0057776a97e7b6172b2c343f36d21", + "rendered_ids_sha256": "35662e48e60c43a39bb5cc46ffb09b9bb2e7fbbafda29b15d9a5d071f7d875ca", + "fixture_body_sha256": "858727a862148967fcfa6563399dd3fd38da3ea8d0038370b952967913991fa8", + "exact_tokenize_event": { + "time_ns": 1793266438830293000, + "tag": "vl-8-0-public", + "prompt_tokens": 127984, + "ids_sha256": "35662e48e60c43a39bb5cc46ffb09b9bb2e7fbbafda29b15d9a5d071f7d875ca" + } + }, + { + "tag": "vl-8-1-public", + "prompt_tokens": 127984, + "usage": { + "prompt_tokens": 127984, + "completion_tokens": 16, + "total_tokens": 128000, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "complete": true, + "ttft_seconds": 423.616200750228, + "elapsed_seconds": 423.7697787499055, + "output_text_sha256": "4f75af35be12d31e8c60561af8cbb489c0fd0f3aad2cbac7b3407dd7f996feb8", + "reply_token_sha256": "53f08a02489cbc1f2693407c451ed9bfae09a86285e9aa1f21b303dbc3662895", + "drafts_acknowledged": true, + "raw_response_sha256": "ea726971c982d765598cbea8b7df29a1ea1a6e837596aca7f92698c5484f4b65", + "actual_tokenize_sha256": "dd6ff738fbd506529512dd819b2fc2d965f0057776a97e7b6172b2c343f36d21", + "rendered_ids_sha256": "35662e48e60c43a39bb5cc46ffb09b9bb2e7fbbafda29b15d9a5d071f7d875ca", + "fixture_body_sha256": "858727a862148967fcfa6563399dd3fd38da3ea8d0038370b952967913991fa8", + "exact_tokenize_event": { + "time_ns": 1793266864146068000, + "tag": "vl-8-1-public", + "prompt_tokens": 127984, + "ids_sha256": "35662e48e60c43a39bb5cc46ffb09b9bb2e7fbbafda29b15d9a5d071f7d875ca" + } + } + ], + "repeat_equality": [ + { + "bits": 4, + "prompt_tokens": 127984, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "2cd3959a4daa4603c6708c8feb5c278ccefe64e1dd6a72ae94c175f08b17f5b9" + }, + { + "bits": 6, + "prompt_tokens": 127984, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "a09f77d7448e7a505d12252536706338fdde6187ee38914677ebddade9c64e4e" + }, + { + "bits": 8, + "prompt_tokens": 127984, + "repeats": 2, + "full_reply_tokens_equal": true, + "reply_token_sha256": "53f08a02489cbc1f2693407c451ed9bfae09a86285e9aa1f21b303dbc3662895" + } + ] + } + ], + "counts": { + "completed_requests": 26, + "completion_tokens": 416, + "cached_tokens": 0, + "repeat_cells": 13, + "fresh_server_cleanups": 16, + "cleanup_exit_zero": 16 + }, + "limits": [ + "Text4 total prompt+completion budget262144 is not a claim of every physical KV slot or simultaneous full-window workload.", + "VL checkpoints here receive text prompts; no image long-context acceptance is asserted.", + "Timings are native context diagnostics without a standard-server comparison or speed qualification.", + "Memory values in server_cleanups are per-process peak physical footprint and maximum RSS, not device allocator peaks.", + "Admission budgets are conservative source-derived estimates; no memory or wired limits were raised.", + "All three packets completed and released their leases; all owned model processes exited." + ], + "outer_controller": { + "execution_events_sha256": "a6206cb70f5f6c3114475b7f23ed2f2686c40a08f82dfcd290ba18bfb1d4872d", + "execution_manifest_sha256": "8f0a532427975f263d8de69a3157fab8dd65b713b7753fc3dec02cb2545bd6e1", + "packet_exits": [ + 0, + 0, + 0 + ], + "controller_exit": 0, + "packets_completed": 3, + "final_readonly_verification": { + "lock_free": true, + "device": 16777233, + "inode": 2622134, + "last_owned_pids_absent": true + } + } +} diff --git a/docs/receipts/lfm3/current-public-long-context.json b/docs/receipts/lfm3/current-public-long-context.json new file mode 100644 index 0000000000..4d22e68544 --- /dev/null +++ b/docs/receipts/lfm3/current-public-long-context.json @@ -0,0 +1,956 @@ +{ + "schema_version": 1, + "scope": "Current public runtime cold long-context HTTP checks. Local48GiB M4 covers text1.7B4bit and VL2B4bit with TEXT prompts; no images in this context experiment.", + "status": "All measured context checks passed; no speed or FP64 fidelity acceptance claimed.", + "runtime_fix_commit": "dd7263c26", + "native_binary_sha256": "3d4a3daeda697420bfbba6d81972896c2bf9e31efaf6f12321ee8487a299354b", + "runs": [ + { + "host": { + "gpu": "Apple M4 Pro", + "gpu_cores": 20, + "physical_memory_bytes": 51539607552, + "os": "ProductName:\t\tmacOS\nProductVersion:\t\t26.5.1\nBuildVersion:\t\t25F80\n" + }, + "status": { + "actual_exit": 0, + "requests_completed": 8, + "server_legs_completed": 4, + "owned_server_exit0": 4, + "all_exact_live_prompt_ids": true, + "all_completion16_cached0": true, + "repeat_cells": 4, + "all_repeated_output_token_hashes_equal": true + }, + "provenance": { + "source_head_at_preparation": "07a7cb4cbc137259c1f53373e59e6d88d8ff8074", + "native_provenance": "Public applied artifact3d4 from attention-range receipt; git diff dd7263c26..07a7cb4 -- zig empty, later commits docs only", + "native_artifact": { + "public": { + "path": "$PACKET/artifacts/tensorfold-native", + "sha256": "3d4a3daeda697420bfbba6d81972896c2bf9e31efaf6f12321ee8487a299354b" + } + }, + "actual_native_version": { + "public": "tensorfold-native 1.1.6" + }, + "client_python_version": "3.13.15 (main, Aug 5 2026, 12:25:43) [Clang 21.0.0 (clang-2100.1.1.101)]", + "client_environment_requirements_sha256": "cdede98457af90bb37dc146204af09dc67059ae7829932d7bae76b9185740ee4", + "checkpoint_files_and_revisions": { + "LFM3-1.7B-MLX-4bit": { + "repository": "LiquidAI/LFM3-1.7B-MLX-4bit", + "revision": "d0ca73dcfa70ce18733754a492cdc1fb0123262d", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 2423, + "sha256": "7df2ea819f9a31b770192a0a9dc1b7953a7dd8c41b816cc3905bbec99ef332bd", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 328, + "sha256": "0095ba7158e938b1e2d3082d412f00d8be85e42cbbd7608b63866f7a081cf270", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 973083386, + "sha256": "1be5bf672013cd91aa62148f77c5eea8d2784a95845d5fe826e08c7b84823815", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 50700, + "sha256": "2c8b88e25f6a1055164dd051436b170872200047bce4b3ec5f2a58f6ef929a75", + "verified_against_pinned_revision": true + }, + "special_tokens_map.json": { + "bytes": 92, + "sha256": "7946f864cbfb62a1b041bda937244210a1a87f4a6cab3a48b791131b2cd57f88", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908634, + "sha256": "41927bd314d17d7d7fadb44e10cd0919f7b71528d7741596d8cbd15c6945ec44", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1859, + "sha256": "b77214b9f12be74b9e5e1dab35d7dace077a31be969314bf38866fd6c8163853", + "verified_against_pinned_revision": true + } + } + }, + "LFM3-VL-2B-MLX-4bit": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "revision": "c823fade98e35a3d6942ad4d38b1ea15f675cd12", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 4419, + "sha256": "0efb7708e0771e3e104ac628954cdc71070dd436e0b7974615bd1b9eb836f9d9", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 323, + "sha256": "30b1115bcdc4e5671567a5b1255a2f3d2f5a912a79292051c6f1e9db81b5e8c4", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 1590959088, + "sha256": "58268dfdf441ae4020d2a52fc97a814645e8a9624946e2d490403d4dcd71a255", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 61256, + "sha256": "38b756dbde167ab7c60756af7d5652414c31afdf4e77700b8d3c99395515d639", + "verified_against_pinned_revision": true + }, + "processor_config.json": { + "bytes": 1254, + "sha256": "f183f5a5e5b3a32e666fce05033e6cb7ca496d26e709a2cee611a13eb7d6b610", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908258, + "sha256": "a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1759, + "sha256": "d63d1ce2f332047e398764b4170352b96629a5194643105bfd6d70c90b750396", + "verified_against_pinned_revision": true + } + } + } + }, + "runner_sha256": "959eda8cc9ebba01f01d0850c3fb59f1bec26a6b9d288581ecc2f95617febe18", + "cold_client_sha256": "7b8ea55aa88834814f95a6f679654e16074b7a8364e34b39a276f009fe0231f2", + "guard_sha256": "0caf67448d401704bd849744584248729decf5fd5b994f669a518cb61891df37" + }, + "protocol": { + "context": 70000, + "configured_parallel": 4, + "actual_simultaneous_requests": 1, + "prompt_cache_gib": 0, + "temperature": 0, + "thinking": false, + "completion_tokens": 16, + "families": [ + "text", + "vl" + ], + "bits": [ + 4 + ], + "cold_prompt_tokens": [ + 32768, + 65536 + ], + "fresh_server_order": [ + "text4", + "text4", + "VL4", + "VL4" + ], + "fresh_KV_each_request_verified_cached0": true, + "draft_field_in_saved_request": "absent; actual runtime draft setting retained per request", + "long_draft_false_comparison": false, + "long_concurrent_solo_comparison": false, + "memory_policy": "Existing native admission unchanged; no memory or wired limits raised. Process footprint measured per server across its32k and64k requests, not per-request GPU allocator peaks.", + "lock_events": [ + { + "event": "lock_wait", + "time_ns": 1793212362725465000, + "path": "$TMPDIR/tensorfold-model.lock" + }, + { + "event": "lock_acquired", + "time_ns": 1793212362725689000, + "path": "$TMPDIR/tensorfold-model.lock", + "pid": 19448, + "device": 16777231, + "inode": 46835039 + }, + { + "event": "complete", + "time_ns": 1793214795988082000, + "legs": 4, + "all_lengths_validated": true, + "performance_gate_claim": false + }, + { + "event": "lock_release", + "time_ns": 1793214795988270000, + "pid": 19448 + } + ], + "owned_load_ready_cleanup_events": [ + { + "event": "owned_model_load_begin", + "time_ns": 1793212364432858000, + "tag": "text-4-0-public", + "pid": 19484, + "command": [ + "$PACKET/artifacts/tensorfold-native", + "serve", + "$MODELS/LFM3-1.7B-MLX-4bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--name", + "lfm3-local", + "--context", + "70000", + "--parallel", + "4", + "--prompt-cache-gib", + "0", + "--no-thinking", + "--temperature", + "0", + "--no-update-check", + "--metrics-open" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "event": "owned_model_ready", + "time_ns": 1793212366601124000, + "tag": "text-4-0-public", + "api_model": "lfm3-local", + "models": [ + { + "id": "lfm3-local", + "object": "model", + "created": 1793212366, + "owned_by": "tensorfold", + "context_length": 70000, + "max_model_len": 70000, + "reasoning": { + "default_enabled": false + } + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793212982412501000, + "tag": "text-4-0-public", + "pid": 19484, + "exit": 0, + "memory": { + "max_rss_bytes": 1231667200, + "peak_physical_footprint_bytes": 2818033392 + } + }, + { + "event": "owned_model_load_begin", + "time_ns": 1793212982639224000, + "tag": "text-4-1-public", + "pid": 29903, + "command": [ + "$PACKET/artifacts/tensorfold-native", + "serve", + "$MODELS/LFM3-1.7B-MLX-4bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--name", + "lfm3-local", + "--context", + "70000", + "--parallel", + "4", + "--prompt-cache-gib", + "0", + "--no-thinking", + "--temperature", + "0", + "--no-update-check", + "--metrics-open" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "event": "owned_model_ready", + "time_ns": 1793212984797712000, + "tag": "text-4-1-public", + "api_model": "lfm3-local", + "models": [ + { + "id": "lfm3-local", + "object": "model", + "created": 1793212984, + "owned_by": "tensorfold", + "context_length": 70000, + "max_model_len": 70000, + "reasoning": { + "default_enabled": false + } + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793213588196102000, + "tag": "text-4-1-public", + "pid": 29903, + "exit": 0, + "memory": { + "max_rss_bytes": 1231994880, + "peak_physical_footprint_bytes": 2822227600 + } + }, + { + "event": "owned_model_load_begin", + "time_ns": 1793213588433067000, + "tag": "vl-4-0-public", + "pid": 40856, + "command": [ + "$PACKET/artifacts/tensorfold-native", + "serve", + "$MODELS/LFM3-VL-2B-MLX-4bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--name", + "lfm3-local", + "--context", + "70000", + "--parallel", + "4", + "--prompt-cache-gib", + "0", + "--no-thinking", + "--temperature", + "0", + "--no-update-check", + "--metrics-open" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "event": "owned_model_ready", + "time_ns": 1793213591598905000, + "tag": "vl-4-0-public", + "api_model": "lfm3-local", + "models": [ + { + "id": "lfm3-local", + "object": "model", + "created": 1793213591, + "owned_by": "tensorfold", + "context_length": 70000, + "max_model_len": 70000, + "reasoning": { + "default_enabled": false + } + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793214192784840000, + "tag": "vl-4-0-public", + "pid": 40856, + "exit": 0, + "memory": { + "max_rss_bytes": 1850359808, + "peak_physical_footprint_bytes": 3478177952 + } + }, + { + "event": "owned_model_load_begin", + "time_ns": 1793214193015609000, + "tag": "vl-4-1-public", + "pid": 51402, + "command": [ + "$PACKET/artifacts/tensorfold-native", + "serve", + "$MODELS/LFM3-VL-2B-MLX-4bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--name", + "lfm3-local", + "--context", + "70000", + "--parallel", + "4", + "--prompt-cache-gib", + "0", + "--no-thinking", + "--temperature", + "0", + "--no-update-check", + "--metrics-open" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "event": "owned_model_ready", + "time_ns": 1793214196162312000, + "tag": "vl-4-1-public", + "api_model": "lfm3-local", + "models": [ + { + "id": "lfm3-local", + "object": "model", + "created": 1793214196, + "owned_by": "tensorfold", + "context_length": 70000, + "max_model_len": 70000, + "reasoning": { + "default_enabled": false + } + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793214795872074000, + "tag": "vl-4-1-public", + "pid": 51402, + "exit": 0, + "memory": { + "max_rss_bytes": 1850261504, + "peak_physical_footprint_bytes": 3486206112 + } + } + ] + }, + "cells": [ + { + "family": "text", + "bits": 4, + "prompt_tokens": 32768, + "fresh_server_repetitions": 2, + "all_output_16_cached0": true, + "native_live_ids_equal_published": true, + "repeated_full_output_token_hash_equal": true, + "output_token_sha256": "e15e7490e640d938a0a846dfc9de1dd7a51e50ae97ca4fd42a213cbb633f66f5" + }, + { + "family": "text", + "bits": 4, + "prompt_tokens": 65536, + "fresh_server_repetitions": 2, + "all_output_16_cached0": true, + "native_live_ids_equal_published": true, + "repeated_full_output_token_hash_equal": true, + "output_token_sha256": "cf6c4062b60dc49e0ca503b7be2aaf3d247d73be037d269cf73e127a5e35aa32" + }, + { + "family": "vl", + "bits": 4, + "prompt_tokens": 32768, + "fresh_server_repetitions": 2, + "all_output_16_cached0": true, + "native_live_ids_equal_published": true, + "repeated_full_output_token_hash_equal": true, + "output_token_sha256": "f35dfbfd86653608bd3aa0fe6672229ecfe9ec17bf45ebcdb91dc7c7e894af4a" + }, + { + "family": "vl", + "bits": 4, + "prompt_tokens": 65536, + "fresh_server_repetitions": 2, + "all_output_16_cached0": true, + "native_live_ids_equal_published": true, + "repeated_full_output_token_hash_equal": true, + "output_token_sha256": "6dc17e16c1bdd265cbfecc43f466db1fb29c9240e7818d912ecc773b568b4856" + } + ], + "requests": [ + { + "family": "text", + "bits": 4, + "leg": 0, + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "native_live_ids_equal_published": true, + "published_ids_file": "docs/receipts/lfm3/cold-prompts/text-32768.ids.json", + "published_ids_json_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f", + "actual_tokenize_response_sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450", + "published_body_file": "docs/receipts/lfm3/cold-prompts/text-32768.body.json", + "published_body_sha256": "6e4c2ecd84a08e202dddce7f9b50bada5c006f9d529a74177f38dc11c08e0bb4", + "actual_sent_body_sha256": "8669dab27e5eb8320e6149804676a48eed58b6f7b2eee972fa92c563e180686d", + "output_token_sha256": "e15e7490e640d938a0a846dfc9de1dd7a51e50ae97ca4fd42a213cbb633f66f5", + "output_text_sha256": "382055acebd5e91b010ffa41b6497ebdb6d22c9bbdd7a2ab220c18df2e11f6b3", + "sampling": "greedy", + "runtime_drafts_setting": true, + "client_ttft_seconds": 139.58678591717035, + "client_elapsed_seconds": 139.8003973751329, + "engine_prefill_seconds": 139.575941459, + "raw_result_sha256": "8439a265a08546347e4935490734747684565af6f7fd96a1662c87ac3cdbc494" + }, + { + "family": "text", + "bits": 4, + "leg": 0, + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "native_live_ids_equal_published": true, + "published_ids_file": "docs/receipts/lfm3/cold-prompts/text-65536.ids.json", + "published_ids_json_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed", + "actual_tokenize_response_sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082", + "published_body_file": "docs/receipts/lfm3/cold-prompts/text-65536.body.json", + "published_body_sha256": "d6dea630f39ee2ddd13e8a88a90050d615d518e21f5a9219e1dc3d2297617b63", + "actual_sent_body_sha256": "5dbd5266aa3cd2635ba020f32f85bdb03ae71155c3114601b55cdd0007f5cfb6", + "output_token_sha256": "cf6c4062b60dc49e0ca503b7be2aaf3d247d73be037d269cf73e127a5e35aa32", + "output_text_sha256": "49b9640f88ffd97f99e1c6a354211b68d7b226874d3d8c1afbc261003f02cb39", + "sampling": "greedy", + "runtime_drafts_setting": true, + "client_ttft_seconds": 474.5117108747363, + "client_elapsed_seconds": 474.7762949159369, + "engine_prefill_seconds": 474.499127333, + "raw_result_sha256": "d09252928059098d7686f05a30c94ae0e3534d6e7905a4b9ec61d6a211fd14b1" + }, + { + "family": "text", + "bits": 4, + "leg": 1, + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "native_live_ids_equal_published": true, + "published_ids_file": "docs/receipts/lfm3/cold-prompts/text-32768.ids.json", + "published_ids_json_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f", + "actual_tokenize_response_sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450", + "published_body_file": "docs/receipts/lfm3/cold-prompts/text-32768.body.json", + "published_body_sha256": "6e4c2ecd84a08e202dddce7f9b50bada5c006f9d529a74177f38dc11c08e0bb4", + "actual_sent_body_sha256": "8669dab27e5eb8320e6149804676a48eed58b6f7b2eee972fa92c563e180686d", + "output_token_sha256": "e15e7490e640d938a0a846dfc9de1dd7a51e50ae97ca4fd42a213cbb633f66f5", + "output_text_sha256": "382055acebd5e91b010ffa41b6497ebdb6d22c9bbdd7a2ab220c18df2e11f6b3", + "sampling": "greedy", + "runtime_drafts_setting": true, + "client_ttft_seconds": 135.97505516605452, + "client_elapsed_seconds": 136.1665180833079, + "engine_prefill_seconds": 135.9630235, + "raw_result_sha256": "58310c03fccf8cea8286c2d2121aa19f7e7b9e062c07277ce56528d63ffc874d" + }, + { + "family": "text", + "bits": 4, + "leg": 1, + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "native_live_ids_equal_published": true, + "published_ids_file": "docs/receipts/lfm3/cold-prompts/text-65536.ids.json", + "published_ids_json_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed", + "actual_tokenize_response_sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082", + "published_body_file": "docs/receipts/lfm3/cold-prompts/text-65536.body.json", + "published_body_sha256": "d6dea630f39ee2ddd13e8a88a90050d615d518e21f5a9219e1dc3d2297617b63", + "actual_sent_body_sha256": "5dbd5266aa3cd2635ba020f32f85bdb03ae71155c3114601b55cdd0007f5cfb6", + "output_token_sha256": "cf6c4062b60dc49e0ca503b7be2aaf3d247d73be037d269cf73e127a5e35aa32", + "output_text_sha256": "49b9640f88ffd97f99e1c6a354211b68d7b226874d3d8c1afbc261003f02cb39", + "sampling": "greedy", + "runtime_drafts_setting": true, + "client_ttft_seconds": 465.85748220793903, + "client_elapsed_seconds": 466.141094957944, + "engine_prefill_seconds": 465.84401475, + "raw_result_sha256": "8877e6ff54c3923bcac03902f521a84967a75ceb72855b6d14318633ec56f671" + }, + { + "family": "vl", + "bits": 4, + "leg": 0, + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "native_live_ids_equal_published": true, + "published_ids_file": "docs/receipts/lfm3/cold-prompts/vl-32768.ids.json", + "published_ids_json_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f", + "actual_tokenize_response_sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450", + "published_body_file": "docs/receipts/lfm3/cold-prompts/vl-32768.body.json", + "published_body_sha256": "6e4c2ecd84a08e202dddce7f9b50bada5c006f9d529a74177f38dc11c08e0bb4", + "actual_sent_body_sha256": "8669dab27e5eb8320e6149804676a48eed58b6f7b2eee972fa92c563e180686d", + "output_token_sha256": "f35dfbfd86653608bd3aa0fe6672229ecfe9ec17bf45ebcdb91dc7c7e894af4a", + "output_text_sha256": "2acefce2656f36e21405907d799ceb7a85e7557924451ed2c622c79474fa0999", + "sampling": "greedy", + "runtime_drafts_setting": true, + "client_ttft_seconds": 145.4195410842076, + "client_elapsed_seconds": 145.65195962507278, + "engine_prefill_seconds": 145.406425, + "raw_result_sha256": "8328d2302a5fcc44b0c25b9794d6307a0e8493144165e8c46a99c1d6030f4627" + }, + { + "family": "vl", + "bits": 4, + "leg": 0, + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "native_live_ids_equal_published": true, + "published_ids_file": "docs/receipts/lfm3/cold-prompts/vl-65536.ids.json", + "published_ids_json_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed", + "actual_tokenize_response_sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082", + "published_body_file": "docs/receipts/lfm3/cold-prompts/vl-65536.body.json", + "published_body_sha256": "d6dea630f39ee2ddd13e8a88a90050d615d518e21f5a9219e1dc3d2297617b63", + "actual_sent_body_sha256": "5dbd5266aa3cd2635ba020f32f85bdb03ae71155c3114601b55cdd0007f5cfb6", + "output_token_sha256": "6dc17e16c1bdd265cbfecc43f466db1fb29c9240e7818d912ecc773b568b4856", + "output_text_sha256": "eb30855e72463651ef28f847b1464ebf1651787ab4add8a37e5bd90565054e97", + "sampling": "greedy", + "runtime_drafts_setting": true, + "client_ttft_seconds": 453.94360912498087, + "client_elapsed_seconds": 454.2238815417513, + "engine_prefill_seconds": 453.929644125, + "raw_result_sha256": "8cc0d5b657dc2494fb1851c41bfb31d86a5fd4795a34bd1ce3d8e62ea7513829" + }, + { + "family": "vl", + "bits": 4, + "leg": 1, + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "native_live_ids_equal_published": true, + "published_ids_file": "docs/receipts/lfm3/cold-prompts/vl-32768.ids.json", + "published_ids_json_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f", + "actual_tokenize_response_sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450", + "published_body_file": "docs/receipts/lfm3/cold-prompts/vl-32768.body.json", + "published_body_sha256": "6e4c2ecd84a08e202dddce7f9b50bada5c006f9d529a74177f38dc11c08e0bb4", + "actual_sent_body_sha256": "8669dab27e5eb8320e6149804676a48eed58b6f7b2eee972fa92c563e180686d", + "output_token_sha256": "f35dfbfd86653608bd3aa0fe6672229ecfe9ec17bf45ebcdb91dc7c7e894af4a", + "output_text_sha256": "2acefce2656f36e21405907d799ceb7a85e7557924451ed2c622c79474fa0999", + "sampling": "greedy", + "runtime_drafts_setting": true, + "client_ttft_seconds": 132.31399954203516, + "client_elapsed_seconds": 132.5132726249285, + "engine_prefill_seconds": 132.301139041, + "raw_result_sha256": "24b7c72f2e09a206610a182551f143a5a04ef5bcc7f4106d626cc4846ee8b15e" + }, + { + "family": "vl", + "bits": 4, + "leg": 1, + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "native_live_ids_equal_published": true, + "published_ids_file": "docs/receipts/lfm3/cold-prompts/vl-65536.ids.json", + "published_ids_json_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed", + "actual_tokenize_response_sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082", + "published_body_file": "docs/receipts/lfm3/cold-prompts/vl-65536.body.json", + "published_body_sha256": "d6dea630f39ee2ddd13e8a88a90050d615d518e21f5a9219e1dc3d2297617b63", + "actual_sent_body_sha256": "5dbd5266aa3cd2635ba020f32f85bdb03ae71155c3114601b55cdd0007f5cfb6", + "output_token_sha256": "6dc17e16c1bdd265cbfecc43f466db1fb29c9240e7818d912ecc773b568b4856", + "output_text_sha256": "eb30855e72463651ef28f847b1464ebf1651787ab4add8a37e5bd90565054e97", + "sampling": "greedy", + "runtime_drafts_setting": true, + "client_ttft_seconds": 465.8405224168673, + "client_elapsed_seconds": 466.1292872498743, + "engine_prefill_seconds": 465.82749725, + "raw_result_sha256": "ed1470fa7ba46b436b662ed9eee87aa39c85634f606c8da363c9935112028e3a" + } + ], + "per_server_footprints": [ + { + "family": "text", + "bits": 4, + "leg": 0, + "max_rss_bytes": 1231667200, + "peak_physical_footprint_bytes": 2818033392 + }, + { + "family": "text", + "bits": 4, + "leg": 1, + "max_rss_bytes": 1231994880, + "peak_physical_footprint_bytes": 2822227600 + }, + { + "family": "vl", + "bits": 4, + "leg": 0, + "max_rss_bytes": 1850359808, + "peak_physical_footprint_bytes": 3478177952 + }, + { + "family": "vl", + "bits": 4, + "leg": 1, + "max_rss_bytes": 1850261504, + "peak_physical_footprint_bytes": 3486206112 + } + ], + "raw_file_manifest": { + "events.json": { + "bytes": 9853, + "sha256": "8e188ba422d7f63b551c8cc1501dfe377ca60b6e159eb71fe611dd09e51741b8" + }, + "manifest.json": { + "bytes": 12583, + "sha256": "acc389270cd5420597c41a50d02e402942ce3ff1c059dac988cce2b909da8984" + }, + "records.json": { + "bytes": 3876, + "sha256": "5f870598f57e044bb2a85d58e498069ea9d2f1250f7ea7dc96558c93db6e9ceb" + }, + "text-4-0-public.after.health": { + "bytes": 298, + "sha256": "9c1b816b0eb1a525cd199e124a13c970db89bfa5eb3feee98a434be7b6699b1e" + }, + "text-4-0-public.after.metrics": { + "bytes": 12200, + "sha256": "9a6ecda2d3f4bdf41b592f05939e48a3c615307b8265f190981a97bb13a1eaa3" + }, + "text-4-0-public.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-0-public.before.health": { + "bytes": 296, + "sha256": "3ecca835aef6168623e5fd09af1a553676471c6c3d76176c12322725ac4e2881" + }, + "text-4-0-public.before.metrics": { + "bytes": 12128, + "sha256": "9cc5e25cd64db521fbd3ce28eeff18076a2f4a0f879a19c235cb03562c224102" + }, + "text-4-0-public.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-0-public.cold-32768.actual-tokenize.json": { + "bytes": 229416, + "sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450" + }, + "text-4-0-public.cold-32768.client.log": { + "bytes": 350, + "sha256": "e41c00cb479e1c2fdc415ee729a6348ab4a7441e2f5cbe1334dab6da33f5a322" + }, + "text-4-0-public.cold-32768.json": { + "bytes": 204611, + "sha256": "8439a265a08546347e4935490734747684565af6f7fd96a1662c87ac3cdbc494" + }, + "text-4-0-public.cold-65536.actual-tokenize.json": { + "bytes": 458792, + "sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082" + }, + "text-4-0-public.cold-65536.client.log": { + "bytes": 349, + "sha256": "65446d1f66f538a305f97b7a18d7c19cb01cf69291c2addc6f64146974149bef" + }, + "text-4-0-public.cold-65536.json": { + "bytes": 401225, + "sha256": "d09252928059098d7686f05a30c94ae0e3534d6e7905a4b9ec61d6a211fd14b1" + }, + "text-4-0-public.server.log": { + "bytes": 2863, + "sha256": "84b0cf8c09526df94ad38acfab7059ce0f86d27539e8cc1ea5ec439bf04297d9" + }, + "text-4-1-public.after.health": { + "bytes": 298, + "sha256": "7e994d2238d049d9c0896261bf62aa264236336a7779efe0b06b110bc6f290db" + }, + "text-4-1-public.after.metrics": { + "bytes": 12195, + "sha256": "68cef663e4ed0b733c6bab028d6707eca56e57b6d00f9a8963d0791cda38341e" + }, + "text-4-1-public.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-1-public.before.health": { + "bytes": 296, + "sha256": "3ecca835aef6168623e5fd09af1a553676471c6c3d76176c12322725ac4e2881" + }, + "text-4-1-public.before.metrics": { + "bytes": 12128, + "sha256": "07dab59d95ce9e3ff675912eda10439f0d5312bf7ff493c2f3b954adb2001ee3" + }, + "text-4-1-public.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-1-public.cold-32768.actual-tokenize.json": { + "bytes": 229416, + "sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450" + }, + "text-4-1-public.cold-32768.client.log": { + "bytes": 350, + "sha256": "9fc3f47eea5f7041ecaa512ffc98f4fc4e113da1a9ecdc235b6fc6434e8cb5b4" + }, + "text-4-1-public.cold-32768.json": { + "bytes": 204608, + "sha256": "58310c03fccf8cea8286c2d2121aa19f7e7b9e062c07277ce56528d63ffc874d" + }, + "text-4-1-public.cold-65536.actual-tokenize.json": { + "bytes": 458792, + "sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082" + }, + "text-4-1-public.cold-65536.client.log": { + "bytes": 349, + "sha256": "2245a3be6b6c94c18056ea94f674285dae8193d649ccbfd06d4bcba3ff838b7b" + }, + "text-4-1-public.cold-65536.json": { + "bytes": 401223, + "sha256": "8877e6ff54c3923bcac03902f521a84967a75ceb72855b6d14318633ec56f671" + }, + "text-4-1-public.server.log": { + "bytes": 2863, + "sha256": "2b70209f2525aca2348c9bba24e97491301f35c1e77b6854791514b3ad4f849b" + }, + "vl-4-0-public.after.health": { + "bytes": 298, + "sha256": "7f5940a84963cdf6dc32d38c332269b420e17dd518e22c258a06f4ca9e637bc4" + }, + "vl-4-0-public.after.metrics": { + "bytes": 12200, + "sha256": "5acd0131f7f835870f392563a1fd7d8085bbd002e0ebddb2a001324cf51f8fad" + }, + "vl-4-0-public.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-0-public.before.health": { + "bytes": 296, + "sha256": "3ecca835aef6168623e5fd09af1a553676471c6c3d76176c12322725ac4e2881" + }, + "vl-4-0-public.before.metrics": { + "bytes": 12128, + "sha256": "07419319a246cf380cab00ccb870a7218a0db42c2a37d6062128ad1b591e1df7" + }, + "vl-4-0-public.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-0-public.cold-32768.actual-tokenize.json": { + "bytes": 229416, + "sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450" + }, + "vl-4-0-public.cold-32768.client.log": { + "bytes": 350, + "sha256": "5819197d4bfa1c0311797a1ada0388dc884658cb8ce00f4192e953abe180a7db" + }, + "vl-4-0-public.cold-32768.json": { + "bytes": 204609, + "sha256": "8328d2302a5fcc44b0c25b9794d6307a0e8493144165e8c46a99c1d6030f4627" + }, + "vl-4-0-public.cold-65536.actual-tokenize.json": { + "bytes": 458792, + "sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082" + }, + "vl-4-0-public.cold-65536.client.log": { + "bytes": 350, + "sha256": "45d3f632481267081e3c53afa0117b8ba63303325432267912b62620973d0074" + }, + "vl-4-0-public.cold-65536.json": { + "bytes": 401229, + "sha256": "8cc0d5b657dc2494fb1851c41bfb31d86a5fd4795a34bd1ce3d8e62ea7513829" + }, + "vl-4-0-public.server.log": { + "bytes": 2864, + "sha256": "5912c925ec3d51d41938cd6fd8b8a44d3565dd7b331bb38935284eaf85f20e64" + }, + "vl-4-1-public.after.health": { + "bytes": 298, + "sha256": "7e994d2238d049d9c0896261bf62aa264236336a7779efe0b06b110bc6f290db" + }, + "vl-4-1-public.after.metrics": { + "bytes": 12198, + "sha256": "d3a1469bb6833f555a20db304f3f6e289e5d8c30160e579eb075a7b5849d1057" + }, + "vl-4-1-public.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-1-public.before.health": { + "bytes": 296, + "sha256": "3ecca835aef6168623e5fd09af1a553676471c6c3d76176c12322725ac4e2881" + }, + "vl-4-1-public.before.metrics": { + "bytes": 12128, + "sha256": "e5434eddfcbbf9f7220903f4888446b01bf95fc4676b2e926e205cb73daf5400" + }, + "vl-4-1-public.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-1-public.cold-32768.actual-tokenize.json": { + "bytes": 229416, + "sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450" + }, + "vl-4-1-public.cold-32768.client.log": { + "bytes": 350, + "sha256": "420ac68f6e0103dab109ab73d7c8b841a9bdf372684bf91972d8aa83e361ce10" + }, + "vl-4-1-public.cold-32768.json": { + "bytes": 204612, + "sha256": "24b7c72f2e09a206610a182551f143a5a04ef5bcc7f4106d626cc4846ee8b15e" + }, + "vl-4-1-public.cold-65536.actual-tokenize.json": { + "bytes": 458792, + "sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082" + }, + "vl-4-1-public.cold-65536.client.log": { + "bytes": 349, + "sha256": "ed726e4880a2b2856ec36560fc411b56ea92a9aba4b11548ff819901f657c26c" + }, + "vl-4-1-public.cold-65536.json": { + "bytes": 401228, + "sha256": "ed1470fa7ba46b436b662ed9eee87aa39c85634f606c8da363c9935112028e3a" + }, + "vl-4-1-public.server.log": { + "bytes": 2864, + "sha256": "ef7bdbe921dcd04489196d7e632ec49b0b833aee8e36a80ad5aca39f835159c1" + } + } + } + ], + "limitations": [ + "VL checkpoint requests contain text only. No long-context image acceptance or image-throughput claim.", + "Repeated replies establish deterministic fresh execution on these exact requests, not an independent FP64 teacher-forced model gate.", + "No same-machine standard or before/after timing comparison here; timings are native context diagnostics only.", + "This local48GiB run covers4bit only. M5 or larger-memory results do not qualify unmeasured6/8bit memory on this host.", + "No128k or maximum native-window/context acceptance from these32k/64k checks.", + "Configured parallel4 reserves the serving setting, but these long requests run sequentially; no long-context four-stream concurrency claim." + ], + "raw_retention": "Full exact native/tokenize responses, public input bodies/arrays, all output/runtime chunks, process logs and model/environment manifests retained privately and hashed. Private paths and request IDs are omitted from publication." +} diff --git a/docs/receipts/lfm3/decode-concurrency.json b/docs/receipts/lfm3/decode-concurrency.json new file mode 100644 index 0000000000..78f991ac5b --- /dev/null +++ b/docs/receipts/lfm3/decode-concurrency.json @@ -0,0 +1,168 @@ +{ + "native_sha256": "5c8a04d4af599879797f4f5bed8e00ab7d9d70732e698bd2a1ec2a01ec762e09", + "variant": "strict row-axis only; original attention", + "families": { + "text": { + "decode": [ + { + "prompt": "fibonacci-raw-chat", + "decode_tps_median": 317.6004846170962, + "ttft_s_median": 0.011276375036686659 + }, + { + "prompt": "gpu-chat-no-think-chat", + "decode_tps_median": 316.8637201461089, + "ttft_s_median": 0.012718458194285631 + } + ], + "concurrency": [ + { + "prompt": "code-chat", + "streams": 1, + "aggregate_tps": 316.9, + "failed": 0, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + } + }, + { + "prompt": "code-chat", + "streams": 4, + "aggregate_tps": 538.4, + "failed": 0, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + } + }, + { + "prompt": "chat-chat", + "streams": 1, + "aggregate_tps": 317.3, + "failed": 0, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + } + }, + { + "prompt": "chat-chat", + "streams": 4, + "aggregate_tps": 530.6, + "failed": 0, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + } + } + ], + "peak_physical_footprint_bytes": 1476495016 + }, + "vl": { + "decode": [ + { + "prompt": "fibonacci-raw-chat", + "decode_tps_median": 302.95882338372087, + "ttft_s_median": 0.011537457816302776 + }, + { + "prompt": "gpu-chat-no-think-chat", + "decode_tps_median": 297.51584456535494, + "ttft_s_median": 0.012835666071623564 + } + ], + "concurrency": [ + { + "prompt": "code-chat", + "streams": 1, + "aggregate_tps": 303.0, + "failed": 0, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + } + }, + { + "prompt": "code-chat", + "streams": 4, + "aggregate_tps": 536.4, + "failed": 0, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + } + }, + { + "prompt": "chat-chat", + "streams": 1, + "aggregate_tps": 296.6, + "failed": 0, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + } + }, + { + "prompt": "chat-chat", + "streams": 4, + "aggregate_tps": 521.8, + "failed": 0, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + } + } + ], + "peak_physical_footprint_bytes": 2136000672 + } + } +} diff --git a/docs/receipts/lfm3/deferred-prompt-head.json b/docs/receipts/lfm3/deferred-prompt-head.json new file mode 100644 index 0000000000..87fba8d9d9 --- /dev/null +++ b/docs/receipts/lfm3/deferred-prompt-head.json @@ -0,0 +1,406 @@ +{ + "status": "Default-off development option; numerical gates passed on M4 and small cold-TTFT reductions observed", + "date": "2026-10-30", + "build_option": "-Dlfm3-defer-prompt-head=true", + "default_enabled": false, + "release_builds": "Enabled option refused with explicit release version; distribution and host-only engine modules remain disabled", + "change": "Skip vocabulary projection and logits copy only on unobserved intermediate prompt chunks; retain transformer work and cache commits. Final heads, marked reuse observers and arbitrary interruption observers retain logits. Recorded/trace forwards reject Head.none.", + "model": { + "family": "LFM3-VL-2B-MLX-4bit", + "weights_sha256": "58268dfdf441ae4020d2a52fc97a814645e8a9624946e2d490403d4dcd71a255", + "config_sha256": "0efb7708e0771e3e104ac628954cdc71070dd436e0b7974615bd1b9eb836f9d9" + }, + "device": "Apple M4", + "model_qualification": { + "prompt_tokens": [ + 2048, + 8192 + ], + "skipped_intermediate_heads": [ + 14, + 62 + ], + "chunk_rows": 128, + "mark_boundaries": [ + 129, + 257 + ], + "checks": [ + "Every chunk active full-addressed KV and convolution state bytes", + "Observed and final full-width BF16 logits", + "Skipped-head cache logits remain unchanged", + "Snapshot bytes at marks", + "Saved prefix257 resume state and final logits", + "16 greedy output IDs and full logits/state per decode step", + "Real backend mark, cancellation, yield, interrupted-cache cleanup, saved-prefix resume and first-token equality" + ], + "observer_events_per_mode": [ + 2, + 3, + 3 + ], + "negative_control": "Deliberately omitting final head rejected with DeferredHeadByteMismatch", + "summary_sha256": "793b02d962098d013878d14768ede9f4d764697fa8ad546482f941f5220877cc", + "prompt_fixture_sha256": { + "vl-2048.npy": "82c7dac5126bf3f7bf789f2a7385214044ad7ace0d151c52935e139ad99bcf2b", + "vl-8192.npy": "b371a7a09adfed00451d2aeba2dc64f5851e7ee62136dddcd8f77e1e07673581" + }, + "prompt_sources": [ + "docs/receipts/lfm3/cold-prompts/vl-2048.ids.json", + "docs/receipts/lfm3/cold-prompts/vl-8192.ids.json" + ] + }, + "image_qualification": { + "scope": "Legacy cat-derived projected rows from earlier preprocessing, not the corrected current processor. Injected70rows into derived512tokens; vision tower not rerun.", + "image_start": 4, + "image_rows": 70, + "prompt_tokens": 512, + "skipped_heads": 2, + "checks": [ + "active KV/conv bytes", + "final logits", + "mark snapshots", + "prefix resume", + "16 decode IDs and logits" + ], + "summary_sha256": "f044d2d922184927cf60e416fcccb0535cb44a23a0b53113553aca9fb0509692" + }, + "served_comparison": { + "scope": "LOCAL M4 VL4 matched private native false/true deferredhead served cold2k8k, two legs each, not MLX comparison", + "order": [ + "baseline", + "candidate", + "candidate", + "baseline" + ], + "samples_per_route": 2, + "prompt_cache": false, + "output_tokens": 16, + "prompt_ids": "Exact pinned token arrays checked through native tokenize endpoint for both routes", + "output_identity": "All response text digests equal. Cold client does not emit token IDs; separate model qualification proves16 IDs/full logits.", + "baseline_binary_sha256": "628c8ef425c003cc4ac22036a46612aada21f8d8bf6dbf2ff382d21fb98389bc", + "candidate_binary_sha256": "65b13cb701af5d30b7d9411352b63972a5e546e48836854671da7cf5ffa37700", + "predeclared_baseline_drift_limit": 0.05, + "cases": [ + { + "prompt_tokens": 2048, + "baseline_ttft_s": [ + 3.3865730413235724, + 3.3919462081976235 + ], + "candidate_ttft_s": [ + 3.3729968750849366, + 3.3754817922599614 + ], + "baseline_median_s": 3.389259624760598, + "candidate_median_s": 3.374239333672449, + "baseline_over_candidate": 1.0044514599003862, + "baseline_first_last_drift": 0.0015866088841098858, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "not emitted by cold client; separate model qualification covers16 IDs/logits", + "prompt_ids": "both routes /tokenize exact pinned arrays" + }, + { + "prompt_tokens": 8192, + "baseline_ttft_s": [ + 17.143353125080466, + 17.1442855829373 + ], + "candidate_ttft_s": [ + 17.07568662473932, + 17.081687083933502 + ], + "baseline_median_s": 17.143819354008883, + "candidate_median_s": 17.07868685433641, + "baseline_over_candidate": 1.0038136714038957, + "baseline_first_last_drift": 5.4391801302289267e-05, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "not emitted by cold client; separate model qualification covers16 IDs/logits", + "prompt_ids": "both routes /tokenize exact pinned arrays" + } + ], + "summary_sha256": "ba05fd65735fa1c6c8dbb10405d8f991d0360e8f71507c7befd84a1597553473", + "limitations": "Small observed gains from two samples per route; no statistical confidence interval, M5, MLX comparison, fresh vision-tower or broad checkpoint claim. Public build option plumbing uses existing lfm3_options; measured pair used equivalent explicit native backend option." + }, + "reproduce": "Build tf-lfm3-deferred-head; run MODEL TOKENS.npy OUTPUT.json, optionally IMAGE_ROWS.bf16 IMAGE_START. Use --bad-final for planted rejection. Inputs require258..8192 tokens. Compare matched native builds with option false/true using the same cold requests, cache disabled and exclusive device admission.", + "public_integration_validation": { + "default_build": "native, tf-lfm3-deferred-head, dev-tools, test-lfm3-prefill-host: passed", + "enabled_build": "native, tf-lfm3-deferred-head, test-lfm3-prefill-host with optiontrue: passed", + "cpu": "apple_m1", + "release_enabled_control": "Rejected with Deferred prompt head is a development-only option", + "lean": { + "baseline": 428, + "current": 428, + "new": 0 + }, + "enabled_native_sha256": "c7229956a5349fa31df3301c4cbe051da2c028c61c761a68a321b885c5b54ef3", + "qualifier_sha256": "32dc59e753e11aa8253e79b028d6aac97f6aa900a8262fdc194785a0627a16ca", + "public_gpu": { + "device": "Apple M4", + "compiled_development_default": true, + "text_prompt_tokens": 2048, + "text_skipped_heads": 14, + "injected_image_prompt_tokens": 512, + "image_skipped_heads": 2, + "full_state_logits_resume_and_16_decode_equal": true, + "actual_backend_mark_cancel_yield_pass": true, + "bad_final_control": "Rejected with DeferredHeadByteMismatch", + "runs_exit_codes": [ + 0, + 0, + 1 + ], + "runner_passed": true, + "receipt_sha256": "3ac964afb306ff2b9cdc174e12980169de8d3a202b3c232480aaf900c24bf999", + "log_sha256": [ + "d799f4634441be1b978e2ceffd7f4032ff312a1f54dc2c8f62b2f93d23cc186c", + "18de5b86c6617d2535e31170c66517cbc26a6a267c020ace5ed891ee1e9bae14", + "1172a8168a0b5e984c0bba5040ff453a87df314216f8ced4a425ddcdda3b1c7e" + ], + "manifest_sha256": "4d299d86188ebf06f6cca420aa0144b722fb58f0bd215bcd7500fba568b19b64" + } + }, + "public_m5_qualification": { + "scope": "Public compiled optiontrue; text2k/8k, legacyinjectedimage512; no M5 timing claim", + "skipped_heads": [ + 14, + 62, + 2 + ], + "full_state_logits_resume_16_decode_equal": true, + "actual_backend_callbacks_pass": true, + "negative_final_control_rejected": true, + "summary_sha256": "f667e2c499b34781fb8b24be3cf45a35a23d1efaa8a0c852fc61f73ae291b96a", + "stage_receipts": { + "smoke": { + "exits": [ + 0 + ], + "receipt_sha256": "38dd8dd4d4df72cf2157a3fa21e66abeb6877bbfefe48e9afc2601dabd1e4aca" + }, + "positive": { + "exits": [ + 0, + 0 + ], + "receipt_sha256": "a901095aa17e91746684785a5fb186154734da0e3f5b4b50201540a80f224c46" + }, + "controls": { + "exits": [ + 1 + ], + "receipt_sha256": "dca51f989b46fc574eaca1df22d70aa33214683e9e3579584c5ad7ca7f9a72d3" + } + } + }, + "corrected_image_qualification": { + "device": "Apple M4", + "scope": "Corrected current-processor cat70projectedrows injected into same512token prompt; tower not rerun in this gate", + "projected_rows_sha256": "ffa0401f452f39a07833d6692ac9eff0d5485cc93a14fb3433bdad74c42ab05e", + "patches_sha256": "9d39c11551f105b8982e3a9acceffcdf687d360c17a7931b7fff21757a6e5473", + "independent_prescribed_native_equal_words": 143360, + "legacy_vs_corrected_different_words": 126898, + "skipped_heads": 2, + "state_logits_snapshots_resume_16_decode_equal": true, + "receipt_sha256": "578d28c60b74a562c647de3bbf8fd62efc828bb46be419c316e1d02155d36db0", + "log_sha256": "18de5b86c6617d2535e31170c66517cbc26a6a267c020ace5ed891ee1e9bae14", + "M5_status": "Passed: same corrected injected rows; full state/logits, two snapshots, resumed prefix257 and16 decode tokens equal; two heads skipped. No fresh tower or timing claim.", + "M5_manifest_sha256": "5f22ce436a487a3571d752c3a8faf153fe70676656e628973f1ea84e425f1a38", + "M5_receipt_sha256": "03dd2eed55e94ab08b1cc709825ac104321111428b569e834c2a2bd27e15d133", + "M5_result_sha256": "bf3ecda81358da9c03799605fffc4fbd2d9fe4140f76737f2b432afad2a65b69", + "M5_log_sha256": "18de5b86c6617d2535e31170c66517cbc26a6a267c020ace5ed891ee1e9bae14" + }, + "composed_attention_deferred_m5": { + "scope": "Same-source native build options false/false versus true/true; VL4 text2k/8k and corrected injected image512. No fresh tower, wider checkpoint or timing claim.", + "manifest_sha256": "aceb3c983a5fbad89672433cbb2c65c6b74682ffe607537af6e0bc30dabd2d5e", + "cross_binary_final_active_kv_conv_logits_sha_and_16_tokens_equal": true, + "per_binary_all_chunk_state_observed_logits_marks_resume_callbacks_pass": true, + "nonvacuous_skipped_heads": [ + 14, + 62, + 2 + ], + "negative_controls": [ + "final head omitted: DeferredHeadByteMismatch", + "planted snapshot hash: ComposedStateMismatch" + ], + "stages": { + "smoke": { + "receipt_sha256": "76913a198435a0319a1f411dd2848da1236f8bb56523d66ba4d0bf4b6fe5b6e3", + "exit_codes": [ + 0, + 0, + 0 + ] + }, + "positive": { + "receipt_sha256": "ee303349a99247bd055609fd7b3c23b526743de5b55cac8e1917bb8cd88df520", + "exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + "controls": { + "receipt_sha256": "e45833c6999f45aa9c425e474d28778b6e4948f1600b1c9d9bca36d9468e8952", + "exit_codes": [ + 1, + 1 + ] + } + } + }, + "composed_m5_serving": { + "scope": "M5 VL4 matched composed attention plus deferred-head native falsefalse/truetrue served cold2k8k, two legs each, not MLX comparison", + "default_off": true, + "protocol": "Fresh native server ABBA, pinned2k then8k IDs, cache0,16 output tokens, no warmup/retry, two legs each.", + "summary_sha256": "664facf3d80af22e2c7146f88fc7c4fe17711a4cad059bbadc5a0657e5358975", + "raw_files_verified": 63, + "cases": [ + { + "prompt_tokens": 2048, + "baseline_ttft_s": [ + 0.8617813750170171, + 0.8616457916796207 + ], + "candidate_ttft_s": [ + 0.8461027080193162, + 0.8467173343524337 + ], + "baseline_median_s": 0.8617135833483189, + "candidate_median_s": 0.8464100211858749, + "baseline_over_candidate": 1.0180805540806366, + "baseline_first_last_drift": -0.00015732915717014961, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "not emitted by cold client; separate model qualification covers16 IDs/logits", + "prompt_ids": "both routes /tokenize exact pinned arrays" + }, + { + "prompt_tokens": 8192, + "baseline_ttft_s": [ + 4.639439832884818, + 4.660513749811798 + ], + "candidate_ttft_s": [ + 4.434221500065178, + 4.424068999942392 + ], + "baseline_median_s": 4.649976791348308, + "candidate_median_s": 4.429145250003785, + "baseline_over_candidate": 1.049858726431321, + "baseline_first_last_drift": 0.004542340818304291, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "not emitted by cold client; separate model qualification covers16 IDs/logits", + "prompt_ids": "both routes /tokenize exact pinned arrays" + } + ], + "server_prefill": { + "metric": "tensorfold:request_prefill_seconds_sum", + "scope": "Per-leg aggregate of two requests (2k+8k); scrape is before/after both, so individual-length prefill duration cannot be recovered. Separate from client TTFT.", + "baseline_sum_s": [ + 5.488429, + 5.509407 + ], + "candidate_sum_s": [ + 5.267624, + 5.257553 + ], + "baseline_median_sum_s": 5.498918, + "candidate_median_sum_s": 5.2625885 + }, + "limitations": "No new ITL/decode benchmark, MLX comparison, fresh vision tower, wider-checkpoint timing or confidence interval. HTTP output text SHA matches; separate numerical gate covers state/logits/token IDs." + }, + "composed_m5_remaining_checkpoints": { + "scope": "Text4/6/8 and VL6/8 at2k/8k, plus corrected injected image512 forVL6/8. VL-rendered text token arrays used as valid model inputs, not a text chat-template parity claim. No fresh vision tower or serving timing.", + "manifest_sha256": "8077a8ff7ba91b0402b2419d89bda748d74a56882a09f6812247ecdadff686a7", + "all_cross_binary_state_logits_16_tokens_equal": true, + "all_per_binary_chunk_marks_resume_checks_pass": true, + "all_five_bad_final_controls_rejected": true, + "cases": [ + { + "case": "LFM3-1.7B-MLX-4bit-vl-2048", + "skipped_heads": 14, + "result_sha256": "5983f83352bd8e216a40fc3b779d90e8a1a0b5cc0dc72ad6d068c34cf032e85c" + }, + { + "case": "LFM3-1.7B-MLX-6bit-vl-2048", + "skipped_heads": 14, + "result_sha256": "01d1e269ba8a7919f4a97c3a30db4dfb46808b514189b82f5e652ce7956c6771" + }, + { + "case": "LFM3-1.7B-MLX-8bit-vl-2048", + "skipped_heads": 14, + "result_sha256": "27b723cec339e4a1c366dd307dd3915c34f9cac8905b6ffc695bfce05e5a04aa" + }, + { + "case": "LFM3-VL-2B-MLX-6bit-vl-2048", + "skipped_heads": 14, + "result_sha256": "7274b0c09df97cc67de9c4764f14e06dbc772650414c098c1f01b0bc1232cf47" + }, + { + "case": "LFM3-VL-2B-MLX-8bit-vl-2048", + "skipped_heads": 14, + "result_sha256": "8db8e859fda9cb72eb18cd7f181694ec104d3148a2888819b7b8bb06db85cdbd" + }, + { + "case": "LFM3-1.7B-MLX-4bit-vl-8192", + "skipped_heads": 62, + "result_sha256": "0e8cbdec06cf2f443745144e96a9230f1c7e1fefcf0d631e386ed2df1c8ac189" + }, + { + "case": "LFM3-1.7B-MLX-6bit-vl-8192", + "skipped_heads": 62, + "result_sha256": "ddecf8ac439b9acf8d3130076440bb3fbac1bfb16b32b01b542262592bf89e4f" + }, + { + "case": "LFM3-1.7B-MLX-8bit-vl-8192", + "skipped_heads": 62, + "result_sha256": "2fc56c8c2af934bddc753cb5738ebc5af4722e11fa4bdd28e20bf1824b7789d2" + }, + { + "case": "LFM3-VL-2B-MLX-6bit-image512", + "skipped_heads": 2, + "result_sha256": "8cc888761ce47b5249ac9d7debbc640fa5524422d9f0d8d29725d6f2e3805038" + }, + { + "case": "LFM3-VL-2B-MLX-6bit-vl-8192", + "skipped_heads": 62, + "result_sha256": "f38fe52957fa7db8e61e0da0438909bf218a6f2227fd7e89b068fe38e882013f" + }, + { + "case": "LFM3-VL-2B-MLX-8bit-image512", + "skipped_heads": 2, + "result_sha256": "6188832c42c9e689dccf8031481fab0593f960a0010edc1bab45bcec645535bd" + }, + { + "case": "LFM3-VL-2B-MLX-8bit-vl-8192", + "skipped_heads": 62, + "result_sha256": "9c50ac37e1bd76b4032c14fac647b67fa596e82f25405f8e5235f8cd7fb43ca6" + } + ], + "stages": { + "smoke": { + "receipt_sha256": "0bbf9951981ff74e47260eba9e44dedce623ebc5c04ad4f7bcab35494b9218f7", + "commands": 15, + "expected_exit": 0 + }, + "positive": { + "receipt_sha256": "dc665bd413262a61ad17e68998880562d2e1b5ffba23384653412e15440edd0d", + "commands": 21, + "expected_exit": 0 + }, + "controls": { + "receipt_sha256": "aa98a8338a64c7bc6bce0fb3e3e131693e4d6baa8e76d7c3f27f82e8796e1bfe", + "commands": 5, + "expected_exit": 1 + } + } + } +} diff --git a/docs/receipts/lfm3/earliest-stage-rounding.json b/docs/receipts/lfm3/earliest-stage-rounding.json new file mode 100644 index 0000000000..3198cf5c0c --- /dev/null +++ b/docs/receipts/lfm3/earliest-stage-rounding.json @@ -0,0 +1,400 @@ +{ + "scope": "First actual divergence on identical input rows, with production reduction rounding and unchanged existing bounds; not whole-model acceptance", + "cases": [ + { + "case": "realcat8", + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-8bit", + "native_diagnostic_sha256": "a6f6d5c4dfa719d9a2e90526a210bae07641da158c39b7772c200d0a1f17b468", + "embedding_equal": 184320, + "first_norm_equal": 184320, + "first_norm_values": 184320, + "projection": { + "values": 552960, + "replay_bf16_equal": 552960, + "replay_vs_fp64_raw_bound_exceedances": 0, + "native_vs_fp64_store_bound_exceedances": 0, + "existing_contract_depth": 36, + "worst_raw_bound_fraction": 0.004461910283213048, + "first_boundary_crossing": { + "row": 0, + "column": 5962, + "fp64": -6.937049874977674e-05, + "replayed_fp32": -6.938725709915161e-05, + "ideal_bf16": -6.914138793945312e-05, + "native_bf16": -6.961822509765625e-05, + "raw_error": 1.6758349374867976e-08, + "existing_raw_bound": 5.486111173345787e-05, + "factorized_magnitude": 25.567076293120408 + }, + "actual_raw_fp32_replay_equal": 552960, + "actual_raw_fp32_values": 552960, + "actual_raw_fp32_max_error_vs_replay": 0.0, + "actual_raw_fp32_sha256": "b0ae65908e77508efa4af926ab951461db3f9e7df00d0f9bd8bd20941a60c4b1", + "helper_source_sha256": "86963d6266a982f4cac7c85265048703841a9545c19f2d2c5271eb8e02a0879b" + }, + "same_native_input_conv": { + "values": 184320, + "fp32_replay_equal": 184320, + "max_error": 0.0 + }, + "guided_first_two_layers": { + "arithmetic": "Independent FP32 affine, RMS norm and convolution reduction replay; other operations retain FP64", + "stages": [ + { + "stage": "layer_00_act", + "values": 691200, + "equal_values": 691200 + }, + { + "stage": "layer_00_conv_gated", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_00_conv_in", + "values": 552960, + "equal_values": 552960 + }, + { + "stage": "layer_00_down", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_00_ffn_norm", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_00_gate", + "values": 691200, + "equal_values": 691200 + }, + { + "stage": "layer_00_mixer", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_00_operator_norm", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_00_output", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_00_post_operator", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_00_up", + "values": 691200, + "equal_values": 691200 + }, + { + "stage": "layer_01_act", + "values": 691200, + "equal_values": 691200 + }, + { + "stage": "layer_01_conv_gated", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_01_conv_in", + "values": 552960, + "equal_values": 552960 + }, + { + "stage": "layer_01_down", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_01_ffn_norm", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_01_gate", + "values": 691200, + "equal_values": 691200 + }, + { + "stage": "layer_01_mixer", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_01_operator_norm", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_01_output", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_01_post_operator", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_01_up", + "values": 691200, + "equal_values": 691200 + } + ], + "values": 7833600, + "equal_values": 7833600 + } + }, + { + "case": "reasoning64-vl6", + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-6bit", + "native_diagnostic_sha256": "a6f6d5c4dfa719d9a2e90526a210bae07641da158c39b7772c200d0a1f17b468", + "embedding_equal": 131072, + "first_norm_equal": 131072, + "first_norm_values": 131072, + "projection": { + "values": 393216, + "replay_bf16_equal": 393216, + "replay_vs_fp64_raw_bound_exceedances": 0, + "native_vs_fp64_store_bound_exceedances": 0, + "existing_contract_depth": 36, + "worst_raw_bound_fraction": 0.0038237892749845067, + "first_boundary_crossing": { + "row": 0, + "column": 246, + "fp64": 0.0001032436775858514, + "replayed_fp32": 0.00010323524475097656, + "ideal_bf16": 0.00010347366333007812, + "native_bf16": 0.000102996826171875, + "raw_error": 8.43283487483859e-09, + "existing_raw_bound": 6.220623788976059e-05, + "factorized_magnitude": 28.990145838870376 + }, + "actual_raw_fp32_replay_equal": 393216, + "actual_raw_fp32_values": 393216, + "actual_raw_fp32_max_error_vs_replay": 0.0, + "actual_raw_fp32_sha256": "9e7b26fa0a4e1b6e77dc7a76c3437a2e4444abcade4fd905a2632acb65300da4", + "helper_source_sha256": "86963d6266a982f4cac7c85265048703841a9545c19f2d2c5271eb8e02a0879b" + }, + "same_native_input_conv": { + "values": 131072, + "fp32_replay_equal": 131072, + "max_error": 0.0 + }, + "guided_first_two_layers": { + "arithmetic": "Independent FP32 affine, RMS norm and convolution reduction replay; other operations retain FP64", + "stages": [ + { + "stage": "layer_00_act", + "values": 491520, + "equal_values": 491520 + }, + { + "stage": "layer_00_conv_gated", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_00_conv_in", + "values": 393216, + "equal_values": 393216 + }, + { + "stage": "layer_00_down", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_00_ffn_norm", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_00_gate", + "values": 491520, + "equal_values": 491520 + }, + { + "stage": "layer_00_mixer", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_00_operator_norm", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_00_output", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_00_post_operator", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_00_up", + "values": 491520, + "equal_values": 491520 + }, + { + "stage": "layer_01_act", + "values": 491520, + "equal_values": 491520 + }, + { + "stage": "layer_01_conv_gated", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_01_conv_in", + "values": 393216, + "equal_values": 393216 + }, + { + "stage": "layer_01_down", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_01_ffn_norm", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_01_gate", + "values": 491520, + "equal_values": 491520 + }, + { + "stage": "layer_01_mixer", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_01_operator_norm", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_01_output", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_01_post_operator", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_01_up", + "values": 491520, + "equal_values": 491520 + } + ], + "values": 5570560, + "equal_values": 5570560 + } + } + ], + "raw_capture": "Diagnostic Metal tap calls unchanged production tf_rp_rows_at<4,2,BITS,false> before BF16 storage; every rounded captured output matches native.", + "query_underflow_observation": { + "case": "realcat8", + "layer": 2, + "values": 184320, + "differing_values": 470, + "columns": [ + 56, + 59, + 108 + ], + "all_differing_native_values_zero": true, + "maximum_absolute_error": 1.1663108012064884e-38, + "examples": [ + { + "row": 0, + "head": 1, + "column": 59, + "reference": -7.071333204165323e-39, + "native": 0.0 + }, + { + "row": 0, + "head": 3, + "column": 59, + "reference": -7.989688165745235e-39, + "native": 0.0 + }, + { + "row": 0, + "head": 14, + "column": 56, + "reference": -2.938735877055719e-39, + "native": 0.0 + }, + { + "row": 0, + "head": 14, + "column": 59, + "reference": 1.0101904577379033e-39, + "native": 0.0 + }, + { + "row": 0, + "head": 14, + "column": 108, + "reference": 1.0285575569695016e-38, + "native": 0.0 + }, + { + "row": 1, + "head": 1, + "column": 56, + "reference": -3.03057137321371e-39, + "native": -0.0 + }, + { + "row": 1, + "head": 1, + "column": 59, + "reference": -1.285696946211877e-39, + "native": 0.0 + }, + { + "row": 1, + "head": 1, + "column": 108, + "reference": 5.969307250269429e-39, + "native": 0.0 + }, + { + "row": 1, + "head": 3, + "column": 59, + "reference": -9.09171411964113e-39, + "native": 0.0 + }, + { + "row": 1, + "head": 14, + "column": 56, + "reference": 4.408103815583578e-39, + "native": 0.0 + } + ], + "scope": "Observed native zeros versus preserved BF16 subnormals in the CPU reference; not an end-to-end acceptance waiver" + } +} diff --git a/docs/receipts/lfm3/ffn-portable-reuse-rejected.json b/docs/receipts/lfm3/ffn-portable-reuse-rejected.json new file mode 100644 index 0000000000..e4e51740bf --- /dev/null +++ b/docs/receipts/lfm3/ffn-portable-reuse-rejected.json @@ -0,0 +1,122 @@ +{ + "scope": "Source-free ledger of rejected standalone public-arithmetic FFN weight-reuse experiments. No production default change.", + "fixture": { + "checkpoint": "LiquidAI/LFM3-1.7B-MLX-4bit", + "operation": "FFN gate", + "rows": 64, + "k": 2048, + "tested_columns": 2048, + "full_columns": 7680, + "column_start": 0 + }, + "variants": { + "cached4": "Four rows, decode fourcolumns into64floatlane cache before originalpairouterloop; preserves oneactivationloadperpair.", + "cached8": "Same decodedlane cache across eightrows.", + "triples": "Existing public three-row BF16-storage helper with original two/one-rowtails.", + "shared-pairs-float-LDS": "Four SIMDgroups each retain2rows/4cols; share decodedFP32codes and duplicatedcoefficients via9216byteLDS, two barriers/K512.", + "compact-transposed-LDS": "Same2rows/SG,128threads4SG, ushortcodes[col][element][lane] and pergroupBF16coefficients;4352byteLDSvs9216; same2barriers/K512. No measured hardwarebankstall attribution." + }, + "gates": "Cached4/8 and triples:25case byteidentity screens including GS32/64 bits4/6/8 tails/NF, actualslice everycolumnFP64no-worse and plantednegativecontrols. Shared-pairs: bounded actualslice canonical/FP64gate. These are not full66 or full-model qualifications. CompactLDS:14case screen actual64+derived65tail and6format9rowfinite/NF, everyactualcolumnFP64no-worse, two plantedcontrols.", + "cost_protocol": { + "balanced_rounds": 5, + "legs": 20, + "min_calibration_seconds": 0.025, + "min_sample_seconds": 0.02, + "max_sample_seconds": 1, + "baseline_drift_and_spread_limit": 0.05, + "same_output_and_guards_every_leg": true, + "no_hidden_retries": true + }, + "results": [ + { + "device": "Apple M4", + "variant": "cached4", + "baseline_seconds_per_op": 0.0002635113924043253, + "candidate_seconds_per_op": 0.00033853125205496326, + "median_candidate_over_public_throughput": 0.7777216816172479, + "stable": true, + "baseline_first_last_drift": 0.00020938915523638357, + "analysis_sha256": "fac52715a1f5af9211fde42ac13e0e1dac02875ed5f8883f8da56e35257dfd31", + "full_output_matches_frozen_canonical": true + }, + { + "device": "Apple M4", + "variant": "cached8", + "baseline_seconds_per_op": 0.00026354947658546735, + "candidate_seconds_per_op": 0.00031502587989962194, + "median_candidate_over_public_throughput": 0.8364290055429738, + "stable": true, + "baseline_first_last_drift": 0.00010066294419197597, + "analysis_sha256": "68f6d3a3b76010080590266b369c26bd879e5c6cf4ce7b0cda4468b464a6b5e3", + "full_output_matches_frozen_canonical": true + }, + { + "device": "Apple M5", + "variant": "cached4", + "baseline_seconds_per_op": 6.72171222504403e-05, + "candidate_seconds_per_op": 0.00010990905730068334, + "median_candidate_over_public_throughput": 0.6113964318541105, + "stable": true, + "baseline_first_last_drift": -0.00010351041271328132, + "analysis_sha256": "e04c8bdbffaff2bdf3d162a9b5a9f49e7001c7c1802e5a903b60c9e50c6eef34", + "full_output_matches_frozen_canonical": true + }, + { + "device": "Apple M5", + "variant": "cached8", + "baseline_seconds_per_op": 6.725671346430317e-05, + "candidate_seconds_per_op": 0.00011073811811002088, + "median_candidate_over_public_throughput": 0.6071178337837835, + "stable": true, + "baseline_first_last_drift": -4.234749437015406e-05, + "analysis_sha256": "08030cd6a0072859a4f2b2bb0038f79ed20d6077ed5f9fb331f2c1e050293bca", + "full_output_matches_frozen_canonical": true + }, + { + "device": "Apple M4", + "variant": "triples", + "baseline_seconds_per_op": 0.0002635327164171031, + "candidate_seconds_per_op": 0.0002824956045515137, + "median_candidate_over_public_throughput": 0.9311926863223228, + "stable": true, + "baseline_first_last_drift": -0.0018862538425180597, + "analysis_sha256": "b684e7ab47319340e32193823afe501917af46b75e9de422859c0bcff1fcf2c1", + "full_output_matches_frozen_canonical": true + }, + { + "device": "Apple M5", + "variant": "shared-pairs-float-LDS", + "baseline_seconds_per_op": 6.718225085933227e-05, + "candidate_seconds_per_op": 8.050610358623089e-05, + "median_candidate_over_public_throughput": 0.8345021465595635, + "stable": true, + "baseline_first_last_drift": 0.0001514320707756145, + "analysis_sha256": "eea69bcd98dc39c4d917eb955437027e252bf6408a48c042cfbf744c258937fd", + "full_output_matches_frozen_canonical": true + }, + { + "device": "Apple M4", + "variant": "compact-transposed-LDS", + "baseline_seconds_per_op": 0.00026341683042119257, + "candidate_seconds_per_op": 0.00034259798121638596, + "median_candidate_over_public_throughput": 0.7688219515629374, + "stable": true, + "baseline_first_last_drift": -0.00029217137416359673, + "analysis_sha256": "16b418f778bd3b66dae4219f5b2b860e1cc018d13d84452177b8f1826b6d5ad1", + "full_output_matches_frozen_canonical": true + }, + { + "device": "Apple M5", + "variant": "compact-transposed-LDS", + "baseline_seconds_per_op": 6.723702017552569e-05, + "candidate_seconds_per_op": 9.397123221788206e-05, + "median_candidate_over_public_throughput": 0.7155009673949159, + "stable": true, + "baseline_first_last_drift": -0.00010045612681630622, + "analysis_sha256": "b12ecc11ddfec20e7b88cff6a3133639c7bdde6c9680a7080fb95214859be5ab", + "full_output_matches_frozen_canonical": true + } + ], + "conclusion": "All listed variants are slower than their matched public row-pair baseline. Do not promote or extrapolate a gain from source-level weight reuse. No measured register-spill, cache-miss or LDS-bank-stall attribution is established.", + "pending": "M5 three-row timing not included. Stop nearby reuse tuning until new profiling evidence." +} diff --git a/docs/receipts/lfm3/fidelity-investigation.json b/docs/receipts/lfm3/fidelity-investigation.json new file mode 100644 index 0000000000..c4e7a8341e --- /dev/null +++ b/docs/receipts/lfm3/fidelity-investigation.json @@ -0,0 +1,184 @@ +{ + "checkpoint_policy_evidence": { + "mlx_version": "0.32.4", + "official_source_commit": "36969c094354086e15d2a327a9c50feb20b9cf11", + "source_tag": "v0.32.4", + "device": { + "device_name": "Apple M4 Pro", + "max_recommended_working_set_size": 40200896512, + "memory_size": 51539607552, + "architecture": "applegpu_g16s", + "max_buffer_length": 30150672384, + "resource_limit": 499000 + }, + "projection": "language_model.model.layers.0.conv.in_proj", + "bits": 8, + "group_size": 64, + "K": 2048, + "N": 6144, + "input": "same first BF16 operator_norm row repeated M times", + "results": [ + { + "rows": 1, + "dtype": "mlx.core.bfloat16", + "same_as_exact_dequant_FP64_rounded": 6144, + "same_as_BF16_weights_FP64_rounded": 3803, + "outputs": 6144, + "rms_vs_exact_FP64": 0.0003453614152093189 + }, + { + "rows": 2, + "dtype": "mlx.core.bfloat16", + "same_as_exact_dequant_FP64_rounded": 6144, + "same_as_BF16_weights_FP64_rounded": 3803, + "outputs": 6144, + "rms_vs_exact_FP64": 0.0003453614152093189 + }, + { + "rows": 4, + "dtype": "mlx.core.bfloat16", + "same_as_exact_dequant_FP64_rounded": 6144, + "same_as_BF16_weights_FP64_rounded": 3803, + "outputs": 6144, + "rms_vs_exact_FP64": 0.0003453614152093189 + }, + { + "rows": 8, + "dtype": "mlx.core.bfloat16", + "same_as_exact_dequant_FP64_rounded": 6144, + "same_as_BF16_weights_FP64_rounded": 3803, + "outputs": 6144, + "rms_vs_exact_FP64": 0.0003453614152093189 + }, + { + "rows": 12, + "dtype": "mlx.core.bfloat16", + "same_as_exact_dequant_FP64_rounded": 6144, + "same_as_BF16_weights_FP64_rounded": 3803, + "outputs": 6144, + "rms_vs_exact_FP64": 0.0003453614152093189 + }, + { + "rows": 13, + "dtype": "mlx.core.bfloat16", + "same_as_exact_dequant_FP64_rounded": 3803, + "same_as_BF16_weights_FP64_rounded": 6143, + "outputs": 6144, + "rms_vs_exact_FP64": 0.0004261891067811766 + }, + { + "rows": 16, + "dtype": "mlx.core.bfloat16", + "same_as_exact_dequant_FP64_rounded": 3803, + "same_as_BF16_weights_FP64_rounded": 6143, + "outputs": 6144, + "rms_vs_exact_FP64": 0.0004261891067811766 + }, + { + "rows": 32, + "dtype": "mlx.core.bfloat16", + "same_as_exact_dequant_FP64_rounded": 3803, + "same_as_BF16_weights_FP64_rounded": 6143, + "outputs": 6144, + "rms_vs_exact_FP64": 0.0004261891067811766 + }, + { + "rows": 64, + "dtype": "mlx.core.bfloat16", + "same_as_exact_dequant_FP64_rounded": 3803, + "same_as_BF16_weights_FP64_rounded": 6144, + "outputs": 6144, + "rms_vs_exact_FP64": 0.0004261898084132233 + } + ], + "M1_M64_exact_equal": 3803, + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-8bit", + "checkpoint_revision": "83336d5be3874a2bdd3297bde77b2114dbd3a816", + "fixture_sha256": "9c3636ee22fb5d1a9d4d6e3cc0b8e47cce20cb4288bb16f2970de8d55ee27708", + "source_links": [ + "https://github.com/ml-explore/mlx/blob/36969c094354086e15d2a327a9c50feb20b9cf11/mlx/backend/metal/quantized.cpp#L89", + "https://github.com/ml-explore/mlx/blob/36969c094354086e15d2a327a9c50feb20b9cf11/mlx/backend/metal/quantized.cpp#L1900", + "https://github.com/ml-explore/mlx/blob/36969c094354086e15d2a327a9c50feb20b9cf11/mlx/backend/metal/kernels/quantized.h#L555", + "https://github.com/ml-explore/mlx/blob/36969c094354086e15d2a327a9c50feb20b9cf11/mlx/backend/metal/kernels/quantized.h#L674", + "https://github.com/ml-explore/mlx/blob/36969c094354086e15d2a327a9c50feb20b9cf11/mlx/backend/metal/kernels/quantized.h#L851" + ], + "interpretation": "Official MLX uses FP32 affine factorization for these vector widths and BF16 reconstructed weights for these matrix widths. Native uses one affine reconstruction policy across prompt and decode. This evidence alone is not a model-fidelity acceptance policy." + }, + "projection_error_audit": { + "cells": 66, + "outputs": 60954624, + "raw_tap_matches_all_native": true, + "one_row_tap_matches_all_native_first_rows": true, + "raw_FP32_bound_exceedances": 0, + "BF16_bound_exceedances": 0, + "max_raw_FP32_bound_fraction": 0.01465208540032716, + "max_BF16_bound_fraction": 0.9807553251550621, + "max_raw_abs_error": 2.555781975388527e-06, + "max_worst_column_raw_rms": 7.7385387436973e-07, + "model_acceptance_threshold": null + }, + "first_actual_miss_primitive": { + "source_sha256": "86963d6266a982f4cac7c85265048703841a9545c19f2d2c5271eb8e02a0879b", + "formula": "shared tf_rp_rows_at<4,2,BITS,false>; FP32 accumulator captured before BF16 store", + "tap_rounds_to_native_bytes": true, + "row": 1, + "column": 4373, + "expected_FP64": -6.1827595345675945e-06, + "actual_FP32": -6.18956983089447e-06, + "error": -6.810296326875687e-09, + "ideal_BF16": -6.16908073425293e-06, + "native_BF16": -6.198883056640625e-06, + "existing_contract_depth": 36, + "factorized_term_magnitude": 25.70957951276796, + "existing_contract_FP32_work_bound": 5.5166891125901165e-05, + "fraction_of_work_bound": 0.00012344897796276604 + }, + "causal_history_isolation": { + "row": 17, + "native_final_norm_FP64_head_maximum_set": [ + 296 + ], + "native_maximum_set": [ + 296 + ], + "whole_FP64_model_maximum_set": [ + 405 + ], + "native_input_FP64_head_matches_native": 127999, + "head_values": 128000, + "same_input_conv": [ + { + "layer": 0, + "same_native_inputs_correctly_rounded_values": 526329, + "values": 526336, + "row17_native_input_output_differences": 0, + "row17_max_abs_error": 0.0, + "max_abs_error": 4.76837158203125e-07 + }, + { + "layer": 1, + "same_native_inputs_correctly_rounded_values": 526328, + "values": 526336, + "row17_native_input_output_differences": 0, + "row17_max_abs_error": 0.0, + "max_abs_error": 1.9073486328125e-06 + } + ] + }, + "whole_model_direct_FP64_comparison": { + "cells": 17, + "rows": 18, + "native_in_oracle_maximum": 9, + "mlx_in_oracle_maximum": 16, + "both_in_maximum": 8, + "neither_in_maximum": 1, + "accepted_threshold": null + }, + "interpretation": [ + "The audited shared projection implementation obeys its existing FP32 and BF16 store error budgets on the measured input classes.", + "The real mismatch traces to a measured FP32 accumulation error crossing a BF16 boundary and propagating through causal history.", + "Official MLX uses different effective reconstruction arithmetic for vector and matrix paths on the measured projection.", + "No new end-to-end model-fidelity tolerance or tie policy is defined by this evidence.", + "Model-fidelity acceptance remains unresolved; a prospective faster projection must be compared independently before adoption." + ] +} diff --git a/docs/receipts/lfm3/fidelity-prompts/code-257.ids.json b/docs/receipts/lfm3/fidelity-prompts/code-257.ids.json new file mode 100644 index 0000000000..b5c9d5c2cf --- /dev/null +++ b/docs/receipts/lfm3/fidelity-prompts/code-257.ids.json @@ -0,0 +1 @@ +[22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129] diff --git a/docs/receipts/lfm3/fidelity-prompts/code-513.ids.json b/docs/receipts/lfm3/fidelity-prompts/code-513.ids.json new file mode 100644 index 0000000000..8c2cb799d6 --- /dev/null +++ b/docs/receipts/lfm3/fidelity-prompts/code-513.ids.json @@ -0,0 +1 @@ +[22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318] diff --git a/docs/receipts/lfm3/fidelity-prompts/code-64.ids.json b/docs/receipts/lfm3/fidelity-prompts/code-64.ids.json new file mode 100644 index 0000000000..0fdc90d682 --- /dev/null +++ b/docs/receipts/lfm3/fidelity-prompts/code-64.ids.json @@ -0,0 +1 @@ +[22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162] diff --git a/docs/receipts/lfm3/fidelity-prompts/manifest.json b/docs/receipts/lfm3/fidelity-prompts/manifest.json new file mode 100644 index 0000000000..2e6e92b679 --- /dev/null +++ b/docs/receipts/lfm3/fidelity-prompts/manifest.json @@ -0,0 +1,119 @@ +[ + { + "name": "prose-64.npy", + "source_text": "Explain why the sky is blue, then describe how the color changes at sunset. Give a clear explanation using ordinary words. ", + "repeat_count": 20, + "special_tokens": false, + "tokenizer": "LiquidAI/LFM3-1.7B-MLX-4bit/tokenizer.json", + "nominal_token_count": 64, + "actual_token_count": 64, + "dtype": " b:\n return a\n else:\n " + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + } + } + ], + "last": [ + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + } + } + ] + } + }, + "speed_qualified": false + } + ], + "counts": { + "passed": 120, + "failed": 0, + "skipped": 0 + }, + "vl_refusals": [ + { + "bits": 4, + "vl_refusal": true, + "exit_status": 1 + }, + { + "bits": 6, + "vl_refusal": true, + "exit_status": 1 + }, + { + "bits": 8, + "vl_refusal": true, + "exit_status": 1 + } + ] + }, + "performance": { + "scope": "M5 4-bit text/VL native old/new/new/old; three repetitions per code/prose prompt, 256 output tokens each. A small measured dispatch gain, not a standard-server gate pass.", + "hardware": "Apple M5 Ultra 256 GiB, macOS 27.0 build 26A428", + "summary": { + "baseline_sha256": "44220f39df174b2364172eac4b8705dbd5f5f47f3175c7ed300a6d47ae3bbcbc", + "candidate_sha256": "e9a1488abb130af39c21ee08068fe490d5926271cebcd2e9d0f815711e3730b4", + "order": [ + "old", + "new", + "new", + "old" + ], + "counts": { + "alone_equal": 240, + "alone_unequal": 0, + "alone_failed": 0, + "serial_equal": 32, + "serial_unequal": 0, + "serial_failed": 0 + }, + "full_hash_requests": 240, + "hash_groups": [ + { + "family": "text", + "prompt": "long-code-chat", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7" + }, + { + "family": "text", + "prompt": "long-prose-chat", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a" + }, + { + "family": "vl", + "prompt": "long-code-chat", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee" + }, + { + "family": "vl", + "prompt": "long-prose-chat", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa" + } + ], + "cells": [ + { + "family": "text", + "mode": "four_stream", + "prompt": "long-code-chat", + "old_run_medians": [ + 837.0, + 838.8 + ], + "new_run_medians": [ + 850.0, + 849.2 + ], + "old_median": 837.9, + "new_median": 849.6, + "rate_ratio": 1.0139634801288937 + }, + { + "family": "text", + "mode": "four_stream", + "prompt": "long-prose-chat", + "old_run_medians": [ + 857.0, + 858.1 + ], + "new_run_medians": [ + 869.5, + 868.0 + ], + "old_median": 857.55, + "new_median": 868.75, + "rate_ratio": 1.013060462946767 + }, + { + "family": "text", + "mode": "solo", + "prompt": "long-code-chat", + "old_run_medians": [ + 317.52504786591186, + 315.9518189760151 + ], + "new_run_medians": [ + 320.5081878043543, + 319.67577794253697 + ], + "old_median": 316.7384334209635, + "new_median": 320.09198287344566, + "rate_ratio": 1.0105877566427979 + }, + { + "family": "text", + "mode": "solo", + "prompt": "long-prose-chat", + "old_run_medians": [ + 333.35962421230994, + 331.1493532644477 + ], + "new_run_medians": [ + 335.87454621476815, + 335.0961656322742 + ], + "old_median": 332.2544887383788, + "new_median": 335.4853559235212, + "rate_ratio": 1.0097240738489657 + }, + { + "family": "vl", + "mode": "four_stream", + "prompt": "long-code-chat", + "old_run_medians": [ + 814.1, + 813.1 + ], + "new_run_medians": [ + 824.5, + 824.3 + ], + "old_median": 813.6, + "new_median": 824.4, + "rate_ratio": 1.0132743362831858 + }, + { + "family": "vl", + "mode": "four_stream", + "prompt": "long-prose-chat", + "old_run_medians": [ + 810.8, + 809.5 + ], + "new_run_medians": [ + 821.7, + 819.8 + ], + "old_median": 810.15, + "new_median": 820.75, + "rate_ratio": 1.0130839967907177 + }, + { + "family": "vl", + "mode": "solo", + "prompt": "long-code-chat", + "old_run_medians": [ + 301.8038551050975, + 302.2026871330548 + ], + "new_run_medians": [ + 305.9170505050233, + 305.175979302901 + ], + "old_median": 302.00327111907615, + "new_median": 305.54651490396213, + "rate_ratio": 1.0117324682337263 + }, + { + "family": "vl", + "mode": "solo", + "prompt": "long-prose-chat", + "old_run_medians": [ + 299.7549943842911, + 300.1215491541061 + ], + "new_run_medians": [ + 303.7439269888273, + 302.96962197420424 + ], + "old_median": 299.9382717691986, + "new_median": 303.3567744815158, + "rate_ratio": 1.0113973541694197 + } + ] + }, + "decode_client_scope": "Single-stream client stores a truncated token digest and full output-text SHA; full token SHA equality is established by concurrent request evidence.", + "clients": { + "_trimmed_value": { + "bytes": 408421, + "sha256": "184b143f38865f9b512f761e6e0f57758881241fb853f38ede05fc69ec51e814", + "keys": null, + "count": 16 + } + } + }, + "_trim": { + "note": "Raw capture replaced by counts, first/last items and SHA-256 of the canonical JSON of each trimmed list or string.", + "original_sha256": "43c0a460bf22219b5bdc1ed352c99d7baa7ea65ee7e292f446e48459b20ad7e2", + "original_bytes": 1191335 + } +} diff --git a/docs/receipts/lfm3/local-standard-phase1.json b/docs/receipts/lfm3/local-standard-phase1.json new file mode 100644 index 0000000000..3984b2161b --- /dev/null +++ b/docs/receipts/lfm3/local-standard-phase1.json @@ -0,0 +1,1493 @@ +{ + "schema_version": 1, + "scope": "Actual locked local Apple M4 Pro phase1: text4 and VL4, code/prose text requests, exact256-token solo/four-stream decode, cold2048/8192 with16 outputs. No performance gate pass.", + "status": { + "actual_exit_code": 0, + "server_legs": 12, + "measured_decode_requests": 360, + "cold_requests": 24, + "all_lengths_validated": true, + "performance_gate_claim": false + }, + "host": { + "os": "ProductName:\t\tmacOS\nProductVersion:\t\t26.5.1\nBuildVersion:\t\t25F80\n", + "gpu": "Apple M4 Pro", + "gpu_cores": 20, + "physical_memory_bytes": 51539607552 + }, + "provenance": { + "source_base": "21728fcd8498c809dc2737e47f4e8b6977d9f5a1", + "native_artifacts": { + "public": { + "path": "$PACKET/artifacts/tensorfold-native-public", + "sha256": "ad3ca5a0d39daa399b5bde058514291ec52f87a50799528127c654d30a139672" + }, + "combined": { + "path": "$PACKET/artifacts/tensorfold-native-combined", + "sha256": "ad72d7ec17b681ede54676ba75f7d4d1b39282aa0dae7cf27d8b201660fce489" + } + }, + "standard": { + "python": "$PACKET/standard-venv/bin/python", + "module": "mlx_vlm.server", + "source_commit": "230fc308efb577d87446306bfb8a3e9f1a129558", + "mlx": "0.32.4", + "mlx_metal": "0.32.4", + "mlx_vlm": "0.7.11", + "dependency_provenance": "Core source/MLX pins match prior standard procedure. Other dependencies are newly resolved and fully frozen; not asserted identical to the old M5 environment.", + "memory_policy": "Unmodified MLX defaults; no memory/wired limit overrides.", + "request_context_limit": 70000, + "context_policy": "Pinned --max-kv-size70000 sets request budget; LFM3 family cache construction remains lazy and is not claimed allocation-identical to native.", + "checkpoint_context_caps": { + "text": 262144, + "vl": 128000 + } + }, + "versions": { + "python": "3.13.15 (main, Aug 5 2026, 12:25:43) [Clang 21.0.0 (clang-2100.1.1.101)]", + "packages": { + "shellingham": "1.5.4", + "uvicorn": "0.54.0", + "xxhash": "4.0.2", + "filelock": "4.2.4", + "mlx": "0.32.4", + "starlette": "1.10.0", + "huggingface_hub": "2.3.0", + "click": "8.5.0", + "packaging": "26.3", + "tokenizers": "0.23.3", + "Pygments": "2.21.0", + "pyarrow": "26.0.0", + "sounddevice": "0.5.6", + "requests": "2.34.2", + "rich": "15.0.0", + "certifi": "2026.10.23", + "yarl": "1.25.1", + "websockets": "17.3", + "mlx-vlm": "0.7.11", + "opentelemetry-api": "1.45.1", + "pandas": "3.1.0", + "datasets": "5.1.0", + "sentencepiece": "0.2.2", + "propcache": "0.5.4", + "llguidance": "1.9.1", + "numpy": "2.5.4", + "aiohappyeyeballs": "2.7.1", + "cryptography": "50.0.2", + "pycparser": "3.12", + "pillow": "13.0.0", + "fsspec": "2026.7.0", + "hf-xet": "1.7.0", + "safetensors": "0.8.0", + "miniaudio": "1.71", + "tqdm": "4.70.1", + "urllib3": "2.8.0", + "h11": "0.16.0", + "pydantic_core": "2.50.1", + "mdurl": "0.1.2", + "python-multipart": "0.0.32", + "mlx-metal": "0.32.4", + "PyYAML": "6.0.3", + "httpcore2": "2.16.0", + "opencv-python": "5.0.0.93", + "truststore": "0.10.5", + "frozenlist": "1.8.0", + "cffi": "2.1.1", + "httpx2": "2.16.0", + "aiosignal": "1.4.0", + "typer": "0.27.3", + "mlx-audio": "0.6.2", + "Jinja2": "3.1.6", + "annotated-types": "0.8.0", + "regex": "2026.9.29", + "markdown-it-py": "4.2.0", + "tzdata": "2026.6", + "annotated-doc": "0.0.5", + "typing-inspection": "0.4.4", + "fastapi": "0.162.0", + "aiohttp": "3.14.5", + "attrs": "26.1.0", + "multidict": "7.2.1", + "typing_extensions": "4.16.0", + "MarkupSafe": "3.0.4", + "pydantic": "2.14.1", + "python-dateutil": "3.0.0", + "multiprocess": "0.70.19", + "charset-normalizer": "3.5.2", + "anyio": "4.15.1", + "six": "1.17.0", + "scipy": "1.18.1", + "dill": "0.4.1", + "transformers": "5.21.0", + "idna": "3.20" + } + }, + "requirements_sha256": "cdede98457af90bb37dc146204af09dc67059ae7829932d7bae76b9185740ee4", + "models": { + "LFM3-1.7B-MLX-4bit": { + "repository": "LiquidAI/LFM3-1.7B-MLX-4bit", + "revision": "d0ca73dcfa70ce18733754a492cdc1fb0123262d", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 2423, + "sha256": "7df2ea819f9a31b770192a0a9dc1b7953a7dd8c41b816cc3905bbec99ef332bd", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 328, + "sha256": "0095ba7158e938b1e2d3082d412f00d8be85e42cbbd7608b63866f7a081cf270", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 973083386, + "sha256": "1be5bf672013cd91aa62148f77c5eea8d2784a95845d5fe826e08c7b84823815", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 50700, + "sha256": "2c8b88e25f6a1055164dd051436b170872200047bce4b3ec5f2a58f6ef929a75", + "verified_against_pinned_revision": true + }, + "special_tokens_map.json": { + "bytes": 92, + "sha256": "7946f864cbfb62a1b041bda937244210a1a87f4a6cab3a48b791131b2cd57f88", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908634, + "sha256": "41927bd314d17d7d7fadb44e10cd0919f7b71528d7741596d8cbd15c6945ec44", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1859, + "sha256": "b77214b9f12be74b9e5e1dab35d7dace077a31be969314bf38866fd6c8163853", + "verified_against_pinned_revision": true + } + } + }, + "LFM3-VL-2B-MLX-4bit": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "revision": "c823fade98e35a3d6942ad4d38b1ea15f675cd12", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 4419, + "sha256": "0efb7708e0771e3e104ac628954cdc71070dd436e0b7974615bd1b9eb836f9d9", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 323, + "sha256": "30b1115bcdc4e5671567a5b1255a2f3d2f5a912a79292051c6f1e9db81b5e8c4", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 1590959088, + "sha256": "58268dfdf441ae4020d2a52fc97a814645e8a9624946e2d490403d4dcd71a255", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 61256, + "sha256": "38b756dbde167ab7c60756af7d5652414c31afdf4e77700b8d3c99395515d639", + "verified_against_pinned_revision": true + }, + "processor_config.json": { + "bytes": 1254, + "sha256": "f183f5a5e5b3a32e666fce05033e6cb7ca496d26e709a2cee611a13eb7d6b610", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908258, + "sha256": "a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1759, + "sha256": "d63d1ce2f332047e398764b4170352b96629a5194643105bfd6d70c90b750396", + "verified_against_pinned_revision": true + } + } + } + }, + "client_sha256": { + "$REPO/tools/lfm3/bench_decode.py": "c011106f4f182db19371e3f91f70bcc374160786dd105f2d713a7fea07ab8c4c", + "$REPO/tools/bench_concurrent.py": "0e72e26769f082459329fc068c25f6f72a9155e422962ea54cb5d1226227752d", + "$REPO/tools/bench_evidence.py": "54c28474f0804dbd120e3fb5859be0d1667ef21ca502880d0bed098ec9bb42ff", + "$PACKET/cold_client.py": "7b8ea55aa88834814f95a6f679654e16074b7a8364e34b39a276f009fe0231f2" + }, + "fixture_sha256": { + "vl-2048.ids.json": "f5edac3447405086d7ec361d6c899410ea7cbbb2ef697c27fd7486ef5ff877fe", + "text-2048.ids.json": "f5edac3447405086d7ec361d6c899410ea7cbbb2ef697c27fd7486ef5ff877fe", + "text-2048.body.json": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "vl-2048.body.json": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "text-8192.body.json": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "vl-8192.body.json": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "vl-8192.ids.json": "a9b3cdf070b9ca5ea523565bcb82c80b6af5aae71f640a8fba0460c9069d22cd", + "text-8192.ids.json": "a9b3cdf070b9ca5ea523565bcb82c80b6af5aae71f640a8fba0460c9069d22cd" + }, + "packet_files": { + "run_existing_clients.py": "43d10d451a76afe6c895802355e87946f8b1680bbf02cdb215fd9c17a3f9ba05", + "config.json": "0cdeabce95cdaf8273f9e30ea59d12a24a936d526d5f82d3ae0e56c3e95711c1", + "guard.py": "0caf67448d401704bd849744584248729decf5fd5b994f669a518cb61891df37", + "requirements.lock": "cdede98457af90bb37dc146204af09dc67059ae7829932d7bae76b9185740ee4", + "setup-standard.sh": "e77b6ce06eda842a3c30a043148db3b543a4204505ccd77d726781a6d743d41c", + "independent-review.json": "fcea5c48ef445a2c59d386ff03ccafd86d22f4a10a734893ec3d787cd3b118c1", + "combined-source-manifest.json": "2c86619e73c86e6d21a99f1276102294c6c538005af393af5ea4bd6178b61ddc", + "combined-source.patch": "58df0a67e783c6d3af89fe18fb878083c9f1055ed09a692340b13612eaf0d9f2" + } + }, + "protocol": { + "phase": { + "families": [ + "text", + "vl" + ], + "bits": [ + 4 + ], + "cold_tokens": [ + 2048, + 8192 + ], + "decode_tokens": 256, + "cold_output_tokens": 16, + "reps": 3, + "levels": [ + 1, + 4 + ], + "order": [ + "public", + "combined", + "standard", + "standard", + "combined", + "public" + ] + }, + "settings": { + "native_context": 70000, + "parallel": 4, + "prompt_cache_gib": 0, + "temperature": 0, + "thinking": false, + "prefix_cache": "disabled on both", + "port": 18085 + }, + "lock_path": "$TMPDIR/tensorfold-model.lock", + "guard_source": { + "origin": "$TMPDIR/lfm3-local-locked-profile.py", + "sha256": "0145717ffc373c5cfed39dd11a9943154d9b61d33badef6cd27c4a866e7c6057", + "change": "admission allows only owned descendants while a measured server is running; recognizes frozen binary names" + }, + "admission": "Exclusive flock held throughout model inventory, all twelve sequential owned server legs, measurement and cleanup. Process admission rejects foreign model jobs; only owned server process groups are cleaned up.", + "lock_events": [ + { + "event": "lock_wait", + "time_ns": 1793194653503933000, + "path": "$TMPDIR/tensorfold-model.lock" + }, + { + "event": "lock_acquired", + "time_ns": 1793195302533622000, + "path": "$TMPDIR/tensorfold-model.lock", + "pid": 47771, + "device": 16777231, + "inode": 46835039 + }, + { + "event": "complete", + "time_ns": 1793195924040963000, + "legs": 12, + "all_lengths_validated": true, + "performance_gate_claim": false + }, + { + "event": "lock_release", + "time_ns": 1793195924041207000, + "pid": 47771 + } + ], + "server_commands": [ + { + "engine": "public", + "family": "text", + "command": { + "_trimmed_list": { + "count": 20, + "sha256": "e6188b408615cec4ad96f16926b608ec98d160ac86b63859c916501bdd6f52ad", + "first": [ + "$PACKET/artifacts/tensorfold-native-public", + "serve" + ], + "last": [ + "--metrics-open" + ] + } + }, + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "engine": "combined", + "family": "text", + "command": { + "_trimmed_list": { + "count": 20, + "sha256": "e3e76f89aada59cc7aa4662bc1bee33202ebae99eb5b14e3ebf181363a9e3206", + "first": [ + "$PACKET/artifacts/tensorfold-native-combined", + "serve" + ], + "last": [ + "--metrics-open" + ] + } + }, + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "engine": "standard", + "family": "text", + "command": { + "_trimmed_list": { + "count": 19, + "sha256": "3b1af5ff83cd61814dbc4c6b793fad72ebd5bb3e36c36dd0c591816b1b0ca35f", + "first": [ + "$PACKET/standard-venv/bin/python", + "-m" + ], + "last": [ + "0" + ] + } + }, + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": "70000" + }, + { + "engine": "public", + "family": "vl", + "command": { + "_trimmed_list": { + "count": 20, + "sha256": "73c923b46433a48007040969642ab8b3feb63443e3e5763f9d3b437e76937d66", + "first": [ + "$PACKET/artifacts/tensorfold-native-public", + "serve" + ], + "last": [ + "--metrics-open" + ] + } + }, + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "engine": "combined", + "family": "vl", + "command": { + "_trimmed_list": { + "count": 20, + "sha256": "5876712ce3133f7e2e4c413bcd464c71640b57aa9e7d2f5310b2d2b146d1d443", + "first": [ + "$PACKET/artifacts/tensorfold-native-combined", + "serve" + ], + "last": [ + "--metrics-open" + ] + } + }, + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "engine": "standard", + "family": "vl", + "command": { + "_trimmed_list": { + "count": 19, + "sha256": "c4a005522ae14df911bdc8fb3d25c26ba994b32a45c7b4ba8b6abfed57d61a52", + "first": [ + "$PACKET/standard-venv/bin/python", + "-m" + ], + "last": [ + "0" + ] + } + }, + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": "70000" + } + ], + "memory_policy": "Native admission unchanged; MLX memory and wired limits unmodified. /usr/bin/time -l captures physical footprint and RSS; available server metrics preserved. No allocator/pinned peak inferred from RSS.", + "replay": [ + "python3 \"$PACKET/run_existing_clients.py\"", + "python3 \"$PACKET/run_existing_clients.py\" --execute --output \"$NEW_OUTPUT\"" + ], + "replay_requirements": "Original frozen packet and artifacts are local retained inputs, not bundled executable tooling in this receipt. Set REPO/MODELS/PACKET paths appropriately; output must not exist. Packet validates pinned dependencies, binaries, client/fixture and model files before load. Reconstruct from the hashes/versions/commands here if the retained packet is unavailable." + }, + "results": { + "legs": { + "_trimmed_list": { + "count": 12, + "sha256": "c11955f53e6de1d8e585e9dabea937c6d83d6ad08d418580e6c3f7b08c9216d2", + "first": [ + { + "family": "text", + "bits": 4, + "leg": 0, + "engine": "public", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 3.1976742083206773, + "output_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 13.849388832692057, + "output_sha256": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 1217773568, + "peak_physical_footprint_bytes": 1514243752 + }, + "tag": "text-4-0-public", + "decode_cells": { + "_trimmed_value": { + "bytes": 13612, + "sha256": "63ca4cca88ba935ca6d4ab261ce88290b38ec17671900ec13183988f30bf2cdd", + "keys": null, + "count": 4 + } + } + }, + { + "family": "text", + "bits": 4, + "leg": 1, + "engine": "combined", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 3.4472638331353664, + "output_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 17.29660391714424, + "output_sha256": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 1218560000, + "peak_physical_footprint_bytes": 1514964696 + }, + "tag": "text-4-1-combined", + "decode_cells": { + "_trimmed_value": { + "bytes": 13598, + "sha256": "0af5d423a7e4b0e2be6dd4ba12f2b4283eb8e8e7234832c7b4decb8387ac4126", + "keys": null, + "count": 4 + } + } + } + ], + "last": [ + { + "family": "vl", + "bits": 4, + "leg": 5, + "engine": "public", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 3.1815165420994163, + "output_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 13.796967291738838, + "output_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 1836122112, + "peak_physical_footprint_bytes": 2173814944 + }, + "tag": "vl-4-5-public", + "decode_cells": { + "_trimmed_value": { + "bytes": 13597, + "sha256": "def12cda3878a113b8e12300ae9dd036d49b92cfffe080f8f74725cbfa944ef9", + "keys": null, + "count": 4 + } + } + } + ] + } + }, + "six_rep_medians": { + "_trimmed_list": { + "count": 24, + "sha256": "1228da6a36b6bc31cf392034f7ac0b94c4cbddde85bc1738d2fd15951565aa1e", + "first": [ + { + "family": "text", + "engine": "public", + "prompt": "long-code-chat", + "streams": 1, + "existing_cell_tps": [ + 175.0, + 175.1 + ], + "six_existing_rep_tps": [ + 174.33357930951772, + 176.54645032063524, + 175.0280052102641, + 175.0803803635659, + 174.57396389141493, + 176.4727407119902 + ], + "median_six_rep_tps": 175.054192786915 + }, + { + "family": "text", + "engine": "public", + "prompt": "long-code-chat", + "streams": 4, + "existing_cell_tps": [ + 388.3, + 389.8 + ], + "six_existing_rep_tps": [ + 388.32442956935364, + 387.38800732228174, + 390.4867598842693, + 385.24300236921965, + 390.4341652635153, + 389.79263659668885 + ], + "median_six_rep_tps": 389.05853308302125 + } + ], + "last": [ + { + "family": "vl", + "engine": "standard", + "prompt": "long-prose-chat", + "streams": 4, + "existing_cell_tps": [ + 439.2, + 445.3 + ], + "six_existing_rep_tps": [ + 432.67965065963, + 439.15798068790735, + 442.19728263942795, + 445.3158422193051, + 438.6344021467109, + 446.9232375053428 + ], + "median_six_rep_tps": 440.6776316636676 + } + ] + } + }, + "cold": { + "_trimmed_list": { + "count": 12, + "sha256": "74f5234410255991be95aeca8b9e9909e0808abc1e617a410128594e6094b2ec", + "first": [ + { + "family": "text", + "engine": "public", + "prompt_tokens": 2048, + "ttft_s": [ + 3.1976742083206773, + 3.1604983750730753 + ], + "output_sha256": [ + "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9" + ] + }, + { + "family": "text", + "engine": "public", + "prompt_tokens": 8192, + "ttft_s": [ + 13.849388832692057, + 14.409123916644603 + ], + "output_sha256": [ + "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349", + "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349" + ] + } + ], + "last": [ + { + "family": "vl", + "engine": "standard", + "prompt_tokens": 8192, + "ttft_s": [ + 4.1006536250934005, + 4.097270667087287 + ], + "output_sha256": [ + "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e" + ] + } + ] + } + }, + "physical_footprint": [ + { + "family": "text", + "engine": "public", + "legs": [ + { + "max_rss_bytes": 1217773568, + "peak_physical_footprint_bytes": 1514243752 + }, + { + "max_rss_bytes": 1217822720, + "peak_physical_footprint_bytes": 1514260112 + } + ] + }, + { + "family": "text", + "engine": "combined", + "legs": [ + { + "max_rss_bytes": 1218560000, + "peak_physical_footprint_bytes": 1514964696 + }, + { + "max_rss_bytes": 1217888256, + "peak_physical_footprint_bytes": 1514325720 + } + ] + }, + { + "family": "text", + "engine": "standard", + "legs": [ + { + "max_rss_bytes": 1509490688, + "peak_physical_footprint_bytes": 2763966528 + }, + { + "max_rss_bytes": 1487306752, + "peak_physical_footprint_bytes": 2758903896 + } + ] + }, + { + "family": "vl", + "engine": "public", + "legs": [ + { + "max_rss_bytes": 1835630592, + "peak_physical_footprint_bytes": 2173503600 + }, + { + "max_rss_bytes": 1836122112, + "peak_physical_footprint_bytes": 2173814944 + } + ] + }, + { + "family": "vl", + "engine": "combined", + "legs": [ + { + "max_rss_bytes": 1836417024, + "peak_physical_footprint_bytes": 2174273720 + }, + { + "max_rss_bytes": 1836220416, + "peak_physical_footprint_bytes": 2173945944 + } + ] + }, + { + "family": "vl", + "engine": "standard", + "legs": [ + { + "max_rss_bytes": 2108932096, + "peak_physical_footprint_bytes": 3283257768 + }, + { + "max_rss_bytes": 2107703296, + "peak_physical_footprint_bytes": 3283946040 + } + ] + } + ] + }, + "limitations": { + "_trimmed_list": { + "count": 10, + "sha256": "5b12c7b55b3b47b8777e284b134420a04efa35bc3f4c244b35efe24b6090ae11", + "first": [ + "Standard omits token hashes: alone/serial token equality unverified; output text digests are not substituted.", + "Both engines use request budget70000, but native reserved cache and standard lazy KV/sliding cache allocation differ." + ], + "last": [ + "Peak physical footprint is the actual /usr/bin/time -l process measurement, not a claimed GPU allocator or pinned-memory peak." + ] + } + }, + "raw_file_manifest": { + "events.json": { + "bytes": 30478, + "sha256": "ddbc001617d38543feb37798395b11dd4ef682e6c546d5805888bbf743d6d65b" + }, + "independent-review.json": { + "bytes": 3624, + "sha256": "9e68ef3a90ea418cf844f1333244a5d46cda6a91c9d738d59178e001cd706a92" + }, + "independent-review.py": { + "bytes": 4080, + "sha256": "9f0282acde5693fa1442b1eab9dcbe29249555ca625e4b42fae880a5408a3420" + }, + "manifest.json": { + "bytes": 12788, + "sha256": "22148de349b615a3b8b72ef17f98e6623f202aa26c0766cf2e476f503f38ab71" + }, + "phase1-summary.json": { + "bytes": 17748, + "sha256": "ed46d838704efdcda3cd097072f088ecad013fd70ccd719f14026d33933caaeb" + }, + "records.json": { + "bytes": 12701, + "sha256": "ce8583512252c2ae87b57b77835530aab44aecea40e231a641a9ee67f4534a94" + }, + "text-4-0-public.after.health": { + "bytes": 298, + "sha256": "421a862e6e4c08711e03c75307d1ad907b9538c77e95145c995ff22ab1e52a98" + }, + "text-4-0-public.after.metrics": { + "bytes": 12299, + "sha256": "3505202fb9381f863cf3ddb40483968079867d175c10265d65bc3305b8116c9c" + }, + "text-4-0-public.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-0-public.before.health": { + "bytes": 296, + "sha256": "8c789a277ca189a0923767fa83532a352de4a11334a37855d824cd2137333df4" + }, + "text-4-0-public.before.metrics": { + "bytes": 12128, + "sha256": "f9da163dbf442d34b0fcb0d7abcf3d4d34ac59c4bbe624dd5d8c4181d856fffc" + }, + "text-4-0-public.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-0-public.cold-2048.client.log": { + "bytes": 348, + "sha256": "84d78b3202f5c276158c6314db0f0d53a1af25c5d3e8093bc38ceff524217c1e" + }, + "text-4-0-public.cold-2048.json": { + "bytes": 20294, + "sha256": "9a054d2222ea0b76f2fc008fb0d38707924d590ca0999d3048452e3a577198cc" + }, + "text-4-0-public.cold-8192.client.log": { + "bytes": 349, + "sha256": "aaec18c32ab79dfdd565b1052483fec760046d31c083bfba955203b96684c7b0" + }, + "text-4-0-public.cold-8192.json": { + "bytes": 57151, + "sha256": "277c3104bf3e7b4911edf6b8e7bd58a914f98d32a1a25d67506758748a63d03f" + }, + "text-4-0-public.concurrent.client.log": { + "bytes": 3232, + "sha256": "8261f032cb35002bba510cd84e96b03808c398ff870adb74a87eacf10b8f91f6" + }, + "text-4-0-public.concurrent.json": { + "bytes": 280772, + "sha256": "e1cfbb2f30e112b8d3cb1202d456abf4aaf94aed9f8362e0d0e37218f5843dda" + }, + "text-4-0-public.server.log": { + "bytes": 13609, + "sha256": "c9ab4413703b0d7e8f41c11c56f34e036832bacf8bdacc8454af0041c12887be" + }, + "text-4-1-combined.after.health": { + "bytes": 298, + "sha256": "9903e3b95f46b71d8186f3abe415a96ef62d0a94ceaa7f3efd7b1208a0cefd36" + }, + "text-4-1-combined.after.metrics": { + "bytes": 12303, + "sha256": "aeec875bc3c6effbda360d32501dbfc3376a2db4c8e58a4c0afb3785845860c3" + }, + "text-4-1-combined.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-1-combined.before.health": { + "bytes": 296, + "sha256": "3ecca835aef6168623e5fd09af1a553676471c6c3d76176c12322725ac4e2881" + }, + "text-4-1-combined.before.metrics": { + "bytes": 12128, + "sha256": "cfd6e0d1abeb10f092e108c9090eb92ff5e62c04adf5a845e34ad44968fec4b3" + }, + "text-4-1-combined.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-1-combined.cold-2048.client.log": { + "bytes": 349, + "sha256": "97526b0326a0fcf617cb4a2cc77fff29599d75524b655e36db0c4121b132129e" + }, + "text-4-1-combined.cold-2048.json": { + "bytes": 20295, + "sha256": "1a889aaf0e6e69e26d777522a3365cdd6729154e300fbb0d89caefdb29ba96db" + }, + "text-4-1-combined.cold-8192.client.log": { + "bytes": 348, + "sha256": "88b8ac9358e175852c77bae8a353f676e663a46702ddf838b275e1ed44f99e45" + }, + "text-4-1-combined.cold-8192.json": { + "bytes": 57150, + "sha256": "06581ab3330b40c0bb2c3482400880f2fe9e68e32abaabb954c07c5e02100cd7" + }, + "text-4-1-combined.concurrent.client.log": { + "bytes": 3232, + "sha256": "1d71a2b74dc478fb756de4371090853d9d2ddd142cdd0adb3fc0dfef4a27d39b" + }, + "text-4-1-combined.concurrent.json": { + "bytes": 281297, + "sha256": "ac39d743350fd459e644f2cb1b8c547d5468ebdb2936ce00e29a2fcb5108f495" + }, + "text-4-1-combined.server.log": { + "bytes": 13603, + "sha256": "1edf3e0031171bc652ad5e0c477aa3151b124617a454d8df4ca91e5d36edc17b" + }, + "text-4-2-standard.after.health": { + "bytes": 449, + "sha256": "17e6666ff1419af1e0f5f493004feab65b09c9d7d8bfd22fbacc102fa8729e6c" + }, + "text-4-2-standard.after.metrics": { + "bytes": 31998, + "sha256": "7504f2b8de52f1e5cc7ce1308e3cca240bd073cbac4a28a20597369ec7c1128a" + }, + "text-4-2-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-4-2-standard.before.health": { + "bytes": 449, + "sha256": "17e6666ff1419af1e0f5f493004feab65b09c9d7d8bfd22fbacc102fa8729e6c" + }, + "text-4-2-standard.before.metrics": { + "bytes": 905, + "sha256": "26b51aaf2c7f37eb81e52b011882c77ed30331e6181ec9a2d924d68ea14c30a1" + }, + "text-4-2-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-4-2-standard.cold-2048.client.log": { + "bytes": 295, + "sha256": "5232bc69406a15fe6603220d1fab78c88288aa16d0e1115fb0d6a8994a3c8f6a" + }, + "text-4-2-standard.cold-2048.json": { + "bytes": 26045, + "sha256": "0d204373fa161d53573b2474654d35008670be3c7ccdaed45f254d19f8aa4f29" + }, + "text-4-2-standard.cold-8192.client.log": { + "bytes": 293, + "sha256": "eded448fa26377b41ac8ef22d7f9d02810b751ac2d21bab59b27ff9a6cbddd5e" + }, + "text-4-2-standard.cold-8192.json": { + "bytes": 62899, + "sha256": "75ba4fdded3c69ad4fb46ce77963ff70340245c645308f7f094fc0830185ad4e" + }, + "text-4-2-standard.concurrent.client.log": { + "bytes": 3238, + "sha256": "080ed7a5a6cb6ca212d6936d1746d94ca6927fb9b49d60b461639a8fa31f5223" + }, + "text-4-2-standard.concurrent.json": { + "bytes": 283021, + "sha256": "e8fa24b3febd6e2716963ad2981697b25b0f9a7fa6f9dddd23be5edbc6c82db0" + }, + "text-4-2-standard.server.log": { + "bytes": 50717, + "sha256": "b9a8190dbd56dc7af5c097ef265e70232ddfe46b3506164338cea5df330e713a" + }, + "text-4-3-standard.after.health": { + "bytes": 449, + "sha256": "17e6666ff1419af1e0f5f493004feab65b09c9d7d8bfd22fbacc102fa8729e6c" + }, + "text-4-3-standard.after.metrics": { + "bytes": 32022, + "sha256": "727d50eb840467c1181ecdb08a26f4263b7c33be916d9692b85d29809f2005ff" + }, + "text-4-3-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-4-3-standard.before.health": { + "bytes": 449, + "sha256": "17e6666ff1419af1e0f5f493004feab65b09c9d7d8bfd22fbacc102fa8729e6c" + }, + "text-4-3-standard.before.metrics": { + "bytes": 904, + "sha256": "52c02d19f03dd438ecfe12fa4c3d4ca92e37ae8de76520d008694f76872cc44d" + }, + "text-4-3-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-4-3-standard.cold-2048.client.log": { + "bytes": 294, + "sha256": "1a818bccb1f78515b6b86bcbd143a047071af7893927540b939d273455132137" + }, + "text-4-3-standard.cold-2048.json": { + "bytes": 26043, + "sha256": "18140744926db72ff2fd3eb2504d19087632a1a01bcf206c5cfddbd544676e02" + }, + "text-4-3-standard.cold-8192.client.log": { + "bytes": 293, + "sha256": "4d95a17179bf0f5f62ce4d042e055b8cc0483248cdb809f34093ab0c5b4aa87b" + }, + "text-4-3-standard.cold-8192.json": { + "bytes": 62907, + "sha256": "13d362f5cfdf2feca0d3a28e0144d52257efc37643fe165a2169d31a55c80380" + }, + "text-4-3-standard.concurrent.client.log": { + "bytes": 3238, + "sha256": "57f25bd1a3669b2772c8b4e6509ce3ee82b1b5a8de7bcc54555f727decbebe11" + }, + "text-4-3-standard.concurrent.json": { + "bytes": 283040, + "sha256": "ef6a2b046061524f59ae6615c7e14d5bce0a33ab51c91a50a9c6a23e53850156" + }, + "text-4-3-standard.server.log": { + "bytes": 50731, + "sha256": "c7dbece91bd53f5ed14b7efe779784a220093f59f8aa53fab624cdaf9cd03cae" + }, + "text-4-4-combined.after.health": { + "bytes": 298, + "sha256": "9903e3b95f46b71d8186f3abe415a96ef62d0a94ceaa7f3efd7b1208a0cefd36" + }, + "text-4-4-combined.after.metrics": { + "bytes": 12303, + "sha256": "4ee4130732a297bb67e316569691cd7a7b5eb04a8fe80a0a721ebf617cfa7fbc" + }, + "text-4-4-combined.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-4-combined.before.health": { + "bytes": 296, + "sha256": "3ecca835aef6168623e5fd09af1a553676471c6c3d76176c12322725ac4e2881" + }, + "text-4-4-combined.before.metrics": { + "bytes": 12128, + "sha256": "b28b48959fb7e2e4e9d2047da1f093a37e48c658ab53ae37b960b4d827c18a89" + }, + "text-4-4-combined.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-4-combined.cold-2048.client.log": { + "bytes": 349, + "sha256": "40872192dae33ec46b14b07f73dc5a2dd90ea4091851523caa7ebd621869185f" + }, + "text-4-4-combined.cold-2048.json": { + "bytes": 20295, + "sha256": "e2650773b810b1fd00486daa5dcfa4a6bfb01cf882b372d9fc6e18031276519c" + }, + "text-4-4-combined.cold-8192.client.log": { + "bytes": 349, + "sha256": "97e0bad59aef3280d02619c611b88dd29933bff598e42f52b4428ffba81f3907" + }, + "text-4-4-combined.cold-8192.json": { + "bytes": 57150, + "sha256": "758bdf35f0f1065f4c37702493e1f36757d39a2502e27cbd34abc3117433ab35" + }, + "text-4-4-combined.concurrent.client.log": { + "bytes": 3239, + "sha256": "2174df350dde3de1704e9b3850ed72dd9c6ca3fde56f799542243c52c7260407" + }, + "text-4-4-combined.concurrent.json": { + "bytes": 280318, + "sha256": "6daa98247532da73a09c12c81806bd355ac8cc8ef884060b71bfd8c6d5b2df21" + }, + "text-4-4-combined.server.log": { + "bytes": 13602, + "sha256": "7e11c8a67d6d286c5d65d2605133d995a226613c7d119ad7ffc18f29e72cbd7d" + }, + "text-4-5-public.after.health": { + "bytes": 298, + "sha256": "82defd35723aee950c60a4c840c6b1adceebb8d1001e61eab8892f003e8b6467" + }, + "text-4-5-public.after.metrics": { + "bytes": 12301, + "sha256": "872a11da70e6d602be6d87b90c91f3f97cb82da071bd55bbdcf49802416e8c2d" + }, + "text-4-5-public.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-5-public.before.health": { + "bytes": 296, + "sha256": "8c789a277ca189a0923767fa83532a352de4a11334a37855d824cd2137333df4" + }, + "text-4-5-public.before.metrics": { + "bytes": 12128, + "sha256": "5f6bdd7fa09ed697c0b795006457190705731c37d83f15a2c0dfca013b3f1879" + }, + "text-4-5-public.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-5-public.cold-2048.client.log": { + "bytes": 348, + "sha256": "51d9fd137de6a4c0ffab39077bfd9d6a7fc9bb81ccdafeafb1bba03357b41931" + }, + "text-4-5-public.cold-2048.json": { + "bytes": 20293, + "sha256": "ac543c8047e3b0943f0c74d4eb3a6efcb967c74396a1574e682fa46fb57d3080" + }, + "text-4-5-public.cold-8192.client.log": { + "bytes": 349, + "sha256": "ff21237fe43ac9c4c13dc90a310a74400671fc236eca825c2138591893475f8b" + }, + "text-4-5-public.cold-8192.json": { + "bytes": 57151, + "sha256": "9e83eb99eedf33e4704193ff6594938c62398a19e39f93f13630fc2444d46172" + }, + "text-4-5-public.concurrent.client.log": { + "bytes": 3235, + "sha256": "5053ac4e028b6b3f2af11be70c86168d603286c6ac8598024def19e4c94d3243" + }, + "text-4-5-public.concurrent.json": { + "bytes": 280185, + "sha256": "3410745c9d258cb5b6505f23addef85b3b28a9ed7e0e3db673dc1b466c2fdda3" + }, + "text-4-5-public.server.log": { + "bytes": 13599, + "sha256": "685a364b7fe651a2b3410f6935d3af4c6ab712b60323766ff4a2c7598908fd48" + }, + "vl-4-0-public.after.health": { + "bytes": 298, + "sha256": "09f638fe46665554902ba68e5e5d2aab248272c8127d38439fc3a532615ee5a5" + }, + "vl-4-0-public.after.metrics": { + "bytes": 12303, + "sha256": "de2917cfeb4470e9304a486309c3f4c7f4043c1d58b1f52a2e31dfc68e2381bc" + }, + "vl-4-0-public.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-0-public.before.health": { + "bytes": 296, + "sha256": "8c789a277ca189a0923767fa83532a352de4a11334a37855d824cd2137333df4" + }, + "vl-4-0-public.before.metrics": { + "bytes": 12128, + "sha256": "79b1c2a5ef34b5dd904fba6de209971719abbf3c707b15890c7f71f8ed9b10ed" + }, + "vl-4-0-public.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-0-public.cold-2048.client.log": { + "bytes": 347, + "sha256": "d35590d9f126609b741c99fe8ee4d0e52956d92a931fa51353d4e692394be02d" + }, + "vl-4-0-public.cold-2048.json": { + "bytes": 20276, + "sha256": "d944cfcb4f29cc2ea6f2a63ff8a643655c6df80f9b985e346135b7061af48448" + }, + "vl-4-0-public.cold-8192.client.log": { + "bytes": 349, + "sha256": "6e806eb7422f152005c179f5b8d331726e90e697f00aa86df94c3fbe68d7e5f2" + }, + "vl-4-0-public.cold-8192.json": { + "bytes": 57149, + "sha256": "38166978d5bc8b52b966f94e288f8eb1d362b3dff7f91be59f723b820ed1bbea" + }, + "vl-4-0-public.concurrent.client.log": { + "bytes": 3226, + "sha256": "08cd1fa665e4ee03b932ac013de823c2b0b3df12e3a237a5622655d76977ff42" + }, + "vl-4-0-public.concurrent.json": { + "bytes": 287529, + "sha256": "052ebbc993b194428d9027ab5a6c39d603f1b4739d1e0d611b206ab1bcdc254f" + }, + "vl-4-0-public.server.log": { + "bytes": 13573, + "sha256": "de45910ec4438e5d824343cac9a14e84604777b396e7ac73a302a7a5ae1dbd31" + }, + "vl-4-1-combined.after.health": { + "bytes": 298, + "sha256": "fb2dc251074dfd4ba9f6b848f9f3528656ae1a5a4d3971bd0c589da581e658b9" + }, + "vl-4-1-combined.after.metrics": { + "bytes": 12303, + "sha256": "5e189b70364a948aa2735cf382a48986244b849f88c068732bdca771554fd091" + }, + "vl-4-1-combined.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-1-combined.before.health": { + "bytes": 296, + "sha256": "3ecca835aef6168623e5fd09af1a553676471c6c3d76176c12322725ac4e2881" + }, + "vl-4-1-combined.before.metrics": { + "bytes": 12128, + "sha256": "d8eba96d174b50600e16310f453c1ae9f0c7162952a9c454db18f5de4ca78d5e" + }, + "vl-4-1-combined.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-1-combined.cold-2048.client.log": { + "bytes": 348, + "sha256": "252af0af9a08eece3d674e80f6524935ca54e858472c31978be732baf7d21e71" + }, + "vl-4-1-combined.cold-2048.json": { + "bytes": 20278, + "sha256": "e7942deea75684f0b048bca65b214a2c9ae152e920968197d14a5ac141895f88" + }, + "vl-4-1-combined.cold-8192.client.log": { + "bytes": 348, + "sha256": "689bd543a30adce4a207841e3ee387798ff63a72c47f74fe6102e339864ee1c1" + }, + "vl-4-1-combined.cold-8192.json": { + "bytes": 57148, + "sha256": "870f7e29aaf89881981dc6fe9e0fb9986d18eeb43bf15e22086e957da6e31fc5" + }, + "vl-4-1-combined.concurrent.client.log": { + "bytes": 3239, + "sha256": "9b4ba7b70a672e1f1f85f158baf525fed1cedd44fd51b68cc32f375483d0fb25" + }, + "vl-4-1-combined.concurrent.json": { + "bytes": 286751, + "sha256": "3efc679a0fcef439bef81a74e1fe750ad5be379900c65f58b4faca6a5b3b7b4b" + }, + "vl-4-1-combined.server.log": { + "bytes": 13569, + "sha256": "b82736585b367f1649769ab2100226a0e1f41a32bbf8570846f5f4dcd0b94a3c" + }, + "vl-4-2-standard.after.health": { + "bytes": 453, + "sha256": "ee45e784aa99341cf066d20507920a28dfdcd5fde13e24adccd716753aa53736" + }, + "vl-4-2-standard.after.metrics": { + "bytes": 32069, + "sha256": "fc1c234db5b1f36962a84a45b04d43d3ae8ae3b7950c0143e6d4788a32922f4c" + }, + "vl-4-2-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-4-2-standard.before.health": { + "bytes": 453, + "sha256": "ee45e784aa99341cf066d20507920a28dfdcd5fde13e24adccd716753aa53736" + }, + "vl-4-2-standard.before.metrics": { + "bytes": 909, + "sha256": "e5a24d4c419a5736bede8ea24c4fba04fcf40ec17f8f404582f65af0cd0dc5d3" + }, + "vl-4-2-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-4-2-standard.cold-2048.client.log": { + "bytes": 295, + "sha256": "54d5c9ee5dd3d043062cce61e44b34cfbb5565f91bfcba2605cda76ce3f5df4b" + }, + "vl-4-2-standard.cold-2048.json": { + "bytes": 26043, + "sha256": "6af7c1da5abf82bcc2fed9bca5d8e9dd638b0b1e06c29a0a208863bb7e46a4ba" + }, + "vl-4-2-standard.cold-8192.client.log": { + "bytes": 294, + "sha256": "e44a9c8b1f48e893522b7890e131d5494befd48da34d6f71c6c4695497871e67" + }, + "vl-4-2-standard.cold-8192.json": { + "bytes": 62916, + "sha256": "67b1b0ad360abae12049d8690d6a46aca76104f08f95aa919c32b47055655fc2" + }, + "vl-4-2-standard.concurrent.client.log": { + "bytes": 3216, + "sha256": "abdbcc7a15c3eb186fc85bc76fa498801472a8b9c2db28d396ea8962b00a3e7e" + }, + "vl-4-2-standard.concurrent.json": { + "bytes": 283051, + "sha256": "ac20c8633194c6cd981758a8198175a31fdbd93c8194fa13c15d1f8e4130e68d" + }, + "vl-4-2-standard.server.log": { + "bytes": 50797, + "sha256": "ce86d937b89953711b96a00f73930805a0d9b6919e97bbf8ee8239c4a92561b7" + }, + "vl-4-3-standard.after.health": { + "bytes": 453, + "sha256": "ee45e784aa99341cf066d20507920a28dfdcd5fde13e24adccd716753aa53736" + }, + "vl-4-3-standard.after.metrics": { + "bytes": 32011, + "sha256": "521bd11ae4eae455a5d6cfc07a508fd1e1dcc090adf9d650777c7e65548f01b8" + }, + "vl-4-3-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-4-3-standard.before.health": { + "bytes": 453, + "sha256": "ee45e784aa99341cf066d20507920a28dfdcd5fde13e24adccd716753aa53736" + }, + "vl-4-3-standard.before.metrics": { + "bytes": 909, + "sha256": "e1ec519ebc5d5824395fadc332516be06266a6cb81de8b1c6739e1c3a848a6a9" + }, + "vl-4-3-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-4-3-standard.cold-2048.client.log": { + "bytes": 295, + "sha256": "dcdc8c4ab5a72f64c566af4af21a88e814cbee69eb53fc36e9b552f81e183974" + }, + "vl-4-3-standard.cold-2048.json": { + "bytes": 26046, + "sha256": "c44814cf4c67055eebe95f0e5bf15bc58886810ece4e1ae4b7626f93bf6fa40d" + }, + "vl-4-3-standard.cold-8192.client.log": { + "bytes": 293, + "sha256": "a0974b13161bfdb36627d6629646d57f8dde103846682f98ebb828278ab6c15a" + }, + "vl-4-3-standard.cold-8192.json": { + "bytes": 62911, + "sha256": "691aa208bd7c11ca53b0f6daa2dbee882900716daf48d57a167e30669291cf26" + }, + "vl-4-3-standard.concurrent.client.log": { + "bytes": 3212, + "sha256": "ddc272d953ace7882a258fda730de7b6ce21d559c1ebb188beffd889cc35838c" + }, + "vl-4-3-standard.concurrent.json": { + "bytes": 283078, + "sha256": "1e7609728da420a901fb21738af418aa5254bea9855efab6e06c158aefdb259a" + }, + "vl-4-3-standard.server.log": { + "bytes": 50803, + "sha256": "2a958023dc60e1536754f850d6b645ac6ed193710511ac5252efe9b557810365" + }, + "vl-4-4-combined.after.health": { + "bytes": 298, + "sha256": "9903e3b95f46b71d8186f3abe415a96ef62d0a94ceaa7f3efd7b1208a0cefd36" + }, + "vl-4-4-combined.after.metrics": { + "bytes": 12303, + "sha256": "410a205658fbdf201e0613fbe59d7fac62308a4631ee871749d1af92df31feae" + }, + "vl-4-4-combined.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-4-combined.before.health": { + "bytes": 296, + "sha256": "3ecca835aef6168623e5fd09af1a553676471c6c3d76176c12322725ac4e2881" + }, + "vl-4-4-combined.before.metrics": { + "bytes": 12128, + "sha256": "f13f0e533029c9905c333fab10297ef89759d5e0531ea3d66d6c2b1a82a78c44" + }, + "vl-4-4-combined.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-4-combined.cold-2048.client.log": { + "bytes": 349, + "sha256": "511f761242e9bfd82f6052009f4b676cc1e516bbfd14f77111fe6ce6daa46439" + }, + "vl-4-4-combined.cold-2048.json": { + "bytes": 20278, + "sha256": "e2e455d5bf249cfa628a8b9301ed667f5d114b7ec3c05f34b04946a374d309ba" + }, + "vl-4-4-combined.cold-8192.client.log": { + "bytes": 348, + "sha256": "8030fa144d76c7d314242dcdfc70dd804854c9ef358d8f1d8ff283563ce4296c" + }, + "vl-4-4-combined.cold-8192.json": { + "bytes": 57148, + "sha256": "872ee644c6d57e1b4dcef93c8dd7fcb1cb68647a4a8494385b5d0d8fc311c031" + }, + "vl-4-4-combined.concurrent.client.log": { + "bytes": 3232, + "sha256": "fb487abdb880c8100d647468a2cf1ebca567d4716787f6cd9c0c4a91e75bf57a" + }, + "vl-4-4-combined.concurrent.json": { + "bytes": 286399, + "sha256": "d1005b226f6a12988c7e3441d5570f188178485eb335ef9e2f9edbd3f704de9d" + }, + "vl-4-4-combined.server.log": { + "bytes": 13571, + "sha256": "330f27864d8bb54bb91a04f5b112da15748dd9af60d97e57aabccfb380b34978" + }, + "vl-4-5-public.after.health": { + "bytes": 298, + "sha256": "071cb00608c4dba9f3c96bf0fe786cbc380d73d9cabfbc2dac30a7f98ca06691" + }, + "vl-4-5-public.after.metrics": { + "bytes": 12303, + "sha256": "f0af2844d4dab7231a07111df4d8716b0e3d577c403fa0269606280bed6374c2" + }, + "vl-4-5-public.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-5-public.before.health": { + "bytes": 296, + "sha256": "8c789a277ca189a0923767fa83532a352de4a11334a37855d824cd2137333df4" + }, + "vl-4-5-public.before.metrics": { + "bytes": 12128, + "sha256": "d104ba544adcc2cfa419f70f318c739fa606ac56eb0f3dfd4040c51d55cd905c" + }, + "vl-4-5-public.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-5-public.cold-2048.client.log": { + "bytes": 348, + "sha256": "ff06617eb5a750af3d6ba6fabccede4ea9cf89359aecd654d82efbbabf06834b" + }, + "vl-4-5-public.cold-2048.json": { + "bytes": 20276, + "sha256": "0314819143b49e4b4382cb8e49df1885a14cf6b5a775b31a9a01e40199a5405d" + }, + "vl-4-5-public.cold-8192.client.log": { + "bytes": 348, + "sha256": "c2191169a10f24ae06903210d14a0da8f4c1f15166722f4f9a44a707172f1ad1" + }, + "vl-4-5-public.cold-8192.json": { + "bytes": 57149, + "sha256": "39ab1a8ca9e37da85f611bf75d71897c9f162b08760dd2cec2b221b9e5b95995" + }, + "vl-4-5-public.concurrent.client.log": { + "bytes": 3218, + "sha256": "8f713283ff444de7cde34b0e422f1c7e1f72ecbb7c4dda676d3d5af6839e5ac6" + }, + "vl-4-5-public.concurrent.json": { + "bytes": 286440, + "sha256": "b826b0e5373ccf1da367fa037707bf6c098506de020210d86abf428d28aed468" + }, + "vl-4-5-public.server.log": { + "bytes": 13576, + "sha256": "1b02113c52425afa7b408f453db6a168d8ab0ae2fc3cdca036d05f1fe434ce78" + } + }, + "raw_retention": "Raw client/server logs, health/metrics/settings captures, manifest, events and records remain in local OUTPUT. This receipt includes their hashes rather than copying large token timestamp arrays or unrelated model catalog entries.", + "_trim": { + "note": "Raw capture replaced by counts, first/last items and SHA-256 of the canonical JSON of each trimmed list or string.", + "original_sha256": "4e62ee38aa7939c1a284017c81d211cd2e1d03ceb15f8a507370355a88f6ef88", + "original_bytes": 396781 + } +} diff --git a/docs/receipts/lfm3/m4-ffn-component-diagnostic-rejected.json b/docs/receipts/lfm3/m4-ffn-component-diagnostic-rejected.json new file mode 100644 index 0000000000..3481c11bb9 --- /dev/null +++ b/docs/receipts/lfm3/m4-ffn-component-diagnostic-rejected.json @@ -0,0 +1,105 @@ +{ + "scope": "Rejected M4 counterfactual decomposition diagnostic on actual TEXT4 FFN gate rows64 K2048 first2048/7680 columns. Not additive attribution to the original kernel and not a production candidate.", + "stable": false, + "stability_limit": 0.05, + "fp64": { + "rms": 0.0005267428383594065, + "max_abs": 0.0075808195397257805, + "worst_column_rms": 0.004595861546563877, + "every_column_no_worse": true, + "words": 131072 + }, + "decoded_code_words_equal": 4194304, + "components": [ + { + "mode": "public", + "median_gpu_seconds_per_op": 0.0002631956376717426, + "median_wall_seconds_per_op": 0.00026455468750000003, + "first_last_drift": 0.0002428999546797428, + "within_round_spreads": [ + 0.0009473604771508943, + 0.0003384318514383975, + 8.752778165050982e-05, + 0.0001538135695963428, + 0.001586878958627219 + ], + "stable": true + }, + { + "mode": "packed_decode", + "median_gpu_seconds_per_op": 4.374155696496018e-05, + "median_wall_seconds_per_op": 4.392578125e-05, + "first_last_drift": 0.0002347722056248447, + "within_round_spreads": [ + 0.001459919626688347, + 0.001842322369902333, + 0.03586655677122039, + 0.06131780008675758, + 0.002390031492256755 + ], + "stable": false + }, + { + "mode": "input_prepare", + "median_gpu_seconds_per_op": 2.7438939298463083e-06, + "median_wall_seconds_per_op": 2.784088134765625e-06, + "first_last_drift": 0.009196062636040114, + "within_round_spreads": [ + 0.009050100160004031, + 0.09418500123531458, + 0.034338642051004904, + 0.10100869005936197, + 0.0026607969687828838 + ], + "stable": false + }, + { + "mode": "dot_affine", + "median_gpu_seconds_per_op": 0.000819583328848239, + "median_wall_seconds_per_op": 0.000825890625, + "first_last_drift": 0.001882662879938879, + "within_round_spreads": [ + 0.001232538869545966, + 0.001871578372259783, + 0.00011273648065029056, + 2.062623227549552e-05, + 0.0014560840447383036 + ], + "stable": true + }, + { + "mode": "reduce_pack", + "median_gpu_seconds_per_op": 4.914851842841017e-05, + "median_wall_seconds_per_op": 4.9447265625e-05, + "first_last_drift": -0.0037752107294741766, + "within_round_spreads": [ + 0.001829691530696208, + 0.04876328099464931, + 0.007236481577243632, + 0.04276429918835022, + 0.023502143823044663 + ], + "stable": true + }, + { + "mode": "whole_decomposed", + "median_gpu_seconds_per_op": 0.0009281731763621792, + "median_wall_seconds_per_op": 0.00093384375, + "first_last_drift": 0.0010547062489185688, + "within_round_spreads": [ + 0.0001979186618144672, + 0.0029819320484378545, + 0.00044058661435578784, + 0.0034756845837579996, + 0.00023276829068437607 + ], + "stable": true + } + ], + "raw_sha256": "e29c79e34a61a14d56699ac1573ff52cfa63eb3c23001663c54385cca4748e95", + "scratch_footprint_bytes": 34111488, + "analysis_sha256": "46711af8f6bcb678d2157f1c7fb2f250888d625cf7e97b689e497c4e249cdced", + "outcome": "Numerical gates passed; complete diagnostic rejected because packed decode and input preparation exceeded predeclared 5% within-round stability limit. No retry.", + "counter_evidence": "M4 public Metal counter API enumerated timestamp/GPUTimestamp only. No measured register, occupancy, cache or spill attribution.", + "cleanup": "GPU host exit0; analyzer exit1 DiagnosticStability; lock released." +} diff --git a/docs/receipts/lfm3/m4-ffn-legacy-matrix-rejected.json b/docs/receipts/lfm3/m4-ffn-legacy-matrix-rejected.json new file mode 100644 index 0000000000..a456ab3bc0 --- /dev/null +++ b/docs/receipts/lfm3/m4-ffn-legacy-matrix-rejected.json @@ -0,0 +1,50 @@ +{ + "scope": "Apple M4 Pro exploratory complete legacy SIMD-matrix FFN gate route, actual TEXT4 rows64 K2048 first2048 of7680 columns. Negative bounded screen; no promotion/model qualification.", + "rows": 64, + "n": 2048, + "group": 64, + "bits": 4, + "equal_useful_matrix_work_per_sample": true, + "common_repetitions": 128, + "median_gpu_seconds_per_op": { + "public_row_pairs": 0.0002627867834235076, + "tile_fullroute": 0.0003518678386171814 + }, + "candidate_throughput_vs_public": { + "tile": 0.7468337670650527 + }, + "baseline_first_last_drift_fraction": 0.0004791987765457966, + "exploratory_drift_limit": 0.05, + "stable_for_exploratory_screen": true, + "actual_selected_outputs": [ + 238, + 0 + ], + "actual_repair_fractions": [ + 0.0018157958984375, + 0.0 + ], + "diagnostics_in_timed_work": { + "centers_bounds_stores": true, + "raw_groupdot_capture": false, + "snapshot": false, + "prepare_each_op": true, + "actual_repair_dispatch": true + }, + "final_performance_qualification": false, + "implementation": "Unchanged pre-MPP fused-finish exact-band producer, ordered compensated combine and cooperative global53 repair; legacy simdgroup matrix operations; no device capability bypass.", + "numerical": { + "primary_all_columns_no_worse": true, + "strict_gate_errors": 0, + "tile_twin_word_differences": 0, + "negative_controls": 4, + "five_cost_output_selector_route_identity": true + }, + "provenance": { + "numeric_manifest_sha256": "098bd80398394934a9d37418ebf055872cb46cb2d0ae24c9b476cf63438f7936", + "cost_manifest_sha256": "c93763920151ef0584ca5be2e28e9311541c002172fd2af78b9d1ef0e1642ed6", + "cost_analysis_sha256": "761615daedff89e0b89831f29eca83473f5e9f57c30c5d37fce7ec05211c1d9f", + "shader_sha256": "a866ccc63d61a9454f078d0255a131a7c7d41c9e9561a65bb3ca818d52b7d44e" + }, + "cleanup": "All stages completed; owned children exited and local lock released." +} diff --git a/docs/receipts/lfm3/m4-fused-swiglu-rejected.json b/docs/receipts/lfm3/m4-fused-swiglu-rejected.json new file mode 100644 index 0000000000..05a790cef0 --- /dev/null +++ b/docs/receipts/lfm3/m4-fused-swiglu-rejected.json @@ -0,0 +1,50 @@ +{ + "scope": "Actual TEXT4 full7680-column gate+up+SwiGLU rows64, matched five-round operator screen, not model/serving qualification", + "modes": { + "public_dual_activation": { + "seconds": 0.0019839440210489556, + "first_last_drift": 0.0008691182443005818, + "within_round_spreads": [ + 0.0023387788655644393, + 0.00015884899138818298, + 7.218093726524266e-05, + 0.00022838467355179581, + 3.807140167411127e-05 + ], + "stable": true + }, + "fused": { + "seconds": 0.002006229173275642, + "first_last_drift": -0.0006157325280637949, + "within_round_spreads": [ + 0.0003530385890222476, + 0.0010237456636403852, + 0.0003246003034091949, + 0.00016615617853843467, + 0.00014673585857649663 + ], + "stable": true + } + }, + "stable": true, + "throughput_ratio": 0.9888920206507112, + "device": "Apple M4 Pro", + "outcome": "Rejected performance candidate; exact public arithmetic retained but complete route slower. No production change.", + "numerical": { + "actual_words": 491520, + "gate_up_each_match_frozen_public_canonical": true, + "fused_final_matches_complete_public_baseline": true, + "five_repeat_identity": true, + "bounded_cases": 5, + "negative_controls": 3, + "guard_regions": 22, + "note": "Literal full output identity establishes unchanged error against the same primary reference in every column; no new independent FP64 computation in this fused screen." + }, + "pins": { + "numerical_manifest": "8a787c4ec45e3aee70b6030057e578e02109d444185f665e008759f11bc523f4", + "cost_manifest": "d7bfdc7c65028bc06a181be170bddfa97ae64270e7f458e520c1bda3eb64fa12", + "analysis": "8f9ed0c017c00caeedec6758f4d4ee0a2b7f3e3091232506a5d1f9123e67fee1", + "shader": "7369ffe684be904e90aa9aba32d65c34d615a09e54d0e148f723845e26e3986b" + }, + "cleanup": "All owned children exited, local GPU lease released." +} diff --git a/docs/receipts/lfm3/m5-2048-prompt-tail-contract.json b/docs/receipts/lfm3/m5-2048-prompt-tail-contract.json new file mode 100644 index 0000000000..e8f5e6fd9d --- /dev/null +++ b/docs/receipts/lfm3/m5-2048-prompt-tail-contract.json @@ -0,0 +1,52 @@ +{ + "scope": "M5 targeted mechanical contract for 2048-mode prompt-tail kernels. Not independent propagated model fidelity or full backend qualification.", + "source_base": "d9c1ea531", + "sources_sha256": { + "zig/tests/contract/metal_lfm3_tiled.zig": "b3885ded26ed103e757691374584db4eb32e245b8ebc6f449c582a34fb90ce51", + "zig/tests/contract/metal_lfm3_residual.zig": "bcb83756775468e2762f7e9a694f16ed5e6322c450e28a262367680f712e98c5", + "zig/kernels/metal/lfm3/attention_tiled.metal": "6b134e9062fd55dc2b04bf5124c54c540e9a74d669cdb88336460093abb24c81", + "zig/kernels/metal/lfm3/residual_ffn.metal": "add7140a9ec0589fe674c2066317b9e511430596892cbeb1b831feadd465c86b" + }, + "binary_sha256": "ca7fd00d0e2d6c6c7ec62e5aa749e4352ad71c2fa6137caad5da791e654195b8", + "commands": [ + "tf-kernel-contract --only LFM3 tiled structural --no-limits", + "tf-kernel-contract --only LFM3 residual FFN --no-limits" + ], + "actual_tail_rows": [ + 1, + 5, + 8 + ], + "nonfinite_fixture_rows": 5, + "results": { + "attention": { + "exit": 0, + "elapsed_seconds": 6.321991208940744, + "summary": "SUMMARY 2 kernels: 2 pass gates 1-4, 0 fail, 0 known findings; 2 controls: 2 red as planted, 0 missed; 0 limit problems; gates 4951 ms, limits 0 ms", + "log_sha256": "f8d9df17c65f6971d48803cdc3cd49eaf6408f3d858ba2fa85b0f51c655795cd", + "nonfinite": "PASS nonfinite metal LFM3 tiled structural full: the stated rule holds: visible-query/key/value nonfinite classes; masked future values must not contaminate", + "shapes": [ + "PASS shapes metal LFM3 tiled structural full: 16 shapes exact by its own check (worst 0.000 of the bound), 0 edges refused by the kernel", + "PASS shapes metal LFM3 tiled structural sliding: 16 shapes exact by its own check (worst 0.000 of the bound), 0 edges refused by the kernel" + ], + "controls_red": 2 + }, + "residual": { + "exit": 0, + "elapsed_seconds": 13.454363541211933, + "summary": "SUMMARY 1 kernels: 1 pass gates 1-4, 0 fail, 0 known findings; 2 controls: 2 red as planted, 0 missed; 0 limit problems; gates 12434 ms, limits 0 ms", + "log_sha256": "75e7038a7c5b4da1942008d68d96962879491691c1aae64a38951046b3a4fd05", + "nonfinite": "PASS nonfinite metal LFM3 residual FFN: 30720 non-finite and 30720 finite outputs match the float64 classes", + "shapes": [ + "PASS shapes metal LFM3 residual FFN: 15 shapes within the float64 bound (worst 0.992 of the bound), 0 edges refused by the kernel" + ], + "controls_red": 2 + } + }, + "limitations": [ + "Targeted --no-limits: zero limit problems in summary is not a launch-limit pass.", + "Attention uses a uniform-score structural fixture; it is not a reduced-accumulation witness or official MLX comparison.", + "Five-row nonfinite fixtures do not establish arbitrary tails or full-model FP32/BF16 acceptance.", + "Residual 15-shape synthetic FP64-bound contract is mechanical evidence, separate from agreed model-fidelity gate." + ] +} diff --git a/docs/receipts/lfm3/m5-attention-chunk-fusion-exploratory.json b/docs/receipts/lfm3/m5-attention-chunk-fusion-exploratory.json new file mode 100644 index 0000000000..588e7d6a8f --- /dev/null +++ b/docs/receipts/lfm3/m5-attention-chunk-fusion-exploratory.json @@ -0,0 +1,80 @@ +{ + "schema_version": 1, + "scope": "Private exact attention chunk fusion; M5 VL4 cold text prefill; bounded runtime diagnostic, production default unchanged", + "candidate": "One128thread group per row/head computes up to3 original4096key chunks sequentially, retains1024key part and chunk merge order, fuses final merge dispatch", + "baseline": "Current fused-risk native attention, samebinary deferredheads enabled", + "numeric": { + "synthetic_cases": 35, + "five_repeat_all_intermediate_bytes_equal": true, + "negative_operator_controls": 2, + "real_model_prompts": [ + 2048, + 8192 + ], + "all_active_state_and_logits_equal": true, + "snapshot_marks": [ + 129, + 257 + ], + "resume_prefix": 257, + "greedy_decode_tokens": 16, + "actual_fused_hits": [ + 660, + 2772 + ], + "actual_zero_store_control_rejected": true + }, + "served": { + "protocol": "FreshserverABBA, samebinary envselector0/1, exactpinnedIDs, promptcache0,16outputs, no retries", + "binary_sha256": "806a30d6e816a4adca45faed72d192be4316b4076224e71910bb147e2b61fe74", + "cases": [ + { + "prompt_tokens": 2048, + "baseline_ttft_s": [ + 0.8451337078586221, + 0.8453265000134706 + ], + "candidate_ttft_s": [ + 0.8443755409680307, + 0.8437215420417488 + ], + "baseline_median_s": 0.8452301039360464, + "candidate_median_s": 0.8440485415048897, + "baseline_over_candidate": 1.001399874975259, + "baseline_first_last_drift": 0.00022812029984820015, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "not emitted by cold client; separate model qualification covers16 IDs/logits", + "prompt_ids": "both routes /tokenize exact pinned arrays" + }, + { + "prompt_tokens": 8192, + "baseline_ttft_s": [ + 4.424025000073016, + 4.418909166939557 + ], + "candidate_ttft_s": [ + 4.239424915984273, + 4.240528000053018 + ], + "baseline_median_s": 4.421467083506286, + "candidate_median_s": 4.239976458018646, + "baseline_over_candidate": 1.042804630470154, + "baseline_first_last_drift": -0.0011563752766710067, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "not emitted by cold client; separate model qualification covers16 IDs/logits", + "prompt_ids": "both routes /tokenize exact pinned arrays" + } + ], + "server_token_digest_equal": true + }, + "limits": [ + "OnlyVL4 text fullmodel/served qualified so far; notimages,otherwidths or M4", + "Two legs pervariant; fivepercentbaseline driftgate retained", + "No MLXcomparison and no10xspeedclaim", + "Coldclient outputtextdigest matched; tokenIDs andfullstate/logits verified separately inmodel harness", + "Sharedmultistream,decode and morethan3chunks retainoriginalpath", + "SingleSG slidingwindow source prototype not yet qualified" + ] +} diff --git a/docs/receipts/lfm3/m5-attention-nonfinite-repair-overhead.json b/docs/receipts/lfm3/m5-attention-nonfinite-repair-overhead.json new file mode 100644 index 0000000000..07b06ea982 --- /dev/null +++ b/docs/receipts/lfm3/m5-attention-nonfinite-repair-overhead.json @@ -0,0 +1,184 @@ +{ + "scope": "M5 TEXT4 cold serving overhead of nonfinite-row attention repair, comparing two combined-enabled builds. Not a default-versus-enabled, MLX, or broad fidelity comparison.", + "source_commits": { + "before": "8df18abc47568246c0d38b70c2e280521a24b31a", + "after": "1e43c6722ad2aa4458413c3ddb1c2fe79c911fe1" + }, + "build": { + "source_commits": { + "before": "8df18abc4", + "after": "1e43c6722" + }, + "command_common": "zig build native -Dcpu=apple_m1 -Dlfm3-residual-ffn=true -Dlfm3-tiled-attention=true -j4 (isolated --cache-dir and --prefix)", + "zig_version": "0.17.0", + "zig_sha256": "1c5f706db0ed6d55451940f31dc05a6177d05696bc1fa5d853696d04c718b523", + "build_exit": { + "before": 0, + "after": 0 + }, + "build_log_sha256": { + "before": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "after": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "finite_identity_packet_sha256": "12acac0296114928c4e232806f342543d357a335cd947dd8c68a8595bb35fe26" + }, + "binary_sha256": { + "before": "ebbe95416279a5d5304e414a496631d0eddbe089b29d239e66d52c0e0c5f3aa3", + "after": "3c987ac2decf021380122a22f2e30c12590045a959e6f160e9ed7570577248bd" + }, + "protocol": { + "order": [ + "before", + "after", + "after", + "before" + ], + "fresh_owned_servers": 4, + "prompt_tokens": [ + 2048, + 8192 + ], + "output_tokens": 16, + "cached_tokens": 0, + "context": 70000, + "parallel_setting": 4, + "exact_tokenize_ids_equal_pinned": true, + "baseline_first_last_max_drift": 0.05, + "candidate_max_relative_spread": 0.05, + "timing_retries": 0, + "allocator_or_memory_limit_raised": false, + "existing_model_lock_preserved": true, + "owned_cleanup_all_exit_zero": true + }, + "results": [ + { + "prompt_tokens": 2048, + "baseline_ttft_s": [ + 0.5555327502079308, + 0.5556167499162257 + ], + "candidate_ttft_s": [ + 0.5581228327937424, + 0.558499707840383 + ], + "baseline_median_s": 0.5555747500620782, + "candidate_median_s": 0.5583112703170627, + "baseline_over_candidate": 0.9950985760086297, + "baseline_first_last_drift": 0.00015120568186732797, + "candidate_relative_spread": 0.0006752546652752933, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "not emitted by cold client; consult separate joint-model observation; full-model fidelity remains open", + "prompt_ids": "both routes /tokenize exact pinned arrays", + "candidate_over_baseline_minus_one": 0.004925566280106652 + }, + { + "prompt_tokens": 8192, + "baseline_ttft_s": [ + 2.413786290679127, + 2.4111012499779463 + ], + "candidate_ttft_s": [ + 2.43808350013569, + 2.439888416789472 + ], + "baseline_median_s": 2.4124437703285366, + "candidate_median_s": 2.438985958462581, + "baseline_over_candidate": 0.9891175313896537, + "baseline_first_last_drift": -0.0011123771443847241, + "candidate_relative_spread": 0.0007403014103830596, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "not emitted by cold client; consult separate joint-model observation; full-model fidelity remains open", + "prompt_ids": "both routes /tokenize exact pinned arrays", + "candidate_over_baseline_minus_one": 0.011002199703261883 + } + ], + "full_model_finite_identity": { + "prompt": 128, + "teacher_forced_continuation": 16, + "observed_logit_rows": 17, + "vocabulary": 128000, + "all_bytes_equal": true, + "all_values_finite": true, + "bytes": 4352000, + "sha256": "583760763110f09b4557d7d66e4c8801ce2f1996f1052214c9bf99db2cf34cf1", + "guarded_run_exit": 0, + "receipt_sha256": "16fb015982b855a9a5e65fe3a9c6b03f3557136e0ac0acbdbf484ee473826a68" + }, + "interpretation": "Repair costs +0.493% median TTFT at2k and +1.100% at8k versus the prior enabled route. Both cells satisfy the predeclared stability gates. No speedup claim.", + "limitations": [ + "Two observations per route per prompt length; not a confidence interval.", + "Serving text digests equal; no new claim of generated-token-ID equality.", + "Actual per-request route counters are not in the served binaries; both flags and production source are pinned, with separate model identity test.", + "No full kernel runner or launch-limit suite is part of this timing receipt.", + "Finite identity is bounded to the pinned128prompt+16teacher-forced observations; broader FP32/official-BF16 qualification remains separate." + ], + "packet_pins_sha256": "28f8ac96f9c832401cb149224934831004406df629fb51132b7ffd15e5e0830e", + "packet_archive_sha256": "b5a4cdb5c8df0713975e208886abf233ad4d066d066cfaa8666ba7a03566938f", + "raw_sha256": { + "text-4-0-public.cold-2048.json": "42f3c8d5a1c527628899c7638163a500d69f90d4a608d964119ef2fdfea3d186", + "text-4-1-candidate.cold-8192.client.log": "3f28a98187cfcd258820dd43746740fca4427e4c64b1bab382c948a18f58a20e", + "text-4-1-candidate.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-0-public.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "text-4-0-public.after.metrics": "77af64b73088c8eb7e87376026210ea16e41529defda7b002d5cb26d76ec78dd", + "text-4-3-public.cold-8192.client.log": "927d83cc152d70b3fec8e660d04e1c8dd1fac690a66237bd1564698ba58cf4e0", + "text-4-3-public.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "text-4-1-candidate.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "text-4-1-candidate.after.metrics": "e522d21eda682d7d49c01eb82a9657df5f9ba7d630483972c5e29523753d82d0", + "text-4-0-public.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "text-4-1-candidate.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "text-4-1-candidate.before.metrics": "87c6f7a1aec5b481154e085cb4afccaa94085b879a25ec4f79e2a0ae02751aeb", + "text-4-0-public.cold-8192.json": "299abc9057ea882e4ca13ae47381c2ea67ec71f124531bc3949f5c7832af31e8", + "events.json": "2b476e74299788cb570207d854c96b861dcb81c064a465b053ba4dc634eaafc1", + "text-4-3-public.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "text-4-0-public.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "text-4-1-candidate.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "text-4-0-public.cold-2048.client.log": "43f8b34131523a069ead62c037d26c404d478fc1ddda5f1536d3a68f0b7b2b45", + "text-4-1-candidate.cold-2048.json": "ea7720c9198cd93be8628c875b6464efc5fcaa4deaf718843e0acdcef300f0e4", + "text-4-2-candidate.after.metrics": "074c5325d47924479cc1844e9ce98ad12aaee4c07cc61cd206360fa73d8419a4", + "text-4-2-candidate.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "text-4-2-candidate.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "text-4-3-public.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-3-public.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "text-4-0-public.before.metrics": "850d5807aaa3b32d412334a2c201d5878023930b830922cb0c251bd4e2e5d9d9", + "text-4-1-candidate.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "text-4-2-candidate.server.log": "2e60a35122446ed99d437ffa53e1f7a3732c53cb2e03c4d7acaa8ea9698473bb", + "text-4-2-candidate.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-2-candidate.cold-2048.client.log": "75bd26d4fcf78def26742d1fb631d21fa34f8c082038ea281870e4adba5127bf", + "text-4-1-candidate.cold-8192.json": "ece0444b55565afb90207660500654c56e64f57d9e66761a9d22bc294a965004", + "text-4-0-public.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "text-4-0-public.after.health": "96185b80b9e6bc456410a78382a59c47e8cdf331b0fc871e538d9c0ca6b75f75", + "text-4-1-candidate.after.health": "359c581fd4470bccce5c1371da6e01f7943847df537cdab096bdcd61862d173b", + "text-4-0-public.server.log": "d2c977d85347b019d36ca9b73d30532a0be488381fb326f8988f373656d9913a", + "text-4-3-public.cold-8192.json": "2eb31179323698e1a80c9cf726cddd0ea648b9696feeeaaec85cca4034038c4c", + "text-4-0-public.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-1-candidate.server.log": "3832c305427ecbd44af2ebc35977b1013ff0896ced878d904633bc58bed94537", + "text-4-3-public.after.health": "ca6f5ab8f24552b9436181c0358b18512a93225b17311efd9b09a21805b15d27", + "text-4-2-candidate.before.metrics": "237b97431ebcc704c60a59294fc10eae1eee497d0aa84ae96b9c6894d6a1c487", + "text-4-2-candidate.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "text-4-1-candidate.cold-2048.client.log": "77e1d8f1266dbe28b7ce935d1c91f6d18baa1b2a8d189a497d9e42b60d6e7382", + "manifest.json": "d5178b371c4576392e5ea7f039e9d28fa00e8a8a2d1c8e4fff7d5308ecae82ed", + "text-4-2-candidate.after.health": "70dcace072d916a0ba6b1700bd1e6cac3e6b2714c52b8dfca11f709c9567063e", + "text-4-3-public.cold-2048.client.log": "fb28a9b2f2412a5d993e216c56c0d108426b129e4c8ff2040e3f64b691a7957e", + "text-4-2-candidate.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "text-4-3-public.cold-2048.json": "3253f3a4103ef69dd865c5147e640fab60e7b0a9fc170591788c19818edf79b5", + "records.json": "c66a77433950fcb6b2afce5d9f28d7b92b49c3388174ad3a2d56d21a10a3c9e8", + "text-4-2-candidate.cold-8192.json": "8687ad013df3dad3af1f933ea6159229d2ffb2c391678d040f3458ad10e19f30", + "text-4-2-candidate.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-1-candidate.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "text-4-2-candidate.cold-8192.client.log": "acd250d72a4201810caf3f0a89973dafa40ba7207e737b629b81ca472ec8bbe3", + "text-4-3-public.before.metrics": "b7e2f9da4dbd325b275cb07c66a50f60672d7ca53dd59025ef8269bc26e82e7b", + "text-4-0-public.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "text-4-0-public.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-0-public.cold-8192.client.log": "fd58f2fe167fb1644e132ca2c6b7ed24e9dcc2eb7571206795efa02221150021", + "text-4-3-public.server.log": "0d4ef6e1fb9d32e5948476b870e9b94a4b3e670f89109be49c073a1be922d8c5", + "text-4-1-candidate.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-3-public.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-3-public.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "text-4-3-public.after.metrics": "b7c374c0d2d0903d4c0e626d19014aff9cae1859d118b92db46ff0fbf9143c15", + "text-4-2-candidate.cold-2048.json": "da11dd38b92c441439f7eda849a6ce0a35615510877434f95c135fc795049dc6", + "text-4-3-public.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "text-4-2-candidate.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd" + } +} diff --git a/docs/receipts/lfm3/m5-cold-stage-profile.json b/docs/receipts/lfm3/m5-cold-stage-profile.json new file mode 100644 index 0000000000..c39bcf7526 --- /dev/null +++ b/docs/receipts/lfm3/m5-cold-stage-profile.json @@ -0,0 +1,559 @@ +{ + "scope": "M5 text4 exactHTTPcold2k8k diagnostic; existing profiler labels only, no kernel/arithmetic variant. Stage instrumentation may perturb schedules; plain wall clocks retained. No performance acceptance.", + "hardware": "M5; exclusive existing model lease; no all-width or serving performance acceptance", + "native_cache_capacity": 70000, + "native_prefill_chunk": 128, + "token_verification": [ + { + "tokens": 2048, + "actual_current_native_tokenize_matches_pinned_http_fixture": true, + "body_sha256": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "npy_sha256": "cc3c5a72ddfc5cd60f83fa046250606cc452a0ac483e49184626dc78e5a45d12" + }, + { + "tokens": 8192, + "actual_current_native_tokenize_matches_pinned_http_fixture": true, + "body_sha256": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "npy_sha256": "92bcc953dc435011aec99e723b25210e105f8b9cc4d370d938722ce3861d690a" + } + ], + "source_sha256": { + "zig/tests/lfm3_profile.zig": "f5620a0ca8565abb3d923c2da6793c1b51eb030fa87214d8a7f933195e17bb32", + "zig/src/core/gpu_profile.zig": "80eb55884ef6d7c9b894f06e3ebfa06a4a485bd3f48895d634ed89d6a638261b", + "zig/src/families/lfm3/forward.zig": "894c9541112518287bc9985db51f7e023b870b234da54acc626e9f2f57533bc9", + "zig/kernels/metal/lfm3/attention.metal": "6dc51acbb45e2e65028e72fed72d318629eb7b382e773c5837db9baa8de5d3b9", + "zig/kernels/metal/lfm3/tile_pair.metal": "e21ba279d7518093dd7a8e74bfb848da7f41d97136a2c2ad9d2a1c1d226be50f", + "zig/src/families/lfm3/prompt_projection.zig": "634d4f1ec738dad4a067410875760a66b790cc318e63b54ecf880f18bf67c235", + "zig/src/families/lfm3/tile_pair.zig": "d98e6de843242f0620b79f65568a0a0a859e00138ad807464f8e1d49f6fbed9e" + }, + "diagnostic_binary_sha256": "f664911c1ad88511208596165604bd142e8d689a6afb8889e1e44f88d04e4fb1", + "same_variant_profiled_and_plain_endpoint_identity": true, + "cross_variant_bit_identity_claimed": false, + "runs": [ + { + "tokens": 2048, + "variant": "current", + "cache_capacity": 70000, + "chunk": 128, + "counter_mode": [ + "stage", + "stage" + ], + "plain_wall_seconds_median": 0.883111, + "plain_gpu_seconds_median": 0.846759352600202, + "profiled_stage_seconds_sum": 0.851895154, + "stages": { + "conv_in": { + "gpu_seconds_median": 0.08091941450000001, + "calls_each": [ + 208, + 208 + ] + }, + "conv_out": { + "gpu_seconds_median": 0.028126116000000003, + "calls_each": [ + 208, + 208 + ] + }, + "attn_q": { + "gpu_seconds_median": 0.0237688785, + "calls_each": [ + 176, + 176 + ] + }, + "attn_kv": { + "gpu_seconds_median": 0.012734998, + "calls_each": [ + 176, + 176 + ] + }, + "attn_gate": { + "gpu_seconds_median": 0.001957206, + "calls_each": [ + 176, + 176 + ] + }, + "attn_out": { + "gpu_seconds_median": 0.0237685195, + "calls_each": [ + 176, + 176 + ] + }, + "ffn_gate_up": { + "gpu_seconds_median": 0.36674050300000005, + "calls_each": [ + 384, + 384 + ] + }, + "ffn_down": { + "gpu_seconds_median": 0.179886763, + "calls_each": [ + 384, + 384 + ] + }, + "vocabulary_head": { + "gpu_seconds_median": 0.0023307285000000004, + "calls_each": [ + 16, + 16 + ] + }, + "norm": { + "gpu_seconds_median": 0.0042161035, + "calls_each": [ + 784, + 784 + ] + }, + "embedding": { + "gpu_seconds_median": 0.00010474999999999998, + "calls_each": [ + 16, + 16 + ] + }, + "activation": { + "gpu_seconds_median": 0.005800944500000001, + "calls_each": [ + 768, + 768 + ] + }, + "attention_prepare": { + "gpu_seconds_median": 0.0012115645, + "calls_each": [ + 176, + 176 + ] + }, + "attention": { + "gpu_seconds_median": 0.12032866449999999, + "calls_each": [ + 704, + 704 + ] + } + }, + "all_profiled_endpoints_equal_plain": true, + "repeat_final_logit_sha256": "8700e9a12456dce4f744caa329ad2013f44df7f2704e58e245e52aa0e9e9ddc5", + "attention_fraction_of_profiled_stage_time": 0.1412482086968181, + "plain_wall_seconds_each": [ + 0.882642, + 0.88358 + ] + }, + { + "tokens": 2048, + "variant": "paired_tiles", + "cache_capacity": 70000, + "chunk": 128, + "counter_mode": [ + "stage", + "stage" + ], + "plain_wall_seconds_median": 0.46441299999999996, + "plain_gpu_seconds_median": 0.42822166811674833, + "profiled_stage_seconds_sum": 0.438255828, + "stages": { + "conv_in": { + "gpu_seconds_median": 0.0322938525, + "calls_each": [ + 208, + 208 + ] + }, + "conv_out": { + "gpu_seconds_median": 0.013225664500000001, + "calls_each": [ + 208, + 208 + ] + }, + "attn_q": { + "gpu_seconds_median": 0.011144940500000002, + "calls_each": [ + 176, + 176 + ] + }, + "attn_kv": { + "gpu_seconds_median": 0.007488039, + "calls_each": [ + 176, + 176 + ] + }, + "attn_gate": { + "gpu_seconds_median": 0.0020158750000000003, + "calls_each": [ + 176, + 176 + ] + }, + "attn_out": { + "gpu_seconds_median": 0.011178397000000003, + "calls_each": [ + 176, + 176 + ] + }, + "ffn_gate_up": { + "gpu_seconds_median": 0.1439456705, + "calls_each": [ + 384, + 384 + ] + }, + "ffn_down": { + "gpu_seconds_median": 0.08293642150000001, + "calls_each": [ + 384, + 384 + ] + }, + "vocabulary_head": { + "gpu_seconds_median": 0.0023206685000000003, + "calls_each": [ + 16, + 16 + ] + }, + "norm": { + "gpu_seconds_median": 0.004304714, + "calls_each": [ + 784, + 784 + ] + }, + "embedding": { + "gpu_seconds_median": 0.000251833, + "calls_each": [ + 16, + 16 + ] + }, + "activation": { + "gpu_seconds_median": 0.005752878499999999, + "calls_each": [ + 768, + 768 + ] + }, + "attention_prepare": { + "gpu_seconds_median": 0.0012094789999999998, + "calls_each": [ + 176, + 176 + ] + }, + "attention": { + "gpu_seconds_median": 0.12018739449999999, + "calls_each": [ + 704, + 704 + ] + } + }, + "all_profiled_endpoints_equal_plain": true, + "repeat_final_logit_sha256": "7ad8423d5e79a75bc4b72f9a0b32b4f2023272d3a1ec04b597fb37cd5438e964", + "attention_fraction_of_profiled_stage_time": 0.2742402652087492, + "plain_wall_seconds_each": [ + 0.4644, + 0.464426 + ] + }, + { + "tokens": 8192, + "variant": "current", + "cache_capacity": 70000, + "chunk": 128, + "counter_mode": [ + "stage", + "stage" + ], + "plain_wall_seconds_median": 4.638943, + "plain_gpu_seconds_median": 4.5858032503165305, + "profiled_stage_seconds_sum": 4.605206704, + "stages": { + "conv_in": { + "gpu_seconds_median": 0.32363039149999995, + "calls_each": [ + 832, + 832 + ] + }, + "conv_out": { + "gpu_seconds_median": 0.11234039850000002, + "calls_each": [ + 832, + 832 + ] + }, + "attn_q": { + "gpu_seconds_median": 0.09514477200000002, + "calls_each": [ + 704, + 704 + ] + }, + "attn_kv": { + "gpu_seconds_median": 0.05088421800000002, + "calls_each": [ + 704, + 704 + ] + }, + "attn_gate": { + "gpu_seconds_median": 0.0079379555, + "calls_each": [ + 704, + 704 + ] + }, + "attn_out": { + "gpu_seconds_median": 0.09525006349999998, + "calls_each": [ + 704, + 704 + ] + }, + "ffn_gate_up": { + "gpu_seconds_median": 1.4671723220000001, + "calls_each": [ + 1536, + 1536 + ] + }, + "ffn_down": { + "gpu_seconds_median": 0.719726452, + "calls_each": [ + 1536, + 1536 + ] + }, + "vocabulary_head": { + "gpu_seconds_median": 0.009268914500000003, + "calls_each": [ + 64, + 64 + ] + }, + "norm": { + "gpu_seconds_median": 0.0169286425, + "calls_each": [ + 3136, + 3136 + ] + }, + "embedding": { + "gpu_seconds_median": 0.00036797999999999996, + "calls_each": [ + 64, + 64 + ] + }, + "activation": { + "gpu_seconds_median": 0.023302213, + "calls_each": [ + 3072, + 3072 + ] + }, + "attention_prepare": { + "gpu_seconds_median": 0.0048518125, + "calls_each": [ + 704, + 704 + ] + }, + "attention": { + "gpu_seconds_median": 1.6784005685000003, + "calls_each": [ + 2816, + 2816 + ] + } + }, + "all_profiled_endpoints_equal_plain": true, + "repeat_final_logit_sha256": "19a2f9b7b0794ef2f2be0edb7d16564e24d33f02e0d01972780ae28d09010bf5", + "attention_fraction_of_profiled_stage_time": 0.36445716259428085, + "plain_wall_seconds_each": [ + 4.63588, + 4.642006 + ] + }, + { + "tokens": 8192, + "variant": "paired_tiles", + "cache_capacity": 70000, + "chunk": 128, + "counter_mode": [ + "stage", + "stage" + ], + "plain_wall_seconds_median": 3.107767, + "plain_gpu_seconds_median": 3.053785268450156, + "profiled_stage_seconds_sum": 3.064434951, + "stages": { + "conv_in": { + "gpu_seconds_median": 0.1291200075, + "calls_each": [ + 832, + 832 + ] + }, + "conv_out": { + "gpu_seconds_median": 0.05291137350000001, + "calls_each": [ + 832, + 832 + ] + }, + "attn_q": { + "gpu_seconds_median": 0.04456602450000001, + "calls_each": [ + 704, + 704 + ] + }, + "attn_kv": { + "gpu_seconds_median": 0.029950899500000003, + "calls_each": [ + 704, + 704 + ] + }, + "attn_gate": { + "gpu_seconds_median": 0.0081318085, + "calls_each": [ + 704, + 704 + ] + }, + "attn_out": { + "gpu_seconds_median": 0.04479231750000002, + "calls_each": [ + 704, + 704 + ] + }, + "ffn_gate_up": { + "gpu_seconds_median": 0.5757702760000001, + "calls_each": [ + 1536, + 1536 + ] + }, + "ffn_down": { + "gpu_seconds_median": 0.331787373, + "calls_each": [ + 1536, + 1536 + ] + }, + "vocabulary_head": { + "gpu_seconds_median": 0.009284855000000002, + "calls_each": [ + 64, + 64 + ] + }, + "norm": { + "gpu_seconds_median": 0.017096581500000003, + "calls_each": [ + 3136, + 3136 + ] + }, + "embedding": { + "gpu_seconds_median": 0.0008800625000000004, + "calls_each": [ + 64, + 64 + ] + }, + "activation": { + "gpu_seconds_median": 0.0231809975, + "calls_each": [ + 3072, + 3072 + ] + }, + "attention_prepare": { + "gpu_seconds_median": 0.004806967500000002, + "calls_each": [ + 704, + 704 + ] + }, + "attention": { + "gpu_seconds_median": 1.7921554069999999, + "calls_each": [ + 2816, + 2816 + ] + } + }, + "all_profiled_endpoints_equal_plain": true, + "repeat_final_logit_sha256": "305e345e11883986154d8b8dde198985589ac1d064517c14bdd46224c46dd074", + "attention_fraction_of_profiled_stage_time": 0.5848240982942633, + "plain_wall_seconds_each": [ + 3.114677, + 3.100857 + ] + } + ], + "raw_logical_limits": [ + "These are prefill-only direct native passes, not16-output HTTP elapsed time.", + "Paired FFN gate/up and K/V remain grouped because each is a single existing dispatch.", + "Stage counter instrumentation can perturb scheduling; matched unprofiled wall clocks retained.", + "Projection class labels are host metadata; all kernels/arithmetic/dispatches unchanged." + ], + "performance_accepted": false, + "dominant_eight_k": { + "paired_tiles_attention_seconds": 1.792155407, + "paired_tiles_attention_fraction": 0.5848240982942633, + "paired_tiles_ffn_gate_up_plus_down_seconds": 0.9075576489999999, + "interpretation": "Shared tiles reduce dense time; scalar attention is the largest remaining8k stage and FFN remains substantial. No unmeasured standard stage breakdown inferred." + }, + "counter_clock_scope": { + "counter_mode": "stage; calibrated GPU counter samples for all8runs", + "profiled_command_buffer_gpu_timestamps": "All8invalid/unavailable clocks discarded (null inraw), never used forstage totals.", + "plain_clocks": "Uninstrumented wall and valid plaincommandGPUtimes retained separately; profiledstage sum is diagnostic, notHTTPelapsedtime." + }, + "raw_evidence_sha256": { + "cold-stage-profile/0-current-2048.json": "d57819311ac26762671e0741fb2652cf606000b0c7036cddc42b6b304504bb8e", + "cold-stage-profile/0-current-2048.log": "3abf7120a89d3ff17e97e5d25061c4199236419aba394e4cada6f084877cc954", + "cold-stage-profile/1-paired-2048.json": "2f17f9e7044183412c7a52d3067c1c761398c9df5bf2499c0aae9622b5b64142", + "cold-stage-profile/1-paired-2048.log": "3abf7120a89d3ff17e97e5d25061c4199236419aba394e4cada6f084877cc954", + "cold-stage-profile/2-paired-8192.json": "6f126f49b939dcbf7207afa1f9bc03696f2661a3a4ce39072923bdbb20c5589b", + "cold-stage-profile/2-paired-8192.log": "5cf360bde5c1a9e0ff68b1cbc385982da901a621793fe54b8ef9fac5763c83ef", + "cold-stage-profile/3-current-8192.json": "98492b420fe61647cef131e6b661e663c51bcfcd3ba8b904ac45c3b913c7fa02", + "cold-stage-profile/3-current-8192.log": "5cf360bde5c1a9e0ff68b1cbc385982da901a621793fe54b8ef9fac5763c83ef", + "cold-stage-profile/4-current-8192.json": "280bac4b3abd0f17d2a233fd7946292e500f970af72fc6b76b09650454b59d13", + "cold-stage-profile/4-current-8192.log": "5cf360bde5c1a9e0ff68b1cbc385982da901a621793fe54b8ef9fac5763c83ef", + "cold-stage-profile/5-paired-8192.json": "a6d2afadee39dd978e5a4b61d3dbceeda905bf09f4bc809c84a4ea9cc378c00b", + "cold-stage-profile/5-paired-8192.log": "5cf360bde5c1a9e0ff68b1cbc385982da901a621793fe54b8ef9fac5763c83ef", + "cold-stage-profile/6-paired-2048.json": "df474ed6d68b6065d36ac6486c5237abb93d86dfdbe0ad356f47c09ab0c126fb", + "cold-stage-profile/6-paired-2048.log": "3abf7120a89d3ff17e97e5d25061c4199236419aba394e4cada6f084877cc954", + "cold-stage-profile/7-current-2048.json": "8028346486e0fd8ef4c736f9d688573edac1b9c2c6166e1a4ab241570b6794aa", + "cold-stage-profile/7-current-2048.log": "3abf7120a89d3ff17e97e5d25061c4199236419aba394e4cada6f084877cc954", + "cold-stage-profile/admission.json": "be0c3a3290919ede0ec7782068bd9eb8582d6d3bc309bac481f40f0814eec171", + "cold-stage-profile/events.json": "5d28399bcaf6fecc70c6559fcae7f4d475f33f0fdecb45503d3dfc1c825b3d76", + "cold-stage-profile/records.json": "59fe919a83146c0ac1d9564f4cbf5805666b08719d9994de1bfe3f23c49f1f17", + "cold-stage-profile/summary.json": "8e939eda43fc2fe0624cfdbf002811e394706d954f6d4fbd3aa35f6e1487bdf7", + "cold-token-capture/events.json": "e40beeedfd304799b8e4ee137d00903873008b03befa7a729f0e39c3f9a68dd9", + "cold-token-capture/receipt.json": "f9c67f355260b8a706147ce1d27d4765c3e18f265d12343d4b877b8ee10cfc53", + "cold-token-capture/text-2048.npy": "cc3c5a72ddfc5cd60f83fa046250606cc452a0ac483e49184626dc78e5a45d12", + "cold-token-capture/text-8192.npy": "92bcc953dc435011aec99e723b25210e105f8b9cc4d370d938722ce3861d690a", + "cold-token-capture/text-capture.server.log": "3792e9007dc40ec42809781d62f6a3f7d8a1c15f96f79129e995a56a83403872" + } +} diff --git a/docs/receipts/lfm3/m5-composed-attention-held-input.json b/docs/receipts/lfm3/m5-composed-attention-held-input.json new file mode 100644 index 0000000000..b8455ce3c4 --- /dev/null +++ b/docs/receipts/lfm3/m5-composed-attention-held-input.json @@ -0,0 +1,702 @@ +{ + "scope": "Genuine TEXT4 baseline/composed128-row post-RoPE query and cached key/value held-input SDPA,11attentionlayers each; finite only; no propagated model or long-history qualification", + "cells": [ + { + "name": "baseline-layer10", + "pass": true, + "native": { + "mean_absolute_error": 0.00010594119835396392, + "rms_error": 0.00021550765407174605, + "relative_rms": 0.0015259195565940598, + "maximum_absolute_error": 0.015355110168457031, + "maximum_coordinate": [ + 2, + 1, + 103 + ] + }, + "reference": { + "mean_absolute_error": 0.00010726264637057415, + "rms_error": 0.00021789633100235262, + "relative_rms": 0.0015428327788111428, + "maximum_absolute_error": 0.015355110168457031, + "maximum_coordinate": [ + 2, + 1, + 103 + ] + }, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true + } + }, + { + "name": "baseline-layer12", + "pass": true, + "native": { + "mean_absolute_error": 7.395126232685473e-05, + "rms_error": 0.0001150114182268993, + "relative_rms": 0.001648756981157906, + "maximum_absolute_error": 0.0013636946678161621, + "maximum_coordinate": [ + 10, + 12, + 117 + ] + }, + "reference": { + "mean_absolute_error": 7.479105307990532e-05, + "rms_error": 0.00011664460840612599, + "relative_rms": 0.0016721697322662021, + "maximum_absolute_error": 0.0013636946678161621, + "maximum_coordinate": [ + 10, + 12, + 117 + ] + }, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true + } + }, + { + "name": "baseline-layer14", + "pass": true, + "native": { + "mean_absolute_error": 7.412607476845953e-05, + "rms_error": 0.00011659142212767856, + "relative_rms": 0.0016626462764450763, + "maximum_absolute_error": 0.0019254684448242188, + "maximum_coordinate": [ + 3, + 4, + 21 + ] + }, + "reference": { + "mean_absolute_error": 7.504565638619641e-05, + "rms_error": 0.00011844840975364292, + "relative_rms": 0.0016891277577184853, + "maximum_absolute_error": 0.0019254684448242188, + "maximum_coordinate": [ + 3, + 4, + 21 + ] + }, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true + } + }, + { + "name": "baseline-layer16", + "pass": true, + "native": { + "mean_absolute_error": 6.565777003919885e-05, + "rms_error": 0.00010105450152199125, + "relative_rms": 0.0016662280876570976, + "maximum_absolute_error": 0.0009753108024597168, + "maximum_coordinate": [ + 84, + 6, + 54 + ] + }, + "reference": { + "mean_absolute_error": 6.641923482766975e-05, + "rms_error": 0.00010248465323955035, + "relative_rms": 0.001689809016022663, + "maximum_absolute_error": 0.001065760850906372, + "maximum_coordinate": [ + 19, + 6, + 54 + ] + }, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true + } + }, + { + "name": "baseline-layer18", + "pass": true, + "native": { + "mean_absolute_error": 1.4001119790051363e-05, + "rms_error": 2.7138564834208702e-05, + "relative_rms": 0.0017797468423647758, + "maximum_absolute_error": 0.0004876852035522461, + "maximum_coordinate": [ + 52, + 3, + 21 + ] + }, + "reference": { + "mean_absolute_error": 1.556294355272247e-05, + "rms_error": 2.809986246218312e-05, + "relative_rms": 0.0018427887323251334, + "maximum_absolute_error": 0.0005100518465042114, + "maximum_coordinate": [ + 60, + 0, + 79 + ] + }, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true + } + }, + { + "name": "baseline-layer2", + "pass": true, + "native": { + "mean_absolute_error": 4.413009259408515e-05, + "rms_error": 7.805294035885002e-05, + "relative_rms": 0.0016863690845297266, + "maximum_absolute_error": 0.0019491910934448242, + "maximum_coordinate": [ + 18, + 0, + 87 + ] + }, + "reference": { + "mean_absolute_error": 4.476193206600276e-05, + "rms_error": 7.927686684144782e-05, + "relative_rms": 0.0017128125698423987, + "maximum_absolute_error": 0.0019491910934448242, + "maximum_coordinate": [ + 18, + 0, + 87 + ] + }, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true + } + }, + { + "name": "baseline-layer20", + "pass": true, + "native": { + "mean_absolute_error": 3.917284894791747e-05, + "rms_error": 6.70369759858605e-05, + "relative_rms": 0.0015679623941835079, + "maximum_absolute_error": 0.0009984374046325684, + "maximum_coordinate": [ + 4, + 6, + 50 + ] + }, + "reference": { + "mean_absolute_error": 3.9700066141687114e-05, + "rms_error": 6.813284616510957e-05, + "relative_rms": 0.001593594266813504, + "maximum_absolute_error": 0.001111745834350586, + "maximum_coordinate": [ + 117, + 7, + 50 + ] + }, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true + } + }, + { + "name": "baseline-layer22", + "pass": true, + "native": { + "mean_absolute_error": 3.3669950547364404e-05, + "rms_error": 5.628150388543914e-05, + "relative_rms": 0.001671012112142587, + "maximum_absolute_error": 0.0009761154651641846, + "maximum_coordinate": [ + 94, + 7, + 94 + ] + }, + "reference": { + "mean_absolute_error": 3.416266679137419e-05, + "rms_error": 5.7299750519919624e-05, + "relative_rms": 0.0017012441127449295, + "maximum_absolute_error": 0.001028984785079956, + "maximum_coordinate": [ + 114, + 7, + 94 + ] + }, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true + } + }, + { + "name": "baseline-layer4", + "pass": true, + "native": { + "mean_absolute_error": 5.230709027901747e-05, + "rms_error": 8.98947544824526e-05, + "relative_rms": 0.0016505846531944756, + "maximum_absolute_error": 0.0012113451957702637, + "maximum_coordinate": [ + 1, + 7, + 109 + ] + }, + "reference": { + "mean_absolute_error": 5.300656258128925e-05, + "rms_error": 9.112962872549464e-05, + "relative_rms": 0.0016732585509755546, + "maximum_absolute_error": 0.0012113451957702637, + "maximum_coordinate": [ + 1, + 7, + 109 + ] + }, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true + } + }, + { + "name": "baseline-layer6", + "pass": true, + "native": { + "mean_absolute_error": 7.294471913596444e-05, + "rms_error": 0.00012866378806297889, + "relative_rms": 0.0016581336303815512, + "maximum_absolute_error": 0.0032798051834106445, + "maximum_coordinate": [ + 3, + 2, + 38 + ] + }, + "reference": { + "mean_absolute_error": 7.400460207339584e-05, + "rms_error": 0.00013074124021856306, + "relative_rms": 0.001684906456959583, + "maximum_absolute_error": 0.0032798051834106445, + "maximum_coordinate": [ + 3, + 2, + 38 + ] + }, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true + } + }, + { + "name": "baseline-layer8", + "pass": true, + "native": { + "mean_absolute_error": 5.9115582082743e-05, + "rms_error": 9.577837283847067e-05, + "relative_rms": 0.001668083770351387, + "maximum_absolute_error": 0.0009766817092895508, + "maximum_coordinate": [ + 55, + 9, + 120 + ] + }, + "reference": { + "mean_absolute_error": 5.980964068350405e-05, + "rms_error": 9.711988195074954e-05, + "relative_rms": 0.0016914475999055265, + "maximum_absolute_error": 0.001072615385055542, + "maximum_coordinate": [ + 88, + 8, + 120 + ] + }, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true + } + }, + { + "name": "candidate-layer10", + "pass": true, + "native": { + "mean_absolute_error": 0.00010553484145395611, + "rms_error": 0.00021304710107183093, + "relative_rms": 0.0015085044325107025, + "maximum_absolute_error": 0.01345968246459961, + "maximum_coordinate": [ + 1, + 2, + 103 + ] + }, + "reference": { + "mean_absolute_error": 0.00010687674114391642, + "rms_error": 0.0002160143358914129, + "relative_rms": 0.0015295142789489807, + "maximum_absolute_error": 0.01345968246459961, + "maximum_coordinate": [ + 1, + 2, + 103 + ] + }, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true + } + }, + { + "name": "candidate-layer12", + "pass": true, + "native": { + "mean_absolute_error": 7.394120734161447e-05, + "rms_error": 0.00011502306016155579, + "relative_rms": 0.0016489296643631783, + "maximum_absolute_error": 0.0010012984275817871, + "maximum_coordinate": [ + 26, + 12, + 95 + ] + }, + "reference": { + "mean_absolute_error": 7.479174766955327e-05, + "rms_error": 0.00011669720624184763, + "relative_rms": 0.00167292962689585, + "maximum_absolute_error": 0.0011237263679504395, + "maximum_coordinate": [ + 120, + 13, + 95 + ] + }, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true + } + }, + { + "name": "candidate-layer14", + "pass": true, + "native": { + "mean_absolute_error": 7.426683930036827e-05, + "rms_error": 0.00011692272740483567, + "relative_rms": 0.0016664167991819168, + "maximum_absolute_error": 0.0018712282180786133, + "maximum_coordinate": [ + 9, + 4, + 21 + ] + }, + "reference": { + "mean_absolute_error": 7.518459540593837e-05, + "rms_error": 0.00011876118757111996, + "relative_rms": 0.0016926190694652233, + "maximum_absolute_error": 0.0018712282180786133, + "maximum_coordinate": [ + 9, + 4, + 21 + ] + }, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true + } + }, + { + "name": "candidate-layer16", + "pass": true, + "native": { + "mean_absolute_error": 6.56544750982954e-05, + "rms_error": 0.00010096562425795954, + "relative_rms": 0.001664785982118596, + "maximum_absolute_error": 0.0009687840938568115, + "maximum_coordinate": [ + 87, + 6, + 54 + ] + }, + "reference": { + "mean_absolute_error": 6.643148362238697e-05, + "rms_error": 0.00010242254834008744, + "relative_rms": 0.0016888086810001547, + "maximum_absolute_error": 0.0010988712310791016, + "maximum_coordinate": [ + 111, + 4, + 54 + ] + }, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true + } + }, + { + "name": "candidate-layer18", + "pass": true, + "native": { + "mean_absolute_error": 1.3927175131937954e-05, + "rms_error": 2.6936330654066946e-05, + "relative_rms": 0.001767880277342061, + "maximum_absolute_error": 0.0004877150058746338, + "maximum_coordinate": [ + 93, + 3, + 79 + ] + }, + "reference": { + "mean_absolute_error": 1.55402708957821e-05, + "rms_error": 2.7944179012469313e-05, + "relative_rms": 0.001834027194613516, + "maximum_absolute_error": 0.0005096942186355591, + "maximum_coordinate": [ + 19, + 2, + 79 + ] + }, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true + } + }, + { + "name": "candidate-layer2", + "pass": true, + "native": { + "mean_absolute_error": 4.411905693170721e-05, + "rms_error": 7.792555527646081e-05, + "relative_rms": 0.0016842156568792179, + "maximum_absolute_error": 0.0019272565841674805, + "maximum_coordinate": [ + 19, + 0, + 87 + ] + }, + "reference": { + "mean_absolute_error": 4.474380736595533e-05, + "rms_error": 7.908844566105766e-05, + "relative_rms": 0.0017093493654042817, + "maximum_absolute_error": 0.0019789934158325195, + "maximum_coordinate": [ + 19, + 0, + 87 + ] + }, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true + } + }, + { + "name": "candidate-layer20", + "pass": true, + "native": { + "mean_absolute_error": 3.926719299077717e-05, + "rms_error": 6.714418695789652e-05, + "relative_rms": 0.0015701435351544247, + "maximum_absolute_error": 0.0014007091522216797, + "maximum_coordinate": [ + 3, + 6, + 50 + ] + }, + "reference": { + "mean_absolute_error": 3.980428101430411e-05, + "rms_error": 6.830684421057062e-05, + "relative_rms": 0.0015973318719501532, + "maximum_absolute_error": 0.0014007091522216797, + "maximum_coordinate": [ + 3, + 6, + 50 + ] + }, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true + } + }, + { + "name": "candidate-layer22", + "pass": true, + "native": { + "mean_absolute_error": 3.3827553837209546e-05, + "rms_error": 5.647171576988976e-05, + "relative_rms": 0.0016724742124102996, + "maximum_absolute_error": 0.0009740591049194336, + "maximum_coordinate": [ + 91, + 7, + 94 + ] + }, + "reference": { + "mean_absolute_error": 3.431406171326983e-05, + "rms_error": 5.74845218685334e-05, + "relative_rms": 0.0017024696191207246, + "maximum_absolute_error": 0.001048445701599121, + "maximum_coordinate": [ + 5, + 7, + 94 + ] + }, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true + } + }, + { + "name": "candidate-layer4", + "pass": true, + "native": { + "mean_absolute_error": 5.255698228174297e-05, + "rms_error": 9.072950274258105e-05, + "relative_rms": 0.0016651707576544098, + "maximum_absolute_error": 0.0018589496612548828, + "maximum_coordinate": [ + 1, + 7, + 109 + ] + }, + "reference": { + "mean_absolute_error": 5.326737022220843e-05, + "rms_error": 9.200325722568675e-05, + "relative_rms": 0.0016885481448722876, + "maximum_absolute_error": 0.0018589496612548828, + "maximum_coordinate": [ + 1, + 7, + 109 + ] + }, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true + } + }, + { + "name": "candidate-layer6", + "pass": true, + "native": { + "mean_absolute_error": 7.304410733427111e-05, + "rms_error": 0.00012859638127734113, + "relative_rms": 0.0016562644199458122, + "maximum_absolute_error": 0.0031279325485229492, + "maximum_coordinate": [ + 3, + 15, + 29 + ] + }, + "reference": { + "mean_absolute_error": 7.412466821033706e-05, + "rms_error": 0.0001306955614176869, + "relative_rms": 0.0016833009301724352, + "maximum_absolute_error": 0.0031279325485229492, + "maximum_coordinate": [ + 3, + 15, + 29 + ] + }, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true + } + }, + { + "name": "candidate-layer8", + "pass": true, + "native": { + "mean_absolute_error": 5.915452997777779e-05, + "rms_error": 9.591084322232719e-05, + "relative_rms": 0.0016681196436167838, + "maximum_absolute_error": 0.0009767115116119385, + "maximum_coordinate": [ + 112, + 8, + 120 + ] + }, + "reference": { + "mean_absolute_error": 5.985122606659514e-05, + "rms_error": 9.728514212980673e-05, + "relative_rms": 0.0016920219984157463, + "maximum_absolute_error": 0.0010822713375091553, + "maximum_coordinate": [ + 93, + 11, + 97 + ] + }, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true + } + } + ], + "passed": 22, + "count": 22, + "bad_scale_rejected": true, + "capture_identity": { + "routes": 2, + "finite_files_per_route": 33, + "selected_full_vocabulary_logit_rows": 17, + "all_logit_bytes_equal_untraced": true, + "query_shape": [ + 128, + 16, + 128 + ], + "kv_shape": [ + 4, + 128, + 128 + ], + "history": 0, + "all_input_pins_verified": true + }, + "reference": "Independent CPU FP32 plus actual installed MLX0.32.4 BF16 SDPA on identical inputs", + "candidate": "Strict QK and BF16high+residual probability PV intoFP32 accumulator", + "zero_added_slack": true, + "processes": { + "all_exit_zero": true, + "guard_lock_released": true, + "timing_claim": false + }, + "raw_archive_sha256": "f72fed61980aa18e6fbb4c63d2ce2e555a17d3915b46cb0643eb47f07a618e74", + "summary_sha256": "1d587ba21ee12fd84cea9b65b55dabae73df954fce6d226435da5712a4da30c9", + "limits": [ + "Whole-cell RMS/max pass, no elementwise-dominance claim", + "Input-boundary test does not erase upstream propagated differences", + "SmallH0R3 failure retained separately; original small-query route remains", + "No allfamily, image, NF, longhistory or concurrency qualification" + ] +} diff --git a/docs/receipts/lfm3/m5-composed-native-mlx.json b/docs/receipts/lfm3/m5-composed-native-mlx.json new file mode 100644 index 0000000000..c04849911f --- /dev/null +++ b/docs/receipts/lfm3/m5-composed-native-mlx.json @@ -0,0 +1,118 @@ +{ + "scope": "Apple M5 matched VL4/6/8 native public, composed fused-risk attention+deferred-head candidate, and MLX-VLM. Default options remain off. Text-only serving; no new image throughput claim.", + "protocol": "Mirrored public/candidate/standard/standard/candidate/public per width; fresh server per leg. Cold pinned2k/8k16outputs no warmup; code/prose256outputs at1/4streams one explicit warmup then3 measured repetitions per leg. Greedy, cache0, context70000.", + "native_build_options": { + "public": [ + false, + false + ], + "candidate": [ + true, + true + ] + }, + "native_binary_sha256": { + "public": "74ecf852d282d67e0961b51b9e65462af6951a46a4736b7c2ba78ea544da015f", + "candidate": "74d882794c32ddf57c4fc69161d1ead22e48021455625404ab931e8a34b60f04" + }, + "standard": { + "mlx": "0.32.4", + "mlx_vlm": "0.7.11", + "source_commit": "230fc308efb577d87446306bfb8a3e9f1a129558" + }, + "metric_definitions": { + "client_ttft": "Request start to first nonempty SSE content; seconds.", + "native_prefill": "Per-request request_prefill_seconds histogram delta.", + "standard_prefill": "Server prompt_eval_time_s = prompt_tokens/prompt_tps from batching response; not an independent GPU clock.", + "decode_delivery": "(256-1)/(last-first content arrival); per-stream delivery proxy.", + "itl_proxy": "1000/decode delivery rate, milliseconds; no true token ITL or token latency percentiles.", + "makespan": "Maximum request completion minus minimum request start for each synchronized group.", + "completion_throughput": "256*streams/makespan, includes prefill and completion overhead.", + "decode_summary": "Median of six per-group medians (three measured repetitions in each mirrored leg)." + }, + "validation": { + "all_usage_lengths_and_cache_zero": true, + "native_cold_and_decode_output_text_sha_equal": true, + "native_prompt_ids_exact_pinned": true, + "mlx_prompt_and_output_token_array_equality": "Unverified; equal counts and text are not token-array proof.", + "separate_native_full_model_state_logits_token_gate": "Passed across six checkpoints; see deferred-prompt-head receipt.", + "cold_drift_limit": 0.05, + "cold_drift_failure": "VL4 standard2k -5.952258470723426%; retain unstable result, no rerun or qualified ratio. All other cold cells pass.", + "decode_drift": "Reported first/last leg median delivery drift; no added post-hoc qualification threshold.", + "owned_cleanup": { + "native_exit0": 12, + "standard_expected_sigterm_minus15": 6, + "final_lock_released": true + } + }, + "artifacts": { + "frozen_packet_pins_sha256": "3ad000ec3fbe748be31618f58ee2b142afd1fec76546ebdf8ea20ac7c010070a", + "summary_sha256": "18fb2720ec5a8c1750c45e6892316a1caf5bb241328d7d4f95dd9eed8b2d1789", + "server_metrics_analysis_sha256": "25cd1d9c9c962434e9a10b9af24f035630de5d9328ad483c6c093d04dcd2bd8f", + "raw_index_sha256": "d437cc534765bd61ce766fa1486ffd40eca1e0382fb0f69ed0bfe13630a815fd", + "raw_index_files": 5111, + "records_sha256": "61ea8ace992218c8bfd225ed30b310158d791fc5bf67f6a51df738bfe0d633a7", + "events_sha256": "09cc44e4c6ae32fdce381170fd42a401562e31c3044cfffa3979930241eabe9b" + }, + "limitations": "Two mirrored legs per engine, no confidence interval. MLX remains faster in observed prefill; these changes do not close the gap. Server prefill definitions differ; use client TTFT for end-to-end comparison. No universal numerical theorem, true token ITL, new image-serving result or production default promotion.", + "cold_columns": ["bits", "prompt_tokens", "engine", "median_client_ttft_s", "median_reported_prefill_s", "reported_prefill_tokens_per_s", "first_last_ttft_drift", "stable_at_5percent"], + "cold_rows": [ + [4, 2048, "public", 0.8618548545055091, 0.8553885, 2394.2337312227132, -0.0008584867040322131, true], + [4, 2048, "candidate", 0.8458011874463409, 0.8395585, 2439.3773632212647, 0.00010261852410486938, true], + [4, 2048, "standard", 0.16349447891116142, 0.0996831669472158, 20545.093647400434, -0.05952258470723426, false], + [4, 8192, "public", 4.651907812571153, 4.6455735, 1763.399072256633, -0.0011344369514318942, true], + [4, 8192, "candidate", 4.42176706250757, 4.4154824999999995, 1855.2898805509933, 0.0011402676019953706, true], + [4, 8192, "standard", 0.3744832498487085, 0.34098479081876576, 24024.531945631756, 0.004764540179620136, true], + [6, 2048, "public", 0.9573665000498295, 0.9510115, 2153.496566550457, -0.0005890297247731491, true], + [6, 2048, "candidate", 0.9423206455539912, 0.9359905, 2188.0563958715393, -0.000441409265524495, true], + [6, 2048, "standard", 0.16602193750441074, 0.10173297952860594, 20131.131610316497, -0.008786620211905016, true], + [6, 8192, "public", 4.973140479531139, 4.9667135, 1649.380420271876, 0.0008262195020374286, true], + [6, 8192, "candidate", 4.767471082974225, 4.761003, 1720.6458387024752, 0.000945129708801451, true], + [6, 8192, "standard", 0.3945427709259093, 0.3599653332494199, 22757.747047612957, 1.911560126943712e-05, true], + [8, 2048, "public", 0.8482776663731784, 0.8419175000000001, 2432.5423809339986, -0.0004089795190296286, true], + [8, 2048, "candidate", 0.8332901874091476, 0.826962, 2476.534592883349, 0.00035473020880671946, true], + [8, 2048, "standard", 0.1660297920461744, 0.0992729994468391, 20629.980069220215, 0.00537388745965095, true], + [8, 8192, "public", 4.512534895678982, 4.506061, 1817.995806093171, -0.0011782529889223792, true], + [8, 8192, "candidate", 4.293734083650634, 4.287327, 1910.7476523250966, 0.0006565594971912336, true], + [8, 8192, "standard", 0.3838677708990872, 0.34983479138463736, 23416.767576421626, 0.0030017335039231874, true] + ], + "decode_columns": ["bits", "kind", "streams", "engine", "median_client_ttft_s", "median_delivery_tokens_per_s", "median_itl_proxy_ms", "median_makespan_s", "median_completion_throughput_tokens_per_s", "first_last_leg_delivery_drift"], + "decode_rows": [ + [4, "code", 1, "public", 0.034554021, 299.6521988301721, 3.337204250980392, 0.884538792, 289.4165095903295, -0.0031520158201207016], + [4, "code", 1, "candidate", 0.0343565, 303.35181493676293, 3.2965048215686275, 0.874815333, 292.63321864652335, -0.004361745708033937], + [4, "code", 1, "standard", 0.031362687, 326.10693563347337, 3.0664788411764707, 0.813544, 314.6725937376211, -0.0003329884097359459], + [4, "code", 4, "public", 0.067814229, 214.69525614241894, 4.65817851372549, 1.256599187, 814.9059661645956, -0.008406436388948002], + [4, "code", 4, "candidate", 0.06766911475000001, 216.41626302249557, 4.621078349019609, 1.2466364165, 821.4104407793853, 0.0020794609679346543], + [4, "code", 4, "standard", 0.06700276050000001, 240.98202849786207, 4.14968778627451, 1.125563646, 909.7666815417474, -0.002776858530897597], + [4, "prose", 1, "public", 0.034827333, 298.7110568899971, 3.3477937098039217, 0.8886755, 288.07486130155803, -0.012639243562539537], + [4, "prose", 1, "candidate", 0.035031916999999996, 301.0656321108558, 3.3215348862745095, 0.8820649995000001, 290.22804558039473, 0.00019878083864610652], + [4, "prose", 1, "standard", 0.031753396, 326.203744780067, 3.065568954901961, 0.8136992915, 314.6125656650063, -0.0012275991969161737], + [4, "prose", 4, "public", 0.0685935, 213.48655520253442, 4.68454424117647, 1.2637062505, 810.3224041997105, -0.008432552136641513], + [4, "prose", 4, "candidate", 0.06860888525, 215.5637403084636, 4.639365604901961, 1.2518997084999999, 817.9574769292135, 0.0026206560220931596], + [4, "prose", 4, "standard", 0.06697753125, 240.99925619414296, 4.149390400980392, 1.125580958, 909.7524369108339, -0.0007577680203710857], + [6, "code", 1, "public", 0.036828729000000004, 272.0443278271382, 3.6759109470588234, 0.974342896, 262.74414743297257, 0.010007612387379128], + [6, "code", 1, "candidate", 0.0367441255, 276.45390858741064, 3.6172542470588236, 0.9594079585, 266.8320074970693, 0.004240330099824607], + [6, "code", 1, "standard", 0.032311520999999996, 285.9927416525064, 3.496592237254902, 0.9240487079999999, 277.04167364366276, -0.00011536736266393177], + [6, "code", 4, "public", 0.07352633325, 203.44092441713823, 4.915842442156863, 1.327536375, 771.3536414823056, -0.0006317265371124092], + [6, "code", 4, "candidate", 0.07349448950000001, 207.45256667408975, 4.820841094117647, 1.3031389785, 785.8003652347915, 0.006811671205279035], + [6, "code", 4, "standard", 0.06898659375, 234.2249175708553, 4.269405678431372, 1.1585147495, 883.8915985192734, -0.002638489416123768], + [6, "prose", 1, "public", 0.0370375415, 267.6468406800777, 3.736266994117647, 0.9900226670000001, 258.57995555801756, 0.0037478108420110345], + [6, "prose", 1, "candidate", 0.037013979, 272.5831445868055, 3.6686486921568626, 0.9724729375000001, 263.24944708482496, 0.008029743454788907], + [6, "prose", 1, "standard", 0.0323412295, 287.016966015801, 3.4841156882352937, 0.921052813, 277.942868489991, 0.004778010528445398], + [6, "prose", 4, "public", 0.0742943435, 202.02346635622234, 4.950337581372549, 1.3371205205, 765.8247590182668, 9.763812828866847e-05], + [6, "prose", 4, "candidate", 0.07412807299999999, 205.9595386304976, 4.85578680490196, 1.312633188, 780.1167390406108, 0.0070411144547481985], + [6, "prose", 4, "standard", 0.06829553125, 234.38959355791636, 4.266404901960785, 1.1571176875, 884.958401088074, 0.0019106418868763964], + [8, "code", 1, "public", 0.0337859375, 262.25521522878523, 3.813079574509804, 1.006557271, 254.33227779306333, 0.0014637284988323618], + [8, "code", 1, "candidate", 0.0341780625, 265.06161114219526, 3.772708005882353, 0.996456771, 256.9102932822417, -0.0006047704107183804], + [8, "code", 1, "standard", 0.032514521000000005, 272.6034505919276, 3.6683321882352944, 0.968332229, 264.37207629640045, 0.0010291489242975072], + [8, "code", 4, "public", 0.06706474975, 206.80834752292935, 4.835720097058823, 1.3005166665, 787.3793958959329, -0.0006952429913269498], + [8, "code", 4, "candidate", 0.06676436475, 209.15845382481902, 4.7813994284313726, 1.286358229, 796.0462964254926, -0.0022837173064849114], + [8, "code", 4, "standard", 0.06888560399999999, 234.24655754716392, 4.2690074764705885, 1.1583798335000002, 883.9933667745996, -0.001103620836617747], + [8, "prose", 1, "public", 0.03434675, 262.7769285170003, 3.8055092313725485, 1.0047247709999998, 254.79615635770887, -0.0011730702246991465], + [8, "prose", 1, "candidate", 0.034352125, 265.0339020530074, 3.7731022058823527, 0.996572791, 256.88038276385595, 0.0002135481598781741], + [8, "prose", 1, "standard", 0.032469146, 272.5399230340055, 3.6691874176470582, 0.968922021, 264.21118241649197, -0.0029753115997911417], + [8, "prose", 4, "public", 0.067618021, 206.70963734495666, 4.838039133333334, 1.301697771, 786.6651520343642, -0.0011685396620899624], + [8, "prose", 4, "candidate", 0.0674831665, 208.90449813747685, 4.787216135294118, 1.2885598335, 794.6856733762716, -0.0007436863303922348], + [8, "prose", 4, "standard", 0.069127854, 234.3142151865785, 4.2677770843137255, 1.158130771, 884.1837017580654, 0.001746839239570086] + ] +} diff --git a/docs/receipts/lfm3/m5-composed-prompt-provisional-serving.json b/docs/receipts/lfm3/m5-composed-prompt-provisional-serving.json new file mode 100644 index 0000000000..af7d19fecf --- /dev/null +++ b/docs/receipts/lfm3/m5-composed-prompt-provisional-serving.json @@ -0,0 +1,162 @@ +{ + "scope": "PROVISIONAL M5 TEXT4 combined FFN-only plus residual-PV tiled-attention prompt path. Bounded logit envelope evidence; not full production qualification.", + "cases": [ + { + "prompt_tokens": 2048, + "baseline_ttft_s": [ + 0.8463148330338299, + 0.8463252908550203 + ], + "candidate_ttft_s": [ + 0.5536723751574755, + 0.5533634996972978 + ], + "baseline_median_s": 0.8463200619444251, + "candidate_median_s": 0.5535179374273866, + "baseline_over_candidate": 1.5289839853752702, + "baseline_first_last_drift": 1.2356892236997652e-05, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "not asserted; output text digest equal", + "prompt_ids": "both routes /tokenize exact pinned arrays", + "candidate_spread": 0.0005581782324757256, + "ttft_reduction_fraction": 0.34597091299516636 + }, + { + "prompt_tokens": 8192, + "baseline_ttft_s": [ + 4.403437749948353, + 4.400628083385527 + ], + "candidate_ttft_s": [ + 2.405512291006744, + 2.4054756248369813 + ], + "baseline_median_s": 4.40203291666694, + "candidate_median_s": 2.4054939579218626, + "baseline_over_candidate": 1.8299912590385026, + "baseline_first_last_drift": -0.0006380620602297604, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "not asserted; output text digest equal", + "prompt_ids": "both routes /tokenize exact pinned arrays", + "candidate_spread": 1.524279414177876e-05, + "ttft_reduction_fraction": 0.4535493024565532 + } + ], + "predeclared_max_baseline_drift": 0.05, + "no_timing_retries": true, + "default_off": true, + "same_binary_sha256": "29f5aa69af84200ffb7d85293f9a40ddb433b4930f6ea4747bb6435edadde359", + "protocol": { + "env_order": [ + 0, + 1, + 1, + 0 + ], + "cache_tokens": 0, + "completion_tokens": 16, + "fresh_servers": 4, + "all_server_exit_zero": true, + "all_client_exit_zero": true, + "same_pinned_prompt_ids": true, + "ffn_calls_per_candidate": [ + 1152, + 4608 + ], + "attention_route_hits_verified": true, + "no_retries": true, + "device_scope": "M5 only" + }, + "bounded_model_evidence": { + "prompt_tokens": 128, + "teacher_forced_continuation_tokens": 16, + "reference": "Independent propagated CPU FP32 and actual pinned official MLX BF16", + "phases": { + "prompt_boundary": { + "native_logit_metrics": { + "mean_absolute_error": 0.03329696540903778, + "rms_error": 0.041121336314284725, + "relative_rms": 0.011827876222482102, + "maximum_absolute_error": 0.20430231094360352, + "maximum_coordinate": [ + 0, + 49445 + ] + }, + "reference_bf16_logit_metrics": { + "mean_absolute_error": 0.03884851189472984, + "rms_error": 0.04764064939088356, + "relative_rms": 0.01370304943028551, + "maximum_absolute_error": 0.20851778984069824, + "maximum_coordinate": [ + 0, + 91107 + ] + }, + "metric_results": { + "rms_error": true, + "maximum_absolute_error": true, + "top20_probability_mean": false, + "top20_probability_max": false + }, + "probability_diagnostics": { + "native_mean": 1.3686212832679892e-05, + "reference_mean": 9.179301337359821e-06, + "native_max": 0.0001479777967422491, + "reference_max": 9.084144305366725e-05 + }, + "native_fp32_top1_misses": 0, + "reference_fp32_top1_misses": 0 + }, + "teacher_forced_continuation": { + "native_logit_metrics": { + "mean_absolute_error": 0.02985543358314638, + "rms_error": 0.037950087085717864, + "relative_rms": 0.010836676663429445, + "maximum_absolute_error": 0.20357131958007812, + "maximum_coordinate": [ + 10, + 26150 + ] + }, + "reference_bf16_logit_metrics": { + "mean_absolute_error": 0.032687712900759155, + "rms_error": 0.0425864734405558, + "relative_rms": 0.01216060036617698, + "maximum_absolute_error": 0.256467342376709, + "maximum_coordinate": [ + 14, + 4438 + ] + }, + "metric_results": { + "rms_error": true, + "maximum_absolute_error": true, + "top20_probability_mean": true, + "top20_probability_max": false + }, + "probability_diagnostics": { + "native_mean": 1.5765819037637397e-05, + "reference_mean": 1.6550785692252214e-05, + "native_max": 0.0005338205920233374, + "reference_max": 0.00043348867067471986 + }, + "native_fp32_top1_misses": 0, + "reference_fp32_top1_misses": 0 + } + } + }, + "limits": [ + "No addition or multiplication of individual optimization gains; combined graph measured directly", + "Bounded prompt and continuation logit RMS/max pass with zero added slack", + "Proposed top20 probability diagnostics still have breaches; retained explicitly", + "No all-family/image/tail/concurrency qualification or nonfinite attention proof", + "H0R3 relaxed and residual operator failures remain excluded by original small-query path", + "No production enablement or10xclaim" + ], + "raw_records_sha256": "76abebe29aed7c1410e4c3614eb4618cc7e1d5380e475d85b7c9ea1ddcf82b45", + "raw_events_sha256": "19000f599395068050297e1c4b81d7c6d38db7b60b5dc40da89109261d1b328a", + "phase_receipt_sha256": "22304036617e4975a851d781c496accf451f9a6250543535bec6cc8d74eaa4c5" +} diff --git a/docs/receipts/lfm3/m5-current-pair-standard.json b/docs/receipts/lfm3/m5-current-pair-standard.json new file mode 100644 index 0000000000..de83b186d8 --- /dev/null +++ b/docs/receipts/lfm3/m5-current-pair-standard.json @@ -0,0 +1,1810 @@ +{ + "schema_version": 1, + "scope": "Fresh exclusive guarded Apple M5 Ultra screen: current public versus paired GS64 tile/twin candidate and pinned standard for TEXT4; current public versus pinned standard for VL4. No GS32 VL candidate in this screen.", + "status": { + "actual_exit": 0, + "server_legs": 10, + "measured_decode_requests": 300, + "measured_decode_repetitions": 120, + "cold_requests": 20, + "strict_decode_tokens": 256, + "strict_cold_output_tokens": 16, + "cached_tokens": 0, + "all_lengths_validated": true, + "owned_server_cleanups": 10, + "performance_accepted": false, + "performance_conclusion": "Neither current public nor paired candidate meets the measured standard throughput/prefill comparison. Paired tiles improve cold prefill versus current public but regress M5 decode. M4 gains are not a M5 decode win." + }, + "correctness_prerequisites": { + "actual_exit": 0, + "current_public_registered_kernels": 70, + "known_findings": 0, + "failures": 0, + "controls_red": 2, + "limit_problems": 0, + "shared_tile_twin_kernels_passed": 6, + "paired_raw_cases": 108 + }, + "host": { + "gpu": "Apple M5 Ultra", + "gpu_cores": 64, + "physical_memory_bytes": 274877906944, + "os": "ProductName:\t\tmacOS\nProductVersion:\t\t27.0\nBuildVersion:\t\t26A428\n" + }, + "provenance": { + "native_source_head_at_preparation": "6a111de5c1b4ef5bcb60d2924446636733beb416", + "native_provenance": "Current artifact3d4 was built before later receipt-only commits, from applied source hashes inattention-range.json; runtime remains identical at preparation HEAD6a111de5. This is not a rebuildat6a. Runtimefixdd7263c26; receiptcommitd3c52154. Pair overlaybasedpublicd3c, exactfilespinned tilepair-manifest.json. No sumcache composition.", + "native_artifacts": { + "current": { + "path": "$RUN/bin/tensorfold-native-current", + "sha256": "3d4a3daeda697420bfbba6d81972896c2bf9e31efaf6f12321ee8487a299354b" + }, + "pair": { + "path": "$RUN/bin/tensorfold-native-pair", + "sha256": "464f472eb8e8bd85316de7122a86f4bcd169c24e86f87d3e469cb181f87b538f" + } + }, + "proof_artifacts": { + "tf-lfm3-dual-project": "dc34ace5c6e0055eb6cb127aabbfdf61fd45245697502dfa0e987ea668d637a2", + "tensorfold-native-current": "3d4a3daeda697420bfbba6d81972896c2bf9e31efaf6f12321ee8487a299354b", + "tf-kernel-contract": "2786719bfe0314754349463721b8e1faa7cd06f6f2efa142db19acacd83d54e6", + "tensorfold-native-pair": "464f472eb8e8bd85316de7122a86f4bcd169c24e86f87d3e469cb181f87b538f" + }, + "standard": { + "python": "$QUALIFICATION/standard-venv/bin/python", + "module": "mlx_vlm.server", + "source_commit": "230fc308efb577d87446306bfb8a3e9f1a129558", + "mlx": "0.32.4", + "mlx_metal": "0.32.4", + "mlx_vlm": "0.7.11", + "dependency_provenance": "Existing remote environment, installedmetadata captured CPUonly; all distributions frozen in requirements.lock. CoreMLX/VLMpins matchsource, dependency differences fromlocalM4retained.", + "memory_policy": "Unmodified MLX defaults; no memory/wired limit overrides.", + "request_context_limit": 70000, + "context_policy": "Pinned --max-kv-size70000 sets request budget; LFM3 family cache construction remains lazy and is not claimed allocation-identical to native.", + "checkpoint_context_caps": { + "text": 262144, + "vl": 128000 + } + }, + "versions": { + "python": "3.13.15 (main, Sep 1 2026, 14:05:45) [Clang 22.1.3 ]", + "packages": { + "shellingham": "1.5.4", + "uvicorn": "0.54.0", + "xxhash": "4.0.2", + "filelock": "4.2.4", + "mlx": "0.32.4", + "mlx-lm": "0.32.2", + "starlette": "1.10.0", + "huggingface_hub": "2.3.0", + "click": "8.5.0", + "packaging": "26.3", + "protobuf": "7.36.2", + "tokenizers": "0.23.3", + "Pygments": "2.21.0", + "pyarrow": "26.0.0", + "sounddevice": "0.5.6", + "requests": "2.34.2", + "rich": "15.0.0", + "certifi": "2026.10.23", + "yarl": "1.25.1", + "websockets": "17.3", + "mlx-vlm": "0.7.11", + "opentelemetry-api": "1.45.1", + "pandas": "3.1.0", + "datasets": "5.1.0", + "sentencepiece": "0.2.2", + "propcache": "0.5.4", + "llguidance": "1.9.1", + "numpy": "2.5.4", + "aiohappyeyeballs": "2.7.1", + "cryptography": "50.0.2", + "pycparser": "3.12", + "tzdata": "2026.5", + "pillow": "13.0.0", + "fsspec": "2026.7.0", + "hf-xet": "1.7.0", + "safetensors": "0.8.0", + "miniaudio": "1.71", + "tqdm": "4.70.1", + "urllib3": "2.8.0", + "h11": "0.16.0", + "pydantic_core": "2.50.1", + "mdurl": "0.1.2", + "python-multipart": "0.0.32", + "mlx-metal": "0.32.4", + "PyYAML": "6.0.3", + "httpcore2": "2.16.0", + "opencv-python": "5.0.0.93", + "truststore": "0.10.5", + "frozenlist": "1.8.0", + "cffi": "2.1.1", + "httpx2": "2.16.0", + "aiosignal": "1.4.0", + "typer": "0.27.3", + "mlx-audio": "0.6.2", + "Jinja2": "3.1.6", + "annotated-types": "0.8.0", + "regex": "2026.9.29", + "markdown-it-py": "4.2.0", + "annotated-doc": "0.0.5", + "typing-inspection": "0.4.4", + "fastapi": "0.162.0", + "aiohttp": "3.14.5", + "attrs": "26.1.0", + "multidict": "7.2.1", + "typing_extensions": "4.16.0", + "MarkupSafe": "3.0.4", + "pydantic": "2.14.1", + "python-dateutil": "3.0.0", + "multiprocess": "0.70.19", + "charset-normalizer": "3.5.2", + "anyio": "4.15.1", + "six": "1.17.0", + "scipy": "1.18.1", + "dill": "0.4.1", + "transformers": "5.21.0", + "idna": "3.20" + } + }, + "requirements_sha256": "275b369ea72d5757b1e3f91e68ed5b746d83b5f7667adc8f5d9d28bb8933ddd6", + "models": { + "LFM3-1.7B-MLX-4bit": { + "repository": "LiquidAI/LFM3-1.7B-MLX-4bit", + "revision": "d0ca73dcfa70ce18733754a492cdc1fb0123262d", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 2423, + "sha256": "7df2ea819f9a31b770192a0a9dc1b7953a7dd8c41b816cc3905bbec99ef332bd", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 328, + "sha256": "0095ba7158e938b1e2d3082d412f00d8be85e42cbbd7608b63866f7a081cf270", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 973083386, + "sha256": "1be5bf672013cd91aa62148f77c5eea8d2784a95845d5fe826e08c7b84823815", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 50700, + "sha256": "2c8b88e25f6a1055164dd051436b170872200047bce4b3ec5f2a58f6ef929a75", + "verified_against_pinned_revision": true + }, + "special_tokens_map.json": { + "bytes": 92, + "sha256": "7946f864cbfb62a1b041bda937244210a1a87f4a6cab3a48b791131b2cd57f88", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908634, + "sha256": "41927bd314d17d7d7fadb44e10cd0919f7b71528d7741596d8cbd15c6945ec44", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1859, + "sha256": "b77214b9f12be74b9e5e1dab35d7dace077a31be969314bf38866fd6c8163853", + "verified_against_pinned_revision": true + } + } + }, + "LFM3-VL-2B-MLX-4bit": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "revision": "c823fade98e35a3d6942ad4d38b1ea15f675cd12", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 4419, + "sha256": "0efb7708e0771e3e104ac628954cdc71070dd436e0b7974615bd1b9eb836f9d9", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 323, + "sha256": "30b1115bcdc4e5671567a5b1255a2f3d2f5a912a79292051c6f1e9db81b5e8c4", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 1590959088, + "sha256": "58268dfdf441ae4020d2a52fc97a814645e8a9624946e2d490403d4dcd71a255", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 61256, + "sha256": "38b756dbde167ab7c60756af7d5652414c31afdf4e77700b8d3c99395515d639", + "verified_against_pinned_revision": true + }, + "processor_config.json": { + "bytes": 1254, + "sha256": "f183f5a5e5b3a32e666fce05033e6cb7ca496d26e709a2cee611a13eb7d6b610", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908258, + "sha256": "a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1759, + "sha256": "d63d1ce2f332047e398764b4170352b96629a5194643105bfd6d70c90b750396", + "verified_against_pinned_revision": true + } + } + } + }, + "clients_sha256": { + "$PACKET/tools/bench_auth.py": "1896120b70d0df05646870a252697e20ce84c63e09dcba88b2a4e5a55ccb3c5b", + "$PACKET/tools/bench_cache.py": "24364dd13c033450f45710fc8685e2c64d6a72d6dd6e223d0d7b19aabbdc1d98", + "$PACKET/tools/bench_concurrent.py": "0e72e26769f082459329fc068c25f6f72a9155e422962ea54cb5d1226227752d", + "$PACKET/tools/bench_evidence.py": "54c28474f0804dbd120e3fb5859be0d1667ef21ca502880d0bed098ec9bb42ff", + "$PACKET/tools/bench_metrics.py": "26f4c3ecbb03df4f12e24d18840fea989d9ef8c63117ca11eb944ef670458a27", + "$PACKET/tools/bench_openai.py": "d5af4fbb756cdfb573546c0e0017652a4bb8f509d1251af683be7a3cbdd426f6", + "$PACKET/tools/bench_timing.py": "158f636ef0b3bed460ce19ef4872b6d8607ff7a8590562d06bae8bd55e6f44ed", + "$PACKET/tools/lfm3/bench_decode.py": "c011106f4f182db19371e3f91f70bcc374160786dd105f2d713a7fea07ab8c4c", + "$PACKET/cold_client.py": "7b8ea55aa88834814f95a6f679654e16074b7a8364e34b39a276f009fe0231f2" + }, + "fixtures_sha256": { + "text-2048.body.json": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "text-2048.ids.json": "f5edac3447405086d7ec361d6c899410ea7cbbb2ef697c27fd7486ef5ff877fe", + "text-8192.body.json": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "text-8192.ids.json": "a9b3cdf070b9ca5ea523565bcb82c80b6af5aae71f640a8fba0460c9069d22cd", + "vl-2048.body.json": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "vl-2048.ids.json": "f5edac3447405086d7ec361d6c899410ea7cbbb2ef697c27fd7486ef5ff877fe", + "vl-8192.body.json": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "vl-8192.ids.json": "a9b3cdf070b9ca5ea523565bcb82c80b6af5aae71f640a8fba0460c9069d22cd" + }, + "provenance_files": { + "tilepair-manifest.json": "a6b5e4cbb9c7f96f6e95bf7d03366b022d123af653bb2e01edd9acd487a02528", + "current-source-receipt.json": "b04753edfcec89c76161e2c3e7e9ea873700179dee7d33c937e2cc53e2f0f654", + "standard-installed-metadata.json": "fb244f662eddced712fac29a25e390dad3d28f0cec79a5284f1c63cfb875b4e8" + }, + "executed_packet_files_sha256": { + "run_existing_clients.py": "8209512004d5c8c19e77a4a1ce1032b54e492e8625faf448cfa34a07e268a2c0", + "guard.py": "b37fb2b0aa20bc29e0d7f7bb4930ce267752c26172c9bb368b2ab18a7392954d", + "config.json": "de8ef0209c0e828118a2aee2442ae1a80a59b2fc5902dc0cec0e3cbbae82aaa7", + "requirements.lock": "275b369ea72d5757b1e3f91e68ed5b746d83b5f7667adc8f5d9d28bb8933ddd6", + "packet-sha256.json": "f3b1b02a9652d83dab8a721d133bbde57d1964c5d7c34f6572c15e9ba6f70e74" + }, + "packet_hash_check": "Read remote packet files after completedrun; hashes exactlymatchfrozenpreparedpacket.", + "public_runtime_source_commit": "dd7263c26", + "public_runtime_same_at_receipt_commit": "6a111de5c1b4ef5bcb60d2924446636733beb416" + }, + "protocol": { + "phase": { + "families": [ + "text", + "vl" + ], + "bits": [ + 4 + ], + "cold_tokens": [ + 2048, + 8192 + ], + "decode_tokens": 256, + "cold_output_tokens": 16, + "reps": 3, + "levels": [ + 1, + 4 + ], + "order": "family_orders", + "family_orders": { + "text": [ + "current", + "pair", + "standard", + "standard", + "pair", + "current" + ], + "vl": [ + "current", + "standard", + "standard", + "current" + ] + } + }, + "settings": { + "native_context": 70000, + "parallel": 4, + "prompt_cache_gib": 0, + "temperature": 0, + "thinking": false, + "prefix_cache": "disabled on both", + "port": 18085 + }, + "lock_path": "/tmp/tensorfold-model.lock", + "guard_source": { + "path": "guard.py", + "sha256": "b37fb2b0aa20bc29e0d7f7bb4930ce267752c26172c9bb368b2ab18a7392954d", + "origin": "ExistingM5guard exactaliases supplied byqualificationowner; no weakened admission." + }, + "http_lock_events": [ + { + "event": "lock_wait", + "time_ns": 1793200491705862000, + "path": "/tmp/tensorfold-model.lock" + }, + { + "event": "lock_acquired", + "time_ns": 1793200491706056000, + "path": "/tmp/tensorfold-model.lock", + "pid": 14813, + "device": 16777233, + "inode": 2622134 + }, + { + "event": "complete", + "time_ns": 1793200708650218000, + "legs": 10, + "all_lengths_validated": true, + "performance_gate_claim": false + }, + { + "event": "lock_release", + "time_ns": 1793200708650385000, + "pid": 14813 + } + ], + "single_model_admission": "Existing exclusive flock plus exact process admission before load and before/after clients and cleanup. Only owned descendants allowed during measurement; no foreign model killed.", + "native_admission": { + "hardware": "M5", + "binary_sha256": { + "tf-lfm3-dual-project": "dc34ace5c6e0055eb6cb127aabbfdf61fd45245697502dfa0e987ea668d637a2", + "tensorfold-native-current": "3d4a3daeda697420bfbba6d81972896c2bf9e31efaf6f12321ee8487a299354b", + "tf-kernel-contract": "2786719bfe0314754349463721b8e1faa7cd06f6f2efa142db19acacd83d54e6", + "tensorfold-native-pair": "464f472eb8e8bd85316de7122a86f4bcd169c24e86f87d3e469cb181f87b538f" + }, + "memory_admission": { + "4": { + "checkpoint_bytes": 973083386, + "conservative_scratch_plus_existing_2GiB_margin_bytes": 4294967296, + "estimated_total_bytes": 5268050682, + "existing_70percent_ram_policy_bytes": 192414534830, + "no_limits_changed": true + }, + "6": { + "checkpoint_bytes": 1405187179, + "conservative_scratch_plus_existing_2GiB_margin_bytes": 4294967296, + "estimated_total_bytes": 5700154475, + "existing_70percent_ram_policy_bytes": 192414534830, + "no_limits_changed": true + }, + "8": { + "checkpoint_bytes": 1837290805, + "conservative_scratch_plus_existing_2GiB_margin_bytes": 4294967296, + "estimated_total_bytes": 6132258101, + "existing_70percent_ram_policy_bytes": 192414534830, + "no_limits_changed": true + } + }, + "model_budget_scope": "Fixed raw projection inputs <=128rows plus conservative2GiB scratch and existing2GiB margin; native serving retains its actual Metal admission", + "guard": "existing exclusive /tmp/tensorfold-model.lock and exact process admission; no prior read-only probe claimed as lease" + }, + "memory_policy": "Native admission unchanged; MLX memory and wired limits unmodified. /usr/bin/time -l captures physical footprint and RSS; available server metrics preserved. No allocator/pinned peak inferred from RSS.", + "server_commands": [ + { + "tag": "text-4-0-current", + "command": { + "_trimmed_list": { + "count": 20, + "sha256": "e15858889bc42a1e8a8472ba3a96bae028c6491a3311473f46180f801b4ca669", + "first": [ + "$RUN/bin/tensorfold-native-current", + "serve" + ], + "last": [ + "--metrics-open" + ] + } + }, + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "tag": "text-4-1-pair", + "command": { + "_trimmed_list": { + "count": 20, + "sha256": "ed21475d7f8eec164a4b5ff52531a441ff22047156e57a7b41182c4c4c4db236", + "first": [ + "$RUN/bin/tensorfold-native-pair", + "serve" + ], + "last": [ + "--metrics-open" + ] + } + }, + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "tag": "text-4-2-standard", + "command": { + "_trimmed_list": { + "count": 19, + "sha256": "62a2f08403b9445e7b9836d4bd9bc0f4635251f927ae49b4c719bbf79b578479", + "first": [ + "$QUALIFICATION/standard-venv/bin/python", + "-m" + ], + "last": [ + "0" + ] + } + }, + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": "70000" + }, + { + "tag": "vl-4-1-standard", + "command": { + "_trimmed_list": { + "count": 19, + "sha256": "83c6f8e797f357f0ad3cbb23bf614481705e282d019b51d2a3c153158abd2b79", + "first": [ + "$QUALIFICATION/standard-venv/bin/python", + "-m" + ], + "last": [ + "0" + ] + } + }, + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": "70000" + } + ], + "actual_dispatch": [ + { + "leg": "text-4-1-pair", + "actual_server_log_line": "info: lfm3 tile qualification: gs64_tiles=true twin=true gs32_affine=false", + "server_log_sha256": "ccf6078e0f3167f216f2cf23ab55175debdcbb9532fa5749d332b1d2e0f4776b" + }, + { + "leg": "text-4-4-pair", + "actual_server_log_line": "info: lfm3 tile qualification: gs64_tiles=true twin=true gs32_affine=false", + "server_log_sha256": "2d899eb63007382b348bb13a70a245792633042bfe2ebc984bece39a752bc072" + } + ], + "replay": [ + "$QUALIFICATION/standard-venv/bin/python \"$PACKET/run_existing_clients.py\"", + "$QUALIFICATION/standard-venv/bin/python \"$PACKET/run_existing_clients.py\" --execute --output \"$NEW_OUTPUT\"" + ], + "replay_scope": "Existing locally retained frozen packet, artifact/model pins and clients are required; output directory must not exist. No new timing framework or throughput formula. Packet source and every retained raw result are hashed below." + }, + "results": { + "reported_summary": { + "text": { + "current": { + "median_of_two_reported_leg_cell_medians_tps": { + "long-code-chat/1": 315.75, + "long-code-chat/4": 861.85, + "long-prose-chat/1": 329.1, + "long-prose-chat/4": 878.75 + }, + "cold_median_ttft_s": { + "2048": 0.8591580830980092, + "8192": 4.615810395916924 + } + }, + "pair": { + "median_of_two_reported_leg_cell_medians_tps": { + "long-code-chat/1": 290.55, + "long-code-chat/4": 803.55, + "long-prose-chat/1": 303.45000000000005, + "long-prose-chat/4": 821.85 + }, + "cold_median_ttft_s": { + "2048": 0.4481715620495379, + "8192": 3.094886645907536 + } + }, + "standard": { + "median_of_two_reported_leg_cell_medians_tps": { + "long-code-chat/1": 344.2, + "long-code-chat/4": 995.0, + "long-prose-chat/1": 345.45, + "long-prose-chat/4": 1008.5 + }, + "cold_median_ttft_s": { + "2048": 0.14974241657182574, + "8192": 0.36902416590601206 + } + } + }, + "vl": { + "current": { + "median_of_two_reported_leg_cell_medians_tps": { + "long-code-chat/1": 298.9, + "long-code-chat/4": 835.65, + "long-prose-chat/1": 296.2, + "long-prose-chat/4": 831.3 + }, + "cold_median_ttft_s": { + "2048": 0.8618560209870338, + "8192": 4.652502874843776 + } + }, + "standard": { + "median_of_two_reported_leg_cell_medians_tps": { + "long-code-chat/1": 325.79999999999995, + "long-code-chat/4": 947.75, + "long-prose-chat/1": 325.25, + "long-prose-chat/4": 948.2 + }, + "cold_median_ttft_s": { + "2048": 0.15912006190046668, + "8192": 0.3741248541045934 + } + } + } + }, + "summary_method": "Each reported aggregate is the median of two existing per-leg cell medians (rounded by the unchanged client); all six original per-repetition TPS values remain below. Cold median uses two fresh-server leg samples per engine.", + "legs": { + "_trimmed_list": { + "count": 10, + "sha256": "07a2506a59892dd5005d672db4fb3da76eb2bff6ced055f983823f97a4f8d997", + "first": [ + { + "_trimmed_value": { + "bytes": 6635, + "sha256": "6a44f0d754c36e5ec8db1efe5bc9171bbb0777ff971c7a89564ec54d1353c87d", + "keys": [ + "family", + "bits", + "leg", + "engine", + "cold", + "decode_validation", + "memory", + "tag", + "decode_cells" + ], + "count": 9 + } + }, + { + "_trimmed_value": { + "bytes": 6629, + "sha256": "6daec8269138f99e568014c22b4f7514b8e176fb25180e5f6c421ae7317b45b4", + "keys": [ + "family", + "bits", + "leg", + "engine", + "cold", + "decode_validation", + "memory", + "tag", + "decode_cells" + ], + "count": 9 + } + } + ], + "last": [ + { + "_trimmed_value": { + "bytes": 6630, + "sha256": "5fee38b82361a8199508eaf1c9b95a99b7fd2e6bd04d69f3488b918403219e19", + "keys": [ + "family", + "bits", + "leg", + "engine", + "cold", + "decode_validation", + "memory", + "tag", + "decode_cells" + ], + "count": 9 + } + } + ] + } + }, + "evidence_pool": { + "8b9be652dd2c88bd": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + "555fe37e25c90f95": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + "877a197cb9bca710": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "760b6cafb707", + "token_sha256": "760b6cafb70745f4d5f53bf8162dbd7ab4ef731fbc4e86ec4085e22e7e341880", + "output_sha256": "e71bf7a65f320c274e70c8c3dc0e56603ca48305ada0390153f7f17b781f1a3a" + }, + "05f89317f1e4d95f": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + "3e1007a53dac19be": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "fc6c1799b10d5ece11495fb9eb854a5562db013147b5382e1e92d4dc95bf5ae7" + }, + "8951a9c3a9d80406": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8f3e436a1e2277cc5f0ba02cc9de357c6286ea1a8d1c78cb277ec1629464af7b" + }, + "13646075c33d9905": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "24f4feefb469f86e1a3fc752cbd6d2404c1530c3c8f87fbf0408b1741da5017a" + }, + "62f3c8997790ce94": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + "9cbca803b391f42b": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + "2b4e83287ef4a0ce": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8f09aba86abc10a48c2aa0da4db383e8a734db540e2ac890f07b5e39578e9502" + }, + "77900a1d84aff413": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9d055e80eba3b89892354174f854afda62cea6fa3621c06ecbc986ae1517ef47" + }, + "430c3620abbb5db4": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8c84c68381968d2a7264db9bdcd270d793ac6ecb227c7272d13158931f57cf17" + }, + "9c50367eb37de1e7": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "fa2d3eed4c2582f3077ad00bb99b4e2db17605f9bcc8dab56f3fa87c696c08e6" + }, + "98395f661f4091e6": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "372fd48a132419570d44767f9832f28ab8e663fbd6d31e8a2db438ef0ca86530" + } + } + }, + "limitations": { + "_trimmed_list": { + "count": 11, + "sha256": "043ae8840618206edd063e160cb71f3395f85832ba105631b655fc1e50ddc798", + "first": [ + "M5 only,4bit text and VL,2k/8k cold prompts, solo/four streams; no6/8bit or32k/64k performance claims.", + "VL requests here contain text only; image correctness and image throughput are separate evidence." + ], + "last": [ + "Fresh remote environment is separately frozen; transitive dependencies are not asserted identical to earlier or local M4 environments." + ] + } + }, + "raw_file_manifest": { + "http-results/events.json": { + "bytes": 29001, + "sha256": "5d94aab893aa6c34a635f1025404ec02654a81e890d86f58846c86cfb0b053e9" + }, + "http-results/manifest.json": { + "bytes": 13718, + "sha256": "2979ff660ee2ba66acaafbbca056a8904eb2e50648a78c4eabd33ccff2b0de55" + }, + "http-results/records.json": { + "bytes": 10615, + "sha256": "33492b85b7e4f1cf88c3ea8ac03f559c094fd73e05831bb93f631c0f69fbbf54" + }, + "http-results/summary.json": { + "bytes": 5889, + "sha256": "332539f66df431b47a9455cecd4a935c0349ef55a949ade6a9c9bdac806e3be5" + }, + "http-results/text-4-0-current.after.health": { + "bytes": 303, + "sha256": "1eb7e69a913a82f6ebdd17f63480c74999d649050645ae34c7f22e35158c294b" + }, + "http-results/text-4-0-current.after.metrics": { + "bytes": 12306, + "sha256": "4a06dfa8b08ab316e9aadce390c57f2647fee84ee92d2a69a33d317434bf61fa" + }, + "http-results/text-4-0-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/text-4-0-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "http-results/text-4-0-current.before.metrics": { + "bytes": 12128, + "sha256": "1e218089fa310167b9ff2a239a738ea7a906d509c583e1fe7e238d2ae8744fe1" + }, + "http-results/text-4-0-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/text-4-0-current.cold-2048.client.log": { + "bytes": 349, + "sha256": "67e69d3a4b0bf1937c21ee32296f5afbb692ecb6666a1aa6e62c52781246925e" + }, + "http-results/text-4-0-current.cold-2048.json": { + "bytes": 20296, + "sha256": "476c961a2df9848be6151d0c562dc08a7c209064747c6f95f34fabd97da03c3b" + }, + "http-results/text-4-0-current.cold-8192.client.log": { + "bytes": 347, + "sha256": "d02f9dac9726aa6aadf83468fa066e48f81a085ef6cc82889b8ad6c134a7554e" + }, + "http-results/text-4-0-current.cold-8192.json": { + "bytes": 57146, + "sha256": "66575cf391fa68ee6b75d5334ca2f457049f83c76ebe5d70dfe1822147e39953" + }, + "http-results/text-4-0-current.concurrent.client.log": { + "bytes": 3239, + "sha256": "f1ea2d553703b4a60da1cbe5ae22c75329cb2b9c179da3a78de681ecbd4b3a52" + }, + "http-results/text-4-0-current.concurrent.json": { + "bytes": 282943, + "sha256": "b72083d1657819e9ab7892bc8cdd7509136dffc521d4b259d445e9cee6134d4b" + }, + "http-results/text-4-0-current.server.log": { + "bytes": 13669, + "sha256": "f5e987c4227f77c6885c14851a2ce47a7a527d6bf7c3660a5c51737d7fe4e0a5" + }, + "http-results/text-4-1-pair.after.health": { + "bytes": 303, + "sha256": "acb548a8d66f97bae9d59037d63d996d818be60c5414283d938bcbab52d14bda" + }, + "http-results/text-4-1-pair.after.metrics": { + "bytes": 12306, + "sha256": "14a55da48d4fc832418b82676457289ab064500abd4de69720b9ba3db725068f" + }, + "http-results/text-4-1-pair.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/text-4-1-pair.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "http-results/text-4-1-pair.before.metrics": { + "bytes": 12128, + "sha256": "bf58eedbdd42df2243693092393a36c9047aa8ce59034c22c79cb6d3f5c66361" + }, + "http-results/text-4-1-pair.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/text-4-1-pair.cold-2048.client.log": { + "bytes": 350, + "sha256": "00153e731229db7d2fb1f720c8361d6620269ad05b765d37b0c3b9183b9a68f0" + }, + "http-results/text-4-1-pair.cold-2048.json": { + "bytes": 20297, + "sha256": "afe080020fda797399fd28f96bd6ee1debb318795130922a07ca26b71e28e130" + }, + "http-results/text-4-1-pair.cold-8192.client.log": { + "bytes": 348, + "sha256": "6f2b529061d69f295347b36873821bfdfd297e8a69cfbd1a602c371e10c97f94" + }, + "http-results/text-4-1-pair.cold-8192.json": { + "bytes": 57144, + "sha256": "69caa7fb69eb933af9c369d1d67c5aed02ed589616c4d9dd3a4e08c798496544" + }, + "http-results/text-4-1-pair.concurrent.client.log": { + "bytes": 3219, + "sha256": "5352d578c2810e60335864fb249e35d8c81b3acdb4aa74137ed5edacab7829e0" + }, + "http-results/text-4-1-pair.concurrent.json": { + "bytes": 282936, + "sha256": "611e0176cebbc07a0d405ab17d00db5b76ee98b647dbd75d0a546421ee2e7b1e" + }, + "http-results/text-4-1-pair.server.log": { + "bytes": 13745, + "sha256": "ccf6078e0f3167f216f2cf23ab55175debdcbb9532fa5749d332b1d2e0f4776b" + }, + "http-results/text-4-2-standard.after.health": { + "bytes": 453, + "sha256": "4b9a34a2eec3165a5c1ef2da3e32af6676938546b56f6c54a64d8d3679bb3540" + }, + "http-results/text-4-2-standard.after.metrics": { + "bytes": 32122, + "sha256": "0e9628073815f37ecba37019f3d2654e7d85ddf68e10bc283a5a786ec09a166d" + }, + "http-results/text-4-2-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "http-results/text-4-2-standard.before.health": { + "bytes": 453, + "sha256": "4b9a34a2eec3165a5c1ef2da3e32af6676938546b56f6c54a64d8d3679bb3540" + }, + "http-results/text-4-2-standard.before.metrics": { + "bytes": 906, + "sha256": "310306af9bef83a55b666256c62c47dbb80c1dd72adc98baca161bc23c348b55" + }, + "http-results/text-4-2-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "http-results/text-4-2-standard.cold-2048.client.log": { + "bytes": 297, + "sha256": "184d6bfd63d825996968201776e52838f7318ccd13ce1a8f2d530e52ac544347" + }, + "http-results/text-4-2-standard.cold-2048.json": { + "bytes": 26076, + "sha256": "2f0e25001ad0a3dfbf93095ed6d398dccbbdd3c61f5950eb197ce3fa4b86b54e" + }, + "http-results/text-4-2-standard.cold-8192.client.log": { + "bytes": 295, + "sha256": "8700ccdd36b9fa65ea90349f0245442e8b9b693560a3081b03ae5ba84dc57363" + }, + "http-results/text-4-2-standard.cold-8192.json": { + "bytes": 62933, + "sha256": "d57c23dc1377da0492d26e5fcf14087c996c0678427302d47302642aac6e2014" + }, + "http-results/text-4-2-standard.concurrent.client.log": { + "bytes": 3236, + "sha256": "f970410e8dd1a8a960111bd7d4b03a3326b50998076204f2e96146e96c74450f" + }, + "http-results/text-4-2-standard.concurrent.json": { + "bytes": 285704, + "sha256": "de508307d2994f7383fac843fc8f951b47a21854036b670af3fa89f2d8ae92ba" + }, + "http-results/text-4-2-standard.server.log": { + "bytes": 50891, + "sha256": "b08e004c53995bd98a69c40d0dbcf71f0f5f5cb2a1590a2af211a99b31d6b1f5" + }, + "http-results/text-4-3-standard.after.health": { + "bytes": 453, + "sha256": "4b9a34a2eec3165a5c1ef2da3e32af6676938546b56f6c54a64d8d3679bb3540" + }, + "http-results/text-4-3-standard.after.metrics": { + "bytes": 32096, + "sha256": "7a2952c94a5c59ab9a56f89376b507e9ef11cb3d0b2246dc54f61c77d6b6bcc2" + }, + "http-results/text-4-3-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "http-results/text-4-3-standard.before.health": { + "bytes": 453, + "sha256": "4b9a34a2eec3165a5c1ef2da3e32af6676938546b56f6c54a64d8d3679bb3540" + }, + "http-results/text-4-3-standard.before.metrics": { + "bytes": 909, + "sha256": "ade7ba6037d5aa112151f57dd04782d956aecb2be47f22cf4fc9c4d083f2413a" + }, + "http-results/text-4-3-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "http-results/text-4-3-standard.cold-2048.client.log": { + "bytes": 296, + "sha256": "96f7200a20c9b35a1d0e0026a5683b9bb63a5dcf47ecccd80751adc0bb98a1a5" + }, + "http-results/text-4-3-standard.cold-2048.json": { + "bytes": 26084, + "sha256": "928120efa4bf266221f356ade434c8e4410768fa2cf290b9ee5c30b0bda54cef" + }, + "http-results/text-4-3-standard.cold-8192.client.log": { + "bytes": 296, + "sha256": "fb594fb0825639c9476210ea3501486ee75eda5db848397d27402bc07fda6f86" + }, + "http-results/text-4-3-standard.cold-8192.json": { + "bytes": 62934, + "sha256": "4c01a217717702166c33ee7857c616aa3532cf0e61bc97c8528262b0efccc018" + }, + "http-results/text-4-3-standard.concurrent.client.log": { + "bytes": 3242, + "sha256": "a02ad302653eb2a160c860cd97940edd484abe41ee5fdba559c2200189252df3" + }, + "http-results/text-4-3-standard.concurrent.json": { + "bytes": 285671, + "sha256": "e4fa010598e6960e5803861d64ff670ac8420eeb929d15b5a139c66d7d22a40c" + }, + "http-results/text-4-3-standard.server.log": { + "bytes": 50891, + "sha256": "9f2095c3c2dd490c74f3ea5484289f35114770b65c51aed23fb86149904a00bf" + }, + "http-results/text-4-4-pair.after.health": { + "bytes": 303, + "sha256": "c6f4ac3c78ea6f6276ceff6e8b7985336950681a1823e616465aae6be499f5b8" + }, + "http-results/text-4-4-pair.after.metrics": { + "bytes": 12306, + "sha256": "69e9b9e0a0e26ab3a79ff750984a01d6b39ff7a96b3c6022956c7467253ade94" + }, + "http-results/text-4-4-pair.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/text-4-4-pair.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "http-results/text-4-4-pair.before.metrics": { + "bytes": 12128, + "sha256": "7a0f69eeb8775379bacfc86cb7f9f5ca661fb411d5981970b034bd74d168d089" + }, + "http-results/text-4-4-pair.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/text-4-4-pair.cold-2048.client.log": { + "bytes": 350, + "sha256": "e2fe1a73e404b85f729f90192fb705340d1d850f3e92bb2ce3dad8b8b5581fcf" + }, + "http-results/text-4-4-pair.cold-2048.json": { + "bytes": 20296, + "sha256": "976ea6aa15e490f9f25fe20056b12def96175307c3fcccf78f7cb259eaffba07" + }, + "http-results/text-4-4-pair.cold-8192.client.log": { + "bytes": 348, + "sha256": "fb432b4276da4abfb875ad84320f6b3b8a59a4b783157b5848c8b742e42128eb" + }, + "http-results/text-4-4-pair.cold-8192.json": { + "bytes": 57147, + "sha256": "2df86a35f6cbf55f16ce80d00ee3e4b55db7bcec60c025bcd9956db0bda24b11" + }, + "http-results/text-4-4-pair.concurrent.client.log": { + "bytes": 3225, + "sha256": "ca0e0ff7a9159534cd408da1462a13f2233718bd1f2a15ea2f3610f5d19ca015" + }, + "http-results/text-4-4-pair.concurrent.json": { + "bytes": 282974, + "sha256": "e400828bd0335e3ecd73f3faa048c4069fdf15c2e12f29ba1f0c831e7355b9af" + }, + "http-results/text-4-4-pair.server.log": { + "bytes": 13745, + "sha256": "2d899eb63007382b348bb13a70a245792633042bfe2ebc984bece39a752bc072" + }, + "http-results/text-4-5-current.after.health": { + "bytes": 303, + "sha256": "602868efc17450aa1967781d4377e8ea2b2a697a8aa46b5975d6927ae9578894" + }, + "http-results/text-4-5-current.after.metrics": { + "bytes": 12306, + "sha256": "6e3890f89c45fa17cb033d64f2d58c909835588e58e8396f7f87a1302e9027e1" + }, + "http-results/text-4-5-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/text-4-5-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "http-results/text-4-5-current.before.metrics": { + "bytes": 12128, + "sha256": "78d5d25064fddeea2fef1c12a8eb507f9f9d20e2b70aebe640ab8e9d10479f1a" + }, + "http-results/text-4-5-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/text-4-5-current.cold-2048.client.log": { + "bytes": 348, + "sha256": "b988118142b64e8c3c2edd33aaab1e5d9763e59514839e3ba4623b6f0db44705" + }, + "http-results/text-4-5-current.cold-2048.json": { + "bytes": 20295, + "sha256": "91dbc90983c79e7f4fc4689b4abf22190817d90c0ef524f933564a00f338842e" + }, + "http-results/text-4-5-current.cold-8192.client.log": { + "bytes": 347, + "sha256": "631738d652676d03ae528ee698e4dc374c8ebb5a3f08b9a9ce77e9f20646a9a5" + }, + "http-results/text-4-5-current.cold-8192.json": { + "bytes": 57143, + "sha256": "e70a5726664b1c2cd377433acb2414412129625f5756046c15b9928672339d22" + }, + "http-results/text-4-5-current.concurrent.client.log": { + "bytes": 3240, + "sha256": "951ad31ea4fe439ec1a0994e7f8fd037caf2eaa16ec7699bb1192860ba8bcafe" + }, + "http-results/text-4-5-current.concurrent.json": { + "bytes": 282915, + "sha256": "f538575e9b2f1fae482a87c5a69742ffd15ed5e7ca92bcec6397453c2bfd4981" + }, + "http-results/text-4-5-current.server.log": { + "bytes": 13669, + "sha256": "9faeafbc875f5f6679351ee0ba05b8347e5c239fd668732d963a6f9d71073a5e" + }, + "http-results/vl-4-0-current.after.health": { + "bytes": 303, + "sha256": "6be7062b427e5dacf8398716128aed6806ff70c753747cebf28d879f91d5c005" + }, + "http-results/vl-4-0-current.after.metrics": { + "bytes": 12306, + "sha256": "6aa607548c1408291794efca5efe852be794d7090912ef23465e68abf0d16bb2" + }, + "http-results/vl-4-0-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/vl-4-0-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "http-results/vl-4-0-current.before.metrics": { + "bytes": 12128, + "sha256": "11865eba546442f6f572b01d99c9eda58ef43dc8e49ce242b289ee14722fb216" + }, + "http-results/vl-4-0-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/vl-4-0-current.cold-2048.client.log": { + "bytes": 349, + "sha256": "c2e93b3c4266d32dffe3fa352860f22d66cd7fabb7cfb664535110ca9f157bae" + }, + "http-results/vl-4-0-current.cold-2048.json": { + "bytes": 20278, + "sha256": "61fe01e1064eb4e60575b24b6365e8fd60cde1ee2e83bdf268351abb5d1d4ae4" + }, + "http-results/vl-4-0-current.cold-8192.client.log": { + "bytes": 347, + "sha256": "6556bd4670562223bf7a1ce5d55721591e585a5ec3b54d16f18a8f6b1c998f8a" + }, + "http-results/vl-4-0-current.cold-8192.json": { + "bytes": 57145, + "sha256": "1363ef6bf24272487c0ae2fbaa26b8fdb5c01ac6efe46c755772e63a93d429ea" + }, + "http-results/vl-4-0-current.concurrent.client.log": { + "bytes": 3228, + "sha256": "9286dadf9f05fa05a6b26f6938ea6c7e60f624bb5f8afd2cab4a225bf4803026" + }, + "http-results/vl-4-0-current.concurrent.json": { + "bytes": 289148, + "sha256": "a595a08c1e7d26d27cfc4e5a937c101f61117f04e18467b00887212fd1ee5bf9" + }, + "http-results/vl-4-0-current.server.log": { + "bytes": 13641, + "sha256": "ef40fe297657b1f1a730f15589c18f871139d84e15ccfe4c9a4beef381807f62" + }, + "http-results/vl-4-1-standard.after.health": { + "bytes": 457, + "sha256": "ddd816f4e9d2c97ca67ca18fc173865751e14813a790ffaf1c1183ffd0e3fac2" + }, + "http-results/vl-4-1-standard.after.metrics": { + "bytes": 32159, + "sha256": "d6dadf4b325194ea1d0df9303c5b19cd28e110f3eeb21f63167d6d95c1e55313" + }, + "http-results/vl-4-1-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "http-results/vl-4-1-standard.before.health": { + "bytes": 457, + "sha256": "ddd816f4e9d2c97ca67ca18fc173865751e14813a790ffaf1c1183ffd0e3fac2" + }, + "http-results/vl-4-1-standard.before.metrics": { + "bytes": 912, + "sha256": "dd4acb9f3449e184831844eb162b1edb74e78b7badea36792c18e334a1a1873b" + }, + "http-results/vl-4-1-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "http-results/vl-4-1-standard.cold-2048.client.log": { + "bytes": 296, + "sha256": "b33d0ae8330bc241718491f3edec228ea7e4c602ad2e45c3a5e8a89b5a6fc993" + }, + "http-results/vl-4-1-standard.cold-2048.json": { + "bytes": 26081, + "sha256": "07a5c7f784a2ea17baa2d804708d06f6c24cec7f1f4a54e55ffb581fe78e55a6" + }, + "http-results/vl-4-1-standard.cold-8192.client.log": { + "bytes": 295, + "sha256": "40f13562c53a6d05295e91dbee87c7506383135960d3dd9c21ad0e63e0ea03e6" + }, + "http-results/vl-4-1-standard.cold-8192.json": { + "bytes": 62952, + "sha256": "73b8c68247b5779c754931beea8d9f031d3298ae51f414549d37b020a66d518e" + }, + "http-results/vl-4-1-standard.concurrent.client.log": { + "bytes": 3219, + "sha256": "bd46d94410ec6af081bed04787e5c8226c1d8f5d21d0aafb14f78dd901323134" + }, + "http-results/vl-4-1-standard.concurrent.json": { + "bytes": 285423, + "sha256": "fe742faf5ec5a6d6150b81cc93d9891e2fbb89cbb5c0dbe7b590d02f49cfd9b4" + }, + "http-results/vl-4-1-standard.server.log": { + "bytes": 50969, + "sha256": "cb0f6f7f0343d4a0c036a07e2b6004fd3dca2eea8bccb8e598848dca9c4b931c" + }, + "http-results/vl-4-2-standard.after.health": { + "bytes": 457, + "sha256": "ddd816f4e9d2c97ca67ca18fc173865751e14813a790ffaf1c1183ffd0e3fac2" + }, + "http-results/vl-4-2-standard.after.metrics": { + "bytes": 32148, + "sha256": "1b9a49e67e7d452f685b4228690e6e62d6216d4afb4e3498b9106dd94d25bfc3" + }, + "http-results/vl-4-2-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "http-results/vl-4-2-standard.before.health": { + "bytes": 457, + "sha256": "ddd816f4e9d2c97ca67ca18fc173865751e14813a790ffaf1c1183ffd0e3fac2" + }, + "http-results/vl-4-2-standard.before.metrics": { + "bytes": 912, + "sha256": "2ca902329cc406ddab4a22917dbfd75ff2dd46ab98a32304cfee627e5e39c472" + }, + "http-results/vl-4-2-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "http-results/vl-4-2-standard.cold-2048.client.log": { + "bytes": 297, + "sha256": "3943f8e96d62cce1fce5866afa0db1ee804a216c86dd4f1d97e99124c86c4ad6" + }, + "http-results/vl-4-2-standard.cold-2048.json": { + "bytes": 26078, + "sha256": "6b0aadd900aecfd94e7edcf41809b1a55efcfd99c5117c6fa921aa8300369b81" + }, + "http-results/vl-4-2-standard.cold-8192.client.log": { + "bytes": 295, + "sha256": "ef0b25d5e93ef2d1edc89e8bce8535a784d0cdc865c63750b35ad6b18450ab25" + }, + "http-results/vl-4-2-standard.cold-8192.json": { + "bytes": 62947, + "sha256": "0f0f1d42041c1b36c15078449ad39eee79c52745b98a5d3dcd6e17610cbe8002" + }, + "http-results/vl-4-2-standard.concurrent.client.log": { + "bytes": 3220, + "sha256": "c692e7879d215df95d388c83672ce27dda29d78f3fd73c470d05eb7fb1a5312b" + }, + "http-results/vl-4-2-standard.concurrent.json": { + "bytes": 285646, + "sha256": "7c810f7d8a77f15d0970b7016a7732e17c9a22122206cfaea3d383f59e862764" + }, + "http-results/vl-4-2-standard.server.log": { + "bytes": 50969, + "sha256": "ed0ea0118772c446dd511feeae5650d2e5618ac353df02e26b5df0e2f46a4222" + }, + "http-results/vl-4-3-current.after.health": { + "bytes": 303, + "sha256": "b76da36250c4d5f6687ee0087d360eb5031c15529606a045e337001d0feec7d2" + }, + "http-results/vl-4-3-current.after.metrics": { + "bytes": 12305, + "sha256": "8f3a02c8d6a74725461341ce74244367f70040e57123bc1c4171dedb1ca9bd49" + }, + "http-results/vl-4-3-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/vl-4-3-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "http-results/vl-4-3-current.before.metrics": { + "bytes": 12128, + "sha256": "01d10c7d9e03abcb4d928582033c5d7107220a4193a498d0974d642835adb1ee" + }, + "http-results/vl-4-3-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/vl-4-3-current.cold-2048.client.log": { + "bytes": 349, + "sha256": "60f435f0abd8537208b684c2a70125f7047c919d7430f5208fa8a92f32ee4931" + }, + "http-results/vl-4-3-current.cold-2048.json": { + "bytes": 20279, + "sha256": "affe00c4a3c4ef931f0413a191b19730bed2dff794aba52d3938486eabb65269" + }, + "http-results/vl-4-3-current.cold-8192.client.log": { + "bytes": 347, + "sha256": "433dff20e6a5f8d2a1892b651e83063603dbe9a73a35cd372f3bca87a5f79672" + }, + "http-results/vl-4-3-current.cold-8192.json": { + "bytes": 57146, + "sha256": "ee50fa386246da4b2f8fc1e4ae690affcbd86f197fe3f955085c4b33b0468e29" + }, + "http-results/vl-4-3-current.concurrent.client.log": { + "bytes": 3223, + "sha256": "2c0d8c7115ad692169368fe0e1957bc37b54490c035a25b665f089f41932cc38" + }, + "http-results/vl-4-3-current.concurrent.json": { + "bytes": 289087, + "sha256": "58cc544b81b0e5b88e7011467dc3946747ac8f062dace116c5921c5ec9368bca" + }, + "http-results/vl-4-3-current.server.log": { + "bytes": 13641, + "sha256": "d853680caa98301903e6b435d100ab71ed2c2a680b58ddbcc6e5d39195332f36" + }, + "http.log": { + "bytes": 22106, + "sha256": "d3e10fb147dd34b1153da5298f1ecf7d95e931561ed278e58575fb925e7070f1" + }, + "proof/4-ffn-1.log": { + "bytes": 68, + "sha256": "0a440fb1c1136a2a05e490b1399243c0853d379a6096c94a1836e58013fd3e94" + }, + "proof/4-ffn-15.log": { + "bytes": 70, + "sha256": "13dcb26bc1cbaef6fd249e13695598178e8d82dd748b759c9195aa1f8ee0666a" + }, + "proof/4-ffn-16.log": { + "bytes": 70, + "sha256": "754e50ddb5f087244dcd950de115031d09a47f03704c67b094fe8c090a98f426" + }, + "proof/4-ffn-17.log": { + "bytes": 70, + "sha256": "516ca1b168bae8415125d0c1080ed69400c5ff0f7f4918d6c7883cf942ddad1f" + }, + "proof/4-ffn-2.log": { + "bytes": 68, + "sha256": "21cb81ddf8af88aee84dd9d8794a8f69bd4b8536e592c916fcfb1cc536dfbffc" + }, + "proof/4-ffn-3.log": { + "bytes": 68, + "sha256": "29a2936aade20e2365cdc224525b4347ecdb32ad0d575faef58f84f50fca22d5" + }, + "proof/4-ffn-31.log": { + "bytes": 70, + "sha256": "5d625695b2974141ea2f3dd10af276908b6c02a6b676a3c597193153e1baad59" + }, + "proof/4-ffn-32.log": { + "bytes": 70, + "sha256": "408d51c9202962b5c06f0e9050248cc77a665d440d70c4a0339f6218c3253034" + }, + "proof/4-ffn-33.log": { + "bytes": 71, + "sha256": "7b49974e82e5df448e60a14c5eaeb218f98c0a51575e70adcf5293b9e40c132d" + }, + "proof/4-ffn-4.log": { + "bytes": 69, + "sha256": "99c6bbb75a119148c3c8f0aaf36834f68e911e40ecc95d4273adb7a510a95c82" + }, + "proof/4-ffn-5.log": { + "bytes": 69, + "sha256": "5956e81029fa558ab5de1cb0fc61de5ef9fdbc6aed8d2f3b1410363e10afd793" + }, + "proof/4-ffn-63.log": { + "bytes": 71, + "sha256": "2985d58dcfc5f0c4ba03d720319b615abc5e27e538d6d9626fb7fce3ad92d9b3" + }, + "proof/4-ffn-64.log": { + "bytes": 71, + "sha256": "37f53a635bd8378fd0adeb6628e41003ccbf9fd887b92fc717bd8bb966c661bc" + }, + "proof/4-ffn-7.log": { + "bytes": 69, + "sha256": "53286a376acebe7869a2ec3442138018914c1339b6360335aaad0c0c531bc396" + }, + "proof/4-ffn-8.log": { + "bytes": 69, + "sha256": "6fc70da8e674c660855f6a4d8c0141eb25e99c38605b9c86ad43abd0b48973ec" + }, + "proof/4-ffn-nf7f80.log": { + "bytes": 71, + "sha256": "7b49974e82e5df448e60a14c5eaeb218f98c0a51575e70adcf5293b9e40c132d" + }, + "proof/4-ffn-nf7fc1.log": { + "bytes": 71, + "sha256": "7b49974e82e5df448e60a14c5eaeb218f98c0a51575e70adcf5293b9e40c132d" + }, + "proof/4-ffn-nfff80.log": { + "bytes": 71, + "sha256": "7b49974e82e5df448e60a14c5eaeb218f98c0a51575e70adcf5293b9e40c132d" + }, + "proof/4-kv-1.log": { + "bytes": 66, + "sha256": "fe43a02ddd7448f7b611e9873ab20f82c9689ba2d69e4d04df0cbf3497101ca5" + }, + "proof/4-kv-15.log": { + "bytes": 68, + "sha256": "b45fbcfcbbd9a66ff0339f08e941f7d1d95b723e197ff74590fdd06d29acd234" + }, + "proof/4-kv-16.log": { + "bytes": 68, + "sha256": "e88de31c2703c133832558c4b4bac7025a89ca8de9736c2703e3e6d95652859e" + }, + "proof/4-kv-17.log": { + "bytes": 68, + "sha256": "e17e850d46238db91997e62ecefe8fdd5be9ee63045d066cb36db620b2226453" + }, + "proof/4-kv-2.log": { + "bytes": 66, + "sha256": "72deb27fffed515c3b278354c14e7a6a5d386fb7858468051585a27333036b0c" + }, + "proof/4-kv-3.log": { + "bytes": 66, + "sha256": "906c0a2325ec3967e3c934db49e327b72ed4a8687bea6be6f7d2d3b9a0708a7a" + }, + "proof/4-kv-31.log": { + "bytes": 68, + "sha256": "d042703ea46aa5eb7eb1bed62d474b63c2a326e14cc8cbdd43cd56fdfa1c295c" + }, + "proof/4-kv-32.log": { + "bytes": 68, + "sha256": "5b766a30832287c2b52617eadc4bc4c1bd94be2cfa89b60b333b5a66da1c48e3" + }, + "proof/4-kv-33.log": { + "bytes": 68, + "sha256": "3c0ebba9793619da4e4b5749367b9af81957c42f280bdba35e99aa3c78fa97f8" + }, + "proof/4-kv-4.log": { + "bytes": 66, + "sha256": "cfd917124f13fd7b8edd0796d8046847f207d7eba3221ae13ffa4109fb4d9695" + }, + "proof/4-kv-5.log": { + "bytes": 67, + "sha256": "4368f4840d1c8aee47a4be5c6f4f10b1bf7f8cf04916e5ca81d4f4590d573de7" + }, + "proof/4-kv-63.log": { + "bytes": 69, + "sha256": "d43613b96c8ef0a476966460e3f087c63bae5206859d0dd339b4ce565b607268" + }, + "proof/4-kv-64.log": { + "bytes": 69, + "sha256": "bd1dcf2577e05ef0d1d2f0a6151b9155d3cfeae3b9217c958c4399794899dd50" + }, + "proof/4-kv-7.log": { + "bytes": 67, + "sha256": "7fd31c93457566ea68aa10deb58716b7dd22f21ebc52ede7b59efda1946cf9ea" + }, + "proof/4-kv-8.log": { + "bytes": 67, + "sha256": "164bf999b45eba6814a03eacd622ed46f23a107806b255e1d870e7c11b3047b4" + }, + "proof/4-kv-nf7f80.log": { + "bytes": 68, + "sha256": "3c0ebba9793619da4e4b5749367b9af81957c42f280bdba35e99aa3c78fa97f8" + }, + "proof/4-kv-nf7fc1.log": { + "bytes": 68, + "sha256": "3c0ebba9793619da4e4b5749367b9af81957c42f280bdba35e99aa3c78fa97f8" + }, + "proof/4-kv-nfff80.log": { + "bytes": 68, + "sha256": "3c0ebba9793619da4e4b5749367b9af81957c42f280bdba35e99aa3c78fa97f8" + }, + "proof/6-ffn-1.log": { + "bytes": 68, + "sha256": "4cdd5b755a373447f583a80f0cf82efbebfc8eaebf8c01985a48a219c38e2cf0" + }, + "proof/6-ffn-15.log": { + "bytes": 70, + "sha256": "7ac345eef2bd92c1c93d1f159c5bea9677bdd357a4a497a08c2277d4e5092349" + }, + "proof/6-ffn-16.log": { + "bytes": 70, + "sha256": "f9ba1a57e3396b5f17bc6b15c1a87a1cb0dd66f0f0e2c79d6bedf6d31b3d9a11" + }, + "proof/6-ffn-17.log": { + "bytes": 70, + "sha256": "07540b5d73f9d424fd6a31aca8f0e4df33fe85d124cf38cf634a4da323d4910e" + }, + "proof/6-ffn-2.log": { + "bytes": 68, + "sha256": "99f6e747a32ef7e4fc956bd602066800364e7a9afee4aaebe48d7cb5e0c1a29b" + }, + "proof/6-ffn-3.log": { + "bytes": 68, + "sha256": "f103fdaba792757c755c1d5b834f6be2b832013074af24079bc46bc9a27eed5f" + }, + "proof/6-ffn-31.log": { + "bytes": 70, + "sha256": "5bfde70efd999647b522d2e6fa96af6592e16525cf6f2f3a0fac19d43476be6d" + }, + "proof/6-ffn-32.log": { + "bytes": 70, + "sha256": "9b8898fdc7512b73c53eda15207264da2d2099b84298ac416d7c069bfec13d79" + }, + "proof/6-ffn-33.log": { + "bytes": 71, + "sha256": "535e05bcff4898948786f8600594bb9606068485ac09bbf9176a06816ffde426" + }, + "proof/6-ffn-4.log": { + "bytes": 69, + "sha256": "5dcd1685047003c6381132cb16c9ad58d1b53a8694d8c918fa2982eb5dc435ee" + }, + "proof/6-ffn-5.log": { + "bytes": 69, + "sha256": "69797df071d57c3b4c7ed419091fe6cbb10c18cb9e5acf0e41208ec1c9fa3d75" + }, + "proof/6-ffn-63.log": { + "bytes": 71, + "sha256": "696ce1f02cc4700ec60ee662df1fe5a3a97a1886ce79e05583226a9bb0e4db3c" + }, + "proof/6-ffn-64.log": { + "bytes": 71, + "sha256": "ca26d37339d7ed3547bca3016793c2f4fae5b3146342cd416c6ff108f4cc7682" + }, + "proof/6-ffn-7.log": { + "bytes": 69, + "sha256": "bb3034e2736a8890a9534d5a1d141e11436f714a7f8f8951b34cd348f0af91ab" + }, + "proof/6-ffn-8.log": { + "bytes": 69, + "sha256": "63c849c01b54ceba39ef753989262cb895e104ff78e46fa6e2ed6657f2d32cd5" + }, + "proof/6-ffn-nf7f80.log": { + "bytes": 71, + "sha256": "535e05bcff4898948786f8600594bb9606068485ac09bbf9176a06816ffde426" + }, + "proof/6-ffn-nf7fc1.log": { + "bytes": 71, + "sha256": "535e05bcff4898948786f8600594bb9606068485ac09bbf9176a06816ffde426" + }, + "proof/6-ffn-nfff80.log": { + "bytes": 71, + "sha256": "535e05bcff4898948786f8600594bb9606068485ac09bbf9176a06816ffde426" + }, + "proof/6-kv-1.log": { + "bytes": 66, + "sha256": "c7ed815d9852caa06b4964504a6d2cb7d0560e6b35c69e4ad45fccad50e638b1" + }, + "proof/6-kv-15.log": { + "bytes": 68, + "sha256": "2a1aac83429e67211279125e2042b4eb09df95b1db78718944b866eb08f5b090" + }, + "proof/6-kv-16.log": { + "bytes": 68, + "sha256": "ff05709a1e79f562fe39d06c8fdd100407295d990add0297d550bdcf2a92a2e0" + }, + "proof/6-kv-17.log": { + "bytes": 68, + "sha256": "a33be246560376d27c2e9b3963b51d166538070294e3d5d0117a86ca3b15f73d" + }, + "proof/6-kv-2.log": { + "bytes": 66, + "sha256": "038e9bb056ea42ee2562f5a27b38bd5857f6cef6ac21e9827625cc8e20611bfd" + }, + "proof/6-kv-3.log": { + "bytes": 66, + "sha256": "6539d824cb0c82f85edc3dde2073fdf269d790ecde4e6a3762f2134ceb416aed" + }, + "proof/6-kv-31.log": { + "bytes": 68, + "sha256": "56571924afe7e83dd2f07ae374777b601f4b0c8f631a4513a86a4c9c060b0b18" + }, + "proof/6-kv-32.log": { + "bytes": 68, + "sha256": "e55cc4efe0fca1197abb18097dab7128f22fc955fde63c892d3ac958c7e0d023" + }, + "proof/6-kv-33.log": { + "bytes": 68, + "sha256": "8b22270452882348b7ca6b1e161369a107118339e8ef09f1906eb8f4bbef8cd2" + }, + "proof/6-kv-4.log": { + "bytes": 66, + "sha256": "35ab4f1a3c67dd907d24efc56b7c1db42ab002d48324f57ca8218c89ff5f0b24" + }, + "proof/6-kv-5.log": { + "bytes": 67, + "sha256": "40e89ffee251fac917ac68dbafe2135a16008f584e83c2f36d2848492c5e49a3" + }, + "proof/6-kv-63.log": { + "bytes": 69, + "sha256": "c599c82ed56ab28559f8bd63bf0828f9e60128411732dafe5d6b191724672304" + }, + "proof/6-kv-64.log": { + "bytes": 69, + "sha256": "b163baa7ac2f82faecec8108f59eebd5a82313d3be554f150188d43ab94b4c43" + }, + "proof/6-kv-7.log": { + "bytes": 67, + "sha256": "fbd0176b1cb83cb6219d2e3c166a937abec874862737767094995165ea8cf6e9" + }, + "proof/6-kv-8.log": { + "bytes": 67, + "sha256": "a28e3cdc0510b03de588a644ca6e3dee19946a0b951bdb64deeccd664730d7fa" + }, + "proof/6-kv-nf7f80.log": { + "bytes": 68, + "sha256": "8b22270452882348b7ca6b1e161369a107118339e8ef09f1906eb8f4bbef8cd2" + }, + "proof/6-kv-nf7fc1.log": { + "bytes": 68, + "sha256": "8b22270452882348b7ca6b1e161369a107118339e8ef09f1906eb8f4bbef8cd2" + }, + "proof/6-kv-nfff80.log": { + "bytes": 68, + "sha256": "8b22270452882348b7ca6b1e161369a107118339e8ef09f1906eb8f4bbef8cd2" + }, + "proof/8-ffn-1.log": { + "bytes": 68, + "sha256": "de95bbd2f027c23b1c54e57943a49f81b07c3a61bafe9dda8d4640a73f8e3aac" + }, + "proof/8-ffn-15.log": { + "bytes": 70, + "sha256": "d839986f7d370640775e8d931cbcab71be7650ba4c3acb3d89dd721d1ad33224" + }, + "proof/8-ffn-16.log": { + "bytes": 70, + "sha256": "0de65ffc42a6eb589bcb6b6ea0235f236dd3eb82a2da87d71d53c12ed5c2f87e" + }, + "proof/8-ffn-17.log": { + "bytes": 70, + "sha256": "b48882c7f0eeca32623877bac20c2f984a62bdfa3e655aa2a7510563cf41b8f7" + }, + "proof/8-ffn-2.log": { + "bytes": 68, + "sha256": "dd52b55f8e06a8873ee915bd5394ae0341d76bf1a7a6f238bc8361aed6be9fda" + }, + "proof/8-ffn-3.log": { + "bytes": 68, + "sha256": "601c40a27ae2ad182df0c0e2f8d16011d91bbfbdf069fd959d7603f948fe576a" + }, + "proof/8-ffn-31.log": { + "bytes": 70, + "sha256": "cc13d09d3e3fdc3fdca68e72abc2cefa5b7b67fa80f398b156e609bc55c0a0ed" + }, + "proof/8-ffn-32.log": { + "bytes": 70, + "sha256": "d35a667f5ccbf6f3cd3cbec99439f45ca885fd7a017dd6a121b2c4b7f47bdb52" + }, + "proof/8-ffn-33.log": { + "bytes": 71, + "sha256": "7dd6fee5464bd09733b39eafb4ca9fc2adcf5fb3a4248b8dfc4cc53a010a3527" + }, + "proof/8-ffn-4.log": { + "bytes": 69, + "sha256": "db029a92c9fb380d714a91d3a765d61d59f33ee2b636533a2b62bb6c138ca4a4" + }, + "proof/8-ffn-5.log": { + "bytes": 69, + "sha256": "793fe7ec32771ae0a800c48fa602c121d479fcc9e16975ae25a03bdc796424d9" + }, + "proof/8-ffn-63.log": { + "bytes": 71, + "sha256": "8b203d4126444e54af1c0b89d31a88784ad0f1656f06f11e0d11de3a1a54b6a0" + }, + "proof/8-ffn-64.log": { + "bytes": 71, + "sha256": "fdc04e0afa37428dfbee97275570a7ed66f59ed45c9a86a7aebe7fc098698f0d" + }, + "proof/8-ffn-7.log": { + "bytes": 69, + "sha256": "3d856d211329c919e21cc957632ea76c25b74c9c71df8bd5c7cff1736ba1284f" + }, + "proof/8-ffn-8.log": { + "bytes": 69, + "sha256": "9cc900edd38de2296509fb9c11d0265e4e47e3c5d669dd92917cb1ce7a74d5d4" + }, + "proof/8-ffn-nf7f80.log": { + "bytes": 71, + "sha256": "7dd6fee5464bd09733b39eafb4ca9fc2adcf5fb3a4248b8dfc4cc53a010a3527" + }, + "proof/8-ffn-nf7fc1.log": { + "bytes": 71, + "sha256": "7dd6fee5464bd09733b39eafb4ca9fc2adcf5fb3a4248b8dfc4cc53a010a3527" + }, + "proof/8-ffn-nfff80.log": { + "bytes": 71, + "sha256": "7dd6fee5464bd09733b39eafb4ca9fc2adcf5fb3a4248b8dfc4cc53a010a3527" + }, + "proof/8-kv-1.log": { + "bytes": 66, + "sha256": "c732a5c57edd7ee8e68c868680f77b6add1833b1c164525feb2a786dfafcdfce" + }, + "proof/8-kv-15.log": { + "bytes": 68, + "sha256": "6c3ec60344fb8df4ae18859fa52782d55958aa959cbb776feabbdebc3b9f62ad" + }, + "proof/8-kv-16.log": { + "bytes": 68, + "sha256": "244e6b7e5b5897a9f94609d77bcf25ffa0533050ff9e185bae0d6d7d67850aba" + }, + "proof/8-kv-17.log": { + "bytes": 68, + "sha256": "148ff59ea5186337575b8c87028c9537bc0eeaa0f15b2472875d5f0eff3270e8" + }, + "proof/8-kv-2.log": { + "bytes": 66, + "sha256": "4a8c39bed3908422ad3d004cc8a51e76c4a867e55d7ceae6eb7889286ff88db6" + }, + "proof/8-kv-3.log": { + "bytes": 66, + "sha256": "62b319f41629686cd3b829270b946d78c2822580fbbc9143ddb0ecacda17b537" + }, + "proof/8-kv-31.log": { + "bytes": 68, + "sha256": "7c32a91554b6eb73dd6af27e2e44b2f9e6a381dfc076f921c41b6cb29982b51e" + }, + "proof/8-kv-32.log": { + "bytes": 68, + "sha256": "fd1da0acd80415a00a8e9f0e275fbc11a66952fdabe8a8b338b7a5177cb32ef1" + }, + "proof/8-kv-33.log": { + "bytes": 68, + "sha256": "dc2fea731fbe954b45e80e0b84559fa1b199f965fa0fe9ed56c590fa57d05086" + }, + "proof/8-kv-4.log": { + "bytes": 66, + "sha256": "ab57f621b838f885b8304a1182feb09cad88f8626e13abc97c9c2ea22bd8c560" + }, + "proof/8-kv-5.log": { + "bytes": 67, + "sha256": "143416f4b1c097a1ae93cc63eab31ccd79160c0e29ce6e938d2baf632a7bb399" + }, + "proof/8-kv-63.log": { + "bytes": 69, + "sha256": "c84325439a391804c025f3827b1531c9837a8859c5fc6ac8070f2eb638d8791e" + }, + "proof/8-kv-64.log": { + "bytes": 69, + "sha256": "d3a38b959675b51e641f19ab657ed2893e0e92948a0f3f4551eb3d728d86547b" + }, + "proof/8-kv-7.log": { + "bytes": 67, + "sha256": "aa2ac2da5ffce884af13e1ac72000ceacd0867e5bf2c16ac61feb568519e881a" + }, + "proof/8-kv-8.log": { + "bytes": 67, + "sha256": "add20ea8714e8fc7edb953d1dca4a7b13582da911aee541d651bca3932d914e7" + }, + "proof/8-kv-nf7f80.log": { + "bytes": 68, + "sha256": "dc2fea731fbe954b45e80e0b84559fa1b199f965fa0fe9ed56c590fa57d05086" + }, + "proof/8-kv-nf7fc1.log": { + "bytes": 68, + "sha256": "dc2fea731fbe954b45e80e0b84559fa1b199f965fa0fe9ed56c590fa57d05086" + }, + "proof/8-kv-nfff80.log": { + "bytes": 68, + "sha256": "dc2fea731fbe954b45e80e0b84559fa1b199f965fa0fe9ed56c590fa57d05086" + }, + "proof/current-normal70.log": { + "bytes": 99027, + "sha256": "d3631ea9b1f37e25ef13a43aabfe9a1ff54875170a04fb2acbdb637cafbbbef6" + }, + "proof/current-shared-tiles.log": { + "bytes": 3139, + "sha256": "2767db9832e7801652aafdb998dde8ff69451fade9aff318b39d18e28d1ad1bd" + }, + "proof/events.json": { + "bytes": 81478, + "sha256": "3604552393d3dc3713d9a29dfb9b6312c3d2116def25b48cb054c96a714133d5" + }, + "proof/manifest.json": { + "bytes": 1557, + "sha256": "dd6f090b528cb6163168810fa321b1177f3ecbbc1c3373f6595996974ccb746e" + }, + "proof.log": { + "bytes": 68771, + "sha256": "82eb11b8d7de79cccf57da5616a1a86716777d1fa60e1465c82bcbac826c8a2c" + }, + "receipt.json": { + "bytes": 2915, + "sha256": "c5b5d34650ece84663955d12263e4274308cccc199bd66f32ec052c0ff3c467d" + } + }, + "raw_retention": "Raw logs, per-request token timing arrays, repeated metrics/comparisons, health/settings/metrics and proof outputs remain in the frozen local result directory. This receipt retains per-leg/per-repetition TPS, end-to-end TPS, makespan, per-stream TPS/TTFT, failure counts, strict token/cache evidence and memory; evidence is deduplicated by reference. Hashes retain the omitted raw provenance.", + "_trim": { + "note": "Raw capture replaced by counts, first/last items and SHA-256 of the canonical JSON of each trimmed list or string.", + "original_sha256": "8609d22ac400f7d10fe3e19041080f46ec739b6c5e93024fc3d543663cc0cd30", + "original_bytes": 217576 + } +} diff --git a/docs/receipts/lfm3/m5-current68-standard.json b/docs/receipts/lfm3/m5-current68-standard.json new file mode 100644 index 0000000000..714431d86f --- /dev/null +++ b/docs/receipts/lfm3/m5-current68-standard.json @@ -0,0 +1,1561 @@ +{ + "schema_version": 1, + "scope": "Current public runtime compared with pinned standard on Apple M5 Ultra: text1.7B and VL2B6/8bit,2k/8k text-only cold prompts,256-token solo/four-stream code/prose. No performance gate pass.", + "status": { + "actual_exit": 0, + "server_legs": 16, + "primary_measured_requests": 480, + "primary_repetitions": 192, + "cold_requests": 32, + "all_primary_256_tokens": true, + "all_cold_16_tokens_cached0": true, + "native_concurrent_to_solo_equal": 240, + "native_serial_equal": 32, + "owned_server_cleanup_waits_completed": 16, + "native_graceful_exit0": 8, + "standard_owned_SIGTERM_exit_minus15": 8, + "performance_accepted": false, + "conclusion": "Current public throughput and cold prefill remain slower than the measured applicable standard in all four model/width cells." + }, + "host": { + "gpu": "Apple M5 Ultra", + "gpu_cores": 64, + "physical_memory_bytes": 274877906944, + "os": "ProductName:\t\tmacOS\nProductVersion:\t\t27.0\nBuildVersion:\t\t26A428\n" + }, + "provenance": { + "public_head_at_preparation": "07a7cb4cbc137259c1f53373e59e6d88d8ff8074", + "native_provenance": "Current3d4 applied artifact built at runtime fixdd7263c26, source hashes recorded attention-range.json; git diff dd7263c26..07a7cb4 -- zig empty; later docs commits do not substitute a rebuild.", + "native_artifact": { + "current": { + "path": "$RUN/bin/tensorfold-native-current", + "sha256": "3d4a3daeda697420bfbba6d81972896c2bf9e31efaf6f12321ee8487a299354b" + } + }, + "versions": { + "python": "3.13.15 (main, Sep 1 2026, 14:05:45) [Clang 22.1.3 ]", + "packages": { + "shellingham": "1.5.4", + "uvicorn": "0.54.0", + "xxhash": "4.0.2", + "filelock": "4.2.4", + "mlx": "0.32.4", + "mlx-lm": "0.32.2", + "starlette": "1.10.0", + "huggingface_hub": "2.3.0", + "click": "8.5.0", + "packaging": "26.3", + "protobuf": "7.36.2", + "tokenizers": "0.23.3", + "Pygments": "2.21.0", + "pyarrow": "26.0.0", + "sounddevice": "0.5.6", + "requests": "2.34.2", + "rich": "15.0.0", + "certifi": "2026.10.23", + "yarl": "1.25.1", + "websockets": "17.3", + "mlx-vlm": "0.7.11", + "opentelemetry-api": "1.45.1", + "pandas": "3.1.0", + "datasets": "5.1.0", + "sentencepiece": "0.2.2", + "propcache": "0.5.4", + "llguidance": "1.9.1", + "numpy": "2.5.4", + "aiohappyeyeballs": "2.7.1", + "cryptography": "50.0.2", + "pycparser": "3.12", + "tzdata": "2026.5", + "pillow": "13.0.0", + "fsspec": "2026.7.0", + "hf-xet": "1.7.0", + "safetensors": "0.8.0", + "miniaudio": "1.71", + "tqdm": "4.70.1", + "urllib3": "2.8.0", + "h11": "0.16.0", + "pydantic_core": "2.50.1", + "mdurl": "0.1.2", + "python-multipart": "0.0.32", + "mlx-metal": "0.32.4", + "PyYAML": "6.0.3", + "httpcore2": "2.16.0", + "opencv-python": "5.0.0.93", + "truststore": "0.10.5", + "frozenlist": "1.8.0", + "cffi": "2.1.1", + "httpx2": "2.16.0", + "aiosignal": "1.4.0", + "typer": "0.27.3", + "mlx-audio": "0.6.2", + "Jinja2": "3.1.6", + "annotated-types": "0.8.0", + "regex": "2026.9.29", + "markdown-it-py": "4.2.0", + "annotated-doc": "0.0.5", + "typing-inspection": "0.4.4", + "fastapi": "0.162.0", + "aiohttp": "3.14.5", + "attrs": "26.1.0", + "multidict": "7.2.1", + "typing_extensions": "4.16.0", + "MarkupSafe": "3.0.4", + "pydantic": "2.14.1", + "python-dateutil": "3.0.0", + "multiprocess": "0.70.19", + "charset-normalizer": "3.5.2", + "anyio": "4.15.1", + "six": "1.17.0", + "scipy": "1.18.1", + "dill": "0.4.1", + "transformers": "5.21.0", + "idna": "3.20" + } + }, + "standard": { + "python": "$QUALIFICATION/standard-venv/bin/python", + "module": "mlx_vlm.server", + "source_commit": "230fc308efb577d87446306bfb8a3e9f1a129558", + "mlx": "0.32.4", + "mlx_metal": "0.32.4", + "mlx_vlm": "0.7.11", + "dependency_provenance": "Existing remote environment, installedmetadata captured CPUonly; all distributions frozen in requirements.lock. CoreMLX/VLMpins matchsource, dependency differences fromlocalM4retained.", + "memory_policy": "Unmodified MLX defaults; no memory/wired limit overrides.", + "request_context_limit": 70000, + "context_policy": "Pinned --max-kv-size70000 sets request budget; LFM3 family cache construction remains lazy and is not claimed allocation-identical to native.", + "checkpoint_context_caps": { + "text": 262144, + "vl": 128000 + } + }, + "checkpoint_files_and_revisions": { + "LFM3-1.7B-MLX-6bit": { + "repository": "LiquidAI/LFM3-1.7B-MLX-6bit", + "revision": "26be898c36b545d625d84ac0761c6c3eb664aef3", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 2423, + "sha256": "2b5879bf6d1d99b43e17726c7ee3fb4868e4c2e29c4a49784803df90f494fc2e", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 328, + "sha256": "0095ba7158e938b1e2d3082d412f00d8be85e42cbbd7608b63866f7a081cf270", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 1405187179, + "sha256": "503d68675793fba64907842e176d066c8c3adca22e14c8b3538f9ffb76bd5e0e", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 50701, + "sha256": "940f7e7b4133162a48fbca232e663c32240364aea9d60f1715b62bed41a81e2c", + "verified_against_pinned_revision": true + }, + "special_tokens_map.json": { + "bytes": 92, + "sha256": "7946f864cbfb62a1b041bda937244210a1a87f4a6cab3a48b791131b2cd57f88", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908634, + "sha256": "41927bd314d17d7d7fadb44e10cd0919f7b71528d7741596d8cbd15c6945ec44", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1859, + "sha256": "b77214b9f12be74b9e5e1dab35d7dace077a31be969314bf38866fd6c8163853", + "verified_against_pinned_revision": true + } + } + }, + "LFM3-1.7B-MLX-8bit": { + "repository": "LiquidAI/LFM3-1.7B-MLX-8bit", + "revision": "ee7667f3cfccb2a41725f91820531823fcabc6b7", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 2423, + "sha256": "4ef95e289e7105a09bf010b3ceaa015d3f01163104c039a4e7948739d5b9035b", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 328, + "sha256": "0095ba7158e938b1e2d3082d412f00d8be85e42cbbd7608b63866f7a081cf270", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 1837290805, + "sha256": "2b06cea98d75527c248a0b04383f7cf0a1c72fb4ed7b45096618fc4e59d015ad", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 50701, + "sha256": "0f4f85710fd22ffb8a1977520a734cb7dc7e1f7858a0f91d9c8b56e6d36b78cc", + "verified_against_pinned_revision": true + }, + "special_tokens_map.json": { + "bytes": 92, + "sha256": "7946f864cbfb62a1b041bda937244210a1a87f4a6cab3a48b791131b2cd57f88", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908634, + "sha256": "41927bd314d17d7d7fadb44e10cd0919f7b71528d7741596d8cbd15c6945ec44", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1859, + "sha256": "b77214b9f12be74b9e5e1dab35d7dace077a31be969314bf38866fd6c8163853", + "verified_against_pinned_revision": true + } + } + }, + "LFM3-VL-2B-MLX-6bit": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-6bit", + "revision": "6d71bfae121abedc75f6d531fc7f6ceb4f5e3501", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 4138, + "sha256": "4c03aa298372cbcb478eb08be0a4a59be917b4949756066ce4eac06033d200bc", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 323, + "sha256": "30b1115bcdc4e5671567a5b1255a2f3d2f5a912a79292051c6f1e9db81b5e8c4", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 1783964482, + "sha256": "838c39147541b12e86a4f1b43d6a7814d7fb4efe9d5ff8a151c85ac06f321eef", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 61255, + "sha256": "abc29f6e9809c65c5ce62e6b1754b3f103d82266adf74d6d661dd3f8cd443cb5", + "verified_against_pinned_revision": true + }, + "processor_config.json": { + "bytes": 1254, + "sha256": "f183f5a5e5b3a32e666fce05033e6cb7ca496d26e709a2cee611a13eb7d6b610", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908258, + "sha256": "a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1759, + "sha256": "d63d1ce2f332047e398764b4170352b96629a5194643105bfd6d70c90b750396", + "verified_against_pinned_revision": true + } + } + }, + "LFM3-VL-2B-MLX-8bit": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-8bit", + "revision": "83336d5be3874a2bdd3297bde77b2114dbd3a816", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 4138, + "sha256": "9f7dc06d04d0610c7e27fb6a110c42a3a09b686a6ec53b0eb383f30a8be5a457", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 323, + "sha256": "30b1115bcdc4e5671567a5b1255a2f3d2f5a912a79292051c6f1e9db81b5e8c4", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 2216068156, + "sha256": "50c9476a7f00f5349e81b42ab88b47635b9eba3c7ab5851c598d270cda4c6857", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 61255, + "sha256": "31fe8faaf68bfc2fbce07e9d97ff47c1abad234d783ed615f2d938392b7cf6f0", + "verified_against_pinned_revision": true + }, + "processor_config.json": { + "bytes": 1254, + "sha256": "f183f5a5e5b3a32e666fce05033e6cb7ca496d26e709a2cee611a13eb7d6b610", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908258, + "sha256": "a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1759, + "sha256": "d63d1ce2f332047e398764b4170352b96629a5194643105bfd6d70c90b750396", + "verified_against_pinned_revision": true + } + } + } + }, + "requirements_sha256": "275b369ea72d5757b1e3f91e68ed5b746d83b5f7667adc8f5d9d28bb8933ddd6", + "packet_artifacts_sha256": { + "README.md": "7ae6f5c44bc39833b7e99921999992890724eb9e268900629bdc0c495078a128", + "cold_client.py": "7b8ea55aa88834814f95a6f679654e16074b7a8364e34b39a276f009fe0231f2", + "config.json": "925df4aa906ae82e2e791d0608005a3808e8fab259844642c725150dbdcbe86d", + "current-source-receipt.json": "b04753edfcec89c76161e2c3e7e9ea873700179dee7d33c937e2cc53e2f0f654", + "current68-source-manifest.json": "edc67b0dd974bce6e7e21bbf315948db434c43aba530b46e4829aa1841a821ee", + "fixtures/text-2048.body.json": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "fixtures/text-2048.ids.json": "f5edac3447405086d7ec361d6c899410ea7cbbb2ef697c27fd7486ef5ff877fe", + "fixtures/text-8192.body.json": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "fixtures/text-8192.ids.json": "a9b3cdf070b9ca5ea523565bcb82c80b6af5aae71f640a8fba0460c9069d22cd", + "fixtures/vl-2048.body.json": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "fixtures/vl-2048.ids.json": "f5edac3447405086d7ec361d6c899410ea7cbbb2ef697c27fd7486ef5ff877fe", + "fixtures/vl-8192.body.json": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "fixtures/vl-8192.ids.json": "a9b3cdf070b9ca5ea523565bcb82c80b6af5aae71f640a8fba0460c9069d22cd", + "guard.py": "3c4118b62c3797e1871032fcbbb1bf7d390f143c2e93ddf564b309e7f49a4664", + "independent-review.json": "931231a1bc03c322be339ae7f017c692609f55209bc5e6345261ca66e8727ae1", + "model-revisions.json": "fbe3ba2fea04e926e5c237044f2c5315b31a3d991e5c9c528521dc597a924f88", + "prepare-paths.sh": "417b12ba889dfd003e4e9a6a9e77e03f9aa5e559810351b303f7bc4c7927e3e2", + "requirements.lock": "275b369ea72d5757b1e3f91e68ed5b746d83b5f7667adc8f5d9d28bb8933ddd6", + "run_existing_clients.py": "8209512004d5c8c19e77a4a1ce1032b54e492e8625faf448cfa34a07e268a2c0", + "standard-installed-metadata.json": "fb244f662eddced712fac29a25e390dad3d28f0cec79a5284f1c63cfb875b4e8", + "tools/bench_auth.py": "1896120b70d0df05646870a252697e20ce84c63e09dcba88b2a4e5a55ccb3c5b", + "tools/bench_cache.py": "24364dd13c033450f45710fc8685e2c64d6a72d6dd6e223d0d7b19aabbdc1d98", + "tools/bench_concurrent.py": "0e72e26769f082459329fc068c25f6f72a9155e422962ea54cb5d1226227752d", + "tools/bench_evidence.py": "54c28474f0804dbd120e3fb5859be0d1667ef21ca502880d0bed098ec9bb42ff", + "tools/bench_metrics.py": "26f4c3ecbb03df4f12e24d18840fea989d9ef8c63117ca11eb944ef670458a27", + "tools/bench_openai.py": "d5af4fbb756cdfb573546c0e0017652a4bb8f509d1251af683be7a3cbdd426f6", + "tools/bench_timing.py": "158f636ef0b3bed460ce19ef4872b6d8607ff7a8590562d06bae8bd55e6f44ed", + "tools/lfm3/bench_decode.py": "c011106f4f182db19371e3f91f70bcc374160786dd105f2d713a7fea07ab8c4c" + }, + "client_sha256": { + "$PACKET/tools/bench_auth.py": "1896120b70d0df05646870a252697e20ce84c63e09dcba88b2a4e5a55ccb3c5b", + "$PACKET/tools/bench_cache.py": "24364dd13c033450f45710fc8685e2c64d6a72d6dd6e223d0d7b19aabbdc1d98", + "$PACKET/tools/bench_concurrent.py": "0e72e26769f082459329fc068c25f6f72a9155e422962ea54cb5d1226227752d", + "$PACKET/tools/bench_evidence.py": "54c28474f0804dbd120e3fb5859be0d1667ef21ca502880d0bed098ec9bb42ff", + "$PACKET/tools/bench_metrics.py": "26f4c3ecbb03df4f12e24d18840fea989d9ef8c63117ca11eb944ef670458a27", + "$PACKET/tools/bench_openai.py": "d5af4fbb756cdfb573546c0e0017652a4bb8f509d1251af683be7a3cbdd426f6", + "$PACKET/tools/bench_timing.py": "158f636ef0b3bed460ce19ef4872b6d8607ff7a8590562d06bae8bd55e6f44ed", + "$PACKET/tools/lfm3/bench_decode.py": "c011106f4f182db19371e3f91f70bcc374160786dd105f2d713a7fea07ab8c4c", + "$PACKET/cold_client.py": "7b8ea55aa88834814f95a6f679654e16074b7a8364e34b39a276f009fe0231f2" + }, + "fixture_sha256": { + "text-2048.body.json": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "text-2048.ids.json": "f5edac3447405086d7ec361d6c899410ea7cbbb2ef697c27fd7486ef5ff877fe", + "text-8192.body.json": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "text-8192.ids.json": "a9b3cdf070b9ca5ea523565bcb82c80b6af5aae71f640a8fba0460c9069d22cd", + "vl-2048.body.json": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "vl-2048.ids.json": "f5edac3447405086d7ec361d6c899410ea7cbbb2ef697c27fd7486ef5ff877fe", + "vl-8192.body.json": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "vl-8192.ids.json": "a9b3cdf070b9ca5ea523565bcb82c80b6af5aae71f640a8fba0460c9069d22cd" + } + }, + "protocol": { + "phase": { + "families": [ + "text", + "vl" + ], + "bits": [ + 6, + 8 + ], + "cold_tokens": [ + 2048, + 8192 + ], + "decode_tokens": 256, + "cold_output_tokens": 16, + "reps": 3, + "levels": [ + 1, + 4 + ], + "order": "family_orders", + "family_orders": { + "text": [ + "current", + "standard", + "standard", + "current" + ], + "vl": [ + "current", + "standard", + "standard", + "current" + ] + } + }, + "settings": { + "native_context": 70000, + "parallel": 4, + "prompt_cache_gib": 0, + "temperature": 0, + "thinking": false, + "prefix_cache": "disabled on both", + "port": 18085 + }, + "lock_path": "/tmp/tensorfold-model.lock", + "guard_source": { + "path": "guard.py", + "sha256": "3c4118b62c3797e1871032fcbbb1bf7d390f143c2e93ddf564b309e7f49a4664", + "origin": "Existing M5 narrow_guard exactaliases, addingknownrowtune/chunkbuilds; no admission rule weakened" + }, + "lock_and_completion_events": [ + { + "event": "lock_wait", + "time_ns": 1793213351158947000, + "path": "/tmp/tensorfold-model.lock" + }, + { + "event": "lock_acquired", + "time_ns": 1793213351159128000, + "path": "/tmp/tensorfold-model.lock", + "pid": 34921, + "device": 16777233, + "inode": 2622134 + }, + { + "event": "complete", + "time_ns": 1793213728179653000, + "legs": 16, + "all_lengths_validated": true, + "performance_gate_claim": false + }, + { + "event": "lock_release", + "time_ns": 1793213728179878000, + "pid": 34921 + } + ], + "owned_model_load_ready_cleanup_events": { + "_trimmed_list": { + "count": 48, + "sha256": "8a8afaca60a0b60f0787708bd5524f9c724a54913e46fe98075265563d28c288", + "first": [ + { + "event": "owned_model_load_begin", + "time_ns": 1793213354237076000, + "tag": "text-6-0-current", + "pid": 34935, + "command": { + "_trimmed_list": { + "count": 20, + "sha256": "848ab0fb02fca83567e5bfb47ceb9be772cb0f33e42926289a2432fa434732d4", + "first": [ + "$RUN/bin/tensorfold-native-current", + "serve" + ], + "last": [ + "--metrics-open" + ] + } + }, + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "event": "owned_model_ready", + "time_ns": 1793213355366020000, + "tag": "text-6-0-current", + "api_model": "lfm3-local", + "models": [ + { + "id": "lfm3-local", + "object": "model", + "created": 1793213355, + "owned_by": "tensorfold", + "context_length": 70000, + "max_model_len": 70000, + "reasoning": { + "default_enabled": false + } + } + ] + } + ], + "last": [ + { + "event": "owned_cleanup_complete", + "time_ns": 1793213728107333000, + "tag": "vl-8-3-current", + "pid": 35502, + "exit": 0, + "memory": { + "max_rss_bytes": 2458812416, + "peak_physical_footprint_bytes": 2915699280 + } + } + ] + } + }, + "admission": "Existing exclusive GPU flock and exact model-process admission; only owned descendants admitted during each measurement. Native admission and MLX limits unchanged. No unrelated owner stopped.", + "cleanup_exit_policy": "Eight native servers exited0; eight standard wrappers exited\u221215 after the existing runner intentionally SIGTERMed only owned descendants; admission checked no model remained.", + "actual_prompt_scope": "Published exact request bodies/official token arrays and pinned family tokenizers retained; usage counts verified. This packet does not add a new live/tokenize comparison.", + "memory": "Native admission unchanged; MLX memory and wired limits unmodified. /usr/bin/time -l captures physical footprint and RSS; available server metrics preserved. No allocator/pinned peak inferred from RSS." + }, + "results": { + "reported_summary": { + "_trimmed_value": { + "bytes": 6199, + "sha256": "8b00d6832bfefb764d6f74b45a8b24fdf1b4265d306adbb5875a11a7c95499ad", + "keys": [ + "text", + "vl" + ], + "count": 2 + } + }, + "summary_method": "Median of two existing per-leg reported cell medians; all three per-leg raw repetitions/evidence are retained below. Cold median uses two mirrored server-leg samples.", + "legs": { + "_trimmed_list": { + "count": 16, + "sha256": "50d3caf95244b7b2b8d4d8de7dbe34412af5adcf6075d89ac2f30602575823de", + "first": [ + { + "family": "text", + "bits": 6, + "leg": 0, + "engine": "current", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.9490549168549478, + "output_sha256": "6cb715bc54d1b0adbe24d918f8f147ac719d55d25abea6c65bc204eb159bebe8", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27", + "token_sha256": "b482722716a27c86c613d49d437feba1fe030ed496fb1141b0498ac017ee32c1" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 4.98626804118976, + "output_sha256": "63b1d289496206389642e98d3a58dffa5a753c8e5e32a1c40ced27643265c352", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0", + "token_sha256": "06efc9b74a3d0b1c7f8b7e69ae2da268963d81d498818ac39f495593a6c6da42" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 1647902720, + "peak_physical_footprint_bytes": 2063468512 + }, + "tag": "text-6-0-current", + "decode_cells": { + "_trimmed_value": { + "bytes": 6351, + "sha256": "54e78d117a423e95df508f4dcc1e8fd5b91eba51b02afdacd78c12ff818564e8", + "keys": null, + "count": 4 + } + } + }, + { + "family": "text", + "bits": 6, + "leg": 1, + "engine": "standard", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.16382091725245118, + "output_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "request_body_sha256": "d186e6168076ea392b899be28eb23440cdc2b8a2ba4c208dcb0729ad83d873fa", + "token_sha256": null + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.3932498749345541, + "output_sha256": "63b1d289496206389642e98d3a58dffa5a753c8e5e32a1c40ced27643265c352", + "request_body_sha256": "e08051c419693c025d7fbce09a821a08bd08bf712b254cce7047c3c09ff1a5eb", + "token_sha256": null + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "unverified if server omits token hashes; text digest is not substituted" + }, + "memory": { + "max_rss_bytes": 1913864192, + "peak_physical_footprint_bytes": 3210103328 + }, + "tag": "text-6-1-standard", + "decode_cells": { + "_trimmed_value": { + "bytes": 6352, + "sha256": "ff570f5127adb5444018ea1bfec1af5b88c1100bdfb21b1da49b4f3527063848", + "keys": null, + "count": 4 + } + } + } + ], + "last": [ + { + "family": "vl", + "bits": 8, + "leg": 3, + "engine": "current", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.8491029171273112, + "output_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27", + "token_sha256": "21c3879cda05ccf4cde74c1f54d326d2c271f673656d166241e6643c21f49d89" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 4.512903375085443, + "output_sha256": "b7b834e8f232d6ff122c84f11c241563a5b5bd6ae97dfaabd0e38994edaff19e", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0", + "token_sha256": "8ed4e05be9f053ca054d2c47326a04f0d7fb0c230374bc8bbafaa9618d59ed38" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 2458812416, + "peak_physical_footprint_bytes": 2915699280 + }, + "tag": "vl-8-3-current", + "decode_cells": { + "_trimmed_value": { + "bytes": 6352, + "sha256": "bba822d9a7621947cb689151acf3332625adade7e853aae2566122ea17a998c4", + "keys": null, + "count": 4 + } + } + } + ] + } + }, + "deduplicated_output_evidence": { + "_trimmed_value": { + "bytes": 14075, + "sha256": "991989b4a2e17e3e4267e26ac56d704332f5202bbd65ab8226c762bae51fbab8", + "keys": [ + "0c826dd2b65f7d4a", + "83aca5c0d9d2501c", + "096c4e95759af816", + "b11dff03a9d60648", + "95cfa5b83dc42bc9", + "80f3f3ef627d7207", + "ce7480621f864d70", + "87ec47f1bb1fecea", + "941c5e009cd44c4e", + "5e308b1934cc926e", + "312e01d99328eead", + "61b9f6421abd73f2" + ], + "count": 66 + } + } + }, + "limitations": [ + "Only M5 hardware and6/8bit2k/8k measured here;4bit is retained in its separate current-public receipt. No32k/64k,128k or native-window performance claim.", + "VL requests contain text only; real-image correctness is separate evidence.", + "Standard omits token-id fingerprints; its token equality is unverified, and output-text digests are not substituted. Native token-equality checks passed.", + "Native and standard use request budget70000, but reserved native KV and lazy standard KV/sliding allocations differ.", + "Peak physical footprint and maximum RSS are process measurements, not inferred device/pinned allocator peaks.", + "Mirrored runs retain per-leg drift; no measured thermal/frequency/bandwidth attribution.", + "Performance receipts do not establish a model FP64 fidelity pass or remove disclosed top-token differences." + ], + "raw_file_manifest": { + "events.json": { + "bytes": 49479, + "sha256": "1e55cb246ab1a054a9ef01c1900d40cb991a50af565e7a172daf79eec39d4abb" + }, + "manifest.json": { + "bytes": 17121, + "sha256": "3461a2cfcb2d32a10d961dd3d0620b212f4aad435d9f5326d441d7a8ab78e8f8" + }, + "records.json": { + "bytes": 17057, + "sha256": "4c2efff58789078dbf5ed5bb74597d93c98bafca556f8a01a06e85d2aec022ff" + }, + "text-6-0-current.after.health": { + "bytes": 303, + "sha256": "a9616be96809061e248add31809cfad79caf07dc1518a7c75f2615e0ffbc11d3" + }, + "text-6-0-current.after.metrics": { + "bytes": 12298, + "sha256": "42127e0babeeb70e345bf58118f60e1ebdab01e84c8979ca676d842081824b7c" + }, + "text-6-0-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-6-0-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "text-6-0-current.before.metrics": { + "bytes": 12128, + "sha256": "80b4086c88850ea24dcb1891a3854774f6f4ff1d69a52474eea6cfa328847789" + }, + "text-6-0-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-6-0-current.cold-2048.client.log": { + "bytes": 349, + "sha256": "2b74b2084920c0be5b369a12bc6afc54ab249d31a0cb625f9ee8c9fe4972a9ac" + }, + "text-6-0-current.cold-2048.json": { + "bytes": 20277, + "sha256": "ebbb0bdc92ccf32fb7d24fa62cc28021efb1dc45f5eb7e5156601de93bce34a0" + }, + "text-6-0-current.cold-8192.client.log": { + "bytes": 346, + "sha256": "657ab06e0dd2b4a3269cbe325ce5d583ad81d22242e12829155c65dbd4791800" + }, + "text-6-0-current.cold-8192.json": { + "bytes": 57153, + "sha256": "61cb0c43da2413ee292742931fbb61277f510e4c9e1669fcf494295514645a95" + }, + "text-6-0-current.concurrent.client.log": { + "bytes": 3221, + "sha256": "dae3db4782b6e73d3c96645947537c3743c93178404675217b5a7b72514a5099" + }, + "text-6-0-current.concurrent.json": { + "bytes": 292701, + "sha256": "36a383f37cfb7b4b4879c1d8521f9e5cd8e4e39a8578835acaa3aa5155519b6a" + }, + "text-6-0-current.server.log": { + "bytes": 13634, + "sha256": "e338f34c3fe8bbbc019f9a73375d3cb2290ac6193a28889cef846eee20ecc090" + }, + "text-6-1-standard.after.health": { + "bytes": 453, + "sha256": "87944493ce57beba80069a2d02208f7b9dbf8683b03ab95ad00b4bffdb7e8e2f" + }, + "text-6-1-standard.after.metrics": { + "bytes": 32126, + "sha256": "9fe294108d7f38dad73053f737f44222381161d0cafb48ad364dba5ec5133b9b" + }, + "text-6-1-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-6-1-standard.before.health": { + "bytes": 453, + "sha256": "87944493ce57beba80069a2d02208f7b9dbf8683b03ab95ad00b4bffdb7e8e2f" + }, + "text-6-1-standard.before.metrics": { + "bytes": 909, + "sha256": "4daf5f787b64b4e4279f019dcf4a297d8b8b46ea30950170ccdd1cabd86dcb26" + }, + "text-6-1-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-6-1-standard.cold-2048.client.log": { + "bytes": 297, + "sha256": "09aff08f271e4eb73bd76884e688150fca657e6d38010b70956e2e7ca7c583c0" + }, + "text-6-1-standard.cold-2048.json": { + "bytes": 26083, + "sha256": "b5bdf7b82002359f7b1c90a32bba6c5a1f0a80fe46118b730cc67f81fb49ad7d" + }, + "text-6-1-standard.cold-8192.client.log": { + "bytes": 295, + "sha256": "de88203e1d7a587072a52b1269804dee3ee79acd6cba34063b0c69d1ad6eda0b" + }, + "text-6-1-standard.cold-8192.json": { + "bytes": 62937, + "sha256": "af2079563e3371bc97bc43471f9242358be38e87137355f70a312cd576ac085f" + }, + "text-6-1-standard.concurrent.client.log": { + "bytes": 3234, + "sha256": "7f8a0dd74d7b923c35d36a797f01551e3d9373065e8221436fc2a831b53cd36b" + }, + "text-6-1-standard.concurrent.json": { + "bytes": 285286, + "sha256": "7f327cf4f2875e1d5cc8df95dea1b024f431ec6678a3853d1779cc2a019f3f9f" + }, + "text-6-1-standard.server.log": { + "bytes": 50891, + "sha256": "21df098eb5f6a2b486d0fd574855d34f890af8675c98c6ca7bb11c846a94d252" + }, + "text-6-2-standard.after.health": { + "bytes": 453, + "sha256": "87944493ce57beba80069a2d02208f7b9dbf8683b03ab95ad00b4bffdb7e8e2f" + }, + "text-6-2-standard.after.metrics": { + "bytes": 32135, + "sha256": "8a61eb4fd923a8eeae821138af8d216b306dd69b79ad784552952680e207736a" + }, + "text-6-2-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-6-2-standard.before.health": { + "bytes": 453, + "sha256": "87944493ce57beba80069a2d02208f7b9dbf8683b03ab95ad00b4bffdb7e8e2f" + }, + "text-6-2-standard.before.metrics": { + "bytes": 909, + "sha256": "988211da7bcc099a3f31f49330649e522e10ebcae345c63e767fb9e0c02f7c50" + }, + "text-6-2-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-6-2-standard.cold-2048.client.log": { + "bytes": 297, + "sha256": "e0b77645d2fb7635574f8c05b137e6b9624d5fe5bc1212adba0d607f91975b2d" + }, + "text-6-2-standard.cold-2048.json": { + "bytes": 26082, + "sha256": "8161742b232abd7ecaab3f3f6aa2b28f30a04ac2f5b9c40d8d7fca096bce2a13" + }, + "text-6-2-standard.cold-8192.client.log": { + "bytes": 297, + "sha256": "34303db632d285f6f4b2389142a10d234c9c9bbe99304a1293a902fafd8e86dc" + }, + "text-6-2-standard.cold-8192.json": { + "bytes": 62945, + "sha256": "da2e72beaf2d0fb3cf8bd6722bfe6b09fad9c5a4987054a5a8a764355ec89e72" + }, + "text-6-2-standard.concurrent.client.log": { + "bytes": 3232, + "sha256": "f76f1e9c08747727cbce7d507c88e5ab2f9a209651a70eafba1f9a718008d066" + }, + "text-6-2-standard.concurrent.json": { + "bytes": 285274, + "sha256": "bc77ae8452f2c07d2c4952974895d6c8ad38140cd6fabb4d1cf61c5f6c1ebc34" + }, + "text-6-2-standard.server.log": { + "bytes": 50891, + "sha256": "482dcca26737756163d84fd1ed1ad8bf278c311aa2e11e1e3ac570a66882ffb7" + }, + "text-6-3-current.after.health": { + "bytes": 303, + "sha256": "047f2a6a99c470c6b7c69a4e5c18cd4f91cca1389f3a96f3fbaa54b01997846b" + }, + "text-6-3-current.after.metrics": { + "bytes": 12298, + "sha256": "62293f3de70dd01adc1591ccf0a4cf73d1a314f3dc3cc48471d3109707df7093" + }, + "text-6-3-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-6-3-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "text-6-3-current.before.metrics": { + "bytes": 12128, + "sha256": "4f61b39294688798540c9efe9cc437c0af1fe9e01fbe0ac1c9b86f96290e005f" + }, + "text-6-3-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-6-3-current.cold-2048.client.log": { + "bytes": 349, + "sha256": "58503dcf48365f99574d308275ba4fa8ebd0bb731b16967496f98050768f7ab0" + }, + "text-6-3-current.cold-2048.json": { + "bytes": 20281, + "sha256": "f271a7ca06400ca2e7cbc9b2c9531b4be80ef039b310d014dbf7b14b377eb458" + }, + "text-6-3-current.cold-8192.client.log": { + "bytes": 347, + "sha256": "9551cc002427b94ae82cacbca7c67c04c7b668f591430ca3272a71ac1a1a1868" + }, + "text-6-3-current.cold-8192.json": { + "bytes": 57157, + "sha256": "2f407f58dbbe86b7aae79893d103f6a64fa7c621b013594cf0e52a15f529c51c" + }, + "text-6-3-current.concurrent.client.log": { + "bytes": 3223, + "sha256": "33823c5f8eeb39cba0f7c3a293d168e49844684f7a882a54138e8c53fbd3a010" + }, + "text-6-3-current.concurrent.json": { + "bytes": 292700, + "sha256": "b646e91b78dda6d5055d29e6c29ce9838a5f299c0ac472954e7a2206d75a5e31" + }, + "text-6-3-current.server.log": { + "bytes": 13634, + "sha256": "28f05be106cc010088beaff8f4a4d433ef129cdbeac20ceb3c5a0acf252000ee" + }, + "text-8-0-current.after.health": { + "bytes": 303, + "sha256": "ec3ee92ddbeb3d245a2f8920eef138b8c05e0cffd45e649de196c5be1f1a4467" + }, + "text-8-0-current.after.metrics": { + "bytes": 12297, + "sha256": "c13ccdbc96d1064f4b2856ea0b5c8d2c7a1e5979cb671f505694337d18930dfd" + }, + "text-8-0-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-8-0-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "text-8-0-current.before.metrics": { + "bytes": 12128, + "sha256": "f3c64235a7216dd0d57beea4a549b4f5328f3216c43259354de19b85438c9c1a" + }, + "text-8-0-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-8-0-current.cold-2048.client.log": { + "bytes": 349, + "sha256": "e037e75904d53c65e438808c12ee57453ca78eb466ec7c9d13920023f9358ccc" + }, + "text-8-0-current.cold-2048.json": { + "bytes": 20295, + "sha256": "430c3c0fb2f7ecf1085c96117ccf82f1039c04b696b224efd3bf5896394c0fe7" + }, + "text-8-0-current.cold-8192.client.log": { + "bytes": 346, + "sha256": "777d5f52ba839dce0629e86026e07a2ce18cd55a391d54db6b7006d90d9ac0ff" + }, + "text-8-0-current.cold-8192.json": { + "bytes": 57143, + "sha256": "20ec44def5324c7e04d07b01fa51fda88e6ede8cd352900ff6ef6caf31ce1e74" + }, + "text-8-0-current.concurrent.client.log": { + "bytes": 3227, + "sha256": "b503c3295a5f4bf60e2900886b65ada637dadded49d9a4224ba8d281a1585d5a" + }, + "text-8-0-current.concurrent.json": { + "bytes": 292830, + "sha256": "adfcc6d543483f3fa0dba6aef9b862a9d6c076fcfaf22ecdf059f2a16ca350be" + }, + "text-8-0-current.server.log": { + "bytes": 13637, + "sha256": "7eb96a88a9973bcb9c2fb54af1cb213bae6172918b14e4861e0c7634323dc722" + }, + "text-8-1-standard.after.health": { + "bytes": 453, + "sha256": "c6a2a7b6cbe67290e76f6275e124f57a57c5d2287f34843e2cbbc535bf9d6671" + }, + "text-8-1-standard.after.metrics": { + "bytes": 32122, + "sha256": "fb68a7a8cace89de3649d36071ff38a0fd0e9f6005b597280089ff16d6933ce8" + }, + "text-8-1-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-8-1-standard.before.health": { + "bytes": 453, + "sha256": "c6a2a7b6cbe67290e76f6275e124f57a57c5d2287f34843e2cbbc535bf9d6671" + }, + "text-8-1-standard.before.metrics": { + "bytes": 908, + "sha256": "ffb4ba8f198d74b923641f3adf8c81fdbda006488c86cd35c1457f1d52551b47" + }, + "text-8-1-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-8-1-standard.cold-2048.client.log": { + "bytes": 295, + "sha256": "310945f7f91fd9107494a77fb89ec56c9e3a671d43b28915011997bd8aeb4a31" + }, + "text-8-1-standard.cold-2048.json": { + "bytes": 26076, + "sha256": "fec629a2bf278fcac13443d520ac50012c9d64474215af8e780ed7ed8bb56fd0" + }, + "text-8-1-standard.cold-8192.client.log": { + "bytes": 295, + "sha256": "f927d951ace3882a0bea7b86d5d2513597bf4e75d023bb459a131a0a94c6671a" + }, + "text-8-1-standard.cold-8192.json": { + "bytes": 62930, + "sha256": "27759fc13fe80b50283c260c6272939297e62eb62fcc2e80fdfaa8725293697c" + }, + "text-8-1-standard.concurrent.client.log": { + "bytes": 3229, + "sha256": "803985e8713a012e55828b1f87a05bfe1732314a1833c686e69d3dda9389bd2e" + }, + "text-8-1-standard.concurrent.json": { + "bytes": 285270, + "sha256": "b4bb0b9701ee62304a717fcedac25deeb9d2fdb5813f983ca44058416760e7db" + }, + "text-8-1-standard.server.log": { + "bytes": 50891, + "sha256": "a7f891f2fe6c4544363ec2df1b463ef41c6ef50296a1eda8b855269915161f1a" + }, + "text-8-2-standard.after.health": { + "bytes": 453, + "sha256": "c6a2a7b6cbe67290e76f6275e124f57a57c5d2287f34843e2cbbc535bf9d6671" + }, + "text-8-2-standard.after.metrics": { + "bytes": 32114, + "sha256": "7aefca52488c20bb1f0db32a7162f82d8311fe5b44764a51ba4ecba623b5924a" + }, + "text-8-2-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-8-2-standard.before.health": { + "bytes": 453, + "sha256": "c6a2a7b6cbe67290e76f6275e124f57a57c5d2287f34843e2cbbc535bf9d6671" + }, + "text-8-2-standard.before.metrics": { + "bytes": 908, + "sha256": "526a57fe3d5be98e32fd8141c5ccdbee098e46ab6f525746780352142a81af1c" + }, + "text-8-2-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-8-2-standard.cold-2048.client.log": { + "bytes": 296, + "sha256": "a15252ef5353e2805633558d943cbce05e72c151417f20e907fc0ae755c899c4" + }, + "text-8-2-standard.cold-2048.json": { + "bytes": 26074, + "sha256": "9626b4b94b4358dcf1393f6b5ca5ccdf5f9e1a77d1f94d1079c99a125ec100df" + }, + "text-8-2-standard.cold-8192.client.log": { + "bytes": 296, + "sha256": "0091e6a0ecc30f8637652ffca697125ad0354ff225f4c8bd0bb93ef09c8f4162" + }, + "text-8-2-standard.cold-8192.json": { + "bytes": 62932, + "sha256": "7f714759b65f0b84ddf76a86e2de622b601a0bc69f321810f6266e8d2dba8991" + }, + "text-8-2-standard.concurrent.client.log": { + "bytes": 3226, + "sha256": "57fe99245159f35425ab66b15cbfc254ddb2f2c93dd4c03cee8dd91f885e3893" + }, + "text-8-2-standard.concurrent.json": { + "bytes": 285340, + "sha256": "fa942f2f3291a1f859154d1d23c4af030bccd72e09b1f485a42334de6d2a5cee" + }, + "text-8-2-standard.server.log": { + "bytes": 50891, + "sha256": "887e63589861a355c26af6aaa3af66c67acf5418425cb774b155511748bf0af7" + }, + "text-8-3-current.after.health": { + "bytes": 303, + "sha256": "7d041e37775a9ec72201faed8a37222e2182010675c273c5e0ae12bf66a2fc4b" + }, + "text-8-3-current.after.metrics": { + "bytes": 12297, + "sha256": "e6077dd7a6b59fdad46611a5577f21dd468e118becb764b37561c7320f8f30d2" + }, + "text-8-3-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-8-3-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "text-8-3-current.before.metrics": { + "bytes": 12128, + "sha256": "05e17ede1e0b6a9abaa950940c9aecfc5e0752a4a6f7fed3c942e3bc786fe85d" + }, + "text-8-3-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-8-3-current.cold-2048.client.log": { + "bytes": 349, + "sha256": "e26a3083742a074fe04f06db81d8b1018a0eabbf3dff2a8880f79f771eadd012" + }, + "text-8-3-current.cold-2048.json": { + "bytes": 20295, + "sha256": "a6cdcd146244875f12c29aa65338bab3ed0f0242d4f2902fb9fa744afe79c1c7" + }, + "text-8-3-current.cold-8192.client.log": { + "bytes": 347, + "sha256": "afe30938da4000bd69f6f8b5d806d418e097acc2662e0092786d7673b7e28a5f" + }, + "text-8-3-current.cold-8192.json": { + "bytes": 57144, + "sha256": "9d5a7e9a378ac1fe1c62b7c7b4d9f13ebadb70858ed797618fe048129796c557" + }, + "text-8-3-current.concurrent.client.log": { + "bytes": 3210, + "sha256": "30d1f636fc782236ff4493822d29df3216027074f06bb4755467d4ce292bb17f" + }, + "text-8-3-current.concurrent.json": { + "bytes": 292731, + "sha256": "84d1615396089cbf0589710963fc2dc3905551e676a9cc0ce13302d4bd2a9e09" + }, + "text-8-3-current.server.log": { + "bytes": 13637, + "sha256": "226343ca4b671ffe3785239ca0ab6dbae6a131c1ff7e37bed05ab30dfe5d7148" + }, + "vl-6-0-current.after.health": { + "bytes": 303, + "sha256": "27092cec238d7c761b5e8453789b995e0514416331bf78f14744f4aeb5d415b0" + }, + "vl-6-0-current.after.metrics": { + "bytes": 12306, + "sha256": "829ad01b034ec790a696168e9ca70ba325fca0303125faede4de51c297f0a43f" + }, + "vl-6-0-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-6-0-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "vl-6-0-current.before.metrics": { + "bytes": 12128, + "sha256": "1654eba5ff03499ae49fe04ed23d5571f36d03df5250442533e53bacaacb4a06" + }, + "vl-6-0-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-6-0-current.cold-2048.client.log": { + "bytes": 349, + "sha256": "8be7e8e09abe7c82f81d31e7be1dc530e4144d567bec27d60c407c49f19abd1d" + }, + "vl-6-0-current.cold-2048.json": { + "bytes": 20279, + "sha256": "011dc52417e9c3f08bd6d0d736fd7fee472611ada26d452091f105bba712ca6a" + }, + "vl-6-0-current.cold-8192.client.log": { + "bytes": 347, + "sha256": "c09af7ce9d1abe7b85379b4f0c979fc43d6857a3d1a2c24f726bcc4982236a35" + }, + "vl-6-0-current.cold-8192.json": { + "bytes": 57145, + "sha256": "c550257dba077a7c25962448c51d033f946be27425823a23105afbfc98c7cb9b" + }, + "vl-6-0-current.concurrent.client.log": { + "bytes": 3230, + "sha256": "34c37d77798060df3c38463509c9ccc5cc17ad518e95149d41b8f7b4f99a838a" + }, + "vl-6-0-current.concurrent.json": { + "bytes": 290243, + "sha256": "912ba7e96f1638c00ee1e1456014517b11fdd0263e61bdd04f1fad464a46f219" + }, + "vl-6-0-current.server.log": { + "bytes": 13642, + "sha256": "5fb5b4339ddf144e557f0df6735b058522c27094d5a0252f7d31597290842e0d" + }, + "vl-6-1-standard.after.health": { + "bytes": 457, + "sha256": "d6e561acb183d0bff082d53d5cb1ad0d93a54531ac634fb00bccda2363411ab8" + }, + "vl-6-1-standard.after.metrics": { + "bytes": 32168, + "sha256": "99e97844a2d9dcbf39764232aff5b1db2b6d20e5130e03782c4835350b5e7ea5" + }, + "vl-6-1-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-6-1-standard.before.health": { + "bytes": 457, + "sha256": "d6e561acb183d0bff082d53d5cb1ad0d93a54531ac634fb00bccda2363411ab8" + }, + "vl-6-1-standard.before.metrics": { + "bytes": 913, + "sha256": "70426c7a4b86e9921db23ba4b0033e566817c59f37835e551a11d6de9c721a1a" + }, + "vl-6-1-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-6-1-standard.cold-2048.client.log": { + "bytes": 297, + "sha256": "b241f0cf7544beea964b47747da23e732d8dccad196089941d719fd50c6849fd" + }, + "vl-6-1-standard.cold-2048.json": { + "bytes": 26077, + "sha256": "ca67f9ecf9607e2ac2ebd2b46075fbde3e548eb3f3d87c5eadf1814c5187388e" + }, + "vl-6-1-standard.cold-8192.client.log": { + "bytes": 297, + "sha256": "146a875633757bfc0617563c15f917bac2571c07e54eeaaf145bf611ca6a4701" + }, + "vl-6-1-standard.cold-8192.json": { + "bytes": 62951, + "sha256": "c2319dd650b4feb05e0ab736856feb21b460e4955e6a3c2461e798ae875ff242" + }, + "vl-6-1-standard.concurrent.client.log": { + "bytes": 3222, + "sha256": "f052af60311c5900bfabd2c857e03b91d5b91fe90599c87aaa19e7c36c6ebe43" + }, + "vl-6-1-standard.concurrent.json": { + "bytes": 285398, + "sha256": "d92d0c1d0378bff888fee108bc6011d790ccc81193bcbf39aafcc6f0a24c0d7f" + }, + "vl-6-1-standard.server.log": { + "bytes": 51083, + "sha256": "b6ac9019fc1600364ebda1110d31e29b8f22447f734f5cf84c11131cd0c108e1" + }, + "vl-6-2-standard.after.health": { + "bytes": 457, + "sha256": "d6e561acb183d0bff082d53d5cb1ad0d93a54531ac634fb00bccda2363411ab8" + }, + "vl-6-2-standard.after.metrics": { + "bytes": 32184, + "sha256": "7595d894ef1d409b6bdba8c4cc49da2afb7ec6144c36b74bae0ecbe0ed5cbbff" + }, + "vl-6-2-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-6-2-standard.before.health": { + "bytes": 457, + "sha256": "d6e561acb183d0bff082d53d5cb1ad0d93a54531ac634fb00bccda2363411ab8" + }, + "vl-6-2-standard.before.metrics": { + "bytes": 912, + "sha256": "ca25563480ae3600ec7b170ab15644e9a42ef2227ffae7719c9f1e762115cb66" + }, + "vl-6-2-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-6-2-standard.cold-2048.client.log": { + "bytes": 296, + "sha256": "609e07f2788126d75a435082ee6215ef297a925db28ca062ab05a4b92913614c" + }, + "vl-6-2-standard.cold-2048.json": { + "bytes": 26078, + "sha256": "34f9c58f325ab5eaccfe3d11b49c77c834ca16062e42740e571a7a5b9111bc1b" + }, + "vl-6-2-standard.cold-8192.client.log": { + "bytes": 297, + "sha256": "0616611f1d3ad0ba8ec6604b66b5bdfb219fd81d26122b8a42ace7567ef4096b" + }, + "vl-6-2-standard.cold-8192.json": { + "bytes": 62953, + "sha256": "1b71d0d3ea2592addcdd343fc57e31a631dfc1332110a26540a0a90929d44eb7" + }, + "vl-6-2-standard.concurrent.client.log": { + "bytes": 3220, + "sha256": "050e709e808bc69668ae76e493f205161317a953d440d52462a20e0b470776fb" + }, + "vl-6-2-standard.concurrent.json": { + "bytes": 285412, + "sha256": "79921717738f9bc6c644159f0c9d1828a69e86993f207e762d1a5603fc9a6c33" + }, + "vl-6-2-standard.server.log": { + "bytes": 51119, + "sha256": "63543991ec6dc5a139891588c915c790381c79c00f2c1fe96befd02f0b1da44f" + }, + "vl-6-3-current.after.health": { + "bytes": 303, + "sha256": "81303458a4718d9abb89de41ec631ba9d702281405b7a56cd10af88fdf33ad0f" + }, + "vl-6-3-current.after.metrics": { + "bytes": 12305, + "sha256": "741c1ecc6b645550111305ea8d3ce825c6bc44c82ec5ec42346738cb1791202b" + }, + "vl-6-3-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-6-3-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "vl-6-3-current.before.metrics": { + "bytes": 12128, + "sha256": "383f94ac56bd67ca477ed6d4d34a029dff950051e4b9b7303d121f10a074f135" + }, + "vl-6-3-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-6-3-current.cold-2048.client.log": { + "bytes": 349, + "sha256": "533815c80da6219676d15f2b4f18abc6ca736606fe3473bf73bb15bc3b935061" + }, + "vl-6-3-current.cold-2048.json": { + "bytes": 20278, + "sha256": "c2c0972f342720cc1605c84ebffbfb66b805fed0df9585f4bfa82cf610cb6c2b" + }, + "vl-6-3-current.cold-8192.client.log": { + "bytes": 347, + "sha256": "84ea4ac7d5ccc2d1ba988083436aca44f13d604f857cf7b58314a1338b063bef" + }, + "vl-6-3-current.cold-8192.json": { + "bytes": 57145, + "sha256": "40afd4ab46755fe12a0009067bc2ef31f2b0a09cf5de8fd5700e61b7974546d9" + }, + "vl-6-3-current.concurrent.client.log": { + "bytes": 3233, + "sha256": "b794a7ef86bde67f0c3ca210504925698291e79a8567fc41a7b6e50d84a7b61f" + }, + "vl-6-3-current.concurrent.json": { + "bytes": 290300, + "sha256": "e6bf43dc03bb1fdc6cb5662a6626c3cc801091794d6567fd1365a9e277ba5cc7" + }, + "vl-6-3-current.server.log": { + "bytes": 13642, + "sha256": "31aac91a44f32e60f49a213bcae64536c2709e0d02ed9aa8a648a50a200b10c9" + }, + "vl-8-0-current.after.health": { + "bytes": 303, + "sha256": "a7b510179c027078a79ba1d1e14ac96e95d33213b7111cd873127c92327ce798" + }, + "vl-8-0-current.after.metrics": { + "bytes": 12304, + "sha256": "5813647737abb033d170a64938985ad06b4e6bbc0ec8302afc9b2b576de511ff" + }, + "vl-8-0-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-8-0-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "vl-8-0-current.before.metrics": { + "bytes": 12128, + "sha256": "301db95b2706ef8748d58c6a95b2e75d48551c9e11d6476cc6fd67010f47477f" + }, + "vl-8-0-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-8-0-current.cold-2048.client.log": { + "bytes": 348, + "sha256": "8f7f61eeeeb080a9e89d749a12e124d3318500cce659e7c07246151263d9a066" + }, + "vl-8-0-current.cold-2048.json": { + "bytes": 20276, + "sha256": "f666c81a25cdcea210246b5b01ecf450e8cdf02bf10edde69dcb1889e2cae87d" + }, + "vl-8-0-current.cold-8192.client.log": { + "bytes": 347, + "sha256": "8cd954da93125b581780d1fdeda4aa3f3fd66c05a35e2d880bad3467e641094c" + }, + "vl-8-0-current.cold-8192.json": { + "bytes": 57141, + "sha256": "94237f24059cbc2c18ee0decc0817b4a01fa7dc815e91b3be2b8bac632706fd5" + }, + "vl-8-0-current.concurrent.client.log": { + "bytes": 3225, + "sha256": "b85243b163154075d156dca6a2e21b5527986053114fa8733d7ae9dde91ad082" + }, + "vl-8-0-current.concurrent.json": { + "bytes": 288866, + "sha256": "efbb0beb927bd45ba030170913ea780d2bd7462256f11263ca399e0705b90085" + }, + "vl-8-0-current.server.log": { + "bytes": 13641, + "sha256": "a89b78f9653448bf13d101666f8f4052ee727b6122073d9e471a9d0eb3963e60" + }, + "vl-8-1-standard.after.health": { + "bytes": 457, + "sha256": "d77972274c900ea75a7c25828e72058554a47ad134d26ab4acb86a59e04ff339" + }, + "vl-8-1-standard.after.metrics": { + "bytes": 32127, + "sha256": "8340257ed87f8940aa9d430bb48249eb5abf1aadd680aca3c4949e8bf61339e8" + }, + "vl-8-1-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-8-1-standard.before.health": { + "bytes": 457, + "sha256": "d77972274c900ea75a7c25828e72058554a47ad134d26ab4acb86a59e04ff339" + }, + "vl-8-1-standard.before.metrics": { + "bytes": 913, + "sha256": "9928f200727b8e4df574a61284adce4dc428150550b12517a834da740952b80d" + }, + "vl-8-1-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-8-1-standard.cold-2048.client.log": { + "bytes": 297, + "sha256": "2572925fb424878917803146fafd82c0686295ad5bc22578796ac11cc8fc140b" + }, + "vl-8-1-standard.cold-2048.json": { + "bytes": 26082, + "sha256": "b939e2c2e006b0b1d0f2b6155fc94fb57e68c4552aad7ee0ba78762203516741" + }, + "vl-8-1-standard.cold-8192.client.log": { + "bytes": 297, + "sha256": "7bef492d469a9a85487158880c032c329bad52b75adfa2c9e7dc04b792c13515" + }, + "vl-8-1-standard.cold-8192.json": { + "bytes": 62948, + "sha256": "da107e7322f082e989ed71edc747e9013154525ebcf8d7d356f2ba784a834dbf" + }, + "vl-8-1-standard.concurrent.client.log": { + "bytes": 3222, + "sha256": "42e9cc77c748a764d500bbe8063e04af7c00ec1a2f207cc84aded9fbef202ab9" + }, + "vl-8-1-standard.concurrent.json": { + "bytes": 285423, + "sha256": "9220f89e6ee7bbb83e06134fa6c6b5f9c64f51d073dc7b724d50dc3f151715f2" + }, + "vl-8-1-standard.server.log": { + "bytes": 51107, + "sha256": "6a75266b138a78b383f54e20ecface5feda4976a8d093b4678f80510c2813523" + }, + "vl-8-2-standard.after.health": { + "bytes": 457, + "sha256": "d77972274c900ea75a7c25828e72058554a47ad134d26ab4acb86a59e04ff339" + }, + "vl-8-2-standard.after.metrics": { + "bytes": 32129, + "sha256": "e51ced6f682aeaeb777422035293ad4549b3975c56711ee3964a0fe00c72b25f" + }, + "vl-8-2-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-8-2-standard.before.health": { + "bytes": 457, + "sha256": "d77972274c900ea75a7c25828e72058554a47ad134d26ab4acb86a59e04ff339" + }, + "vl-8-2-standard.before.metrics": { + "bytes": 913, + "sha256": "dae43aa6527a30ee6197dd8b93a22e68df989f0814509b42da5b72fe2ce07b39" + }, + "vl-8-2-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-8-2-standard.cold-2048.client.log": { + "bytes": 297, + "sha256": "c76ab5799f24c678f15504fd1782bbaec0ea99e9e6322b9781132f9e3844ca74" + }, + "vl-8-2-standard.cold-2048.json": { + "bytes": 26082, + "sha256": "3b4d451fa5b93dc2ce412e1e624d0f21f8e3c53fa5dd87c2e099abeef52af397" + }, + "vl-8-2-standard.cold-8192.client.log": { + "bytes": 295, + "sha256": "9ab2a6d260383bae8035ceccb1e4f6f5ac56d5c9fe3802cba32097b8a54e0e94" + }, + "vl-8-2-standard.cold-8192.json": { + "bytes": 62943, + "sha256": "e053e787d01d3f26aed84f37612f465d6c1ddd1104e3f94ebb390ceec5558ac0" + }, + "vl-8-2-standard.concurrent.client.log": { + "bytes": 3225, + "sha256": "48eb513e02d146767a7f14511c0f59c35774a3f8e6edba4e2b3c1916a5563f2b" + }, + "vl-8-2-standard.concurrent.json": { + "bytes": 285287, + "sha256": "804f1992d6a3ba4e2374d8dfcea874051945fbc578610214fd126790a8490abf" + }, + "vl-8-2-standard.server.log": { + "bytes": 51125, + "sha256": "0de141346dc31c081e8826aa34fd0f7368e4b24818940f60e2c6ea8f69c07e09" + }, + "vl-8-3-current.after.health": { + "bytes": 303, + "sha256": "5df60fcd4d7708232d8ae02e246efe009d332f0d57e3b9caca24da59642a76e7" + }, + "vl-8-3-current.after.metrics": { + "bytes": 12304, + "sha256": "5a059e416184dfe60c6b2859c6f2754340b3d3218835d8615d255e70d7851a82" + }, + "vl-8-3-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-8-3-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "vl-8-3-current.before.metrics": { + "bytes": 12128, + "sha256": "e1d8ba708e061e938244dabc86a9e46659fbb9b72c9aa13155ce2ed175fc3f5e" + }, + "vl-8-3-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-8-3-current.cold-2048.client.log": { + "bytes": 349, + "sha256": "770e170bd841b964b8e09a3be820c1dee1ae817938722f2e587fa8816f3d310f" + }, + "vl-8-3-current.cold-2048.json": { + "bytes": 20279, + "sha256": "c61a88909900ffdcc9236362342e0b4ba54cf0ca6b14f329614f3f48acf0c7f1" + }, + "vl-8-3-current.cold-8192.client.log": { + "bytes": 347, + "sha256": "67b7266943b354ce74059abf755e9d067d943a299813c2480a9734f89f26d165" + }, + "vl-8-3-current.cold-8192.json": { + "bytes": 57140, + "sha256": "5ff41c6c9ead50864385713c36790645a5d737852981a630d56f7a81b840e90e" + }, + "vl-8-3-current.concurrent.client.log": { + "bytes": 3228, + "sha256": "1c069b5bb6676f88489a66061569ca6b55c4768c0bd2dfed49caa538aa20c0b9" + }, + "vl-8-3-current.concurrent.json": { + "bytes": 288853, + "sha256": "41799d6c8f51105a0d998dc4fcef6a16ccd825402e8be7631b878e488c8f092b" + }, + "vl-8-3-current.server.log": { + "bytes": 13641, + "sha256": "7fc3fd5c630e654744bfd4e13f7f59c51f3fcef770a261cb626672c071517aec" + } + }, + "raw_retention": "Immutable raw HTTP results, logs, environment and manifests retained privately; relative names and SHA256 supplied. Private paths and request IDs omitted from publication.", + "_trim": { + "note": "Raw capture replaced by counts, first/last items and SHA-256 of the canonical JSON of each trimmed list or string.", + "original_sha256": "42c6215c5fe58be690c08d3157e5fce1c71a55fd71476962843c72d47b741415", + "original_bytes": 388034 + } +} diff --git a/docs/receipts/lfm3/m5-dev-default-2048-smoke.json b/docs/receipts/lfm3/m5-dev-default-2048-smoke.json new file mode 100644 index 0000000000..2e3303a7b3 --- /dev/null +++ b/docs/receipts/lfm3/m5-dev-default-2048-smoke.json @@ -0,0 +1,45 @@ +{ + "scope": "Unversioned development build default options, M5 Ultra TEXT4, bounded 2k/8k serving smoke. No release or complete fidelity claim.", + "parent_revision": "65ae5f75d", + "source_sha256": { + "build.zig": "094d54b752b57cdec616d7b7ceca2c48ee24077d6f30e76d7a7c1f6160da8f45", + "zig/src/native/lfm3.zig": "d10de60063de6a7d3abcc197b8e04af75a112d5babdd6dc697656cc51d4e09c2", + "zig/src/families/lfm3/attention_tiled.zig": "c048143311a4dabd056089d5317a0401c9ee1f46da3f3501488b2768cf710cf6" + }, + "binary_sha256": "de1935f09c7be56e025fb3d94c9c6eb992688dde274effebc6806c1cb19946c3", + "build": "zig build native -Dcpu=apple_m1 (exit 0; no LFM3 options)", + "generated_options": { + "residual_ffn": true, + "tiled_attention": true, + "prompt_rows": 2048 + }, + "release_guard": "Versioned build defaults candidates off. Explicit candidate with -Dversion=1.0.0 rejected by build graph (exit 1).", + "non_tensor_unit_fallback": "Source selects 128 rows when the device has no tensor units; no new M4 model run.", + "cases": [ + { + "prompt_tokens": 2048, + "ttft_s": 0.47437008284032345, + "cached_tokens": 0, + "completion_tokens": 16, + "output_text_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9" + }, + { + "prompt_tokens": 8192, + "ttft_s": 2.0478952089324594, + "cached_tokens": 0, + "completion_tokens": 16, + "output_text_sha256": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349" + } + ], + "matches_prior_opt_in_2048_output_text": true, + "all_clients_and_server_exit_zero": true, + "lock_released": true, + "records_sha256": "e37e2f77288f166e4d7779b03b41563f06dfc09e36a80f286937b625f4166511", + "events_sha256": "40f9cc4985a1a157e530cff4d38284834f840c661172ec5645fc7a9291e580b7", + "limitations": [ + "One observation per length, not a matched speed comparison.", + "Source default enables experimental tiled attention for all M5 LFM3 checkpoint widths; width/image full fidelity remains unqualified.", + "Proposed probability diagnostics fail in the retained bounded text comparison.", + "The 2048 HTTP cancellation test is one sample, not a latency bound." + ] +} diff --git a/docs/receipts/lfm3/m5-fast-bf16-nax-packed-observation.json b/docs/receipts/lfm3/m5-fast-bf16-nax-packed-observation.json new file mode 100644 index 0000000000..325a51c392 --- /dev/null +++ b/docs/receipts/lfm3/m5-fast-bf16-nax-packed-observation.json @@ -0,0 +1,86 @@ +{ + "schema": 1, + "status": "bounded-operator-pass-fullmodel-open", + "acceptance": "User-authorized independentFP32/referenceBF16 envelope; historicalFP64ledgers remain diagnostic. Zero-added-slack RMS and maximumerror; actualMLXBF16 reference on identical held input.", + "scope": "M5 actual TEXT4 layer 0 gate R128 K2048 N7680 GS64 Q4 only. Standalone operator; full-model fidelity and prefill speed remain open.", + "arithmetic": "BF16activation. Exactcodes/scales/bias promotedFP32; explicit fma(q,scale,bias) roundedBF16 weights; strictMPP FP32 destination acrossascendingK; finalBF16. InlineNF detection+tileflagged unchangedpublicfallback.", + "normal": { + "normal_all_MLX_words_equal": true, + "nf_flagged_tile_public_words_equal": false, + "finite_unaffected_rms": 0.0007270719176299991, + "reference_rms": 0.0007270719176299991, + "finite_unaffected_max": 0.008344173431396484, + "reference_max": 0.008344173431396484, + "flags_set": 0, + "fp32_envelope_pass": true + }, + "mixed_nonfinite": { + "normal_all_MLX_words_equal": false, + "nf_flagged_tile_public_words_equal": true, + "finite_unaffected_rms": 0.0006944474587162433, + "reference_rms": 0.0007285995888639391, + "finite_unaffected_max": 0.007863044738769531, + "reference_max": 0.008344173431396484, + "flags_set": 120, + "fp32_envelope_pass": true + }, + "controls": { + "wrong_bias_exit": 1, + "wrong_NF_repair_exit": 1, + "missing_tile_flag_exit": 1, + "error": "TileFlagNumericalMismatch", + "wrong_packed_code_exit": 1 + }, + "diagnostic_old_3pass_stage_us": { + "matrix": 436.68760918080807, + "mask": 233.85416716337204, + "fallback": 83.16664025187492, + "complete": 720.3752174973488, + "caveat": "Singleton separateCB lowclock diagnostics perturb scheduling; not additive or performance qualification." + }, + "provenance": { + "numerical_manifest_sha256": "43c5e440d7ac107485970251ac6dd1064d6060230c6a9869e77d0caea24ae4dd", + "cost_manifest_sha256": "0de739cf0d404c898c1510cf7bedb1b764fcdd3a1afb092fdb4548c08e706146", + "normal_analysis_sha256": "306dc68b117629410581ed5e36f12cd7cf1e3d6e8cad9bfb06801405d3722d96", + "NF_analysis_sha256": "c1b260519dbd343eb4273e333bda71f963f2897ad3a1e5a1dbbf4bb3c6963560", + "cost_raw_sha256": "ee1f2aa8c81cc150243a3a417892977dfac9518258902e800431958c88b229ce", + "all_raw_logs_verified": true, + "all_leases_released": true + }, + "implementation": "Packed Q4 vector load and scale/bias reuse: one uint4 supplies 32 codes per thread/group, preserving each reconstructed weight and matrix accumulation. Inline nonfinite detection and public row-pair fallback for flagged 64x64 output tiles.", + "previous_tileflag_cost": { + "scope": "GenuineR128fullgate complete2pass fastBF16dequantNAX vspublic; fivebalancedround operator only, nofullforwardqualification", + "public_seconds": 0.000478620606372715, + "candidate_seconds": 0.0002579306637926493, + "throughput_ratio": 1.8556173172084671, + "drift": { + "0": -0.00011153008576125778, + "1": -0.0006045067231486012 + }, + "max_within_round_spread": 0.0007058914216269141, + "stable_at_5percent": true, + "all_outputs_match_FP32_envelope_screen": true, + "repeats": 128, + "raw_sha256": "aff1254dddc1c30c5421b14aa8325d019da500a0266d8b3f2846920779130fc5" + }, + "complete_cost": { + "scope": "GenuineR128fullgate complete2pass fastBF16dequantNAX vspublic; fivebalancedround operator only, nofullforwardqualification", + "public_seconds": 0.00047858015932433773, + "candidate_seconds": 9.947525995812612e-05, + "throughput_ratio": 4.8110470837250885, + "drift": { + "0": -5.645172011958799e-05, + "1": 0.0007756388573689765 + }, + "max_within_round_spread": 0.0011559775749903078, + "stable_at_5percent": true, + "all_outputs_match_FP32_envelope_screen": true, + "repeats": 256, + "raw_sha256": "ee1f2aa8c81cc150243a3a417892977dfac9518258902e800431958c88b229ce" + }, + "limitations": [ + "Held-input operator envelope uses independent FP32 and actual MLX 0.32.4 BF16 reference; it is not a full-forward qualification.", + "Only this genuine layer-0 Q4 shape has measured performance; no extrapolation to other layers, widths, shapes, chips, or serving.", + "Candidate matrix plus fallback is two dispatches; baseline is one. Complete graph is timed." + ] +} diff --git a/docs/receipts/lfm3/m5-fast-ffn-propagation-trace.json b/docs/receipts/lfm3/m5-fast-ffn-propagation-trace.json new file mode 100644 index 0000000000..2ec6dfc8dd --- /dev/null +++ b/docs/receipts/lfm3/m5-fast-ffn-propagation-trace.json @@ -0,0 +1,102 @@ +{ + "schema": 1, + "status": "diagnostic-only-no-fidelity-waiver", + "scope": "TEXT4 M5 FFN-only residual prompt matrix, actual128-row prompt then16 fixed teacher-forced steps. Four selected positions127,135,136,142. Public decode and nonFFN projections unchanged.", + "trace_validation": { + "native_baseline_and_candidate_logits_exactly_match_saved_untraced_outputs": true, + "independent_FP32_logits_exactly_match_saved_untraced_F": true, + "candidate_prompt_matrix_calls": 72, + "all_native_and_FP32_capture_hashes_verified": true, + "controller_and_all_three_children_exit_zero": true, + "lock_released": true + }, + "first_selected_prompt_divergence": { + "same_genuine_input_row": true, + "changed_gate_words": [ + { + "column": 2154, + "public": -0.36328125, + "candidate": -0.361328125, + "held_input_FP32": -0.3623046875, + "public_error": 0.0009765625, + "candidate_error": 0.0009765625 + }, + { + "column": 2295, + "public": -0.302734375, + "candidate": -0.30078125, + "held_input_FP32": -0.3017578125, + "public_error": 0.0009765625, + "candidate_error": 0.0009765625 + } + ] + }, + "interpretation": "At selected prompt row127 two layer0 gate words choose opposite BF16 neighbors at exact held-input FP32 midpoints. Their absolute operator errors are equal. At selected continuation rows135/136/142 first baseline/candidate difference appears at layer2 post-operator output, the first full-attention layer. This is consistent with changed prompt history propagating through the cache; selected-row traces do not establish every contributing source across the full history.", + "policy": "No tie waiver, threshold change, coordinate-specific patch or full-model acceptance follows. Better/equal held-input operator metrics do not guarantee better propagated model metrics.", + "phases": { + "prompt_boundary": { + "native": { + "mean_absolute_error": 0.02342551249439329, + "rms_error": 0.029859538470363673, + "relative_rms": 0.008588605253210563, + "maximum_absolute_error": 0.15762969851493835, + "maximum_coordinate": [ + 0, + 36890 + ] + }, + "reference_bf16": { + "mean_absolute_error": 0.03884851189472984, + "rms_error": 0.04764064939088356, + "relative_rms": 0.01370304943028551, + "maximum_absolute_error": 0.20851778984069824, + "maximum_coordinate": [ + 0, + 91107 + ] + }, + "proposed_metric_results": { + "rms_error": true, + "maximum_absolute_error": true, + "top20_probability_mean": false, + "top20_probability_max": false + } + }, + "teacher_forced_continuation": { + "native": { + "mean_absolute_error": 0.03108298368196589, + "rms_error": 0.039936400975985005, + "relative_rms": 0.01140387012815813, + "maximum_absolute_error": 0.26453685760498047, + "maximum_coordinate": [ + 7, + 6292 + ] + }, + "reference_bf16": { + "mean_absolute_error": 0.032687712900759155, + "rms_error": 0.0425864734405558, + "relative_rms": 0.01216060036617698, + "maximum_absolute_error": 0.256467342376709, + "maximum_coordinate": [ + 14, + 4438 + ] + }, + "proposed_metric_results": { + "rms_error": true, + "maximum_absolute_error": false, + "top20_probability_mean": true, + "top20_probability_max": true + } + } + }, + "provenance": { + "tokens_npy_sha256": "1dcb73fe69abc9d3042d528514b912a18c3e5f9cec5a5ff275c6fed58a4ffa70", + "native_trace_manifest_sha256": "2bc6db07654bfc07faba3d8c70978346a6c106461ebc903e1a039b70159e0794", + "native_trace_identity_sha256": "5e6caff5dd9ec6c35eb539888f2f0dd7a7e58d3c6379a76a64a1da5a63054d5d", + "FP32_trace_receipt_sha256": "29f040ff54ed4524a595c465d92b7d3caabd7d3d821bf75b1255d822ac367a39", + "analysis_sha256": "85a588b79afb291fc5cb2e3c1cafd93935c86ac741e0be0e2abc6b87df069184", + "analysis_script_sha256": "6a8c5610f0eb98e12783b58bc0dd6b430076a1ac08b634552972a544d482c9c0" + } +} diff --git a/docs/receipts/lfm3/m5-fast-residual-prompt-provisional.json b/docs/receipts/lfm3/m5-fast-residual-prompt-provisional.json new file mode 100644 index 0000000000..9878a10487 --- /dev/null +++ b/docs/receipts/lfm3/m5-fast-residual-prompt-provisional.json @@ -0,0 +1,239 @@ +{ + "schema": 1, + "status": "PROVISIONAL-serving-gain-numerical-rejected", + "scope": "M5 TEXT4 Q4/GS64 residual-weight tiled prompt projections, 128-row chunks; public decode projections; private developer option only. No production promotion.", + "authorization": "User requested provisional end-to-end timing after finite-output and nonvacuous route checks, before broader fidelity acceptance.", + "implementation": "BF16 high and residual affine weight components, separate strict matrix accumulations, FP32 sum then BF16; nonfinite tiles use public row-pair fallback.", + "held_input_operator": { + "gate": { + "native": { + "mean_absolute_error": 0.0004282208804823995, + "rms_error": 0.0006554408840321345, + "relative_rms": 0.0016702313459241976, + "maximum_absolute_error": 0.007761955261230469, + "maximum_coordinate": [ + 19, + 6325 + ] + }, + "reference_bf16": { + "mean_absolute_error": 0.0005146326388109894, + "rms_error": 0.0007270719176299991, + "relative_rms": 0.0018527655768072338, + "maximum_absolute_error": 0.008344173431396484, + "maximum_coordinate": [ + 4, + 445 + ] + }, + "passed": true + }, + "up": { + "native": { + "mean_absolute_error": 0.00023988126580718472, + "rms_error": 0.00036742669405024123, + "relative_rms": 0.0016653896392712529, + "maximum_absolute_error": 0.0049169063568115234, + "maximum_coordinate": [ + 12, + 2119 + ] + }, + "reference_bf16": { + "mean_absolute_error": 0.00033850177936471936, + "rms_error": 0.0004556513123757718, + "relative_rms": 0.0020652744807028024, + "maximum_absolute_error": 0.0049169063568115234, + "maximum_coordinate": [ + 12, + 2119 + ] + }, + "passed": true + }, + "down": { + "native": { + "mean_absolute_error": 8.849430627116961e-05, + "rms_error": 0.00019918425001084545, + "relative_rms": 0.0017976343353374538, + "maximum_absolute_error": 0.015062332153320312, + "maximum_coordinate": [ + 0, + 1688 + ] + }, + "reference_bf16": { + "mean_absolute_error": 0.00011842676151635478, + "rms_error": 0.0002245361179170129, + "relative_rms": 0.0020264344950417707, + "maximum_absolute_error": 0.015062332153320312, + "maximum_coordinate": [ + 0, + 1688 + ] + }, + "passed": true + } + }, + "full_model": { + "passed": false, + "fixed_prompt_tokens": 128, + "teacher_forced_continuations": 16, + "selected_full_vocabulary_rows": 17, + "top1_matches_fp32_all_rows": true, + "failures": { + "prompt_boundary": { + "native": { + "mean_absolute_error": 0.05981908342488724, + "rms_error": 0.06732795929188548, + "relative_rms": 0.019365780399993968, + "maximum_absolute_error": 0.2148294448852539, + "maximum_coordinate": [ + 0, + 30059 + ] + }, + "reference_bf16": { + "mean_absolute_error": 0.03884851189472984, + "rms_error": 0.04764064939088356, + "relative_rms": 0.01370304943028551, + "maximum_absolute_error": 0.20851778984069824, + "maximum_coordinate": [ + 0, + 91107 + ] + }, + "metric_results": { + "rms_error": false, + "maximum_absolute_error": false, + "top20_probability_mean": true, + "top20_probability_max": true + } + }, + "teacher_forced_continuation": { + "native": { + "mean_absolute_error": 0.03918649825569497, + "rms_error": 0.04971042433487815, + "relative_rms": 0.014194850043484667, + "maximum_absolute_error": 0.28858280181884766, + "maximum_coordinate": [ + 14, + 1399 + ] + }, + "reference_bf16": { + "mean_absolute_error": 0.032687712900759155, + "rms_error": 0.0425864734405558, + "relative_rms": 0.01216060036617698, + "maximum_absolute_error": 0.256467342376709, + "maximum_coordinate": [ + 14, + 4438 + ] + }, + "metric_results": { + "rms_error": false, + "maximum_absolute_error": false, + "top20_probability_mean": true, + "top20_probability_max": true + } + } + } + }, + "basic": { + "scope": "PROVISIONAL basic sanity only, not FP32 fidelity", + "encoded_calls": 142, + "fallback_calls": 2476, + "finite_words": 2176000, + "different_baseline_words": 1924929, + "logits_sha256": "a8dfee0f08dbe93d1ccf7b7cc128d7018294baf0b11e0e68fc339466890b2aeb" + }, + "gate_complete_cost": { + "scope": "GenuineR128fullgate complete2pass fastBF16dequantNAX vspublic; fivebalancedround operator only, nofullforwardqualification", + "public_seconds": 0.00047865055421425495, + "candidate_seconds": 0.0002224270829174202, + "throughput_ratio": 2.1519436749164313, + "drift": { + "0": 6.121026473859814e-05, + "1": 0.040339739716438894 + }, + "max_within_round_spread": 0.03977839188106569, + "stable_at_5percent": true, + "all_outputs_match_FP32_envelope_screen": true, + "repeats": 128, + "raw_sha256": "5d184ab92ecf1cb71355f63be9d955b39ca78c666e9e54614adaf075758c2125" + }, + "served_cases": [ + { + "prompt_tokens": 2048, + "baseline_ttft_s": [ + 0.8585991663858294, + 0.8589464169926941 + ], + "candidate_ttft_s": [ + 0.617787292227149, + 0.6173734590411186 + ], + "baseline_median_s": 0.8587727916892618, + "candidate_median_s": 0.6175803756341338, + "baseline_over_candidate": 1.3905441713679303, + "baseline_first_last_drift": 0.0004044385558006347, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "not emitted by cold client; candidate basic finite/hits pass; full-model FP32 RMS/max gates FAIL", + "prompt_ids": "both routes /tokenize exact pinned arrays" + }, + { + "prompt_tokens": 8192, + "baseline_ttft_s": [ + 4.614732750225812, + 4.618652666918933 + ], + "candidate_ttft_s": [ + 3.6667009578086436, + 3.6655994160100818 + ], + "baseline_median_s": 4.616692708572373, + "candidate_median_s": 3.6661501869093627, + "baseline_over_candidate": 1.2592753905874043, + "baseline_first_last_drift": 0.0008494352555799001, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "not emitted by cold client; candidate basic finite/hits pass; full-model FP32 RMS/max gates FAIL", + "prompt_ids": "both routes /tokenize exact pinned arrays" + } + ], + "serving_protocol": "Same-source build option off/on; ABBA fresh servers; exact pinned 2048/8192 prompt IDs; prompt cache disabled and cached0; temperature0/no-thinking;16 outputs; no hidden retry; predeclared5% baseline drift.", + "route_counts": { + "baseline": [ + 0, + 0 + ], + "candidate_per_leg": [ + 2272, + 9088 + ] + }, + "cleanup": { + "server_exits": [ + 0, + 0, + 0, + 0 + ], + "lock_released": true, + "all_served_raw_hashes_verified": true + }, + "provenance": { + "source_base": "7d8b6141a", + "served_pins_sha256": "dc42e943b526ab2a2a24bdb63567739b1463d13bd7825fd83f2f8bb2033d4bb0", + "served_summary_sha256": "d7b79a1395ea0b134010b8827d323db00d5d49a1460e8b8b4b66ef2bcf07634c", + "full_model_phases_sha256": "1a53f8865714abd877e619e628803eae70af9b606e6007415bc6260e5585075e", + "native_basic_manifest_sha256": "134727a06ae3f8af5f2d002d01ec6615e33041527ea03418b796aac2f4d1f8a6" + }, + "remaining": [ + "Full-model prompt and continuation RMS/max gates fail; equal top1 and output text do not waive failure.", + "No qualified prefill improvement, decode/ITL gain, or10x claim.", + "Measured route changes other prompt projections as well as FFN; next attribution is an explicit FFN-only route." + ] +} diff --git a/docs/receipts/lfm3/m5-ffn-exploratory-ledger.json b/docs/receipts/lfm3/m5-ffn-exploratory-ledger.json new file mode 100644 index 0000000000..07c35fa347 --- /dev/null +++ b/docs/receipts/lfm3/m5-ffn-exploratory-ledger.json @@ -0,0 +1,201 @@ +{ + "status": "Private exploratory operator results only; no production or final performance qualification", + "shape": { + "rows": 64, + "K": 2048, + "N_tested": 2048, + "N_full_gate": 7680, + "group_size": 64, + "quantization_bits": 4, + "output_elements": 131072 + }, + "method": [ + "Same actual checkpoint activation and weight slice, equal matrix work and output shape for both routes.", + "Complete candidate cost includes preparation, producer, selection/combine and repair; persistent scratch allocated before timing.", + "Five complete output/selector/route identity checks precede timing; fused routes additionally check the exact timed graph.", + "Two mirrored rounds per experiment; 512 common repetitions per sample; >=25 ms calibration minimum, >=20 ms measured samples, 1 s cap, unchanged 5% baseline drift limit. These are bounded exploratory screens, not final five-round qualification.", + "All timing rows below passed their exploratory drift gate. Cross-experiment differences are observations, not a single randomized comparison of all variants." + ], + "measurements": [ + { + "variant": "Exact-band producer + cooperative integer repair", + "baseline_us": 67.18440772601753, + "candidate_us": 360.50899279871373, + "candidate_over_baseline": 5.365962207613608, + "candidate_throughput_fraction": 0.1863598663779497, + "selected_outputs": 137, + "repetitions": 512, + "baseline_first_last_drift_fraction": 0.0003585299460018923, + "exploratory_stability_pass": true, + "raw_measurement_sha256": "b61ffc27084fcdb1c5832b6337a83ecce4dfd0e81b9a4aaeb1a46e20cb3e4d3c" + }, + { + "variant": "Shared weight decoding and coefficient metadata", + "baseline_us": 67.23319484081003, + "candidate_us": 353.1779789227585, + "candidate_over_baseline": 5.2530298427583, + "candidate_throughput_fraction": 0.19036632761159272, + "selected_outputs": 137, + "repetitions": 512, + "baseline_first_last_drift_fraction": 0.00032798360352506625, + "exploratory_stability_pass": true, + "raw_measurement_sha256": "9d709506c736a63edd1de302668ae6a7fcb456ec724d9e4c30e683bfc08e1833" + }, + { + "variant": "16 x 16 producer tile", + "baseline_us": 67.10314937663497, + "candidate_us": 255.05448456897284, + "candidate_over_baseline": 3.8009316543014853, + "candidate_throughput_fraction": 0.26309339155528033, + "selected_outputs": 137, + "repetitions": 512, + "baseline_first_last_drift_fraction": 0.0001540187112675435, + "exploratory_stability_pass": true, + "raw_measurement_sha256": "6808393a249c2468b3b64ee1ab1438284c52a580c8dbcd26120c962481122bb2" + }, + { + "variant": "Four ordered split-K chunks", + "baseline_us": 67.26973424520111, + "candidate_us": 249.81339447549544, + "candidate_over_baseline": 3.71360757223917, + "candidate_throughput_fraction": 0.2692799334737021, + "selected_outputs": 121, + "repetitions": 512, + "baseline_first_last_drift_fraction": 7.864628450250831e-05, + "exploratory_stability_pass": true, + "raw_measurement_sha256": "b2f6bfdb997d1e558465b0cf9ffeaac28a49e5a22a2d95c87a9442f9cb81e15d" + }, + { + "variant": "Compensated products + bucket repair scheduling", + "baseline_us": 67.31465646225843, + "candidate_us": 142.85559109339374, + "candidate_over_baseline": 2.1222063455599622, + "candidate_throughput_fraction": 0.4712077136571475, + "selected_outputs": 120, + "repetitions": 512, + "baseline_first_last_drift_fraction": 6.045342473718198e-05, + "exploratory_stability_pass": true, + "raw_measurement_sha256": "8e40ea0e61ff573324d9a0ab8c6d91fa6d0c91941d0c96bbf7fa425441aa2d91" + }, + { + "variant": "Affine double-double group accumulation", + "baseline_us": 67.31860321451677, + "candidate_us": 141.2617589267029, + "candidate_over_baseline": 2.098405970732334, + "candidate_throughput_fraction": 0.4765522086515054, + "selected_outputs": 120, + "repetitions": 512, + "baseline_first_last_drift_fraction": 0.0002393863397320616, + "exploratory_stability_pass": true, + "raw_measurement_sha256": "f247ccee601d04715de88191421a802efd9f6522472eb8a967a11b97f31ef392" + }, + { + "variant": "Shared conservative row/column budgets", + "baseline_us": 67.25435423504678, + "candidate_us": 127.99157684639795, + "candidate_over_baseline": 1.9030972537343986, + "candidate_throughput_fraction": 0.5254592207716793, + "selected_outputs": 238, + "repetitions": 512, + "baseline_first_last_drift_fraction": 8.34901218501411e-05, + "exploratory_stability_pass": true, + "raw_measurement_sha256": "7367abb02bc0dade8e3ad4eac3e41f5240a5a303b955bb4db7a0b79794cb5575" + }, + { + "variant": "Fused combine, selection, and repair", + "baseline_us": 67.14213077430031, + "candidate_us": 123.46203584456816, + "candidate_over_baseline": 1.838816171318548, + "candidate_throughput_fraction": 0.5438281518282148, + "selected_outputs": 238, + "repetitions": 512, + "baseline_first_last_drift_fraction": 0.00014179077468150984, + "exploratory_stability_pass": true, + "raw_measurement_sha256": "02cb19f9f7111794904d0a77c18300b7e4671e6da92981cde66d67fa7e8cb5c3" + }, + { + "variant": "Materialized activation bands", + "baseline_us": 67.15637209708802, + "candidate_us": 127.79545068042353, + "candidate_over_baseline": 1.9029534605542648, + "candidate_throughput_fraction": 0.5254989261317691, + "selected_outputs": 238, + "repetitions": 512, + "baseline_first_last_drift_fraction": 0.0008676486827685353, + "exploratory_stability_pass": true, + "raw_measurement_sha256": "1d8c9ebaedb22f4e44fe5f0c35a4088df8269a9a32550aa22ff1cc2d6598b3fc" + }, + { + "variant": "Explicit strict MPP, serial band chains", + "baseline_us": 67.25260391249321, + "candidate_us": 117.20515885826899, + "candidate_over_baseline": 1.7427601615362334, + "candidate_throughput_fraction": 0.5738024210505854, + "selected_outputs": 238, + "repetitions": 512, + "baseline_first_last_drift_fraction": 8.712257596004663e-05, + "exploratory_stability_pass": true, + "raw_measurement_sha256": "58449d9cad27133a849e9bbe92a9e16d64a1b365449de4253265fb0fae8e144d" + }, + { + "variant": "Explicit strict MPP, parallel band chains", + "baseline_us": 67.27791333105415, + "candidate_us": 110.96643083874369, + "candidate_over_baseline": 1.6493738486315952, + "candidate_throughput_fraction": 0.6062906846920431, + "selected_outputs": 238, + "repetitions": 512, + "baseline_first_last_drift_fraction": 5.322268134651951e-05, + "exploratory_stability_pass": true, + "raw_measurement_sha256": "dfa1568c7e0dc3a7e76e226e64ba22f380d4cbb3e28cf194c2c4e9bb152931e8" + }, + { + "variant": "Four SIMD groups with exact half-K sums", + "baseline_us": 67.25386538164457, + "candidate_us": 114.54093373686192, + "candidate_over_baseline": 1.703113019406068, + "candidate_throughput_fraction": 0.5871600936670269, + "selected_outputs": 238, + "repetitions": 512, + "baseline_first_last_drift_fraction": 0.0001427854323591979, + "exploratory_stability_pass": true, + "raw_measurement_sha256": "3ee51193facfd9079e20277261fa40d24265655e16791aa19612be484b58ec2c" + } + ], + "numerical": [ + "Every listed cost candidate passed independent streamed FP64 primary checks on the entire tested output slice, every-column no-worse gates, aggregate/worst-column/max-error checks and planted fault controls.", + "Latest four-SIMD-group result is byte-identical to the two-SIMD-group predecessor for all captured producer/final arrays, centers, budgets, selectors and activation bands. Eight fault controls were rejected.", + "The latest output slice matches rounded FP64 primary words; the public comparison differs in five words on this fixture. No tolerance was increased.", + "Exact-band integer admission and the retained 128*d*d*u^2 radius are conditional on the stated full-FP32 MPP/RNE/lattice assumptions. Universal Apple device reduction/FTZ guarantees remain open; finite-fixture success does not establish them." + ], + "early_results": [ + "Fixed two-band admission selected 16502/131072 outputs; three-band admission selected 228; actual-mass two-band admission selected 137.", + "Serial repair variant reached approximately 2162 us per operation in calibration, versus approximately 67 us public baseline. It failed bounded calibration and is not a qualified timing row.", + "Earlier stage diagnostics attributed approximately 1990 us to serial repair and 249 us to its producer; cooperative repair and bucket scheduling addressed that observed bottleneck." + ], + "latest_stage_diagnostic_us": { + "baseline": 71.6252252459526, + "full_route": 135.5626154690981, + "prepare": 18.500257283449173, + "producer": 79.02085781097412, + "fused_finish": 43.00009459257126, + "scope": "20 singleton command-buffer samples of the serial-band explicit MPP route; all endpoint identities passed; not throughput qualification or hardware utilization evidence" + }, + "conclusions": [ + "Best current private route is two parallel MPP band chains: 110.966 us versus its 67.278 us baseline, still 1.6494x slower. Do not integrate or enable it.", + "Materialized activation bands regressed relative to fused finish, and four-SIMD-group half-K splitting regressed by approximately 3.22% relative to two parallel bands. Neither is a promotion candidate.", + "The executed tile call graph explicitly reaches the strict MPP 16x32x16 BF16/BF16/FP32 primitive. This is compiler call-graph evidence, not a whole-device tensor-unit disable or hardware utilization counter.", + "Available hardware audit exposed no useful compute/occupancy/bandwidth counters. No saturation, cache or spill claim is supported." + ], + "audit_notes": [ + "Earlier shared-budget/fused/preband host source accidentally changed a calibration repeat ceiling from 4096 to 4608 during an unrelated reflection edit. Every affected measured run used 512 repetitions, so neither ceiling was reached. Historical raw artifacts are preserved; later hosts restore the original 4096 ceiling.", + "The candidate dispatch-count description remained stale at four after fusion; inspected executed code uses three dispatches. Later stage metadata records three. This does not change recorded timings.", + "A local NumPy 2 verifier required an explicit unsigned mask in place of a negative Python integer mask. Shader outputs were unchanged and the corrected verifier passed every activation-band word." + ], + "scope_limits": [ + "One checkpoint class and 2048-column gate slice only; no full-width 7680-column gate/up/down sweep.", + "No whole-model, HTTP serving, image-path or end-to-end prefill qualification for these private FFN kernels.", + "No public runtime/default changes, no installed MLX mutation, no promotion of a non-winning kernel." + ], + "baseline": "Existing public native row-pair projection kernel" +} diff --git a/docs/receipts/lfm3/m5-ffn-r128-full-gate-confirmation.json b/docs/receipts/lfm3/m5-ffn-r128-full-gate-confirmation.json new file mode 100644 index 0000000000..a6736d4974 --- /dev/null +++ b/docs/receipts/lfm3/m5-ffn-r128-full-gate-confirmation.json @@ -0,0 +1,228 @@ +{ + "schema_version": 1, + "scope": "Five balanced rounds confirm a private fullwidth R128 FFN gate operator improvement; not model or serving qualification", + "production_default_changed": false, + "candidate_source_published": false, + "previous_exploratory_receipt": "m5-ffn-r128-full-gate-exploratory.json", + "shape": { + "rows": 128, + "n": 7680, + "k": 2048, + "group": 64, + "bits": 4 + }, + "numerical_evidence": "Same unchanged kernel/fixtures as exploratory receipt: all983040outputs match roundedFP64; no worse column; captured publicgate identity; eight controls rejected. Conditional MPP device proof remains open.", + "protocol": { + "rounds": 5, + "seed": 20261030, + "order": [ + "BAAB", + "ABBA", + "BAAB", + "ABBA", + "BAAB" + ], + "common_repetitions": 64, + "calibration_min_seconds": 0.025, + "sample_min_seconds": 0.02, + "sample_max_seconds": 1, + "warmup_min_seconds": 0.03, + "baseline_drift_limit": 0.05, + "within_round_spread_limit": 0.05, + "five_complete_graph_identity_checks": true + }, + "measurements": { + "public_median_us": 478.0979834322352, + "candidate_median_us": 448.58496403321624, + "latency_reduction_fraction": 0.06173006459292474, + "throughput_ratio": 1.0657913701200952, + "baseline_first_last_drift_fraction": 4.631285169915659e-05, + "within_round_spread_fractions": [ + 2.859697800255466e-05, + 9.576890201813259e-05, + 5.311136172707313e-05, + 0.00025536887435493796, + 6.53614877954694e-05, + 0.0013203176020051394, + 1.906802421766906e-05, + 0.0005197275782504725, + 8.44372820552071e-05, + 0.0003338785731539611 + ], + "all_stability_gates_passed": true, + "selected_global53_repairs": 1068, + "serial_fallback": 0, + "raw_records": [ + { + "round": 0, + "leg": 0, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.028715624939650297, + "warm_gpu_seconds": 0.05741754174232483 + }, + { + "round": 0, + "leg": 1, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.03059641644358635, + "warm_gpu_seconds": 0.030598874669522047 + }, + { + "round": 0, + "leg": 2, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.0305972914211452, + "warm_gpu_seconds": 0.030608208384364843 + }, + { + "round": 0, + "leg": 3, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.028718375135213137, + "warm_gpu_seconds": 0.05742962518706918 + }, + { + "round": 1, + "leg": 0, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.03059987537562847, + "warm_gpu_seconds": 0.030622000340372324 + }, + { + "round": 1, + "leg": 1, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.028718083165585995, + "warm_gpu_seconds": 0.05741345835849643 + }, + { + "round": 1, + "leg": 2, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.02871075039729476, + "warm_gpu_seconds": 0.05741658341139555 + }, + { + "round": 1, + "leg": 3, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.030598250217735767, + "warm_gpu_seconds": 0.030600249767303467 + }, + { + "round": 2, + "leg": 0, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.028737000189721584, + "warm_gpu_seconds": 0.05747545836493373 + }, + { + "round": 2, + "leg": 1, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.030600291676819324, + "warm_gpu_seconds": 0.030598000157624483 + }, + { + "round": 2, + "leg": 2, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.030598291661590338, + "warm_gpu_seconds": 0.030605874955654144 + }, + { + "round": 2, + "leg": 3, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.028699083253741264, + "warm_gpu_seconds": 0.0574368336237967 + }, + { + "round": 3, + "leg": 0, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.030599874909967184, + "warm_gpu_seconds": 0.03059862507507205 + }, + { + "round": 3, + "leg": 1, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.02870812499895692, + "warm_gpu_seconds": 0.05741104204207659 + }, + { + "round": 3, + "leg": 2, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.02869320847094059, + "warm_gpu_seconds": 0.05738841649144888 + }, + { + "round": 3, + "leg": 3, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.030599291436374187, + "warm_gpu_seconds": 0.030598416924476624 + }, + { + "round": 4, + "leg": 0, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.028696791734546423, + "warm_gpu_seconds": 0.057453916408121586 + }, + { + "round": 4, + "leg": 1, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.030595249962061644, + "warm_gpu_seconds": 0.030596999917179346 + }, + { + "round": 4, + "leg": 2, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.030597833450883627, + "warm_gpu_seconds": 0.030606666579842567 + }, + { + "round": 4, + "leg": 3, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.02870637457817793, + "warm_gpu_seconds": 0.05742625007405877 + } + ] + }, + "limits": [ + "Native public row-pair baseline, not MLX.", + "Gate operator only; no inference TTFT or end-to-end improvement established.", + "Other projections, checkpoints, widths, images and prompt lengths remain unqualified.", + "Persistent partialscratch60MiB; practical integration memory admission remains mandatory." + ], + "evidence_sha256": { + "raw_cost": "d42183d36e52b5ae07f380c5fb62c0e553ebf6bb0d3582d5e26701e92f22763b", + "analysis": "a09b0994f40b2247c3f673799f76fec194ff414a56e80f13c8ebd38e3f5e6b96" + } +} diff --git a/docs/receipts/lfm3/m5-ffn-r128-full-gate-exploratory.json b/docs/receipts/lfm3/m5-ffn-r128-full-gate-exploratory.json new file mode 100644 index 0000000000..17fd01f743 --- /dev/null +++ b/docs/receipts/lfm3/m5-ffn-r128-full-gate-exploratory.json @@ -0,0 +1,119 @@ +{ + "schema_version": 1, + "scope": "Private exact-band MPP full gate exploratory operator screen; source-free aggregate receipt", + "production_default_changed": false, + "candidate_source_published": false, + "shape": { + "rows": 128, + "k": 2048, + "n": 7680, + "group": 64, + "bits": 4, + "checkpoint": "LFM3 TEXT 1.7B 4-bit", + "layer": 0, + "activation": "Genuine first128 contiguous cold prompt rows; not repeated64-row input" + }, + "comparison": "Public native row-pair projection versus private complete prepare/producer/fused-finish route; not an MLX comparison", + "numeric": { + "outputs": 983040, + "strict_gate_errors": 0, + "worse_columns": 0, + "candidate_word_differences_from_rounded_fp64": 0, + "baseline_word_differences_from_rounded_fp64": 125, + "captured_public_gate_word_differences": 0, + "selected_global53_repairs": 1068, + "serial_fallback": 0, + "all_eight_controls_rejected": true, + "control_verification": "Every final word compared to SHA-pinned positive proved exact roundedFP64, plus original selector-validity and combine-order-field gates. Complete raw control FP64 analysis not claimed. Initial checker omitted selector gate and correctly failed; revised checker preserved that failure and applied the original gate." + }, + "timing": { + "rounds": 2, + "order_each_round": "ABBA", + "samples_per_variant": 4, + "common_repetitions": 64, + "public_median_us": 478.0885428772308, + "candidate_median_us": 449.91634422331117, + "candidate_latency_reduction_fraction": 0.058926738725788685, + "candidate_throughput_ratio": 1.0626165264179348, + "baseline_first_last_drift_fraction": 3.2676115740963116e-05, + "five_exact_timed_graph_identity_checks": true, + "persistent_partial_scratch_bytes": 62914560, + "raw_samples": [ + { + "round": 0, + "leg": 0, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.030596500262618065, + "warm_gpu_seconds": 0.030599375255405903 + }, + { + "round": 0, + "leg": 1, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.02879870869219303, + "warm_gpu_seconds": 0.057600791566073895 + }, + { + "round": 0, + "leg": 2, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.028804750181734562, + "warm_gpu_seconds": 0.05757950013503432 + }, + { + "round": 0, + "leg": 3, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.030601374804973602, + "warm_gpu_seconds": 0.030597624834626913 + }, + { + "round": 1, + "leg": 0, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.030597833450883627, + "warm_gpu_seconds": 0.030608083587139845 + }, + { + "round": 1, + "leg": 1, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.0287905833683908, + "warm_gpu_seconds": 0.057665541768074036 + }, + { + "round": 1, + "leg": 2, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.028774041682481766, + "warm_gpu_seconds": 0.05762825021520257 + }, + { + "round": 1, + "leg": 3, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.030597500037401915, + "warm_gpu_seconds": 0.030597875360399485 + } + ] + }, + "limits": [ + "Exploratory two-round result; five balanced confirmation rounds pending.", + "Only this actual full-width gate tested; no fullmodel, up/down projection, HTTP or image performance qualification.", + "Conditional device MPP exactness/reduction assumptions remain open; empirical primary and every-column gates unchanged.", + "Small64-row/2048-column slice remained slower than native. No general speedup extrapolation." + ], + "evidence_sha256": { + "raw_cost": "d585b26f6bf1a01e6ab6abd7eb8891191e08b13c29a9c3f2ab84072f729968eb", + "primary_analysis": "a230b2cc51dae65b638fb38d95cd9e4f0c372a863988f9c027098780eab8844a", + "bounded_control_summary": "cf157d45c9bb28a7aa89d2090d2117c04101f400909b9e2f9edd59f8aacbab99" + } +} diff --git a/docs/receipts/lfm3/m5-ffn-r128-tile32-observation.json b/docs/receipts/lfm3/m5-ffn-r128-tile32-observation.json new file mode 100644 index 0000000000..bbaffe2abe --- /dev/null +++ b/docs/receipts/lfm3/m5-ffn-r128-tile32-observation.json @@ -0,0 +1,116 @@ +{ + "schema": 1, + "status": "numerically-passed-no-material-incremental-gain", + "scope": "M5 actual TEXT4 layer0 FFN gate R128 K2048 N7680 GS64 q4; standalone operator only; no public runtime change or serving qualification.", + "change": "Two independent original16x32 row halves share a32x32 threadgroup; all4SIMDgroups active; original per-output MPP, group and DD order retained.", + "geometry": { + "prior_threadgroups": 7680, + "candidate_threadgroups": 3840, + "threads": 128, + "prior_static_lds_bytes": 4864, + "candidate_static_lds_bytes": 9216, + "prior_live_output_pairs_per_thread": 4, + "candidate_live_output_pairs_per_thread": 8 + }, + "numerical": { + "full_raw_arrays_byte_identical_to_prior_qualified_route": 24, + "final_primary_qualified_words": 983040, + "metadata_reflection_pass": true, + "wrong_rowhalf_control_exit": 1, + "wrong_band_control_exit": 1, + "negative_control_error": "Tile32IdentityMismatch", + "strict_primary_gate_inherited_by_full_output_identity": true, + "universal_MPP_contract_claim": false + }, + "stage_diagnostic": { + "scope": "20singleton GPUstage diagnostics; no throughput/occupancy qualification", + "median_us": { + "baseline": 612.4377250671387, + "fullroute": 585.3958427906036, + "prepare": 22.979220375418663, + "producer": 469.43756751716137, + "fused_finish": 103.99986058473587, + "sum_stages": 597.9789420962334 + }, + "min_us": { + "baseline": 483.91660675406456, + "fullroute": 466.0417325794697, + "prepare": 18.500257283449173, + "producer": 374.62497130036354, + "fused_finish": 85.87492629885674, + "sum_stages": 479.58362847566605 + }, + "max_us": { + "baseline": 661.1663848161697, + "fullroute": 634.0001709759235, + "prepare": 24.79134127497673, + "producer": 509.08373668789864, + "fused_finish": 113.58316987752914, + "sum_stages": 644.2503072321415 + }, + "all_final_identities": true, + "selected": 1068 + }, + "stage_caveats": [ + "Separate command buffers perturb scheduling; stage times are nonadditive diagnostics.", + "Singleton sample clocks drift substantially; no stable throughput qualification from stage medians." + ], + "complete_cost": { + "scope": "Exploratory actual R128 fullwidth gate operator screen, five balanced rounds; operator confirmation only, not model/HTTP qualification", + "rows": 128, + "n": 7680, + "group": 64, + "bits": 4, + "equal_useful_matrix_work_per_sample": true, + "common_repetitions": 64, + "median_gpu_seconds_per_op": { + "public_row_pairs": 0.0004781292300322093, + "tile_fullroute": 0.00044869270641356707 + }, + "candidate_throughput_vs_public": { + "tile": 1.0656050860597457 + }, + "baseline_first_last_drift_fraction": 7.079625654597148e-05, + "exploratory_drift_limit": 0.05, + "stable_for_exploratory_screen": true, + "within_round_spreads": [ + 5.311100043867556e-05, + 0.00011321098934207741, + 2.7238328266015176e-05, + 0.0005251213227473937, + 4.630868883476131e-05, + 0.0005439663894102429, + 9.541302781807912e-06, + 0.00018286797618239498, + 2.1761088027618503e-05, + 0.0008134824366681932 + ], + "all_stability_gates_passed": true, + "actual_selected_outputs": [ + 1068, + 0 + ], + "actual_repair_fractions": [ + 0.00108642578125, + 0.0 + ], + "diagnostics_in_timed_work": { + "centers_bounds_stores": true, + "raw_groupdot_capture": false, + "snapshot": false, + "prepare_each_op": true, + "actual_repair_dispatch": true + }, + "final_performance_qualification": false + }, + "interpretation": "448.693us candidate versus478.129us public confirms existing roughly6% standalone gain. Prior16x32 candidate about448.6us; no meaningful incremental gain established. Half the threadgroup count did not produce a large speedup. No serving promotion.", + "provenance": { + "numerical_manifest_sha256": "588e8c651cc2f26b698568a5d584f9461e2f416ab92b7ba2e36232c0bb92d74f", + "cost_manifest_sha256": "ecc58f84fabf040e39d8ffd71dd90bd72023bfdadfca38fb005998f35fb45269", + "cost_raw_sha256": "efca3ab94ff357e2395a49d066fdc812619c83e699a0c734331c048e8430a634", + "cost_analysis_sha256": "b86012b8f459221a03710b67d63e40ac39f4608652a35b0efb8c97bca0c22fd3", + "stage_analysis_sha256": "75a18e951cae2b1fbd249cdf92a9703e016c0e73c43986bdd54d19ad5eacdefa", + "numerical_identity_sha256": "e6d3158b1aaa1baa0d1d26529f62be327468c184311a4692da3ca2ccb2c84912", + "all_jobs_exited_and_leases_released": true + } +} diff --git a/docs/receipts/lfm3/m5-ffn-r128-up-negative.json b/docs/receipts/lfm3/m5-ffn-r128-up-negative.json new file mode 100644 index 0000000000..4aee4ba6f3 --- /dev/null +++ b/docs/receipts/lfm3/m5-ffn-r128-up-negative.json @@ -0,0 +1,215 @@ +{ + "scope": "Private TEXT4 layer00 genuineR128 fullwidth UP projection; five balanced rounds; negative performance result", + "production_default_changed": false, + "shape": { + "rows": 128, + "n": 7680, + "k": 2048, + "group": 64, + "bits": 4 + }, + "numerical": { + "candidate_rounded_fp64_word_differences": 0, + "public_rounded_fp64_word_differences": 16, + "worse_columns": 0, + "output_count": 983040, + "captured_public_up_identity": true, + "selected_g53_repairs": 2297, + "serial_fallback": 0, + "all_eight_controls_rejected": true + }, + "timing": { + "public_median_us": 478.09114403207786, + "candidate_median_us": 480.5013049917761, + "candidate_throughput_ratio": 0.9949840698981255, + "candidate_latency_increase_fraction": 0.005041216491423972, + "rounds": 5, + "common_repetitions": 64, + "baseline_drift_fraction": 6.263831105199126e-05, + "within_round_spread_fractions": [ + 6.944327925683365e-05, + 0.00022094080929065723, + 6.817742877562606e-06, + 0.0011866840284097755, + 4.9033397572359836e-05, + 0.00019503463908279985, + 1.225191647940069e-05, + 0.0008984324228322081, + 2.3148138107818685e-05, + 0.0010228061269593458 + ], + "all_stability_gates_passed": true, + "raw_records": [ + { + "round": 0, + "leg": 0, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.030736375134438276, + "warm_gpu_seconds": 0.03074458334594965 + }, + { + "round": 0, + "leg": 1, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.030598875135183334, + "warm_gpu_seconds": 0.03059916663914919 + }, + { + "round": 0, + "leg": 2, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.03059675032272935, + "warm_gpu_seconds": 0.030608583241701126 + }, + { + "round": 0, + "leg": 3, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.03074316680431366, + "warm_gpu_seconds": 0.03080249996855855 + }, + { + "round": 1, + "leg": 0, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.03059912519529462, + "warm_gpu_seconds": 0.030600666534155607 + }, + { + "round": 1, + "leg": 1, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.030739666894078255, + "warm_gpu_seconds": 0.03081958368420601 + }, + { + "round": 1, + "leg": 2, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.030776166822761297, + "warm_gpu_seconds": 0.030767499934881926 + }, + { + "round": 1, + "leg": 3, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.030598916579037905, + "warm_gpu_seconds": 0.03059904184192419 + }, + { + "round": 2, + "leg": 0, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.030761000234633684, + "warm_gpu_seconds": 0.030730499885976315 + }, + { + "round": 2, + "leg": 1, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.0305995000526309, + "warm_gpu_seconds": 0.030598958488553762 + }, + { + "round": 2, + "leg": 2, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.030597999691963196, + "warm_gpu_seconds": 0.030620625242590904 + }, + { + "round": 2, + "leg": 3, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.030767000280320644, + "warm_gpu_seconds": 0.0307437083683908 + }, + { + "round": 3, + "leg": 0, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.03059599967673421, + "warm_gpu_seconds": 0.03060012497007847 + }, + { + "round": 3, + "leg": 1, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.030734583269804716, + "warm_gpu_seconds": 0.030730499885976315 + }, + { + "round": 3, + "leg": 2, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.030762208625674248, + "warm_gpu_seconds": 0.03075666632503271 + }, + { + "round": 3, + "leg": 3, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.030595624819397926, + "warm_gpu_seconds": 0.030600874684751034 + }, + { + "round": 4, + "leg": 0, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.030741041991859674, + "warm_gpu_seconds": 0.030798583291471004 + }, + { + "round": 4, + "leg": 1, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.03059766674414277, + "warm_gpu_seconds": 0.03059862507507205 + }, + { + "round": 4, + "leg": 2, + "variant": 0, + "repeats": 64, + "gpu_seconds": 0.030596958473324776, + "warm_gpu_seconds": 0.030605708248913288 + }, + { + "round": 4, + "leg": 3, + "variant": 1, + "repeats": 64, + "gpu_seconds": 0.030772500205785036, + "warm_gpu_seconds": 0.030771125108003616 + } + ] + }, + "decision": "Do not promote standalone UP route. Joint gate/up must be measured as a complete graph against native dual projection; adding separate medians is not joint evidence.", + "limits": [ + "Conditional MPP/DD device proof remains open.", + "No fullmodel/HTTP/image or other-layer qualification.", + "Comparison is native row-pair operator, not MLX." + ], + "evidence_sha256": { + "raw_cost": "b0a05487404e7036f197d2ff4c391c6cfa1bb876a8cad8013c8347db404136a9", + "analysis": "98cf925e826a0c15a2d2949617125e78a60204dfceac785d425701f16f2a1cd2" + } +} diff --git a/docs/receipts/lfm3/m5-full-contract-tail.json b/docs/receipts/lfm3/m5-full-contract-tail.json new file mode 100644 index 0000000000..48847b7220 --- /dev/null +++ b/docs/receipts/lfm3/m5-full-contract-tail.json @@ -0,0 +1,24 @@ +{ + "scope": "Full M5 Metal contract runner, built from 65ae5f75d before the later development-default host/build change; not an exact-head or model-fidelity qualification.", + "source_revision": "65ae5f75d", + "hardware": "Apple M5 Ultra", + "command": "tf-kernel-contract --rotary-accuracy rotary.json", + "exit": 1, + "elapsed_seconds": 1393.7855796250515, + "binary_sha256": "ca7fd00d0e2d6c6c7ec62e5aa749e4352ad71c2fa6137caad5da791e654195b8", + "raw_log_sha256": "2406e57623a1277c27841a2ac8a106d322ca9b470a7079882561152d1e2d22fd", + "summary": "SUMMARY 321 kernels: 320 pass gates 1-4, 1 fail, 2 known findings; 325 controls: 325 red as planted, 0 missed; 0 limit problems; gates 1388992 ms, limits 2982 ms", + "production_failure": "FAIL shapes metal Qwen4 native HC act production: at 128x14336x1 output 218116 is -3.997803e-3 where float64 gives -4.028320e-3: 1.94 of its bound", + "baseline_note": "The same Qwen4 HC case was previously reproduced on pinned base b30d5b9d; see final-current-backend-m5.json.", + "limits_summary": "PASS limits: 364 registered Metal launches against M1-M5, 0 problems", + "lfm3_new_kernels": { + "residual_ffn": true, + "attention_full": true, + "attention_sliding": true + }, + "limitations": [ + "Runner binary predates 765538940 development-default host/build selection.", + "Attention sums fixture explicitly does not establish a reduced-accumulator stress witness.", + "Full backend exit 1; no global pass claimed." + ] +} diff --git a/docs/receipts/lfm3/m5-fused-swiglu-observation.json b/docs/receipts/lfm3/m5-fused-swiglu-observation.json new file mode 100644 index 0000000000..70b21816b3 --- /dev/null +++ b/docs/receipts/lfm3/m5-fused-swiglu-observation.json @@ -0,0 +1,50 @@ +{ + "scope": "Actual TEXT4 full7680-column gate+up+SwiGLU rows64, matched five-round operator screen, not model/serving qualification", + "modes": { + "public_dual_activation": { + "seconds": 0.0004813688137801364, + "first_last_drift": -5.411151046130769e-06, + "within_round_spreads": [ + 2.4335360861993394e-05, + 5.140975960693517e-05, + 2.569575760480447e-05, + 0.0006370785072513652, + 1.6233672745613603e-05 + ], + "stable": true + }, + "fused": { + "seconds": 0.00047985742276068777, + "first_last_drift": 0.00018995511613772997, + "within_round_spreads": [ + 9.091387751891666e-05, + 5.413057285741374e-06, + 0.0, + 0.0002469324900127301, + 0.00019672310616303257 + ], + "stable": true + } + }, + "stable": true, + "throughput_ratio": 1.0031496668546949, + "device": "Apple M5", + "observation": "Within this five-round operator run fused was0.314% faster in time. This tiny observation does not establish a material full-model gain; M4 samecase was1.1%slower. No default promotion.", + "numerical": { + "bounded_cases": 5, + "actual_rows": 64, + "full_output_columns": 7680, + "final_words": 491520, + "all_final_words_equal_public": true, + "gate_up_equal_frozen_canonical": true, + "negative_controls_actual_exit1": 3, + "five_repeat_identity": true, + "guards": 22 + }, + "provenance": { + "cost_analysis_sha256": "1634a410b850e4b62abd18ffff5c09fa06fd88d443763fba6a356a2273fc0860", + "numeric_receipt_sha256": "4a8efa351aa30d319bbeca0ca6a8fb6c6d01dead83e11c63e71c40b3287f0569", + "control_receipt_sha256": "03e35e0156f13950f228b5d4f31ab420c75cad6dd89bb9e80ff7f1b96e357eb9" + }, + "cleanup": "All owned numerical and cost stages completed, controls failed as intended, leases released." +} diff --git a/docs/receipts/lfm3/m5-joint-gate-up-model-served-rejected.json b/docs/receipts/lfm3/m5-joint-gate-up-model-served-rejected.json new file mode 100644 index 0000000000..3fe18d272f --- /dev/null +++ b/docs/receipts/lfm3/m5-joint-gate-up-model-served-rejected.json @@ -0,0 +1,273 @@ +{ + "schema": 1, + "status": "rejected-serving-regression", + "scope": "Private pinned TEXT4 layer0 joint gate/up, R128 prompt chunks, M5. Default off; no full-model FP64 fidelity qualification.", + "implementation": "One shared prepare; sequential gate and UP tiled producers and repairs reuse scratch; sticky inline failure checked before cache/logit publication.", + "binding": { + "scope": "Joint integrated firstgate/up binding and individual projectionprimary only; no nonlinear/fullmodelfidelity/timing claim", + "projections": { + "gate": { + "baseline": { + "rms": 0.0006554460990912766, + "max_abs": 0.007761517306789756, + "worst_column_rms": 0.007279313909585347 + }, + "candidate": { + "rms": 0.0006554460950282121, + "max_abs": 0.007761517306789756, + "worst_column_rms": 0.007279313909585347 + }, + "all_columns_no_worse": true, + "public_words_corrected": 125 + }, + "up": { + "baseline": { + "rms": 0.00036742454786637454, + "max_abs": 0.004916278412565589, + "worst_column_rms": 0.003449675882687866 + }, + "candidate": { + "rms": 0.00036742454771191796, + "max_abs": 0.004916278412565589, + "worst_column_rms": 0.003449675882687866 + }, + "all_columns_no_worse": true, + "public_words_corrected": 16 + } + }, + "checks": { + "baseline_norm": true, + "candidate_norm": true, + "baseline_gate": true, + "candidate_gate": true, + "baseline_up": true, + "candidate_up": true + } + }, + "model_observations": { + "128": { + "analysis": { + "integration_checks_pass": true, + "scope": "downstream observation only; no full-model FP64 no-worse claim", + "tokens_equal": true, + "candidate_completed": 1, + "buffers": { + "state": { + "words": 1601536, + "different_words": 1316127, + "rms_difference": 0.01117040315198572, + "max_abs_difference": 0.34375, + "baseline_sha256": "203d561363fca5dd17ff1b1febda88b21a9641620ded5af9b27193c09526a10b", + "candidate_sha256": "eac3f4413b16c06520d9c086f38ecb7fbd241e43b847efd2d4d386cbdbde0ffb" + }, + "logits": { + "words": 128000, + "different_words": 113854, + "rms_difference": 0.04541529547282464, + "max_abs_difference": 0.203125, + "baseline_sha256": "0a4eb9bdec658d09c27da35983f67b796b9ec488f53e718e1f0ce080ce76497e", + "candidate_sha256": "0a61af3a09d2b2987ea44378651920d0904f0f23534e007411b2ad8f21056514" + } + } + }, + "ineligible_hits": 0, + "scan_clean_and_last_flag_pass": true, + "poison_checks_pass": true, + "serialization_check_pass": true, + "observation_sha256": "bb37c67a204a6e400ef264ec85af7b4b7d1bd98ad1ce5e670692234e2d7c11f2" + }, + "2048": { + "analysis": { + "integration_checks_pass": true, + "scope": "downstream observation only; no full-model FP64 no-worse claim", + "tokens_equal": true, + "candidate_completed": 16, + "buffers": { + "state": { + "words": 23228416, + "different_words": 19721571, + "rms_difference": 0.015290566636436524, + "max_abs_difference": 0.9609375, + "baseline_sha256": "7f7e23060107d5252391778f46727022f8ee157fa34eb1753c424cac2f3f771f", + "candidate_sha256": "c7ea4ab6ce9f69942022db28ffe28bad048a836c59febf949b8ba50d677c3a1f" + }, + "logits": { + "words": 128000, + "different_words": 120785, + "rms_difference": 0.09709622568538484, + "max_abs_difference": 0.59375, + "baseline_sha256": "8700e9a12456dce4f744caa329ad2013f44df7f2704e58e245e52aa0e9e9ddc5", + "candidate_sha256": "f89f839965ed09bf664585528cb387528f6006d5e5a3eea3ea8b961dec96a702" + } + } + }, + "ineligible_hits": 0, + "scan_clean_and_last_flag_pass": true, + "poison_checks_pass": true, + "serialization_check_pass": true, + "observation_sha256": "6a680ccd17f51cb2ea56ecb87d7ef15a938651e702c672f3dda64e242c41f9e8" + }, + "8192": { + "analysis": { + "integration_checks_pass": true, + "scope": "downstream observation only; no full-model FP64 no-worse claim", + "tokens_equal": true, + "candidate_completed": 64, + "buffers": { + "state": { + "words": 92434432, + "different_words": 78317322, + "rms_difference": 0.015156337250765232, + "max_abs_difference": 0.9609375, + "baseline_sha256": "5524361e72b466c99d75fbf958f1faa0c715920b3456f01532907c47a76cb27b", + "candidate_sha256": "878021016a5addcb47e6da52e573bd3252aa0f20f3f22c7a9c9a712731873d00" + }, + "logits": { + "words": 128000, + "different_words": 123543, + "rms_difference": 0.12498101481535923, + "max_abs_difference": 0.650390625, + "baseline_sha256": "19a2f9b7b0794ef2f2be0edb7d16564e24d33f02e0d01972780ae28d09010bf5", + "candidate_sha256": "45dcff1c51646d0ecb33f3a281c8099a4c4eeb0c8443947df70ea2f56db861a2" + } + } + }, + "ineligible_hits": 0, + "scan_clean_and_last_flag_pass": true, + "poison_checks_pass": true, + "serialization_check_pass": true, + "observation_sha256": "3f979d1cb5d08f59b7398e481b1ddff4dc8432b22fd86c7ecd505654f52d5276" + } + }, + "serving": { + "protocol": "Fresh-server public/candidate/candidate/public ABBA; exact pinned prompt IDs; cache disabled; 16 output tokens; no hidden retries; 5% baseline drift gate.", + "cases": [ + { + "prompt_tokens": 2048, + "baseline_ttft_s": [ + 0.8587004579603672, + 0.8591408748179674 + ], + "candidate_ttft_s": [ + 0.8685380830429494, + 0.8663557921536267 + ], + "baseline_median_s": 0.8589206663891673, + "candidate_median_s": 0.8674469375982881, + "baseline_over_candidate": 0.9901708440717681, + "baseline_first_last_drift": 0.000512887647278415, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "not emitted by cold client; consult separate joint-model observation; full-model fidelity remains open", + "prompt_ids": "both routes /tokenize exact pinned arrays" + }, + { + "prompt_tokens": 8192, + "baseline_ttft_s": [ + 4.616734708659351, + 4.616750834044069 + ], + "candidate_ttft_s": [ + 4.646457958035171, + 4.642253666184843 + ], + "baseline_median_s": 4.61674277135171, + "candidate_median_s": 4.644355812110007, + "baseline_over_candidate": 0.9940544949880247, + "baseline_first_last_drift": 3.4928116376953255e-06, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "not emitted by cold client; consult separate joint-model observation; full-model fidelity remains open", + "prompt_ids": "both routes /tokenize exact pinned arrays" + } + ], + "candidate_hits_per_leg": { + "2048": 16, + "8192": 64 + }, + "public_hits_per_leg": 0, + "owned_servers_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "lease_released": true, + "interpretation": "Standalone complete-graph 3.575% lower GPU time did not improve serving. No promotion. TTFT is measured, not isolated prefill GPU time; no new decode/ITL claim." + }, + "provenance": { + "model_manifest_sha256": "ffed1d112542b17e6cde3f0a38350090b1155203545ceb4ff6d7485e35eadf2f", + "remote_model_manifest_sha256": "6e8345364daa90e66d26956694af69a32f9e96206388ff4437c7d6d05811715f", + "served_pins_sha256": "f71a6b43fb10f13670e33226b24b494c148f9a93466e3c93e4c6037d12e53eed", + "remote_served_pins_sha256": "4355539b0abefc9c47e4d27b13b382661d4ee7896b984ca3ee35718239cb148b", + "baseline_binary_sha256": "a5c7ce0c302c2369916e653c84c11f08c965e56d3364ab097c9ed8beb3a3fef3", + "candidate_binary_sha256": "020d8174ccd3e6168a01410a1e43dec39965ce070e3985c7a930c92eb40bfbf4", + "served_summary_sha256": "51b62c08346386b9d27048b3ec684639c654932becb4106cb13d5e48f798fb3f", + "independent_local_analyzer_and_raw_hash_verification": true + }, + "raw_sha256": { + "text-4-0-public.cold-2048.json": "1e0c1aefcfdf72a2bb14942b9f3bac0183f3fb7adc9662e5d5538a98c0883537", + "text-4-1-candidate.cold-8192.client.log": "a2e7562d826df69cac54040bb1c4aff888e680aac865fb9960c768bb62a67ef8", + "text-4-1-candidate.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-0-public.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "text-4-0-public.after.metrics": "5c956284f785cb4613349f8821b380747142d0d103d51427c0ef2d89477677fe", + "text-4-3-public.cold-8192.client.log": "007de4e0eeae458e0cc1bcf9c962e36375fe10c3ef4171ee501df4eafb3a5f61", + "text-4-3-public.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "text-4-1-candidate.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "text-4-1-candidate.after.metrics": "aa587d91bbdba0c39e0a40bab1aeb1f49495458331ce985c65dc381200b1e969", + "text-4-0-public.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "text-4-1-candidate.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "text-4-1-candidate.before.metrics": "f9474ca9a7b603f2f589902dde4c2037daa6f66cf46d95622be6039467b84f0e", + "text-4-0-public.cold-8192.json": "b6812869c0e8d39fd49253390ce8086aaeb6731c11409fab83baa991188f158b", + "events.json": "8c2f27541a14f32e11e7eea4538ca99012156603ce6d030b68f78a3449a0488c", + "text-4-3-public.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "text-4-0-public.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "text-4-1-candidate.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "text-4-0-public.cold-2048.client.log": "994f4b5484beed502a9008a934c5337e2dc8fcc433c28f9856beacde93553c35", + "text-4-1-candidate.cold-2048.json": "6cbcb2f82d55ccf27ef88ff6606e0d48daf99641a2661f70d63c49542180d121", + "text-4-2-candidate.after.metrics": "cea09bc04bd17f98bc14e8ae629d071623e6a5fc6e280077265b3de9014ffb28", + "text-4-2-candidate.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "text-4-2-candidate.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "text-4-3-public.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-3-public.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "text-4-0-public.before.metrics": "d9035deceb9c059ec781b7ea20548d05ed23cccefdd30b1f405a8edfab787e3d", + "text-4-1-candidate.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "text-4-2-candidate.server.log": "6153271eae942c8191e813b3ada570381d6b90359f98dada6efdeb293f32c0d1", + "text-4-2-candidate.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-2-candidate.cold-2048.client.log": "6dcf5bd92f6326d9e962e4de1122692704866a552e11246d7c00e8a8677ad180", + "text-4-1-candidate.cold-8192.json": "1ec60f0fc2c8a71bcf82cdec37e15d97290e71d5f2897a304a3aa1176fa2fc88", + "text-4-0-public.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "text-4-0-public.after.health": "7a1a671cc130e55cc7caaa09232e5bbe9646fa9a7248d6042e2ef104728b6b17", + "text-4-1-candidate.after.health": "f7f177aacdf9f2ab68d8c33eb701819d22e479fe83400b93d0de384d302bd0a2", + "text-4-0-public.server.log": "4f5536625dd64f13eefabe2ddf537f5b6ddad4fbd0645687ad2bda424717f2b6", + "text-4-3-public.cold-8192.json": "36f8b051740fcfc86f6021571699effe35ba57bae79489983d70010dc489b600", + "text-4-0-public.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-1-candidate.server.log": "5eb09201f2fa7ea025ae31104019e4890df26d2b415b7f4baba6b2c019f84358", + "text-4-3-public.after.health": "7a1a671cc130e55cc7caaa09232e5bbe9646fa9a7248d6042e2ef104728b6b17", + "text-4-2-candidate.before.metrics": "83d60c3c180319ec940ec69b97ddff683ff8b83084d4b889b9d7d7bcfd4733fe", + "text-4-2-candidate.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "text-4-1-candidate.cold-2048.client.log": "3f008684bbf16f2067f8cdbf74e72f85e9b8b7287e0682f55e5fb94ded1dc64c", + "manifest.json": "5d293087b161c9d5d8f5eeb740061de9fd1ba801651b7aa3f3e668728f65e2a7", + "text-4-2-candidate.after.health": "35eefb4783a26e99e546ddbc35081c4648b4ab9cb76ad125f3955bf4ec3f8532", + "text-4-3-public.cold-2048.client.log": "ee26de02197a999ed70a348640af7aa3172c94480cdbf399087920aaa0de7915", + "text-4-2-candidate.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "text-4-3-public.cold-2048.json": "e6859112274e12e214b9257bf73a2e2b940f6413e5ec2278ad02a4b1a0fb9f99", + "records.json": "f7d6888000811d2ef4fb602a9046047c08d3e8b67754f8a18836914f3bda25d8", + "text-4-2-candidate.cold-8192.json": "2beb29a21d6c1d122ec6453dedcaa411b7ec96a459d82e12a4fdf1696659ae90", + "text-4-2-candidate.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-1-candidate.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "text-4-2-candidate.cold-8192.client.log": "e05974231c26b4c9931054a37d9ba71e9104aa9822a1fe579306b5e35d57e542", + "text-4-3-public.before.metrics": "31821c4aeb4540b13a03146b4875fec48f1a6a5bb81c0fc80ff9fbfd5ebcf5aa", + "text-4-0-public.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "text-4-0-public.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-0-public.cold-8192.client.log": "a84d9197964da7cca27a5cdf0b76687b661af0c79b52488b58a0e8696ab2fdf4", + "text-4-3-public.server.log": "cd17931d157f619515b205da1fe26f2dcae49150d2e91ebadfd295081eda7fe7", + "text-4-1-candidate.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-3-public.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-3-public.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "text-4-3-public.after.metrics": "703750e8f18472592a4b7fe180db7d5612e09dc35e0eb6e59e41db4b31f34d1e", + "text-4-2-candidate.cold-2048.json": "dedce1dcea5ddceef847d9b1f805169a7cc9d55bc3314ca334c61aafcd08be0f", + "text-4-3-public.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "text-4-2-candidate.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd" + } +} diff --git a/docs/receipts/lfm3/m5-joint-gate-up-shared-prepare.json b/docs/receipts/lfm3/m5-joint-gate-up-shared-prepare.json new file mode 100644 index 0000000000..8674321e82 --- /dev/null +++ b/docs/receipts/lfm3/m5-joint-gate-up-shared-prepare.json @@ -0,0 +1,139 @@ +{ + "scope": "M5 genuine TEXT4 layer00 R128 full7680 gate+up+BF16activation standalone completegraph. No model integration, serving win, or nonlinear/full-model FP64 fidelity claim.", + "fixture": { + "checkpoint": "LiquidAI/LFM3-1.7B-MLX-4bit", + "rows": 128, + "k": 2048, + "n": 7680, + "group": 64, + "bits": 4, + "source": "Genuine contiguous128-row layer00 normalized input; no duplicated64 rows" + }, + "candidate": "One prepare reused by sequential gate and up producers/inline-error repairs; same scratch reused under encoder barriers; original BF16activation kernel.", + "dispatches": { + "public_dual_plus_activation": 2, + "candidate_shared_prepare_gate_up_plus_activation": 6 + }, + "persistent_candidate_scratch_bytes": 71892996, + "numerical": { + "five_repeat_full_outputs": true, + "words_each_projection": 983040, + "public_three_outputs_match_native_capture": true, + "candidate_gate_up_match_independently_primary_qualified_words": true, + "candidate_activation_reference": "Unchanged production activation kernel on frozen corrected gate/up; not independent nonlinear FP64 oracle", + "input_output_guards": 26, + "internal_scratch_guard_padded": false, + "unresolved_error_checked": true, + "wrong_up_and_omitted_activation_controls": "Both actual exit1 CanonicalMismatch" + }, + "protocol": { + "balanced_rounds": 5, + "legs_per_round": 4, + "common_repetitions": true, + "minimum_calibration_seconds": 0.025, + "minimum_recorded_gpu_seconds": 0.02, + "maximum_recorded_gpu_seconds": 1, + "five_correctness_passes_and_calibration_warm_gpu": true, + "explicit_per_leg_warmup": false, + "host_error_check": "Once per repeated timed batch; not per operator", + "no_hidden_retries": true + }, + "cost": { + "scope": "Public dual+activation versus shared-prepare certified gate+up+activation; fixedR128TEXT4; no model fidelity or serving claim", + "complete_graph": true, + "candidate_scratch_bytes": 71892996, + "rounds": [ + { + "False": { + "median": 0.0009558476522215642, + "spread": 4.0877529885241515e-05 + }, + "True": { + "median": 0.0009216764301527292, + "spread": 7.073286096170861e-06 + } + }, + { + "False": { + "median": 0.0009558418023516424, + "spread": 4.09531493850146e-06 + }, + "True": { + "median": 0.0009218072955263779, + "spread": 0.0018606278883588345 + } + }, + { + "False": { + "median": 0.0009558587189530954, + "spread": 6.820338743329657e-06 + }, + "True": { + "median": 0.0009217154947691597, + "spread": 0.0009851264653157887 + } + }, + { + "False": { + "median": 0.000955830080783926, + "spread": 4.080140723505821e-06 + }, + "True": { + "median": 0.0009215664103976451, + "spread": 0.0003589331458864198 + } + }, + { + "False": { + "median": 0.0009558821693644859, + "spread": 9.126931718150821e-05 + }, + "True": { + "median": 0.0009214635356329381, + "spread": 0.0005597366075464905 + } + } + ], + "limits": { + "round_spread": 0.05, + "round_median_spread": 0.05, + "first_last_drift": 0.05 + }, + "baseline": { + "seconds_per_op": 0.0009558476522215642, + "first_last_drift": 3.6111552757844834e-05, + "median_spread": 5.4495648972752164e-05, + "max_round_spread": 9.126931718150821e-05 + }, + "candidate": { + "seconds_per_op": 0.0009216764301527292, + "first_last_drift": -0.00023098618216343958, + "median_spread": 0.00037305859661995733, + "max_round_spread": 0.0018606278883588345 + }, + "candidate_over_public_throughput": 1.037075074235296, + "stable": true, + "raw_sha256": "0be31dab07b41e5cc36695be2918fb05b53cd53b7c978f570e66ee83091e72f5" + }, + "gpu_latency_reduction_fraction": 0.03574965318941248, + "limitations": [ + "Prior gate-only model integration regressed TTFT despite a standalone projection gain. Joint operator gain does not establish a serving gain.", + "Joint gate-only sticky-error control passed; private full-model integration still requires actual binding, publication/poison and downstream observation before serving timing.", + "Full-model FP64no-worse remains open; prior model observation matched16tokens but had substantial logits/state differences." + ], + "raw_receipt_sha256": { + "positive": "b75de4b91ae664d98299b062f91e1caa184d0e920c7b12b6572ecf9029bb800b", + "controls": "dfbe06d45513e84c18dab9be0a2b7c5c84fb78f05e4921a7f2cd4d5e42fb5f5b", + "cost": "761d49511ebd89f630ced5d2ed44e66c30be9e8f23e6c87e6d26cbf2a1261376" + }, + "independent_verification": "All raw log hashes, actual exits, final lease release and frozen costanalyzer verified locally.", + "joint_sticky_error_control": { + "positive": "Five repetitions inject actual final repairflag2 in gate only; up succeeds and all its final flags are zero, all outputs canonical, shared failure remains1 and check rejects.", + "negative": "Planted error reset in UP actualexit1 MissingStickyFailure", + "receipt_sha256": "77ec655d82c1a8415ed14d5e030dfe21ea3fbd01935f9798787ef924e2810e4e", + "events_sha256": "aaed52a6cbc4ca558d47d1ebfae800211492e363ca9df64beb7ff0166809e865", + "positive_log_sha256": "d12c3e65d67abc17e4de9d204a97f3bf2862510f8771191d92d5b305a07d6903", + "negative_log_sha256": "732fca853c6f7e81872b3e25d36efb49925b5763d7e302861277734fcb645ec2", + "independent_actual_exits_log_hashes_and_lease_release_verified": true + } +} diff --git a/docs/receipts/lfm3/m5-nax-32k-fidelity.json b/docs/receipts/lfm3/m5-nax-32k-fidelity.json new file mode 100644 index 0000000000..c7180883df --- /dev/null +++ b/docs/receipts/lfm3/m5-nax-32k-fidelity.json @@ -0,0 +1,447 @@ +{ + "prompt_tokens": 32768, + "continuation_rows": 16, + "device": "Apple M5 Ultra", + "text": "real prose, one fixed token sequence for all widths", + "reference": "independent FP32 decoder and official MLX BF16, both fed the same tokens", + "routes": { + "nax_tiled_prompt": { + "rows": 102, + "pooled_rms_native": 0.032296981555119966, + "pooled_rms_official_bf16": 0.03373134369347903, + "pooled_ratio": 0.9574768751759998, + "rows_above_1p5x_official": 6, + "per_model": { + "LFM3-1.7B-MLX-4bit": { + "prompt_boundary": { + "rows": 1, + "native_rms": 0.024311248185334228, + "official_bf16_rms": 0.026831738891510262, + "native_max": 0.14638233184814453, + "official_bf16_max": 0.15340590476989746, + "native_top1_misses": 0, + "official_bf16_top1_misses": 0, + "proposed_metrics_not_met": [ + "top20_probability_mean", + "top20_probability_max" + ], + "top1_differences": [] + }, + "teacher_forced_continuation": { + "rows": 16, + "native_rms": 0.03531880628829018, + "official_bf16_rms": 0.03997996631472334, + "native_max": 0.28801822662353516, + "official_bf16_max": 0.3483619689941406, + "native_top1_misses": 0, + "official_bf16_top1_misses": 0, + "proposed_metrics_not_met": [], + "top1_differences": [] + } + }, + "LFM3-1.7B-MLX-6bit": { + "prompt_boundary": { + "rows": 1, + "native_rms": 0.019839066560119717, + "official_bf16_rms": 0.024457985529238054, + "native_max": 0.14065933227539062, + "official_bf16_max": 0.19359827041625977, + "native_top1_misses": 0, + "official_bf16_top1_misses": 0, + "proposed_metrics_not_met": [], + "top1_differences": [] + }, + "teacher_forced_continuation": { + "rows": 16, + "native_rms": 0.029129037699508874, + "official_bf16_rms": 0.03326967166162887, + "native_max": 0.24056923389434814, + "official_bf16_max": 0.2304544448852539, + "native_top1_misses": 0, + "official_bf16_top1_misses": 0, + "proposed_metrics_not_met": [ + "maximum_absolute_error", + "top20_probability_mean", + "top20_probability_max" + ], + "top1_differences": [] + } + }, + "LFM3-1.7B-MLX-8bit": { + "prompt_boundary": { + "rows": 1, + "native_rms": 0.021103950548045463, + "official_bf16_rms": 0.019152764769749354, + "native_max": 0.13100147247314453, + "official_bf16_max": 0.11677169799804688, + "native_top1_misses": 0, + "official_bf16_top1_misses": 0, + "proposed_metrics_not_met": [ + "rms_error", + "maximum_absolute_error", + "top20_probability_mean", + "top20_probability_max" + ], + "top1_differences": [] + }, + "teacher_forced_continuation": { + "rows": 16, + "native_rms": 0.031197310500633017, + "official_bf16_rms": 0.03113037185178176, + "native_max": 0.2583197355270386, + "official_bf16_max": 0.23815441131591797, + "native_top1_misses": 0, + "official_bf16_top1_misses": 0, + "proposed_metrics_not_met": [ + "rms_error", + "maximum_absolute_error", + "top20_probability_mean", + "top20_probability_max" + ], + "top1_differences": [] + } + }, + "LFM3-VL-2B-MLX-4bit": { + "prompt_boundary": { + "rows": 1, + "native_rms": 0.03887785904676225, + "official_bf16_rms": 0.025605707372078373, + "native_max": 0.13653945922851562, + "official_bf16_max": 0.13260555267333984, + "native_top1_misses": 1, + "official_bf16_top1_misses": 1, + "proposed_metrics_not_met": [ + "rms_error", + "maximum_absolute_error" + ], + "top1_differences": [ + { + "row": 0, + "native_top": 654, + "reference_bf16_top": 654, + "fp32_top": 1131, + "native_maximum_set": [ + 654, + 1131 + ], + "reference_maximum_set": [ + 654, + 1131 + ], + "fp32_maximum_set": [ + 1131 + ], + "native_margin": 0.0, + "reference_bf16_margin": 0.0, + "fp32_margin": 0.08532047271728516, + "fp32_native_contender_gap": 0.08532047271728516, + "fp32_reference_contender_gap": 0.08532047271728516, + "native_only_miss": false + } + ] + }, + "teacher_forced_continuation": { + "rows": 16, + "native_rms": 0.03666612632209707, + "official_bf16_rms": 0.0348104796607303, + "native_max": 0.19969844818115234, + "official_bf16_max": 0.19331121444702148, + "native_top1_misses": 0, + "official_bf16_top1_misses": 0, + "proposed_metrics_not_met": [ + "rms_error", + "maximum_absolute_error", + "top20_probability_max" + ], + "top1_differences": [] + } + }, + "LFM3-VL-2B-MLX-6bit": { + "prompt_boundary": { + "rows": 1, + "native_rms": 0.023632405489822744, + "official_bf16_rms": 0.03372439317436025, + "native_max": 0.10165882110595703, + "official_bf16_max": 0.14469528198242188, + "native_top1_misses": 0, + "official_bf16_top1_misses": 0, + "proposed_metrics_not_met": [], + "top1_differences": [] + }, + "teacher_forced_continuation": { + "rows": 16, + "native_rms": 0.03433726153643044, + "official_bf16_rms": 0.030513009797015433, + "native_max": 0.22870779037475586, + "official_bf16_max": 0.2049427032470703, + "native_top1_misses": 1, + "official_bf16_top1_misses": 0, + "proposed_metrics_not_met": [ + "rms_error", + "maximum_absolute_error", + "top20_probability_mean", + "top20_probability_max" + ], + "top1_differences": [ + { + "row": 0, + "native_top": 589, + "reference_bf16_top": 58884, + "fp32_top": 58884, + "native_maximum_set": [ + 589, + 58884 + ], + "reference_maximum_set": [ + 58884 + ], + "fp32_maximum_set": [ + 58884 + ], + "native_margin": 0.0, + "reference_bf16_margin": 0.125, + "fp32_margin": 0.13377094268798828, + "fp32_native_contender_gap": 0.13377094268798828, + "fp32_reference_contender_gap": 0.0, + "native_only_miss": true + } + ] + } + }, + "LFM3-VL-2B-MLX-8bit": { + "prompt_boundary": { + "rows": 1, + "native_rms": 0.026211339611638483, + "official_bf16_rms": 0.03482085233001276, + "native_max": 0.10755634307861328, + "official_bf16_max": 0.1343708038330078, + "native_top1_misses": 0, + "official_bf16_top1_misses": 0, + "proposed_metrics_not_met": [], + "top1_differences": [] + }, + "teacher_forced_continuation": { + "rows": 16, + "native_rms": 0.02820281649131737, + "official_bf16_rms": 0.03381069201792409, + "native_max": 0.1860060691833496, + "official_bf16_max": 0.2198195457458496, + "native_top1_misses": 0, + "official_bf16_top1_misses": 0, + "proposed_metrics_not_met": [ + "top20_probability_mean", + "top20_probability_max" + ], + "top1_differences": [] + } + } + } + }, + "baseline_prompt": { + "rows": 102, + "pooled_rms_native": 0.03167307407601195, + "pooled_rms_official_bf16": 0.03373134369347903, + "pooled_ratio": 0.9389805032325176, + "rows_above_1p5x_official": 9, + "per_model": { + "LFM3-1.7B-MLX-4bit": { + "prompt_boundary": { + "rows": 1, + "native_rms": 0.028130729673664197, + "official_bf16_rms": 0.026831738891510262, + "native_max": 0.13915395736694336, + "official_bf16_max": 0.15340590476989746, + "native_top1_misses": 0, + "official_bf16_top1_misses": 0, + "proposed_metrics_not_met": [ + "rms_error", + "top20_probability_mean", + "top20_probability_max" + ], + "top1_differences": [] + }, + "teacher_forced_continuation": { + "rows": 16, + "native_rms": 0.031360554044810064, + "official_bf16_rms": 0.03997996631472334, + "native_max": 0.22579622268676758, + "official_bf16_max": 0.3483619689941406, + "native_top1_misses": 0, + "official_bf16_top1_misses": 0, + "proposed_metrics_not_met": [], + "top1_differences": [] + } + }, + "LFM3-1.7B-MLX-6bit": { + "prompt_boundary": { + "rows": 1, + "native_rms": 0.02278565423070546, + "official_bf16_rms": 0.024457985529238054, + "native_max": 0.15340805053710938, + "official_bf16_max": 0.19359827041625977, + "native_top1_misses": 0, + "official_bf16_top1_misses": 0, + "proposed_metrics_not_met": [], + "top1_differences": [] + }, + "teacher_forced_continuation": { + "rows": 16, + "native_rms": 0.03099310502574918, + "official_bf16_rms": 0.03326967166162887, + "native_max": 0.24349021911621094, + "official_bf16_max": 0.2304544448852539, + "native_top1_misses": 0, + "official_bf16_top1_misses": 0, + "proposed_metrics_not_met": [ + "maximum_absolute_error", + "top20_probability_mean", + "top20_probability_max" + ], + "top1_differences": [] + } + }, + "LFM3-1.7B-MLX-8bit": { + "prompt_boundary": { + "rows": 1, + "native_rms": 0.019917083098697524, + "official_bf16_rms": 0.019152764769749354, + "native_max": 0.14476776123046875, + "official_bf16_max": 0.11677169799804688, + "native_top1_misses": 0, + "official_bf16_top1_misses": 0, + "proposed_metrics_not_met": [ + "rms_error", + "maximum_absolute_error", + "top20_probability_mean", + "top20_probability_max" + ], + "top1_differences": [] + }, + "teacher_forced_continuation": { + "rows": 16, + "native_rms": 0.03594687433707124, + "official_bf16_rms": 0.03113037185178176, + "native_max": 0.2350451946258545, + "official_bf16_max": 0.23815441131591797, + "native_top1_misses": 0, + "official_bf16_top1_misses": 0, + "proposed_metrics_not_met": [ + "rms_error", + "top20_probability_mean", + "top20_probability_max" + ], + "top1_differences": [] + } + }, + "LFM3-VL-2B-MLX-4bit": { + "prompt_boundary": { + "rows": 1, + "native_rms": 0.040038664624215764, + "official_bf16_rms": 0.025605707372078373, + "native_max": 0.13653945922851562, + "official_bf16_max": 0.13260555267333984, + "native_top1_misses": 0, + "official_bf16_top1_misses": 1, + "proposed_metrics_not_met": [ + "rms_error", + "maximum_absolute_error" + ], + "top1_differences": [ + { + "row": 0, + "native_top": 1131, + "reference_bf16_top": 654, + "fp32_top": 1131, + "native_maximum_set": [ + 1131 + ], + "reference_maximum_set": [ + 654, + 1131 + ], + "fp32_maximum_set": [ + 1131 + ], + "native_margin": 0.0625, + "reference_bf16_margin": 0.0, + "fp32_margin": 0.08532047271728516, + "fp32_native_contender_gap": 0.0, + "fp32_reference_contender_gap": 0.08532047271728516, + "native_only_miss": false + } + ] + }, + "teacher_forced_continuation": { + "rows": 16, + "native_rms": 0.030018120393483384, + "official_bf16_rms": 0.0348104796607303, + "native_max": 0.18657350540161133, + "official_bf16_max": 0.19331121444702148, + "native_top1_misses": 0, + "official_bf16_top1_misses": 0, + "proposed_metrics_not_met": [ + "top20_probability_max" + ], + "top1_differences": [] + } + }, + "LFM3-VL-2B-MLX-6bit": { + "prompt_boundary": { + "rows": 1, + "native_rms": 0.042030211884171775, + "official_bf16_rms": 0.03372439317436025, + "native_max": 0.14401626586914062, + "official_bf16_max": 0.14469528198242188, + "native_top1_misses": 0, + "official_bf16_top1_misses": 0, + "proposed_metrics_not_met": [ + "rms_error" + ], + "top1_differences": [] + }, + "teacher_forced_continuation": { + "rows": 16, + "native_rms": 0.026109912298379927, + "official_bf16_rms": 0.030513009797015433, + "native_max": 0.21293854713439941, + "official_bf16_max": 0.2049427032470703, + "native_top1_misses": 0, + "official_bf16_top1_misses": 0, + "proposed_metrics_not_met": [ + "maximum_absolute_error", + "top20_probability_mean" + ], + "top1_differences": [] + } + }, + "LFM3-VL-2B-MLX-8bit": { + "prompt_boundary": { + "rows": 1, + "native_rms": 0.06112402311211287, + "official_bf16_rms": 0.03482085233001276, + "native_max": 0.15047073364257812, + "official_bf16_max": 0.1343708038330078, + "native_top1_misses": 0, + "official_bf16_top1_misses": 0, + "proposed_metrics_not_met": [ + "rms_error", + "maximum_absolute_error" + ], + "top1_differences": [] + }, + "teacher_forced_continuation": { + "rows": 16, + "native_rms": 0.03202239404238794, + "official_bf16_rms": 0.03381069201792409, + "native_max": 0.2106175422668457, + "official_bf16_max": 0.2198195457458496, + "native_top1_misses": 0, + "official_bf16_top1_misses": 0, + "proposed_metrics_not_met": [], + "top1_differences": [] + } + } + } + } + } +} diff --git a/docs/receipts/lfm3/m5-nax-cold-prefill-vs-mlx.json b/docs/receipts/lfm3/m5-nax-cold-prefill-vs-mlx.json new file mode 100644 index 0000000000..52eb58087c --- /dev/null +++ b/docs/receipts/lfm3/m5-nax-cold-prefill-vs-mlx.json @@ -0,0 +1,319 @@ +{ + "description": "Cold prefill, client time to first token, one request at a time, prompt cache disabled, tensor-unit route versus mlx-vlm on the same machine and checkpoints. Medians of 2 runs after a warm-up, with the final kernels (BF16 projections with windowed scales and biases, whole-segment tiled prompt attention, key-parallel decode attention). Output hashes of the native route match the previous dense tile byte for byte.", + "machine": "M5 Ultra", + "server_flags": "--prompt-cache-gib 0", + "rows": [ + { + "checkpoint": "text4", + "prompt_tokens": 2048, + "native_ttft_s": 0.106, + "mlx_ttft_s": 0.1156, + "native_over_mlx": 0.917, + "native_prefill_tok_s": 19319, + "mlx_prefill_tok_s": 17714, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "text4", + "prompt_tokens": 8192, + "native_ttft_s": 0.3626, + "mlx_ttft_s": 0.3494, + "native_over_mlx": 1.038, + "native_prefill_tok_s": 22593, + "mlx_prefill_tok_s": 23445, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "text4", + "prompt_tokens": 32768, + "native_ttft_s": 1.6494, + "mlx_ttft_s": 1.6246, + "native_over_mlx": 1.015, + "native_prefill_tok_s": 19867, + "mlx_prefill_tok_s": 20169, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "text4", + "prompt_tokens": 65536, + "native_ttft_s": 3.9508, + "mlx_ttft_s": 3.967, + "native_over_mlx": 0.996, + "native_prefill_tok_s": 16588, + "mlx_prefill_tok_s": 16520, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "text6", + "prompt_tokens": 2048, + "native_ttft_s": 0.11, + "mlx_ttft_s": 0.1233, + "native_over_mlx": 0.892, + "native_prefill_tok_s": 18618, + "mlx_prefill_tok_s": 16607, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "text6", + "prompt_tokens": 8192, + "native_ttft_s": 0.3768, + "mlx_ttft_s": 0.376, + "native_over_mlx": 1.002, + "native_prefill_tok_s": 21742, + "mlx_prefill_tok_s": 21787, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "text6", + "prompt_tokens": 32768, + "native_ttft_s": 1.7048, + "mlx_ttft_s": 1.7188, + "native_over_mlx": 0.992, + "native_prefill_tok_s": 19221, + "mlx_prefill_tok_s": 19065, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "text6", + "prompt_tokens": 65536, + "native_ttft_s": 4.079, + "mlx_ttft_s": 4.1614, + "native_over_mlx": 0.98, + "native_prefill_tok_s": 16067, + "mlx_prefill_tok_s": 15748, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "text8", + "prompt_tokens": 2048, + "native_ttft_s": 0.1113, + "mlx_ttft_s": 0.1212, + "native_over_mlx": 0.918, + "native_prefill_tok_s": 18396, + "mlx_prefill_tok_s": 16897, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "text8", + "prompt_tokens": 8192, + "native_ttft_s": 0.382, + "mlx_ttft_s": 0.3649, + "native_over_mlx": 1.047, + "native_prefill_tok_s": 21445, + "mlx_prefill_tok_s": 22449, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "text8", + "prompt_tokens": 32768, + "native_ttft_s": 1.7115, + "mlx_ttft_s": 1.6813, + "native_over_mlx": 1.018, + "native_prefill_tok_s": 19145, + "mlx_prefill_tok_s": 19489, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "text8", + "prompt_tokens": 65536, + "native_ttft_s": 4.0766, + "mlx_ttft_s": 4.0866, + "native_over_mlx": 0.998, + "native_prefill_tok_s": 16076, + "mlx_prefill_tok_s": 16037, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "vl4", + "prompt_tokens": 2048, + "native_ttft_s": 0.1094, + "mlx_ttft_s": 0.1241, + "native_over_mlx": 0.881, + "native_prefill_tok_s": 18728, + "mlx_prefill_tok_s": 16504, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "vl4", + "prompt_tokens": 8192, + "native_ttft_s": 0.3766, + "mlx_ttft_s": 0.3556, + "native_over_mlx": 1.059, + "native_prefill_tok_s": 21750, + "mlx_prefill_tok_s": 23038, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "vl4", + "prompt_tokens": 32768, + "native_ttft_s": 1.6871, + "mlx_ttft_s": 1.646, + "native_over_mlx": 1.025, + "native_prefill_tok_s": 19422, + "mlx_prefill_tok_s": 19908, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "vl4", + "prompt_tokens": 65536, + "native_ttft_s": 4.0321, + "mlx_ttft_s": 4.0183, + "native_over_mlx": 1.003, + "native_prefill_tok_s": 16254, + "mlx_prefill_tok_s": 16309, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "vl6", + "prompt_tokens": 2048, + "native_ttft_s": 0.11, + "mlx_ttft_s": 0.1303, + "native_over_mlx": 0.844, + "native_prefill_tok_s": 18626, + "mlx_prefill_tok_s": 15721, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "vl6", + "prompt_tokens": 8192, + "native_ttft_s": 0.3772, + "mlx_ttft_s": 0.3764, + "native_over_mlx": 1.002, + "native_prefill_tok_s": 21720, + "mlx_prefill_tok_s": 21766, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "vl6", + "prompt_tokens": 32768, + "native_ttft_s": 1.7048, + "mlx_ttft_s": 1.7205, + "native_over_mlx": 0.991, + "native_prefill_tok_s": 19222, + "mlx_prefill_tok_s": 19046, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "vl6", + "prompt_tokens": 65536, + "native_ttft_s": 4.0689, + "mlx_ttft_s": 4.1684, + "native_over_mlx": 0.976, + "native_prefill_tok_s": 16106, + "mlx_prefill_tok_s": 15722, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "vl8", + "prompt_tokens": 2048, + "native_ttft_s": 0.1112, + "mlx_ttft_s": 0.1295, + "native_over_mlx": 0.859, + "native_prefill_tok_s": 18424, + "mlx_prefill_tok_s": 15818, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "vl8", + "prompt_tokens": 8192, + "native_ttft_s": 0.3801, + "mlx_ttft_s": 0.3657, + "native_over_mlx": 1.039, + "native_prefill_tok_s": 21550, + "mlx_prefill_tok_s": 22399, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "vl8", + "prompt_tokens": 32768, + "native_ttft_s": 1.7117, + "mlx_ttft_s": 1.6833, + "native_over_mlx": 1.017, + "native_prefill_tok_s": 19143, + "mlx_prefill_tok_s": 19466, + "runs": [ + 2, + 2 + ] + }, + { + "checkpoint": "vl8", + "prompt_tokens": 65536, + "native_ttft_s": 4.0871, + "mlx_ttft_s": 4.0915, + "native_over_mlx": 0.999, + "native_prefill_tok_s": 16035, + "mlx_prefill_tok_s": 16018, + "runs": [ + 2, + 2 + ] + } + ] +} diff --git a/docs/receipts/lfm3/m5-nax-exactness.json b/docs/receipts/lfm3/m5-nax-exactness.json new file mode 100644 index 0000000000..a20737f718 --- /dev/null +++ b/docs/receipts/lfm3/m5-nax-exactness.json @@ -0,0 +1,85 @@ +{ + "tool": "tf-lfm3-nax-exact", + "device": "Apple M5 Ultra", + "routes": "lfm3-nax-prompt + lfm3-tiled-attention + split decode attention", + "comparison": "byte-identical logits and state", + "cases": { + "chunk_sizes_equal_chunks_of_2048_on_4500_tokens": [ + 1, + 8, + 9, + 63, + 64, + 65, + 127, + 128, + 129, + 1000, + 2047 + ], + "cut_sets_on_4500_tokens": 5, + "cut_at_9_and_2048_prompt_lengths": [ + 2047, + 2048, + 2049, + 2051, + 4097, + 8193 + ], + "resumed_from_snapshot_prefixes": [ + 1, + 9, + 128, + 1000, + 2048, + 2051 + ], + "window_rows_vs_one_row_decode": { + "rows": [ + 2, + 8, + 9, + 16, + 32 + ], + "prefixes": [ + 127, + 1000, + 4096 + ] + } + }, + "before_fix": { + "LFM3-1.7B-MLX-4bit": { + "passed": 34, + "failed": 9, + "failing": "window rows 9, 16, 32 at every prefix: tiled prompt attention ran on verification windows" + } + }, + "after_fix": { + "LFM3-1.7B-MLX-4bit": { + "passed": 43, + "failed": 0 + }, + "LFM3-1.7B-MLX-6bit": { + "passed": 43, + "failed": 0 + }, + "LFM3-1.7B-MLX-8bit": { + "passed": 43, + "failed": 0 + }, + "LFM3-VL-2B-MLX-4bit": { + "passed": 43, + "failed": 0 + }, + "LFM3-VL-2B-MLX-6bit": { + "passed": 43, + "failed": 0 + }, + "LFM3-VL-2B-MLX-8bit": { + "passed": 43, + "failed": 0 + } + } +} diff --git a/docs/receipts/lfm3/m5-nax-launch-fusion-rejected.json b/docs/receipts/lfm3/m5-nax-launch-fusion-rejected.json new file mode 100644 index 0000000000..e10390be28 --- /dev/null +++ b/docs/receipts/lfm3/m5-nax-launch-fusion-rejected.json @@ -0,0 +1,87 @@ +{ + "scope": "Decode launch reduction candidate on one M5 Ultra: the FFN gate and up projections and SwiGLU in one dispatch (the paired row projection's arithmetic, each product rounded to BF16 before the activation), and the attention query and head-gate projections as one paired dispatch. 26 fewer dispatches a token of about 250.", + "method": "Same harness, prompts and settings for both builds, greedy, max_tokens 128, prompt cache disabled, builds alternated for two rounds of three runs, medians of six.", + "rows": [ + { + "checkpoint": "text4", + "prompt_tokens_target": 0, + "streams": 1, + "decode_tok_s_aggregate_base": 438.1, + "decode_tok_s_aggregate_candidate": 444.3, + "candidate_over_base": 1.014, + "itl_mean_ms_base": 2.282, + "itl_mean_ms_candidate": 2.25 + }, + { + "checkpoint": "text4", + "prompt_tokens_target": 2048, + "streams": 1, + "decode_tok_s_aggregate_base": 431.3, + "decode_tok_s_aggregate_candidate": 438.3, + "candidate_over_base": 1.016, + "itl_mean_ms_base": 2.279, + "itl_mean_ms_candidate": 2.242 + }, + { + "checkpoint": "text4", + "prompt_tokens_target": 0, + "streams": 4, + "decode_tok_s_aggregate_base": 1075.0, + "decode_tok_s_aggregate_candidate": 1055.4, + "candidate_over_base": 0.982, + "itl_mean_ms_base": 3.897, + "itl_mean_ms_candidate": 3.966 + }, + { + "checkpoint": "text4", + "prompt_tokens_target": 2048, + "streams": 4, + "decode_tok_s_aggregate_base": 1069.1, + "decode_tok_s_aggregate_candidate": 1052.3, + "candidate_over_base": 0.984, + "itl_mean_ms_base": 6.146, + "itl_mean_ms_candidate": 6.207 + }, + { + "checkpoint": "vl4", + "prompt_tokens_target": 0, + "streams": 1, + "decode_tok_s_aggregate_base": 410.7, + "decode_tok_s_aggregate_candidate": 415.1, + "candidate_over_base": 1.011, + "itl_mean_ms_base": 2.434, + "itl_mean_ms_candidate": 2.408 + }, + { + "checkpoint": "vl4", + "prompt_tokens_target": 2048, + "streams": 1, + "decode_tok_s_aggregate_base": 397.6, + "decode_tok_s_aggregate_candidate": 402.7, + "candidate_over_base": 1.013, + "itl_mean_ms_base": 2.484, + "itl_mean_ms_candidate": 2.452 + }, + { + "checkpoint": "vl4", + "prompt_tokens_target": 0, + "streams": 4, + "decode_tok_s_aggregate_base": 1074.5, + "decode_tok_s_aggregate_candidate": 1052.6, + "candidate_over_base": 0.98, + "itl_mean_ms_base": 3.888, + "itl_mean_ms_candidate": 3.966 + }, + { + "checkpoint": "vl4", + "prompt_tokens_target": 2048, + "streams": 4, + "decode_tok_s_aggregate_base": 1021.5, + "decode_tok_s_aggregate_candidate": 1001.8, + "candidate_over_base": 0.981, + "itl_mean_ms_base": 5.715, + "itl_mean_ms_candidate": 5.795 + } + ], + "outcome": "Rejected. One stream gains 1.1 to 1.6 percent, four streams lose 1.6 to 2.0 percent. No production change." +} diff --git a/docs/receipts/lfm3/m5-nax-real-image-rows.json b/docs/receipts/lfm3/m5-nax-real-image-rows.json new file mode 100644 index 0000000000..7aa3763388 --- /dev/null +++ b/docs/receipts/lfm3/m5-nax-real-image-rows.json @@ -0,0 +1,113 @@ +{ + "device": "Apple M5 Ultra", + "width": 2048, + "comparison": "native tower and projector rows against an independent FP64 graph (BF16 stage rounding) and official MLX BF16 rows, same FP32 patches, five images", + "images": { + "cat": "photograph 300x200", + "hero": "wide banner 2172x724", + "logo": "square RGBA 1254x1254", + "speeds": "wide banner 2172x724", + "banner": "wide banner 2172x724" + }, + "summary": { + "4-bit": { + "pooled_native_over_official_rms": 0.9464499527820002, + "images_native_better": 4, + "images": 5, + "worst_image_ratio": 1.2307257512424175 + }, + "6-bit": { + "pooled_native_over_official_rms": 0.9409219038850172, + "images_native_better": 3, + "images": 5, + "worst_image_ratio": 1.1834671920246838 + } + }, + "per_image": { + "4-bit": { + "cat": { + "rows": 70, + "native_rms": 0.0005607575567752688, + "official_bf16_rms": 0.0006883479542354507, + "native_over_official": 0.8146425849381702, + "native_max": 0.005889892578125, + "official_bf16_max": 0.01220703125 + }, + "hero": { + "rows": 243, + "native_rms": 0.0007350691218118744, + "official_bf16_rms": 0.0008488824078536666, + "native_over_official": 0.8659257336601424, + "native_max": 0.01556396484375, + "official_bf16_max": 0.0369873046875 + }, + "logo": { + "rows": 256, + "native_rms": 0.0007009128255279245, + "official_bf16_rms": 0.0008589674420230505, + "native_over_official": 0.8159946363940479, + "native_max": 0.02294921875, + "official_bf16_max": 0.030029296875 + }, + "speeds": { + "rows": 243, + "native_rms": 0.0009210555023401223, + "official_bf16_rms": 0.0007483840339005802, + "native_over_official": 1.2307257512424175, + "native_max": 0.02783203125, + "official_bf16_max": 0.0224609375 + }, + "banner": { + "rows": 243, + "native_rms": 0.0006093903407848782, + "official_bf16_rms": 0.0006702709013569544, + "native_over_official": 0.9091702169244937, + "native_max": 0.0185546875, + "official_bf16_max": 0.021484375 + } + }, + "6-bit": { + "cat": { + "rows": 70, + "native_rms": 0.0006593466304518665, + "official_bf16_rms": 0.0005571313128873913, + "native_over_official": 1.1834671920246838, + "native_max": 0.015380859375, + "official_bf16_max": 0.009765625 + }, + "hero": { + "rows": 243, + "native_rms": 0.0006466961528856249, + "official_bf16_rms": 0.0007456945528259121, + "native_over_official": 0.8672400119256348, + "native_max": 0.01025390625, + "official_bf16_max": 0.024169921875 + }, + "logo": { + "rows": 256, + "native_rms": 0.0008679905243361081, + "official_bf16_rms": 0.0008087146100048135, + "native_over_official": 1.0732964553848505, + "native_max": 0.025634765625, + "official_bf16_max": 0.02685546875 + }, + "speeds": { + "rows": 243, + "native_rms": 0.0008073823003217872, + "official_bf16_rms": 0.0009493058472850353, + "native_over_official": 0.8504975531656715, + "native_max": 0.029296875, + "official_bf16_max": 0.0390625 + }, + "banner": { + "rows": 243, + "native_rms": 0.0005997713149630199, + "official_bf16_rms": 0.0006459829484743785, + "native_over_official": 0.9284630753482009, + "native_max": 0.017578125, + "official_bf16_max": 0.013671875 + } + } + }, + "note": "6- and 8-bit checkpoints share one tower, so their rows are identical; teacher-forced text from real-image rows is in m5-nax-route-fidelity.json" +} diff --git a/docs/receipts/lfm3/m5-nax-route-fidelity.json b/docs/receipts/lfm3/m5-nax-route-fidelity.json new file mode 100644 index 0000000000..ef3f5ef93d --- /dev/null +++ b/docs/receipts/lfm3/m5-nax-route-fidelity.json @@ -0,0 +1,1029 @@ +{ + "description": "Native routes versus an independent FP32 forward, with official MLX BF16 for scale (criterion of #1491 comment 6537143418: no further from FP32 than the reference BF16, every top-1 difference listed with margins). No pass/fail rule applied. Sweeps score many causal prompt rows; canonical phases score one prompt row and 17 teacher-forced continuation rows. nax = tensor-unit prompt projections plus the new attention kernels; tiled_prompt = the retired residual-weight FFN plus tiled attention.", + "checkpoints": { + "text4": { + "sweep_rows": 25, + "sweep": { + "official_bf16": { + "rms": 0.039185014622058915, + "max": 0.19534470200538634, + "top20_probability_max": 0.002291574169078311, + "top1_misses": 0 + }, + "baseline": { + "rms": 0.03677548280046484, + "max": 0.1755135664343834, + "top20_probability_max": 0.0017174991858559005, + "top1_misses": 0 + }, + "tiled_prompt": { + "rms": 0.0398973873720482, + "max": 0.18047382175922394, + "top20_probability_max": 0.002747923413640312, + "top1_misses": 0 + }, + "nax": { + "rms": 0.03899665272065138, + "max": 0.18821845144033433, + "top20_probability_max": 0.0012001136422634781, + "top1_misses": 0 + } + }, + "paired_nax_minus_baseline_rms": { + "mean": 0.0022211699201865427, + "se": 0.0029403607058800007, + "nax_worse": 14, + "rows": 25 + }, + "nax_rms_exceeds_bf16_rows": 12, + "top1_differences": [], + "canonical": { + "baseline": { + "prompt_boundary": { + "native_rms": 0.029707890093412955, + "bf16_rms": 0.04764064939088356, + "native_max": 0.1516423225402832, + "bf16_max": 0.20851778984069824, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "top20_probability_mean", + "top20_probability_max" + ], + "top_pick_differences": [] + }, + "teacher_forced_continuation": { + "native_rms": 0.03402449617886581, + "bf16_rms": 0.0425864734405558, + "native_max": 0.2016282081604004, + "bf16_max": 0.256467342376709, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [], + "top_pick_differences": [] + } + }, + "tiled": { + "prompt_boundary": { + "native_rms": 0.041121336314284725, + "bf16_rms": 0.04764064939088356, + "native_max": 0.20430231094360352, + "bf16_max": 0.20851778984069824, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "top20_probability_mean", + "top20_probability_max" + ], + "top_pick_differences": [] + }, + "teacher_forced_continuation": { + "native_rms": 0.037950087085717864, + "bf16_rms": 0.0425864734405558, + "native_max": 0.20357131958007812, + "bf16_max": 0.256467342376709, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "top20_probability_max" + ], + "top_pick_differences": [] + } + }, + "nax": { + "prompt_boundary": { + "native_rms": 0.05130358349267955, + "bf16_rms": 0.04764064939088356, + "native_max": 0.18304729461669922, + "bf16_max": 0.20851778984069824, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "rms_error", + "top20_probability_mean", + "top20_probability_max" + ], + "top_pick_differences": [] + }, + "teacher_forced_continuation": { + "native_rms": 0.043351195781572385, + "bf16_rms": 0.0425864734405558, + "native_max": 0.23638872802257538, + "bf16_max": 0.256467342376709, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "rms_error" + ], + "top_pick_differences": [] + } + } + } + }, + "text6": { + "sweep_rows": 25, + "sweep": { + "official_bf16": { + "rms": 0.04018031004122828, + "max": 0.19565048962831497, + "top20_probability_max": 0.0004705272920054893, + "top1_misses": 0 + }, + "baseline": { + "rms": 0.03904510857050718, + "max": 0.1850430727005005, + "top20_probability_max": 0.00039090498853548635, + "top1_misses": 0 + }, + "tiled_prompt": { + "rms": 0.03889914475997641, + "max": 0.18597806632518768, + "top20_probability_max": 0.0004109389619873649, + "top1_misses": 0 + }, + "nax": { + "rms": 0.037617605755720825, + "max": 0.1804419305920601, + "top20_probability_max": 0.0004057423967945152, + "top1_misses": 0 + } + }, + "paired_nax_minus_baseline_rms": { + "mean": -0.0014275028147863484, + "se": 0.003342043572813961, + "nax_worse": 11, + "rows": 25 + }, + "nax_rms_exceeds_bf16_rows": 10, + "top1_differences": [], + "canonical": { + "baseline": { + "prompt_boundary": { + "native_rms": 0.04901861875813257, + "bf16_rms": 0.031214103195262688, + "native_max": 0.2050304412841797, + "bf16_max": 0.19215011596679688, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "rms_error", + "maximum_absolute_error", + "top20_probability_mean", + "top20_probability_max" + ], + "top_pick_differences": [] + }, + "teacher_forced_continuation": { + "native_rms": 0.04184006094548536, + "bf16_rms": 0.05064403338575528, + "native_max": 0.28766918182373047, + "bf16_max": 0.3306717872619629, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "top20_probability_mean", + "top20_probability_max" + ], + "top_pick_differences": [] + } + }, + "tiled": { + "prompt_boundary": { + "native_rms": 0.05813316250926237, + "bf16_rms": 0.031214103195262688, + "native_max": 0.22626113891601562, + "bf16_max": 0.19215011596679688, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "rms_error", + "maximum_absolute_error", + "top20_probability_mean", + "top20_probability_max" + ], + "top_pick_differences": [] + }, + "teacher_forced_continuation": { + "native_rms": 0.04758823452986163, + "bf16_rms": 0.05064403338575528, + "native_max": 0.24394440650939941, + "bf16_max": 0.3306717872619629, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "top20_probability_mean", + "top20_probability_max" + ], + "top_pick_differences": [] + } + }, + "nax": { + "prompt_boundary": { + "native_rms": 0.04648981800947816, + "bf16_rms": 0.031214103195262688, + "native_max": 0.20114850997924805, + "bf16_max": 0.19215011596679688, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "rms_error", + "maximum_absolute_error", + "top20_probability_mean", + "top20_probability_max" + ], + "top_pick_differences": [] + }, + "teacher_forced_continuation": { + "native_rms": 0.04375156694726434, + "bf16_rms": 0.05064403338575528, + "native_max": 0.26974207162857056, + "bf16_max": 0.3306717872619629, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [], + "top_pick_differences": [] + } + } + } + }, + "text8": { + "sweep_rows": 25, + "sweep": { + "official_bf16": { + "rms": 0.04279252546116228, + "max": 0.19843310952186585, + "top20_probability_max": 0.0003955591587628029, + "top1_misses": 0 + }, + "baseline": { + "rms": 0.03895808404525676, + "max": 0.1796414351463318, + "top20_probability_max": 0.0004932884817942959, + "top1_misses": 0 + }, + "tiled_prompt": { + "rms": 0.03619301455482047, + "max": 0.17607753872871398, + "top20_probability_max": 0.00043222382258892154, + "top1_misses": 0 + }, + "nax": { + "rms": 0.04546231104884131, + "max": 0.20278984904289246, + "top20_probability_max": 0.0004754118558599478, + "top1_misses": 0 + } + }, + "paired_nax_minus_baseline_rms": { + "mean": 0.006504227003584551, + "se": 0.002425139774216567, + "nax_worse": 19, + "rows": 25 + }, + "nax_rms_exceeds_bf16_rows": 15, + "top1_differences": [], + "canonical": { + "baseline": { + "prompt_boundary": { + "native_rms": 0.05756910058878599, + "bf16_rms": 0.05299046310657806, + "native_max": 0.21836501359939575, + "bf16_max": 0.23966991901397705, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "rms_error", + "top20_probability_mean", + "top20_probability_max" + ], + "top_pick_differences": [] + }, + "teacher_forced_continuation": { + "native_rms": 0.041127697666286024, + "bf16_rms": 0.04531972721085561, + "native_max": 0.24574851989746094, + "bf16_max": 0.2324376106262207, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "maximum_absolute_error" + ], + "top_pick_differences": [] + } + }, + "tiled": { + "prompt_boundary": { + "native_rms": 0.028363209962736325, + "bf16_rms": 0.05299046310657806, + "native_max": 0.1697826385498047, + "bf16_max": 0.23966991901397705, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "top20_probability_max" + ], + "top_pick_differences": [] + }, + "teacher_forced_continuation": { + "native_rms": 0.03825201234654284, + "bf16_rms": 0.04531972721085561, + "native_max": 0.20448875427246094, + "bf16_max": 0.2324376106262207, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "top20_probability_max" + ], + "top_pick_differences": [] + } + }, + "nax": { + "prompt_boundary": { + "native_rms": 0.036124218746433207, + "bf16_rms": 0.05299046310657806, + "native_max": 0.19423675537109375, + "bf16_max": 0.23966991901397705, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [], + "top_pick_differences": [] + }, + "teacher_forced_continuation": { + "native_rms": 0.04544825195567689, + "bf16_rms": 0.04531972721085561, + "native_max": 0.248479425907135, + "bf16_max": 0.2324376106262207, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "rms_error", + "maximum_absolute_error" + ], + "top_pick_differences": [] + } + } + } + }, + "vl4": { + "sweep_rows": 9, + "sweep": { + "official_bf16": { + "rms": 0.04035298264210717, + "max": 0.20450172159406874, + "top20_probability_max": 0.005705231316648891, + "top1_misses": 0 + }, + "baseline": { + "rms": 0.05491078416293803, + "max": 0.2325352562798394, + "top20_probability_max": 0.0029790308247212146, + "top1_misses": 0 + }, + "tiled_prompt": { + "rms": 0.05732688706944902, + "max": 0.2529616620805528, + "top20_probability_max": 0.004892909396824816, + "top1_misses": 0 + }, + "nax": { + "rms": 0.057623806743936176, + "max": 0.21610930893156263, + "top20_probability_max": 0.004060037395009934, + "top1_misses": 0 + } + }, + "paired_nax_minus_baseline_rms": { + "mean": 0.002713022580998143, + "se": 0.010366206592222104, + "nax_worse": 5, + "rows": 9 + }, + "nax_rms_exceeds_bf16_rows": 6, + "top1_differences": [], + "canonical": { + "baseline": { + "prompt_boundary": { + "native_rms": 0.045021395748451196, + "bf16_rms": 0.05724186334479262, + "native_max": 0.22849583625793457, + "bf16_max": 0.336777925491333, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [], + "top_pick_differences": [] + }, + "teacher_forced_continuation": { + "native_rms": 0.034404836903271833, + "bf16_rms": 0.03698408052111559, + "native_max": 0.29196321964263916, + "bf16_max": 0.23212289810180664, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "maximum_absolute_error", + "top20_probability_mean", + "top20_probability_max" + ], + "top_pick_differences": [] + } + }, + "tiled": { + "prompt_boundary": { + "native_rms": 0.05484708159668184, + "bf16_rms": 0.05724186334479262, + "native_max": 0.26920086145401, + "bf16_max": 0.336777925491333, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [], + "top_pick_differences": [] + }, + "teacher_forced_continuation": { + "native_rms": 0.03335499915133132, + "bf16_rms": 0.03698408052111559, + "native_max": 0.20049190521240234, + "bf16_max": 0.23212289810180664, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "top20_probability_max" + ], + "top_pick_differences": [] + } + }, + "nax": { + "prompt_boundary": { + "native_rms": 0.055050762659088454, + "bf16_rms": 0.05724186334479262, + "native_max": 0.29662036895751953, + "bf16_max": 0.336777925491333, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [], + "top_pick_differences": [] + }, + "teacher_forced_continuation": { + "native_rms": 0.041001585076833616, + "bf16_rms": 0.03698408052111559, + "native_max": 0.38673388957977295, + "bf16_max": 0.23212289810180664, + "native_top1_misses": 1, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "rms_error", + "maximum_absolute_error", + "top20_probability_max" + ], + "top_pick_differences": [ + { + "row": 3, + "native_top": 207, + "reference_bf16_top": 278, + "fp32_top": 278, + "native_maximum_set": [ + 207, + 278 + ], + "reference_maximum_set": [ + 278 + ], + "fp32_maximum_set": [ + 278 + ], + "native_margin": 0.0, + "reference_bf16_margin": 0.21875, + "fp32_margin": 0.10959959030151367, + "fp32_native_contender_gap": 0.10959959030151367, + "fp32_reference_contender_gap": 0.0, + "native_only_miss": true + } + ] + } + } + } + }, + "vl4long": { + "sweep_rows": 45, + "sweep": { + "official_bf16": { + "rms": 0.03339460902858266, + "max": 0.15890198623140653, + "top20_probability_max": 0.006149748862220873, + "top1_misses": 0 + }, + "baseline": { + "rms": 0.029617537877033515, + "max": 0.1420389109187656, + "top20_probability_max": 0.006166903703145467, + "top1_misses": 0 + }, + "tiled_prompt": { + "rms": 0.02945682283839453, + "max": 0.14509857760535347, + "top20_probability_max": 0.006421012368764529, + "top1_misses": 0 + }, + "nax": { + "rms": 0.02993816489575661, + "max": 0.1438522865012702, + "top20_probability_max": 0.005473988140061671, + "top1_misses": 0 + } + }, + "paired_nax_minus_baseline_rms": { + "mean": 0.0003206270187230935, + "se": 0.0018903250784857888, + "nax_worse": 23, + "rows": 45 + }, + "nax_rms_exceeds_bf16_rows": 18, + "top1_differences": [] + }, + "vl6": { + "sweep_rows": 9, + "sweep": { + "official_bf16": { + "rms": 0.06791185993219352, + "max": 0.31771284341812134, + "top20_probability_max": 0.014008899926185113, + "top1_misses": 1 + }, + "baseline": { + "rms": 0.05036021844438576, + "max": 0.19850130875905356, + "top20_probability_max": 0.005867781359692831, + "top1_misses": 0 + }, + "tiled_prompt": { + "rms": 0.044764633684090026, + "max": 0.2005244361029731, + "top20_probability_max": 0.005781203911422671, + "top1_misses": 0 + }, + "nax": { + "rms": 0.06407420885568303, + "max": 0.3047619279887941, + "top20_probability_max": 0.015415731072864912, + "top1_misses": 1 + } + }, + "paired_nax_minus_baseline_rms": { + "mean": 0.013713990411297276, + "se": 0.014512850628419553, + "nax_worse": 7, + "rows": 9 + }, + "nax_rms_exceeds_bf16_rows": 4, + "top1_differences": [ + { + "N": 90, + "route": "official_bf16", + "fp32_margin": 0.04569435119628906, + "route_margin": 0.25, + "fp32_gap": -0.04569435119628906 + }, + { + "N": 90, + "route": "nax", + "fp32_margin": 0.04569435119628906, + "route_margin": 0.0, + "fp32_gap": -0.04569435119628906 + } + ], + "canonical": { + "baseline": { + "prompt_boundary": { + "native_rms": 0.0525786548534892, + "bf16_rms": 0.1442952219059758, + "native_max": 0.30329906940460205, + "bf16_max": 0.42031431198120117, + "native_top1_misses": 0, + "bf16_top1_misses": 1, + "failed_proposed_metrics": [], + "top_pick_differences": [ + { + "row": 0, + "native_top": 6066, + "reference_bf16_top": 440, + "fp32_top": 6066, + "native_maximum_set": [ + 6066 + ], + "reference_maximum_set": [ + 440 + ], + "fp32_maximum_set": [ + 6066 + ], + "native_margin": 0.125, + "reference_bf16_margin": 0.25, + "fp32_margin": 0.04569435119628906, + "fp32_native_contender_gap": 0.0, + "fp32_reference_contender_gap": 0.04569435119628906, + "native_only_miss": false + } + ] + }, + "teacher_forced_continuation": { + "native_rms": 0.043509245553194, + "bf16_rms": 0.05030189013781307, + "native_max": 0.4391179084777832, + "bf16_max": 0.27240753173828125, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "maximum_absolute_error", + "top20_probability_max" + ], + "top_pick_differences": [] + } + }, + "tiled": { + "prompt_boundary": { + "native_rms": 0.05682561171501801, + "bf16_rms": 0.1442952219059758, + "native_max": 0.2940101623535156, + "bf16_max": 0.42031431198120117, + "native_top1_misses": 0, + "bf16_top1_misses": 1, + "failed_proposed_metrics": [], + "top_pick_differences": [ + { + "row": 0, + "native_top": 6066, + "reference_bf16_top": 440, + "fp32_top": 6066, + "native_maximum_set": [ + 6066 + ], + "reference_maximum_set": [ + 440 + ], + "fp32_maximum_set": [ + 6066 + ], + "native_margin": 0.125, + "reference_bf16_margin": 0.25, + "fp32_margin": 0.04569435119628906, + "fp32_native_contender_gap": 0.0, + "fp32_reference_contender_gap": 0.04569435119628906, + "native_only_miss": false + } + ] + }, + "teacher_forced_continuation": { + "native_rms": 0.04545417399614853, + "bf16_rms": 0.05030189013781307, + "native_max": 0.3274260461330414, + "bf16_max": 0.27240753173828125, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "maximum_absolute_error", + "top20_probability_max" + ], + "top_pick_differences": [] + } + }, + "nax": { + "prompt_boundary": { + "native_rms": 0.14238507718717022, + "bf16_rms": 0.1442952219059758, + "native_max": 0.5386929512023926, + "bf16_max": 0.42031431198120117, + "native_top1_misses": 1, + "bf16_top1_misses": 1, + "failed_proposed_metrics": [ + "maximum_absolute_error", + "top20_probability_mean", + "top20_probability_max" + ], + "top_pick_differences": [ + { + "row": 0, + "native_top": 440, + "reference_bf16_top": 440, + "fp32_top": 6066, + "native_maximum_set": [ + 440, + 124900 + ], + "reference_maximum_set": [ + 440 + ], + "fp32_maximum_set": [ + 6066 + ], + "native_margin": 0.0, + "reference_bf16_margin": 0.25, + "fp32_margin": 0.04569435119628906, + "fp32_native_contender_gap": 0.04569435119628906, + "fp32_reference_contender_gap": 0.04569435119628906, + "native_only_miss": false + } + ] + }, + "teacher_forced_continuation": { + "native_rms": 0.04301288288539518, + "bf16_rms": 0.05030189013781307, + "native_max": 0.25221991539001465, + "bf16_max": 0.27240753173828125, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "top20_probability_max" + ], + "top_pick_differences": [] + } + } + } + }, + "vl6long": { + "sweep_rows": 45, + "sweep": { + "official_bf16": { + "rms": 0.04184474775150998, + "max": 0.21715606045391825, + "top20_probability_max": 0.009298263377207374, + "top1_misses": 1 + }, + "baseline": { + "rms": 0.0365732936782363, + "max": 0.1759725825654136, + "top20_probability_max": 0.006888464530860934, + "top1_misses": 0 + }, + "tiled_prompt": { + "rms": 0.03274911813885285, + "max": 0.18221158120367262, + "top20_probability_max": 0.008384905066112513, + "top1_misses": 0 + }, + "nax": { + "rms": 0.037204068291938126, + "max": 0.19664776275555293, + "top20_probability_max": 0.008582534491770467, + "top1_misses": 1 + } + }, + "paired_nax_minus_baseline_rms": { + "mean": 0.0006307746137018353, + "se": 0.0032078173743502456, + "nax_worse": 25, + "rows": 45 + }, + "nax_rms_exceeds_bf16_rows": 20, + "top1_differences": [ + { + "N": 90, + "route": "official_bf16", + "fp32_margin": 0.04569435119628906, + "route_margin": 0.125, + "fp32_gap": -0.04569435119628906 + }, + { + "N": 90, + "route": "nax", + "fp32_margin": 0.04569435119628906, + "route_margin": 0.0, + "fp32_gap": -0.04569435119628906 + } + ] + }, + "vl8": { + "sweep_rows": 9, + "sweep": { + "official_bf16": { + "rms": 0.055668635305161786, + "max": 0.32083530558480156, + "top20_probability_max": 0.01261331459321191, + "top1_misses": 0 + }, + "baseline": { + "rms": 0.060090205909432294, + "max": 0.3198500759899616, + "top20_probability_max": 0.011049715148431711, + "top1_misses": 0 + }, + "tiled_prompt": { + "rms": 0.06984953204430809, + "max": 0.350392265452279, + "top20_probability_max": 0.011304059793175636, + "top1_misses": 0 + }, + "nax": { + "rms": 0.0695384214930725, + "max": 0.35048242410024005, + "top20_probability_max": 0.01924658697190347, + "top1_misses": 0 + } + }, + "paired_nax_minus_baseline_rms": { + "mean": 0.009448215583640219, + "se": 0.006935443550997549, + "nax_worse": 7, + "rows": 9 + }, + "nax_rms_exceeds_bf16_rows": 9, + "top1_differences": [], + "canonical": { + "baseline": { + "prompt_boundary": { + "native_rms": 0.08996466833747349, + "bf16_rms": 0.06759950572101424, + "native_max": 0.41560298204421997, + "bf16_max": 0.3477163314819336, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "rms_error", + "maximum_absolute_error", + "top20_probability_max" + ], + "top_pick_differences": [] + }, + "teacher_forced_continuation": { + "native_rms": 0.043630560381030116, + "bf16_rms": 0.038167574377201724, + "native_max": 0.27512621879577637, + "bf16_max": 0.3014151304960251, + "native_top1_misses": 1, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "rms_error", + "top20_probability_mean", + "top20_probability_max" + ], + "top_pick_differences": [ + { + "row": 5, + "native_top": 5831, + "reference_bf16_top": 6840, + "fp32_top": 6840, + "native_maximum_set": [ + 5831, + 6840 + ], + "reference_maximum_set": [ + 6840 + ], + "fp32_maximum_set": [ + 6840 + ], + "native_margin": 0.0, + "reference_bf16_margin": 0.09375, + "fp32_margin": 0.08093595504760742, + "fp32_native_contender_gap": 0.08093595504760742, + "fp32_reference_contender_gap": 0.0, + "native_only_miss": true + } + ] + } + }, + "tiled": { + "prompt_boundary": { + "native_rms": 0.0659144404390306, + "bf16_rms": 0.06759950572101424, + "native_max": 0.5613763630390167, + "bf16_max": 0.3477163314819336, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "maximum_absolute_error" + ], + "top_pick_differences": [] + }, + "teacher_forced_continuation": { + "native_rms": 0.0470500589571398, + "bf16_rms": 0.038167574377201724, + "native_max": 0.3434346914291382, + "bf16_max": 0.3014151304960251, + "native_top1_misses": 1, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "rms_error", + "maximum_absolute_error", + "top20_probability_mean" + ], + "top_pick_differences": [ + { + "row": 5, + "native_top": 5831, + "reference_bf16_top": 6840, + "fp32_top": 6840, + "native_maximum_set": [ + 5831 + ], + "reference_maximum_set": [ + 6840 + ], + "fp32_maximum_set": [ + 6840 + ], + "native_margin": 0.09375, + "reference_bf16_margin": 0.09375, + "fp32_margin": 0.08093595504760742, + "fp32_native_contender_gap": 0.08093595504760742, + "fp32_reference_contender_gap": 0.0, + "native_only_miss": true + } + ] + } + }, + "nax": { + "prompt_boundary": { + "native_rms": 0.06953168047410944, + "bf16_rms": 0.06759950572101424, + "native_max": 0.4206857681274414, + "bf16_max": 0.3477163314819336, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "rms_error", + "maximum_absolute_error", + "top20_probability_mean", + "top20_probability_max" + ], + "top_pick_differences": [] + }, + "teacher_forced_continuation": { + "native_rms": 0.047767095388658705, + "bf16_rms": 0.038167574377201724, + "native_max": 0.3506971001625061, + "bf16_max": 0.3014151304960251, + "native_top1_misses": 0, + "bf16_top1_misses": 0, + "failed_proposed_metrics": [ + "rms_error", + "maximum_absolute_error" + ], + "top_pick_differences": [] + } + } + } + }, + "vl8long": { + "sweep_rows": 45, + "sweep": { + "official_bf16": { + "rms": 0.03768535399633972, + "max": 0.1984032659067048, + "top20_probability_max": 0.007951154274390698, + "top1_misses": 0 + }, + "baseline": { + "rms": 0.037341594878442755, + "max": 0.18874870638052624, + "top20_probability_max": 0.0069587672008757685, + "top1_misses": 0 + }, + "tiled_prompt": { + "rms": 0.04092342794136572, + "max": 0.19802138892312846, + "top20_probability_max": 0.0068949121302810295, + "top1_misses": 0 + }, + "nax": { + "rms": 0.03957463988828597, + "max": 0.19738083117538027, + "top20_probability_max": 0.009192611729662039, + "top1_misses": 0 + } + }, + "paired_nax_minus_baseline_rms": { + "mean": 0.002233045009843206, + "se": 0.0025672377001328436, + "nax_worse": 29, + "rows": 45 + }, + "nax_rms_exceeds_bf16_rows": 22, + "top1_differences": [] + } + }, + "rms_to_fp32_ratio_vs_official_bf16_all_sweep_rows": { + "baseline": { + "rows": 237, + "median": 0.916, + "mean": 1.025, + "p90": 1.569, + "max": 5.087, + "fraction_above_1.5": 0.114 + }, + "tiled_prompt": { + "rows": 237, + "median": 0.927, + "mean": 1.026, + "p90": 1.548, + "max": 4.997, + "fraction_above_1.5": 0.118 + }, + "nax": { + "rows": 237, + "median": 0.987, + "mean": 1.042, + "p90": 1.478, + "max": 2.954, + "fraction_above_1.5": 0.089 + } + } +} \ No newline at end of file diff --git a/docs/receipts/lfm3/m5-nax-serving-lifecycle.json b/docs/receipts/lfm3/m5-nax-serving-lifecycle.json new file mode 100644 index 0000000000..e8bbfc86c3 --- /dev/null +++ b/docs/receipts/lfm3/m5-nax-serving-lifecycle.json @@ -0,0 +1,317 @@ +{ + "tool": "tools/lfm3/serve_check.py --cancel --sampled", + "server": "--context 2048 --parallel 4 --prompt-cache-gib 0", + "device": "Apple M5 Ultra", + "scheduling": "prompts fill one 2048-row chunk at a time; the engine runs the shortest remaining prompt first and decodes between chunks", + "widths": { + "LFM3-1.7B-MLX-4bit": { + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "binary_sha256": "b99ed8e30333822c28c21b45ebb2b35feb4b9b3d97c64b19b00d540aae5a2f2f", + "checks": { + "fresh_plain": 6, + "length_over_256": 1, + "plain_equals_fresh": 6, + "drafted_equals_fresh": 6, + "warm_reuses_and_matches_fresh": 1, + "no_stale_prefix_reuse": 1, + "matches_cache_off": 1, + "concurrent_1_equals_solo": 4, + "concurrent_2_equals_solo": 4, + "concurrent_4_equals_solo": 4, + "cancel_server_confirmed": 2, + "cancel_next_plain_equals_fresh": 2, + "cancel_next_drafted_equals_fresh": 2 + }, + "long_context": { + "server_context": 40000, + "prompt_tokens": 31178, + "cold_prefill_s": 1.557, + "idle_small_request_s": 0.031, + "cancel_mid_prefill": [ + { + "cancel_after_s": 0.3, + "next_request_s": 0.04, + "next_output_equals_fresh": true + }, + { + "cancel_after_s": 1.0, + "next_request_s": 0.04, + "next_output_equals_fresh": true + }, + { + "cancel_after_s": 2.0, + "next_request_s": 0.04, + "next_output_equals_fresh": true + } + ], + "small_request_during_three_30k_prefills_s": 0.67, + "that_output_equals_fresh": true, + "three_30k_prefills_wall_s": 4.64, + "server_exit_clean": true + } + }, + "LFM3-1.7B-MLX-6bit": { + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "binary_sha256": "b99ed8e30333822c28c21b45ebb2b35feb4b9b3d97c64b19b00d540aae5a2f2f", + "checks": { + "fresh_plain": 6, + "length_over_256": 1, + "plain_equals_fresh": 6, + "drafted_equals_fresh": 6, + "warm_reuses_and_matches_fresh": 1, + "no_stale_prefix_reuse": 1, + "matches_cache_off": 1, + "concurrent_1_equals_solo": 4, + "concurrent_2_equals_solo": 4, + "concurrent_4_equals_solo": 4, + "cancel_server_confirmed": 2, + "cancel_next_plain_equals_fresh": 2, + "cancel_next_drafted_equals_fresh": 2 + }, + "long_context": { + "server_context": 40000, + "prompt_tokens": 31178, + "cold_prefill_s": 1.608, + "idle_small_request_s": 0.036, + "cancel_mid_prefill": [ + { + "cancel_after_s": 0.3, + "next_request_s": 0.045, + "next_output_equals_fresh": true + }, + { + "cancel_after_s": 1.0, + "next_request_s": 0.045, + "next_output_equals_fresh": true + }, + { + "cancel_after_s": 2.0, + "next_request_s": 0.045, + "next_output_equals_fresh": true + } + ], + "small_request_during_three_30k_prefills_s": 0.67, + "that_output_equals_fresh": true, + "three_30k_prefills_wall_s": 4.81, + "server_exit_clean": true + } + }, + "LFM3-1.7B-MLX-8bit": { + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "binary_sha256": "b99ed8e30333822c28c21b45ebb2b35feb4b9b3d97c64b19b00d540aae5a2f2f", + "checks": { + "fresh_plain": 6, + "length_over_256": 1, + "plain_equals_fresh": 6, + "drafted_equals_fresh": 6, + "warm_reuses_and_matches_fresh": 1, + "no_stale_prefix_reuse": 1, + "matches_cache_off": 1, + "concurrent_1_equals_solo": 4, + "concurrent_2_equals_solo": 4, + "concurrent_4_equals_solo": 4, + "cancel_server_confirmed": 2, + "cancel_next_plain_equals_fresh": 2, + "cancel_next_drafted_equals_fresh": 2 + }, + "long_context": { + "server_context": 40000, + "prompt_tokens": 31178, + "cold_prefill_s": 1.621, + "idle_small_request_s": 0.037, + "cancel_mid_prefill": [ + { + "cancel_after_s": 0.3, + "next_request_s": 0.045, + "next_output_equals_fresh": true + }, + { + "cancel_after_s": 1.0, + "next_request_s": 0.046, + "next_output_equals_fresh": true + }, + { + "cancel_after_s": 2.0, + "next_request_s": 0.046, + "next_output_equals_fresh": true + } + ], + "small_request_during_three_30k_prefills_s": 0.68, + "that_output_equals_fresh": true, + "three_30k_prefills_wall_s": 4.84, + "server_exit_clean": true + } + }, + "LFM3-VL-2B-MLX-4bit": { + "counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "binary_sha256": "b99ed8e30333822c28c21b45ebb2b35feb4b9b3d97c64b19b00d540aae5a2f2f", + "checks": { + "fresh_plain": 10, + "length_over_256": 1, + "plain_equals_fresh": 10, + "drafted_equals_fresh": 10, + "warm_reuses_and_matches_fresh": 1, + "no_stale_prefix_reuse": 1, + "matches_cache_off": 1, + "concurrent_1_equals_solo": 4, + "concurrent_2_equals_solo": 4, + "concurrent_4_equals_solo": 4, + "color_semantics": 4, + "concurrent_equals_solos": 1, + "corrupt_png_rejected": 1, + "cancel_server_confirmed": 3, + "cancel_next_plain_equals_fresh": 3, + "cancel_next_drafted_equals_fresh": 3 + }, + "long_context": { + "server_context": 40000, + "prompt_tokens": 31178, + "cold_prefill_s": 1.599, + "idle_small_request_s": 0.033, + "cancel_mid_prefill": [ + { + "cancel_after_s": 0.3, + "next_request_s": 0.042, + "next_output_equals_fresh": true + }, + { + "cancel_after_s": 1.0, + "next_request_s": 0.042, + "next_output_equals_fresh": true + }, + { + "cancel_after_s": 2.0, + "next_request_s": 0.042, + "next_output_equals_fresh": true + } + ], + "small_request_during_three_30k_prefills_s": 0.73, + "that_output_equals_fresh": true, + "three_30k_prefills_wall_s": 4.77, + "server_exit_clean": true + } + }, + "LFM3-VL-2B-MLX-6bit": { + "counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "binary_sha256": "b99ed8e30333822c28c21b45ebb2b35feb4b9b3d97c64b19b00d540aae5a2f2f", + "checks": { + "fresh_plain": 10, + "length_over_256": 1, + "plain_equals_fresh": 10, + "drafted_equals_fresh": 10, + "warm_reuses_and_matches_fresh": 1, + "no_stale_prefix_reuse": 1, + "matches_cache_off": 1, + "concurrent_1_equals_solo": 4, + "concurrent_2_equals_solo": 4, + "concurrent_4_equals_solo": 4, + "color_semantics": 4, + "concurrent_equals_solos": 1, + "corrupt_png_rejected": 1, + "cancel_server_confirmed": 3, + "cancel_next_plain_equals_fresh": 3, + "cancel_next_drafted_equals_fresh": 3 + }, + "long_context": { + "server_context": 40000, + "prompt_tokens": 31178, + "cold_prefill_s": 1.61, + "idle_small_request_s": 0.036, + "cancel_mid_prefill": [ + { + "cancel_after_s": 0.3, + "next_request_s": 0.045, + "next_output_equals_fresh": true + }, + { + "cancel_after_s": 1.0, + "next_request_s": 0.044, + "next_output_equals_fresh": true + }, + { + "cancel_after_s": 2.0, + "next_request_s": 0.044, + "next_output_equals_fresh": true + } + ], + "small_request_during_three_30k_prefills_s": 0.76, + "that_output_equals_fresh": true, + "three_30k_prefills_wall_s": 4.81, + "server_exit_clean": true + } + }, + "LFM3-VL-2B-MLX-8bit": { + "counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "binary_sha256": "b99ed8e30333822c28c21b45ebb2b35feb4b9b3d97c64b19b00d540aae5a2f2f", + "checks": { + "fresh_plain": 10, + "length_over_256": 1, + "plain_equals_fresh": 10, + "drafted_equals_fresh": 10, + "warm_reuses_and_matches_fresh": 1, + "no_stale_prefix_reuse": 1, + "matches_cache_off": 1, + "concurrent_1_equals_solo": 4, + "concurrent_2_equals_solo": 4, + "concurrent_4_equals_solo": 4, + "color_semantics": 4, + "concurrent_equals_solos": 1, + "corrupt_png_rejected": 1, + "cancel_server_confirmed": 3, + "cancel_next_plain_equals_fresh": 3, + "cancel_next_drafted_equals_fresh": 3 + }, + "long_context": { + "server_context": 40000, + "prompt_tokens": 31178, + "cold_prefill_s": 1.618, + "idle_small_request_s": 0.037, + "cancel_mid_prefill": [ + { + "cancel_after_s": 0.3, + "next_request_s": 0.046, + "next_output_equals_fresh": true + }, + { + "cancel_after_s": 1.0, + "next_request_s": 0.046, + "next_output_equals_fresh": true + }, + { + "cancel_after_s": 2.0, + "next_request_s": 0.046, + "next_output_equals_fresh": true + } + ], + "small_request_during_three_30k_prefills_s": 0.68, + "that_output_equals_fresh": true, + "three_30k_prefills_wall_s": 4.83, + "server_exit_clean": true + } + } + } +} diff --git a/docs/receipts/lfm3/m5-nax-serving-metrics.json b/docs/receipts/lfm3/m5-nax-serving-metrics.json new file mode 100644 index 0000000000..b8df5aa1b8 --- /dev/null +++ b/docs/receipts/lfm3/m5-nax-serving-metrics.json @@ -0,0 +1,1236 @@ +{ + "description": "Serving metrics on one M5 Ultra, same harness and prompts for both routes, cache disabled. Streaming chat completions, greedy, max_tokens 128. Prompt 0 is a short chat prompt; the others are the cold fixtures of about 2k and 8k tokens. Medians over 3 runs after a warm-up. TTFT is the client time to the first streamed token. Inter-token latency is the gap between streamed chunks; with 4 streams it includes other requests prefill. Decode aggregate counts tokens after the last first token. Completion lengths differ where a route stopped earlier (listed).", + "rows": [ + { + "checkpoint": "text4", + "route": "native", + "prompt_tokens_target": 0, + "streams": 1, + "runs": 3, + "ttft_s": 0.0144, + "prefill_tok_s_per_request": 2567, + "prefill_tok_s_aggregate": 2567, + "decode_tok_s_per_request": 437.4346, + "decode_tok_s_aggregate": 437.3684, + "itl_mean_ms": 2.2861, + "itl_p95_ms": 2.257, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "text4", + "route": "native", + "prompt_tokens_target": 0, + "streams": 4, + "runs": 3, + "ttft_s": 0.0355, + "prefill_tok_s_per_request": 1082, + "prefill_tok_s_aggregate": 2618, + "decode_tok_s_per_request": 256.2229, + "decode_tok_s_aggregate": 1073.2, + "itl_mean_ms": 3.9037, + "itl_p95_ms": 3.609, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "text4", + "route": "native", + "prompt_tokens_target": 2048, + "streams": 1, + "runs": 3, + "ttft_s": 0.1, + "prefill_tok_s_per_request": 20474, + "prefill_tok_s_aggregate": 20474, + "decode_tok_s_per_request": 438.1291, + "decode_tok_s_aggregate": 430.6169, + "itl_mean_ms": 2.2824, + "itl_p95_ms": 2.3005, + "completion_tokens": [ + 63 + ] + }, + { + "checkpoint": "text4", + "route": "native", + "prompt_tokens_target": 2048, + "streams": 4, + "runs": 3, + "ttft_s": 0.2572, + "prefill_tok_s_per_request": 8311, + "prefill_tok_s_aggregate": 19697, + "decode_tok_s_per_request": 164.9347, + "decode_tok_s_aggregate": 1066.5639, + "itl_mean_ms": 6.1699, + "itl_p95_ms": 10.7286, + "completion_tokens": [ + 63 + ] + }, + { + "checkpoint": "text4", + "route": "native", + "prompt_tokens_target": 8192, + "streams": 1, + "runs": 3, + "ttft_s": 0.363, + "prefill_tok_s_per_request": 22567, + "prefill_tok_s_aggregate": 22567, + "decode_tok_s_per_request": 391.5345, + "decode_tok_s_aggregate": 384.8315, + "itl_mean_ms": 2.5541, + "itl_p95_ms": 2.5685, + "completion_tokens": [ + 66 + ] + }, + { + "checkpoint": "text4", + "route": "native", + "prompt_tokens_target": 8192, + "streams": 4, + "runs": 3, + "ttft_s": 1.0552, + "prefill_tok_s_per_request": 7913, + "prefill_tok_s_aggregate": 21927, + "decode_tok_s_per_request": 96.7656, + "decode_tok_s_aggregate": 941.6207, + "itl_mean_ms": 10.7704, + "itl_p95_ms": 45.7911, + "completion_tokens": [ + 66 + ] + }, + { + "checkpoint": "text4", + "route": "mlx_vlm", + "prompt_tokens_target": 0, + "streams": 1, + "runs": 3, + "ttft_s": 0.0196, + "prefill_tok_s_per_request": 1885, + "prefill_tok_s_aggregate": 1885, + "decode_tok_s_per_request": 394.8801, + "decode_tok_s_aggregate": 394.6517, + "itl_mean_ms": 2.5324, + "itl_p95_ms": 2.5682, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "text4", + "route": "mlx_vlm", + "prompt_tokens_target": 0, + "streams": 4, + "runs": 3, + "ttft_s": 0.0515, + "prefill_tok_s_per_request": 718, + "prefill_tok_s_aggregate": 2859, + "decode_tok_s_per_request": 289.1714, + "decode_tok_s_aggregate": 1155.8826, + "itl_mean_ms": 3.4582, + "itl_p95_ms": 3.5429, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "text4", + "route": "mlx_vlm", + "prompt_tokens_target": 2048, + "streams": 1, + "runs": 3, + "ttft_s": 0.0948, + "prefill_tok_s_per_request": 21603, + "prefill_tok_s_aggregate": 21603, + "decode_tok_s_per_request": 343.1265, + "decode_tok_s_aggregate": 337.2602, + "itl_mean_ms": 2.9144, + "itl_p95_ms": 2.9837, + "completion_tokens": [ + 80 + ] + }, + { + "checkpoint": "text4", + "route": "mlx_vlm", + "prompt_tokens_target": 2048, + "streams": 4, + "runs": 3, + "ttft_s": 0.3581, + "prefill_tok_s_per_request": 5719, + "prefill_tok_s_aggregate": 22830, + "decode_tok_s_per_request": 239.1232, + "decode_tok_s_aggregate": 903.1616, + "itl_mean_ms": 4.1819, + "itl_p95_ms": 4.5323, + "completion_tokens": [ + 80, + 89 + ] + }, + { + "checkpoint": "text4", + "route": "mlx_vlm", + "prompt_tokens_target": 8192, + "streams": 1, + "runs": 3, + "ttft_s": 0.3495, + "prefill_tok_s_per_request": 23441, + "prefill_tok_s_aggregate": 23441, + "decode_tok_s_per_request": 328.1078, + "decode_tok_s_aggregate": 321.1313, + "itl_mean_ms": 3.0478, + "itl_p95_ms": 3.1148, + "completion_tokens": [ + 66 + ] + }, + { + "checkpoint": "text4", + "route": "mlx_vlm", + "prompt_tokens_target": 8192, + "streams": 4, + "runs": 3, + "ttft_s": 1.3992, + "prefill_tok_s_per_request": 5855, + "prefill_tok_s_aggregate": 23405, + "decode_tok_s_per_request": 217.2057, + "decode_tok_s_aggregate": 760.1654, + "itl_mean_ms": 4.6039, + "itl_p95_ms": 5.1065, + "completion_tokens": [ + 65, + 128 + ] + }, + { + "checkpoint": "text6", + "route": "native", + "prompt_tokens_target": 0, + "streams": 1, + "runs": 3, + "ttft_s": 0.0152, + "prefill_tok_s_per_request": 2437, + "prefill_tok_s_aggregate": 2437, + "decode_tok_s_per_request": 363.7858, + "decode_tok_s_aggregate": 363.7457, + "itl_mean_ms": 2.7489, + "itl_p95_ms": 2.7716, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "text6", + "route": "native", + "prompt_tokens_target": 0, + "streams": 4, + "runs": 3, + "ttft_s": 0.0375, + "prefill_tok_s_per_request": 1023, + "prefill_tok_s_aggregate": 2480, + "decode_tok_s_per_request": 244.6688, + "decode_tok_s_aggregate": 1021.9796, + "itl_mean_ms": 4.0879, + "itl_p95_ms": 3.946, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "text6", + "route": "native", + "prompt_tokens_target": 2048, + "streams": 1, + "runs": 3, + "ttft_s": 0.1042, + "prefill_tok_s_per_request": 19662, + "prefill_tok_s_aggregate": 19662, + "decode_tok_s_per_request": 356.1311, + "decode_tok_s_aggregate": 352.0042, + "itl_mean_ms": 2.808, + "itl_p95_ms": 2.83, + "completion_tokens": [ + 91 + ] + }, + { + "checkpoint": "text6", + "route": "native", + "prompt_tokens_target": 2048, + "streams": 4, + "runs": 3, + "ttft_s": 0.2683, + "prefill_tok_s_per_request": 7966, + "prefill_tok_s_aggregate": 18854, + "decode_tok_s_per_request": 172.6065, + "decode_tok_s_aggregate": 968.7211, + "itl_mean_ms": 5.85, + "itl_p95_ms": 4.1756, + "completion_tokens": [ + 91 + ] + }, + { + "checkpoint": "text6", + "route": "native", + "prompt_tokens_target": 8192, + "streams": 1, + "runs": 3, + "ttft_s": 0.3769, + "prefill_tok_s_per_request": 21735, + "prefill_tok_s_aggregate": 21735, + "decode_tok_s_per_request": 324.1667, + "decode_tok_s_aggregate": 324.1325, + "itl_mean_ms": 3.0848, + "itl_p95_ms": 3.1061, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "text6", + "route": "native", + "prompt_tokens_target": 8192, + "streams": 4, + "runs": 3, + "ttft_s": 1.0997, + "prefill_tok_s_per_request": 7594, + "prefill_tok_s_aggregate": 21023, + "decode_tok_s_per_request": 125.493, + "decode_tok_s_aggregate": 855.1495, + "itl_mean_ms": 8.1272, + "itl_p95_ms": 13.5898, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "text6", + "route": "mlx_vlm", + "prompt_tokens_target": 0, + "streams": 1, + "runs": 3, + "ttft_s": 0.0182, + "prefill_tok_s_per_request": 2031, + "prefill_tok_s_aggregate": 2031, + "decode_tok_s_per_request": 319.4436, + "decode_tok_s_aggregate": 319.2695, + "itl_mean_ms": 3.1304, + "itl_p95_ms": 3.1656, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "text6", + "route": "mlx_vlm", + "prompt_tokens_target": 0, + "streams": 4, + "runs": 3, + "ttft_s": 0.0511, + "prefill_tok_s_per_request": 725, + "prefill_tok_s_aggregate": 2873, + "decode_tok_s_per_request": 260.3955, + "decode_tok_s_aggregate": 1041.6694, + "itl_mean_ms": 3.8403, + "itl_p95_ms": 3.9017, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "text6", + "route": "mlx_vlm", + "prompt_tokens_target": 2048, + "streams": 1, + "runs": 3, + "ttft_s": 0.1042, + "prefill_tok_s_per_request": 19652, + "prefill_tok_s_aggregate": 19652, + "decode_tok_s_per_request": 283.3802, + "decode_tok_s_aggregate": 279.1259, + "itl_mean_ms": 3.5288, + "itl_p95_ms": 3.6206, + "completion_tokens": [ + 91 + ] + }, + { + "checkpoint": "text6", + "route": "mlx_vlm", + "prompt_tokens_target": 2048, + "streams": 4, + "runs": 3, + "ttft_s": 0.3824, + "prefill_tok_s_per_request": 5356, + "prefill_tok_s_aggregate": 21396, + "decode_tok_s_per_request": 222.7973, + "decode_tok_s_aggregate": 869.9491, + "itl_mean_ms": 4.4884, + "itl_p95_ms": 4.7148, + "completion_tokens": [ + 117, + 128 + ] + }, + { + "checkpoint": "text6", + "route": "mlx_vlm", + "prompt_tokens_target": 8192, + "streams": 1, + "runs": 3, + "ttft_s": 0.3744, + "prefill_tok_s_per_request": 21880, + "prefill_tok_s_aggregate": 21880, + "decode_tok_s_per_request": 277.2166, + "decode_tok_s_aggregate": 273.1473, + "itl_mean_ms": 3.6073, + "itl_p95_ms": 3.6661, + "completion_tokens": [ + 99 + ] + }, + { + "checkpoint": "text6", + "route": "mlx_vlm", + "prompt_tokens_target": 8192, + "streams": 4, + "runs": 3, + "ttft_s": 1.4888, + "prefill_tok_s_per_request": 5502, + "prefill_tok_s_aggregate": 22000, + "decode_tok_s_per_request": 198.4092, + "decode_tok_s_aggregate": 751.6804, + "itl_mean_ms": 5.0401, + "itl_p95_ms": 5.4671, + "completion_tokens": [ + 102, + 128 + ] + }, + { + "checkpoint": "text8", + "route": "native", + "prompt_tokens_target": 0, + "streams": 1, + "runs": 3, + "ttft_s": 0.0158, + "prefill_tok_s_per_request": 2339, + "prefill_tok_s_aggregate": 2339, + "decode_tok_s_per_request": 345.1931, + "decode_tok_s_aggregate": 345.1463, + "itl_mean_ms": 2.8969, + "itl_p95_ms": 2.9177, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "text8", + "route": "native", + "prompt_tokens_target": 0, + "streams": 4, + "runs": 3, + "ttft_s": 0.039, + "prefill_tok_s_per_request": 986, + "prefill_tok_s_aggregate": 2383, + "decode_tok_s_per_request": 246.8891, + "decode_tok_s_aggregate": 1034.0095, + "itl_mean_ms": 4.0512, + "itl_p95_ms": 3.8989, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "text8", + "route": "native", + "prompt_tokens_target": 2048, + "streams": 1, + "runs": 3, + "ttft_s": 0.1053, + "prefill_tok_s_per_request": 19450, + "prefill_tok_s_aggregate": 19450, + "decode_tok_s_per_request": 339.1324, + "decode_tok_s_aggregate": 335.865, + "itl_mean_ms": 2.9487, + "itl_p95_ms": 2.9649, + "completion_tokens": [ + 110 + ] + }, + { + "checkpoint": "text8", + "route": "native", + "prompt_tokens_target": 2048, + "streams": 4, + "runs": 3, + "ttft_s": 0.2718, + "prefill_tok_s_per_request": 7864, + "prefill_tok_s_aggregate": 18630, + "decode_tok_s_per_request": 182.8018, + "decode_tok_s_aggregate": 979.6462, + "itl_mean_ms": 5.5121, + "itl_p95_ms": 4.1289, + "completion_tokens": [ + 110 + ] + }, + { + "checkpoint": "text8", + "route": "native", + "prompt_tokens_target": 8192, + "streams": 1, + "runs": 3, + "ttft_s": 0.3801, + "prefill_tok_s_per_request": 21553, + "prefill_tok_s_aggregate": 21553, + "decode_tok_s_per_request": 310.1153, + "decode_tok_s_aggregate": 310.0801, + "itl_mean_ms": 3.2246, + "itl_p95_ms": 3.2438, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "text8", + "route": "native", + "prompt_tokens_target": 8192, + "streams": 4, + "runs": 3, + "ttft_s": 1.1094, + "prefill_tok_s_per_request": 7528, + "prefill_tok_s_aggregate": 20851, + "decode_tok_s_per_request": 126.0276, + "decode_tok_s_aggregate": 864.7598, + "itl_mean_ms": 8.0961, + "itl_p95_ms": 13.7797, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "text8", + "route": "mlx_vlm", + "prompt_tokens_target": 0, + "streams": 1, + "runs": 3, + "ttft_s": 0.0184, + "prefill_tok_s_per_request": 2010, + "prefill_tok_s_aggregate": 2010, + "decode_tok_s_per_request": 300.8181, + "decode_tok_s_aggregate": 300.6726, + "itl_mean_ms": 3.3243, + "itl_p95_ms": 3.3428, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "text8", + "route": "mlx_vlm", + "prompt_tokens_target": 0, + "streams": 4, + "runs": 3, + "ttft_s": 0.0512, + "prefill_tok_s_per_request": 723, + "prefill_tok_s_aggregate": 2877, + "decode_tok_s_per_request": 260.6986, + "decode_tok_s_aggregate": 1042.4128, + "itl_mean_ms": 3.8358, + "itl_p95_ms": 3.888, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "text8", + "route": "mlx_vlm", + "prompt_tokens_target": 2048, + "streams": 1, + "runs": 3, + "ttft_s": 0.1025, + "prefill_tok_s_per_request": 19974, + "prefill_tok_s_aggregate": 19974, + "decode_tok_s_per_request": 266.7817, + "decode_tok_s_aggregate": 263.0735, + "itl_mean_ms": 3.7484, + "itl_p95_ms": 3.8235, + "completion_tokens": [ + 97 + ] + }, + { + "checkpoint": "text8", + "route": "mlx_vlm", + "prompt_tokens_target": 2048, + "streams": 4, + "runs": 3, + "ttft_s": 0.3702, + "prefill_tok_s_per_request": 5533, + "prefill_tok_s_aggregate": 22115, + "decode_tok_s_per_request": 220.5053, + "decode_tok_s_aggregate": 873.6418, + "itl_mean_ms": 4.535, + "itl_p95_ms": 4.7323, + "completion_tokens": [ + 105, + 106 + ] + }, + { + "checkpoint": "text8", + "route": "mlx_vlm", + "prompt_tokens_target": 8192, + "streams": 1, + "runs": 3, + "ttft_s": 0.3645, + "prefill_tok_s_per_request": 22473, + "prefill_tok_s_aggregate": 22473, + "decode_tok_s_per_request": 261.6299, + "decode_tok_s_aggregate": 261.5216, + "itl_mean_ms": 3.8222, + "itl_p95_ms": 3.8897, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "text8", + "route": "mlx_vlm", + "prompt_tokens_target": 8192, + "streams": 4, + "runs": 3, + "ttft_s": 1.4395, + "prefill_tok_s_per_request": 5691, + "prefill_tok_s_aggregate": 22750, + "decode_tok_s_per_request": 196.2076, + "decode_tok_s_aggregate": 784.7262, + "itl_mean_ms": 5.0966, + "itl_p95_ms": 5.4036, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "vl4", + "route": "native", + "prompt_tokens_target": 0, + "streams": 1, + "runs": 3, + "ttft_s": 0.0145, + "prefill_tok_s_per_request": 2548, + "prefill_tok_s_aggregate": 2548, + "decode_tok_s_per_request": 410.7544, + "decode_tok_s_aggregate": 410.6933, + "itl_mean_ms": 2.4345, + "itl_p95_ms": 2.4551, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "vl4", + "route": "native", + "prompt_tokens_target": 0, + "streams": 4, + "runs": 3, + "ttft_s": 0.0358, + "prefill_tok_s_per_request": 1073, + "prefill_tok_s_aggregate": 2598, + "decode_tok_s_per_request": 257.126, + "decode_tok_s_aggregate": 1074.6147, + "itl_mean_ms": 3.8898, + "itl_p95_ms": 3.7549, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "vl4", + "route": "native", + "prompt_tokens_target": 2048, + "streams": 1, + "runs": 3, + "ttft_s": 0.1032, + "prefill_tok_s_per_request": 19851, + "prefill_tok_s_aggregate": 19851, + "decode_tok_s_per_request": 402.6415, + "decode_tok_s_aggregate": 397.6913, + "itl_mean_ms": 2.4836, + "itl_p95_ms": 2.5026, + "completion_tokens": [ + 86 + ] + }, + { + "checkpoint": "vl4", + "route": "native", + "prompt_tokens_target": 2048, + "streams": 4, + "runs": 3, + "ttft_s": 0.2653, + "prefill_tok_s_per_request": 8058, + "prefill_tok_s_aggregate": 19088, + "decode_tok_s_per_request": 176.4562, + "decode_tok_s_aggregate": 1019.7377, + "itl_mean_ms": 5.7316, + "itl_p95_ms": 3.9861, + "completion_tokens": [ + 86 + ] + }, + { + "checkpoint": "vl4", + "route": "native", + "prompt_tokens_target": 8192, + "streams": 1, + "runs": 3, + "ttft_s": 0.3737, + "prefill_tok_s_per_request": 21920, + "prefill_tok_s_aggregate": 21920, + "decode_tok_s_per_request": 362.5058, + "decode_tok_s_aggregate": 362.4574, + "itl_mean_ms": 2.7586, + "itl_p95_ms": 2.7805, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "vl4", + "route": "native", + "prompt_tokens_target": 8192, + "streams": 4, + "runs": 3, + "ttft_s": 1.0892, + "prefill_tok_s_per_request": 7667, + "prefill_tok_s_aggregate": 21247, + "decode_tok_s_per_request": 129.6036, + "decode_tok_s_aggregate": 896.5086, + "itl_mean_ms": 7.8767, + "itl_p95_ms": 12.9431, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "vl4", + "route": "mlx_vlm", + "prompt_tokens_target": 0, + "streams": 1, + "runs": 3, + "ttft_s": 0.0179, + "prefill_tok_s_per_request": 2069, + "prefill_tok_s_aggregate": 2069, + "decode_tok_s_per_request": 364.5744, + "decode_tok_s_aggregate": 364.3469, + "itl_mean_ms": 2.7429, + "itl_p95_ms": 2.7935, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "vl4", + "route": "mlx_vlm", + "prompt_tokens_target": 0, + "streams": 4, + "runs": 3, + "ttft_s": 0.0501, + "prefill_tok_s_per_request": 738, + "prefill_tok_s_aggregate": 2920, + "decode_tok_s_per_request": 268.5677, + "decode_tok_s_aggregate": 1074.574, + "itl_mean_ms": 3.7235, + "itl_p95_ms": 3.8148, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "vl4", + "route": "mlx_vlm", + "prompt_tokens_target": 2048, + "streams": 1, + "runs": 3, + "ttft_s": 0.0964, + "prefill_tok_s_per_request": 21245, + "prefill_tok_s_aggregate": 21245, + "decode_tok_s_per_request": 322.1681, + "decode_tok_s_aggregate": 317.4103, + "itl_mean_ms": 3.104, + "itl_p95_ms": 3.175, + "completion_tokens": [ + 94 + ] + }, + { + "checkpoint": "vl4", + "route": "mlx_vlm", + "prompt_tokens_target": 2048, + "streams": 4, + "runs": 3, + "ttft_s": 0.3631, + "prefill_tok_s_per_request": 5641, + "prefill_tok_s_aggregate": 22547, + "decode_tok_s_per_request": 224.782, + "decode_tok_s_aggregate": 890.1179, + "itl_mean_ms": 4.4488, + "itl_p95_ms": 4.6277, + "completion_tokens": [ + 95 + ] + }, + { + "checkpoint": "vl4", + "route": "mlx_vlm", + "prompt_tokens_target": 8192, + "streams": 1, + "runs": 3, + "ttft_s": 0.3556, + "prefill_tok_s_per_request": 23037, + "prefill_tok_s_aggregate": 23037, + "decode_tok_s_per_request": 307.6455, + "decode_tok_s_aggregate": 307.48, + "itl_mean_ms": 3.2505, + "itl_p95_ms": 3.3025, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "vl4", + "route": "mlx_vlm", + "prompt_tokens_target": 8192, + "streams": 4, + "runs": 3, + "ttft_s": 1.4198, + "prefill_tok_s_per_request": 5770, + "prefill_tok_s_aggregate": 23071, + "decode_tok_s_per_request": 201.6863, + "decode_tok_s_aggregate": 806.862, + "itl_mean_ms": 4.9582, + "itl_p95_ms": 5.2536, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "vl6", + "route": "native", + "prompt_tokens_target": 0, + "streams": 1, + "runs": 3, + "ttft_s": 0.0152, + "prefill_tok_s_per_request": 2434, + "prefill_tok_s_aggregate": 2434, + "decode_tok_s_per_request": 356.993, + "decode_tok_s_aggregate": 356.9411, + "itl_mean_ms": 2.8012, + "itl_p95_ms": 2.7755, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "vl6", + "route": "native", + "prompt_tokens_target": 0, + "streams": 4, + "runs": 3, + "ttft_s": 0.0375, + "prefill_tok_s_per_request": 1026, + "prefill_tok_s_aggregate": 2481, + "decode_tok_s_per_request": 241.7563, + "decode_tok_s_aggregate": 1009.5613, + "itl_mean_ms": 4.1371, + "itl_p95_ms": 3.9543, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "vl6", + "route": "native", + "prompt_tokens_target": 2048, + "streams": 1, + "runs": 3, + "ttft_s": 0.1041, + "prefill_tok_s_per_request": 19680, + "prefill_tok_s_aggregate": 19680, + "decode_tok_s_per_request": 355.9406, + "decode_tok_s_aggregate": 351.1796, + "itl_mean_ms": 2.8095, + "itl_p95_ms": 2.8265, + "completion_tokens": [ + 79 + ] + }, + { + "checkpoint": "vl6", + "route": "native", + "prompt_tokens_target": 2048, + "streams": 4, + "runs": 3, + "ttft_s": 0.2684, + "prefill_tok_s_per_request": 7963, + "prefill_tok_s_aggregate": 18833, + "decode_tok_s_per_request": 165.2874, + "decode_tok_s_aggregate": 968.8092, + "itl_mean_ms": 6.123, + "itl_p95_ms": 4.1994, + "completion_tokens": [ + 79 + ] + }, + { + "checkpoint": "vl6", + "route": "native", + "prompt_tokens_target": 8192, + "streams": 1, + "runs": 3, + "ttft_s": 0.3774, + "prefill_tok_s_per_request": 21704, + "prefill_tok_s_aggregate": 21704, + "decode_tok_s_per_request": 324.5213, + "decode_tok_s_aggregate": 321.1728, + "itl_mean_ms": 3.0815, + "itl_p95_ms": 3.1028, + "completion_tokens": [ + 107 + ] + }, + { + "checkpoint": "vl6", + "route": "native", + "prompt_tokens_target": 8192, + "streams": 4, + "runs": 3, + "ttft_s": 1.1004, + "prefill_tok_s_per_request": 7589, + "prefill_tok_s_aggregate": 21013, + "decode_tok_s_per_request": 116.013, + "decode_tok_s_aggregate": 851.0129, + "itl_mean_ms": 8.8322, + "itl_p95_ms": 13.8596, + "completion_tokens": [ + 107 + ] + }, + { + "checkpoint": "vl6", + "route": "mlx_vlm", + "prompt_tokens_target": 0, + "streams": 1, + "runs": 3, + "ttft_s": 0.0184, + "prefill_tok_s_per_request": 2010, + "prefill_tok_s_aggregate": 2010, + "decode_tok_s_per_request": 318.993, + "decode_tok_s_aggregate": 318.8389, + "itl_mean_ms": 3.1349, + "itl_p95_ms": 3.1753, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "vl6", + "route": "mlx_vlm", + "prompt_tokens_target": 0, + "streams": 4, + "runs": 3, + "ttft_s": 0.0512, + "prefill_tok_s_per_request": 723, + "prefill_tok_s_aggregate": 2874, + "decode_tok_s_per_request": 260.3008, + "decode_tok_s_aggregate": 1040.6574, + "itl_mean_ms": 3.8417, + "itl_p95_ms": 3.9175, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "vl6", + "route": "mlx_vlm", + "prompt_tokens_target": 2048, + "streams": 1, + "runs": 3, + "ttft_s": 0.1034, + "prefill_tok_s_per_request": 19803, + "prefill_tok_s_aggregate": 19803, + "decode_tok_s_per_request": 283.1789, + "decode_tok_s_aggregate": 278.1845, + "itl_mean_ms": 3.5313, + "itl_p95_ms": 3.6268, + "completion_tokens": [ + 79 + ] + }, + { + "checkpoint": "vl6", + "route": "mlx_vlm", + "prompt_tokens_target": 2048, + "streams": 4, + "runs": 3, + "ttft_s": 0.3817, + "prefill_tok_s_per_request": 5365, + "prefill_tok_s_aggregate": 21443, + "decode_tok_s_per_request": 219.1956, + "decode_tok_s_aggregate": 866.354, + "itl_mean_ms": 4.5621, + "itl_p95_ms": 4.7513, + "completion_tokens": [ + 79 + ] + }, + { + "checkpoint": "vl6", + "route": "mlx_vlm", + "prompt_tokens_target": 8192, + "streams": 1, + "runs": 3, + "ttft_s": 0.3748, + "prefill_tok_s_per_request": 21859, + "prefill_tok_s_aggregate": 21859, + "decode_tok_s_per_request": 275.2769, + "decode_tok_s_aggregate": 271.7302, + "itl_mean_ms": 3.6327, + "itl_p95_ms": 3.6848, + "completion_tokens": [ + 112 + ] + }, + { + "checkpoint": "vl6", + "route": "mlx_vlm", + "prompt_tokens_target": 8192, + "streams": 4, + "runs": 3, + "ttft_s": 1.4894, + "prefill_tok_s_per_request": 5500, + "prefill_tok_s_aggregate": 21992, + "decode_tok_s_per_request": 197.7493, + "decode_tok_s_aggregate": 730.3677, + "itl_mean_ms": 5.0569, + "itl_p95_ms": 5.3947, + "completion_tokens": [ + 81, + 110 + ] + }, + { + "checkpoint": "vl8", + "route": "native", + "prompt_tokens_target": 0, + "streams": 1, + "runs": 3, + "ttft_s": 0.0157, + "prefill_tok_s_per_request": 2351, + "prefill_tok_s_aggregate": 2351, + "decode_tok_s_per_request": 345.3645, + "decode_tok_s_aggregate": 345.3213, + "itl_mean_ms": 2.8955, + "itl_p95_ms": 2.9137, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "vl8", + "route": "native", + "prompt_tokens_target": 0, + "streams": 4, + "runs": 3, + "ttft_s": 0.0388, + "prefill_tok_s_per_request": 990, + "prefill_tok_s_aggregate": 2389, + "decode_tok_s_per_request": 247.0022, + "decode_tok_s_aggregate": 1033.7255, + "itl_mean_ms": 4.0494, + "itl_p95_ms": 3.9019, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "vl8", + "route": "native", + "prompt_tokens_target": 2048, + "streams": 1, + "runs": 3, + "ttft_s": 0.1052, + "prefill_tok_s_per_request": 19468, + "prefill_tok_s_aggregate": 19468, + "decode_tok_s_per_request": 339.9374, + "decode_tok_s_aggregate": 335.4299, + "itl_mean_ms": 2.9417, + "itl_p95_ms": 2.9597, + "completion_tokens": [ + 80 + ] + }, + { + "checkpoint": "vl8", + "route": "native", + "prompt_tokens_target": 2048, + "streams": 4, + "runs": 3, + "ttft_s": 0.2717, + "prefill_tok_s_per_request": 7868, + "prefill_tok_s_aggregate": 18628, + "decode_tok_s_per_request": 167.0059, + "decode_tok_s_aggregate": 980.9704, + "itl_mean_ms": 6.0615, + "itl_p95_ms": 4.1485, + "completion_tokens": [ + 80 + ] + }, + { + "checkpoint": "vl8", + "route": "native", + "prompt_tokens_target": 8192, + "streams": 1, + "runs": 3, + "ttft_s": 0.38, + "prefill_tok_s_per_request": 21558, + "prefill_tok_s_aggregate": 21558, + "decode_tok_s_per_request": 310.9826, + "decode_tok_s_aggregate": 308.2366, + "itl_mean_ms": 3.2156, + "itl_p95_ms": 3.2342, + "completion_tokens": [ + 124 + ] + }, + { + "checkpoint": "vl8", + "route": "native", + "prompt_tokens_target": 8192, + "streams": 4, + "runs": 3, + "ttft_s": 1.1099, + "prefill_tok_s_per_request": 7524, + "prefill_tok_s_aggregate": 20836, + "decode_tok_s_per_request": 123.9454, + "decode_tok_s_aggregate": 858.5224, + "itl_mean_ms": 8.24, + "itl_p95_ms": 13.7739, + "completion_tokens": [ + 124 + ] + }, + { + "checkpoint": "vl8", + "route": "mlx_vlm", + "prompt_tokens_target": 0, + "streams": 1, + "runs": 3, + "ttft_s": 0.0185, + "prefill_tok_s_per_request": 2000, + "prefill_tok_s_aggregate": 2000, + "decode_tok_s_per_request": 300.5965, + "decode_tok_s_aggregate": 300.4695, + "itl_mean_ms": 3.3267, + "itl_p95_ms": 3.3485, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "vl8", + "route": "mlx_vlm", + "prompt_tokens_target": 0, + "streams": 4, + "runs": 3, + "ttft_s": 0.0513, + "prefill_tok_s_per_request": 721, + "prefill_tok_s_aggregate": 2849, + "decode_tok_s_per_request": 260.6862, + "decode_tok_s_aggregate": 1042.2428, + "itl_mean_ms": 3.836, + "itl_p95_ms": 3.9063, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "vl8", + "route": "mlx_vlm", + "prompt_tokens_target": 2048, + "streams": 1, + "runs": 3, + "ttft_s": 0.1016, + "prefill_tok_s_per_request": 20163, + "prefill_tok_s_aggregate": 20163, + "decode_tok_s_per_request": 269.0528, + "decode_tok_s_aggregate": 264.2491, + "itl_mean_ms": 3.7167, + "itl_p95_ms": 3.8012, + "completion_tokens": [ + 80 + ] + }, + { + "checkpoint": "vl8", + "route": "mlx_vlm", + "prompt_tokens_target": 2048, + "streams": 4, + "runs": 3, + "ttft_s": 0.3695, + "prefill_tok_s_per_request": 5542, + "prefill_tok_s_aggregate": 22129, + "decode_tok_s_per_request": 219.2983, + "decode_tok_s_aggregate": 866.1247, + "itl_mean_ms": 4.56, + "itl_p95_ms": 4.8287, + "completion_tokens": [ + 80 + ] + }, + { + "checkpoint": "vl8", + "route": "mlx_vlm", + "prompt_tokens_target": 8192, + "streams": 1, + "runs": 3, + "ttft_s": 0.3652, + "prefill_tok_s_per_request": 22431, + "prefill_tok_s_aggregate": 22431, + "decode_tok_s_per_request": 261.1833, + "decode_tok_s_aggregate": 261.0778, + "itl_mean_ms": 3.8287, + "itl_p95_ms": 3.8787, + "completion_tokens": [ + 128 + ] + }, + { + "checkpoint": "vl8", + "route": "mlx_vlm", + "prompt_tokens_target": 8192, + "streams": 4, + "runs": 3, + "ttft_s": 1.439, + "prefill_tok_s_per_request": 5693, + "prefill_tok_s_aggregate": 22749, + "decode_tok_s_per_request": 197.4901, + "decode_tok_s_aggregate": 763.4242, + "itl_mean_ms": 5.0635, + "itl_p95_ms": 5.3301, + "completion_tokens": [ + 116, + 128 + ] + } + ] +} diff --git a/docs/receipts/lfm3/m5-nax-split-full-contract.json b/docs/receipts/lfm3/m5-nax-split-full-contract.json new file mode 100644 index 0000000000..0f88b9a233 --- /dev/null +++ b/docs/receipts/lfm3/m5-nax-split-full-contract.json @@ -0,0 +1,70 @@ +{ + "scope": "Full Metal kernel contract run with the LFM3 NAX prompt projections, the tiled prompt attention (BF16 probabilities for PV) and the key-parallel split attention registered.", + "hardware": "M5 Ultra", + "command": "tf-kernel-contract --rotary-accuracy rotary.json", + "source_revision": "9b23b48f2", + "exit": 1, + "raw_log_sha256": "98cae33f18329887b08ba4d7d1a7fa9b6d89b896650860345fdf91cb371c150f", + "summary": "SUMMARY 326 kernels: 325 pass gates 1-4, 1 fail, 2 known findings; 327 controls: 327 red as planted, 0 missed; 0 limit problems; gates 2381208 ms, limits 1398 ms", + "production_failure": "FAIL shapes metal Qwen4 native HC act production: at 128x14336x1 output 218116 is -3.997803e-3 where float64 gives -4.028320e-3: 1.94 of its bound", + "baseline_note": "The same Qwen4 HC case was reproduced on pinned base b30d5b9d (see m5-full-contract-tail.json); it is not an LFM3 kernel.", + "known_findings": [ + "KNOWN nonfinite metal affine_mm q4 g64: an infinite input gives NaN where MLX's s*sum(q x) + b*sum(x) keeps the infinity (the +128 code offset moves the bias to b - 128 s)", + "KNOWN nonfinite metal affine_mm q4 g64 fp32 out: an infinite input gives NaN where MLX's s*sum(q x) + b*sum(x) keeps the infinity (the +128 code offset moves the bias to b - 128 s)" + ], + "lfm3_new_kernels": [ + "PASS nonfinite metal LFM3 NAX prompt q4g64: 30720 non-finite and 30720 finite outputs match the float64 classes", + "PASS repeat metal LFM3 NAX prompt q4g64: five runs byte-identical (983040 bytes), inputs and guards untouched, no overlap", + "PASS shapes metal LFM3 NAX prompt q4g64: 14 shapes within the float64 bound (worst 0.314 of the bound), 0 edges refused by the kernel", + "PASS sums metal LFM3 NAX prompt q4g64: exact sums at k 7680, where fp16 sums would miss by 12840x and bf16 by 256x", + "FAIL shapes control: Flash Next FSR q6 wrong slot map: at 1x64x64 its bytes differ from the resident expert bodies on the same stored bytes, routes and input rows's from byte 0", + "FAIL nonfinite control: fmax row max: 1 of 8 outputs differ in class; first at 0: finite where float64 gives nan", + "PASS nonfinite metal LFM3 NAX prompt q6g64: 30720 non-finite and 30720 finite outputs match the float64 classes", + "PASS repeat metal LFM3 NAX prompt q6g64: five runs byte-identical (983040 bytes), inputs and guards untouched, no overlap", + "PASS shapes metal LFM3 NAX prompt q6g64: 14 shapes within the float64 bound (worst 0.360 of the bound), 0 edges refused by the kernel", + "PASS sums metal LFM3 NAX prompt q6g64: exact sums at k 7680, where fp16 sums would miss by 4343x and bf16 by 23575x", + "PASS nonfinite metal LFM3 NAX prompt q8g64: 30720 non-finite and 30720 finite outputs match the float64 classes", + "PASS repeat metal LFM3 NAX prompt q8g64: five runs byte-identical (983040 bytes), inputs and guards untouched, no overlap", + "PASS shapes metal LFM3 NAX prompt q8g64: 14 shapes within the float64 bound (worst 0.374 of the bound), 0 edges refused by the kernel", + "PASS sums metal LFM3 NAX prompt q8g64: exact sums at k 7680, where fp16 sums would miss by 1133x and bf16 by 7658x", + "PASS nonfinite metal LFM3 NAX prompt q4g32: 30720 non-finite and 30720 finite outputs match the float64 classes", + "PASS repeat metal LFM3 NAX prompt q4g32: five runs byte-identical (983040 bytes), inputs and guards untouched, no overlap", + "PASS shapes metal LFM3 NAX prompt q4g32: 14 shapes within the float64 bound (worst 0.416 of the bound), 0 edges refused by the kernel", + "PASS sums metal LFM3 NAX prompt q4g32: exact sums at k 7680, where fp16 sums would miss by 13767x and bf16 by 256x", + "PASS nonfinite metal LFM3 tiled structural full: the stated rule holds: visible-query/key/value nonfinite classes; masked future values must not contaminate", + "PASS repeat metal LFM3 tiled structural full: five runs byte-identical (4096 bytes), inputs and guards untouched, no overlap", + "PASS shapes metal LFM3 tiled structural full: 16 shapes exact by its own check (worst 0.000 of the bound), 0 edges refused by the kernel", + "PASS sums metal LFM3 tiled structural full: within its bound at k 8192; fp16 sums would miss 0x, bf16 0.0x (uniform-score structural fixture; no claimed 100x reduced-accumulator witness)", + "FAIL shapes metal LFM3 tiled structural bad-tail: at 5x2048x2043: visible/masked nonfinite class mismatch", + "FAIL nonfinite metal LFM3 tiled structural bad-nonfinite: visible/masked nonfinite class mismatch", + "PASS nonfinite metal LFM3 tiled structural sliding: the stated rule holds: visible-query/key/value nonfinite classes; masked future values must not contaminate", + "PASS repeat metal LFM3 tiled structural sliding: five runs byte-identical (4096 bytes), inputs and guards untouched, no overlap", + "PASS shapes metal LFM3 tiled structural sliding: 16 shapes exact by its own check (worst 0.000 of the bound), 0 edges refused by the kernel", + "PASS sums metal LFM3 tiled structural sliding: within its bound at k 8192; fp16 sums would miss 0x, bf16 0.0x (uniform-score structural fixture; no claimed 100x reduced-accumulator witness)", + "PASS nonfinite metal LFM3 split structural full: the stated rule holds: visible-query/key/value nonfinite classes; masked future values must not contaminate", + "PASS repeat metal LFM3 split structural full: five runs byte-identical (4096 bytes), inputs and guards untouched, no overlap", + "PASS shapes metal LFM3 split structural full: 16 shapes exact by its own check (worst 0.000 of the bound), 0 edges refused by the kernel", + "PASS sums metal LFM3 split structural full: within its bound at k 8192; fp16 sums would miss 0x, bf16 0.0x (uniform-score structural fixture; no claimed 100x reduced-accumulator witness)", + "FAIL shapes metal LFM3 split structural bad-tail: at 5x2048x2043: visible/masked nonfinite class mismatch", + "FAIL nonfinite metal LFM3 split structural bad-nonfinite: visible/masked nonfinite class mismatch", + "PASS nonfinite metal LFM3 split structural sliding: the stated rule holds: visible-query/key/value nonfinite classes; masked future values must not contaminate", + "PASS repeat metal LFM3 split structural sliding: five runs byte-identical (4096 bytes), inputs and guards untouched, no overlap", + "PASS shapes metal LFM3 split structural sliding: 16 shapes exact by its own check (worst 0.000 of the bound), 0 edges refused by the kernel", + "PASS sums metal LFM3 split structural sliding: within its bound at k 8192; fp16 sums would miss 0x, bf16 0.0x (uniform-score structural fixture; no claimed 100x reduced-accumulator witness)" + ], + "lfm3_new_kernel_limits": [ + "PASS limits here LFM3 tiled attention lfm3_attn_tiled_bf16: 128 threads launched, the pipeline takes 128; 0 threadgroup bytes and 0 set (declared at most 0, the GPU's most 32768)", + "PASS limits here LFM3 split attention lfm3_attn_split: 128 threads launched, the pipeline takes 128; 2592 threadgroup bytes and 0 set (declared at most 2592, the GPU's most 32768)", + "PASS limits here LFM3 split attention lfm3_attn_split_merge: 32 threads launched, the pipeline takes 32; 0 threadgroup bytes and 0 set (declared at most 0, the GPU's most 32768)", + "PASS limits here LFM3 split attention lfm3_attn_split_segments: 128 threads launched, the pipeline takes 128; 2592 threadgroup bytes and 0 set (declared at most 2592, the GPU's most 32768)", + "PASS limits here LFM3 split attention lfm3_attn_split_segments_merge: 32 threads launched, the pipeline takes 32; 0 threadgroup bytes and 0 set (declared at most 0, the GPU's most 32768)", + "PASS limits here LFM3 NAX prompt q4g64 tf_qmmw_t_nax_win: 128 threads launched, the pipeline takes 1024; 11264 threadgroup bytes and 0 set (declared at most 11264, the GPU's most 32768)", + "PASS limits here LFM3 NAX prompt q6g64 tf_qmmw_t_nax_win: 128 threads launched, the pipeline takes 1024; 11264 threadgroup bytes and 0 set (declared at most 11264, the GPU's most 32768)", + "PASS limits here LFM3 NAX prompt q8g64 tf_qmmw_t_nax_win: 128 threads launched, the pipeline takes 1024; 11264 threadgroup bytes and 0 set (declared at most 11264, the GPU's most 32768)", + "PASS limits here LFM3 NAX prompt q4g32 tf_qmmw_t_nax_win: 128 threads launched, the pipeline takes 1024; 11264 threadgroup bytes and 0 set (declared at most 11264, the GPU's most 32768)" + ], + "notes": [ + "Lines beginning FAIL on a planted control are the expected red result; the SUMMARY counts 327 controls red as planted and 0 missed.", + "The NAX prompt probes use the dequantized-weight float64 form and a depth that includes the BF16 rounding of each dequantized weight (2^-9 relative); nonfinite classes match float64." + ] +} diff --git a/docs/receipts/lfm3/m5-prefill-batching.json b/docs/receipts/lfm3/m5-prefill-batching.json new file mode 100644 index 0000000000..2e0bbf3db8 --- /dev/null +++ b/docs/receipts/lfm3/m5-prefill-batching.json @@ -0,0 +1,441 @@ +{ + "scope": "Private M5 text4 batching-only causal screen; not enabled publicly and no all-width/final standard performance acceptance", + "change": "Only local native compute chunk and LaneHost prefill_step128/512/1024; globalcacheRules chunk128/repeat_min256 unchanged; all numerical kernels unchanged", + "builds": [ + { + "chunk": 128, + "native_source_sha256": "defb0b6ef8eee2fa72015575c70ae007cfb182b03a6ca73011cedc1082b08d13", + "binary_sha256": "9bee163d44bb3f2aa27d6d06fa791734e043d403a08f48b1dc3ba313cc4fd2c9", + "cacheRules_prompt_chunk_unchanged": 128, + "changed_compute_and_LaneHost_prefill_step": true + }, + { + "chunk": 512, + "native_source_sha256": "0ee5614230af82f626f45d81dab48271e18b329eeab9a2e9423d6a50abb967de", + "binary_sha256": "ed1b7601d65d520822f4208365d389ffe5e10353e8377932da76338442c849db", + "cacheRules_prompt_chunk_unchanged": 128, + "changed_compute_and_LaneHost_prefill_step": true + }, + { + "chunk": 1024, + "native_source_sha256": "d5360c03c851878169e04e1530126e05d248dd8c3a77e6768a7c4c6458d68df8", + "binary_sha256": "72ac6ca09394400a0d2afe6a6d27a40245ca93e8c57d2758bc742744c39aa3e3", + "cacheRules_prompt_chunk_unchanged": 128, + "changed_compute_and_LaneHost_prefill_step": true + } + ], + "environment": { + "host": "ProductName:\t\tmacOS\nProductVersion:\t\t27.0\nBuildVersion:\t\t26A428\n", + "physical_memory_bytes": 274877906944, + "requirements_sha256": "275b369ea72d5757b1e3f91e68ed5b746d83b5f7667adc8f5d9d28bb8933ddd6", + "clients": { + "$PACKET/tools/bench_auth.py": "1896120b70d0df05646870a252697e20ce84c63e09dcba88b2a4e5a55ccb3c5b", + "$PACKET/tools/bench_cache.py": "24364dd13c033450f45710fc8685e2c64d6a72d6dd6e223d0d7b19aabbdc1d98", + "$PACKET/tools/bench_concurrent.py": "0e72e26769f082459329fc068c25f6f72a9155e422962ea54cb5d1226227752d", + "$PACKET/tools/bench_evidence.py": "54c28474f0804dbd120e3fb5859be0d1667ef21ca502880d0bed098ec9bb42ff", + "$PACKET/tools/bench_metrics.py": "26f4c3ecbb03df4f12e24d18840fea989d9ef8c63117ca11eb944ef670458a27", + "$PACKET/tools/bench_openai.py": "d5af4fbb756cdfb573546c0e0017652a4bb8f509d1251af683be7a3cbdd426f6", + "$PACKET/tools/bench_timing.py": "158f636ef0b3bed460ce19ef4872b6d8607ff7a8590562d06bae8bd55e6f44ed", + "$PACKET/tools/lfm3/bench_decode.py": "c011106f4f182db19371e3f91f70bcc374160786dd105f2d713a7fea07ab8c4c", + "$PACKET/cold_client.py": "7b8ea55aa88834814f95a6f679654e16074b7a8364e34b39a276f009fe0231f2" + } + }, + "numerical_proof": { + "cases": 12, + "all_valid_KV_convstate_and_logits_exact_across_chunks": true, + "all_plain_profiled_state_and_logits_exact": true, + "timing_fields": "DISCARDED for entire numerical-only proof because ownedCPUcompilation overlapped; no plain,profiled,counter orGPUclock fromthatproof used forcost. Fresh guardedHTTPscreen supplied measuredclocks.", + "identities": [ + { + "variant": "current", + "tokens": 2048, + "chunk": 128, + "state_sha256": "7f7e23060107d5252391778f46727022f8ee157fa34eb1753c424cac2f3f771f", + "logits_sha256": "8700e9a12456dce4f744caa329ad2013f44df7f2704e58e245e52aa0e9e9ddc5" + }, + { + "variant": "current", + "tokens": 2048, + "chunk": 512, + "state_sha256": "7f7e23060107d5252391778f46727022f8ee157fa34eb1753c424cac2f3f771f", + "logits_sha256": "8700e9a12456dce4f744caa329ad2013f44df7f2704e58e245e52aa0e9e9ddc5" + }, + { + "variant": "current", + "tokens": 2048, + "chunk": 1024, + "state_sha256": "7f7e23060107d5252391778f46727022f8ee157fa34eb1753c424cac2f3f771f", + "logits_sha256": "8700e9a12456dce4f744caa329ad2013f44df7f2704e58e245e52aa0e9e9ddc5" + }, + { + "variant": "paired_tiles", + "tokens": 2048, + "chunk": 128, + "state_sha256": "a311bb2c6496dc8bbe04a40b8e3d592a3d48c869fdd36b37f6a09dc7b10be651", + "logits_sha256": "7ad8423d5e79a75bc4b72f9a0b32b4f2023272d3a1ec04b597fb37cd5438e964" + }, + { + "variant": "paired_tiles", + "tokens": 2048, + "chunk": 512, + "state_sha256": "a311bb2c6496dc8bbe04a40b8e3d592a3d48c869fdd36b37f6a09dc7b10be651", + "logits_sha256": "7ad8423d5e79a75bc4b72f9a0b32b4f2023272d3a1ec04b597fb37cd5438e964" + }, + { + "variant": "paired_tiles", + "tokens": 2048, + "chunk": 1024, + "state_sha256": "a311bb2c6496dc8bbe04a40b8e3d592a3d48c869fdd36b37f6a09dc7b10be651", + "logits_sha256": "7ad8423d5e79a75bc4b72f9a0b32b4f2023272d3a1ec04b597fb37cd5438e964" + }, + { + "variant": "current", + "tokens": 8192, + "chunk": 128, + "state_sha256": "5524361e72b466c99d75fbf958f1faa0c715920b3456f01532907c47a76cb27b", + "logits_sha256": "19a2f9b7b0794ef2f2be0edb7d16564e24d33f02e0d01972780ae28d09010bf5" + }, + { + "variant": "current", + "tokens": 8192, + "chunk": 512, + "state_sha256": "5524361e72b466c99d75fbf958f1faa0c715920b3456f01532907c47a76cb27b", + "logits_sha256": "19a2f9b7b0794ef2f2be0edb7d16564e24d33f02e0d01972780ae28d09010bf5" + }, + { + "variant": "current", + "tokens": 8192, + "chunk": 1024, + "state_sha256": "5524361e72b466c99d75fbf958f1faa0c715920b3456f01532907c47a76cb27b", + "logits_sha256": "19a2f9b7b0794ef2f2be0edb7d16564e24d33f02e0d01972780ae28d09010bf5" + }, + { + "variant": "paired_tiles", + "tokens": 8192, + "chunk": 128, + "state_sha256": "28e0d790935a448a8d724bd4136b9de38e237afb8dcd545b708e9de5f5b974dd", + "logits_sha256": "305e345e11883986154d8b8dde198985589ac1d064517c14bdd46224c46dd074" + }, + { + "variant": "paired_tiles", + "tokens": 8192, + "chunk": 512, + "state_sha256": "28e0d790935a448a8d724bd4136b9de38e237afb8dcd545b708e9de5f5b974dd", + "logits_sha256": "305e345e11883986154d8b8dde198985589ac1d064517c14bdd46224c46dd074" + }, + { + "variant": "paired_tiles", + "tokens": 8192, + "chunk": 1024, + "state_sha256": "28e0d790935a448a8d724bd4136b9de38e237afb8dcd545b708e9de5f5b974dd", + "logits_sha256": "305e345e11883986154d8b8dde198985589ac1d064517c14bdd46224c46dd074" + } + ], + "state_hash_scope": "Perlayer: complete convolutionstate and validcache.lenpositions ofkeys+values foreachKVhead usinghead*capacity stride; excludesunusedKVcapacity/padding. Final128000BF16logits hashed separately. Equality is within each arithmeticvariant across128/512/1024chunks, notbetween current andpaired variants.", + "diagnostic_artifact_and_source_sha256": { + "zig-out/bin/tf-lfm3-profile": "6545301dec2f8bbcb79193b8560bcae847ecb0f0011e990d2d1162264b14729b", + "zig/tests/lfm3_profile.zig": "1d12a9f4e181635bb4446dd100ac3c28c388500be13a1bed79e843a18b74167c", + "zig/src/core/gpu_profile.zig": "80eb55884ef6d7c9b894f06e3ebfa06a4a485bd3f48895d634ed89d6a638261b", + "zig/src/families/lfm3/forward.zig": "894c9541112518287bc9985db51f7e023b870b234da54acc626e9f2f57533bc9" + }, + "artifact_provenance": "Separate state-enhanced diagnostic6545301d usedfor12proofcases; distinctfrom initialcold-stagef664911c. Qualificationownerreadremoteunchangedartifact/source afterproof andconfirmed no interveningchanges." + }, + "actual_guarded_http": { + "legs": 6, + "all_legs_exited_zero": true, + "primary_requests": 180, + "all_cold_requests_cache0": true, + "all_owned_servers_exited": true, + "lock_acquired": [ + { + "event": "lock_acquired", + "time_ns": 1793202901164008000, + "path": "/tmp/tensorfold-model.lock", + "pid": 20531, + "device": 16777233, + "inode": 2622134 + } + ], + "lock_release": [ + { + "event": "lock_release", + "time_ns": 1793203043859929000, + "pid": 20531 + } + ], + "all_primary_decode_requests_exact256": true, + "cold_output_tokens": 16 + }, + "measured": { + "scope": "M5 TEXT4 current-public batching-only; same remote toolchain/clients/tokens; globalcacheRules128 unchanged; no final performance acceptance", + "engines": { + "batch128": { + "medians": { + "long-code-chat/1": 315.4, + "long-code-chat/4": 861.95, + "long-prose-chat/1": 329.1, + "long-prose-chat/4": 879.4 + }, + "cold_medians": { + "2048": 0.8591289380565286, + "8192": 4.614509062608704 + }, + "legs": [ + { + "tag": "text-4-0-batch128", + "cells": { + "long-code-chat/1": 315.8, + "long-code-chat/4": 861.3, + "long-prose-chat/1": 329.8, + "long-prose-chat/4": 879.0 + }, + "cold": { + "2048": 0.8592449170537293, + "8192": 4.614482833072543 + }, + "cold_hash": { + "2048": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "8192": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349" + } + }, + { + "tag": "text-4-5-batch128", + "cells": { + "long-code-chat/1": 315.0, + "long-code-chat/4": 862.6, + "long-prose-chat/1": 328.4, + "long-prose-chat/4": 879.8 + }, + "cold": { + "2048": 0.8590129590593278, + "8192": 4.614535292144865 + }, + "cold_hash": { + "2048": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "8192": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349" + } + } + ] + }, + "batch512": { + "medians": { + "long-code-chat/1": 315.55, + "long-code-chat/4": 860.45, + "long-prose-chat/1": 329.9, + "long-prose-chat/4": 877.3499999999999 + }, + "cold_medians": { + "2048": 0.8431003752630204, + "8192": 4.529019312467426 + }, + "legs": [ + { + "tag": "text-4-1-batch512", + "cells": { + "long-code-chat/1": 316.0, + "long-code-chat/4": 860.9, + "long-prose-chat/1": 330.3, + "long-prose-chat/4": 878.3 + }, + "cold": { + "2048": 0.8432092503644526, + "8192": 4.529814667068422 + }, + "cold_hash": { + "2048": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "8192": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349" + } + }, + { + "tag": "text-4-4-batch512", + "cells": { + "long-code-chat/1": 315.1, + "long-code-chat/4": 860.0, + "long-prose-chat/1": 329.5, + "long-prose-chat/4": 876.4 + }, + "cold": { + "2048": 0.8429915001615882, + "8192": 4.52822395786643 + }, + "cold_hash": { + "2048": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "8192": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349" + } + } + ] + }, + "batch1024": { + "medians": { + "long-code-chat/1": 314.1, + "long-code-chat/4": 859.9, + "long-prose-chat/1": 327.95, + "long-prose-chat/4": 877.55 + }, + "cold_medians": { + "2048": 0.8394205416552722, + "8192": 4.506114687537774 + }, + "legs": [ + { + "tag": "text-4-2-batch1024", + "cells": { + "long-code-chat/1": 314.2, + "long-code-chat/4": 860.0, + "long-prose-chat/1": 327.9, + "long-prose-chat/4": 877.2 + }, + "cold": { + "2048": 0.8395577501505613, + "8192": 4.506872625090182 + }, + "cold_hash": { + "2048": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "8192": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349" + } + }, + { + "tag": "text-4-3-batch1024", + "cells": { + "long-code-chat/1": 314.0, + "long-code-chat/4": 859.8, + "long-prose-chat/1": 328.0, + "long-prose-chat/4": 877.9 + }, + "cold": { + "2048": 0.8392833331599832, + "8192": 4.505356749985367 + }, + "cold_hash": { + "2048": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "8192": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349" + } + } + ] + } + }, + "cold_response_hashes_equal_across_chunks": true, + "cross_chunk_full_256_token_hashes": { + "long-code-chat": [ + "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7" + ], + "long-prose-chat": [ + "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a" + ] + }, + "all_cross_chunk_decode_hashes_equal_per_prompt": true, + "primary_measured_requests": 180 + }, + "conclusion": "1024 reduces median8kTTFT2.35percent with flatdecode; this does not resolve the measured standard-server performance gap.", + "performance_accepted": false, + "raw_evidence_sha256": { + "batch-proof-cpu-overlap/0-current-2048-128.json": "d7c086e19e408df07d1328804f7cedf1afcf977b69604ce7920c8adbf8227730", + "batch-proof-cpu-overlap/0-current-2048-128.log": "3abf7120a89d3ff17e97e5d25061c4199236419aba394e4cada6f084877cc954", + "batch-proof-cpu-overlap/1-current-2048-512.json": "ac686900ba9848b537684ac8b821f902bcff39916c9a2233178320479b6963d3", + "batch-proof-cpu-overlap/1-current-2048-512.log": "027e330fce708cd3666505e90436b8b18d6d7a977b16d760bc3e34194ec8f325", + "batch-proof-cpu-overlap/10-paired-8192-512.json": "e07ca97c54659b36bb6c241ba25d5f10903dd12a01cee159ce19a1e27aaa5a8a", + "batch-proof-cpu-overlap/10-paired-8192-512.log": "e1b2a45a79d96cdab9d8cc6f875d25100ac33ba777cf7639f6b2ee460514311c", + "batch-proof-cpu-overlap/11-paired-8192-1024.json": "5642e80e5851847bc5feb4e57985ce9ae34ccd5c7bad35fd647f038d5febbfa0", + "batch-proof-cpu-overlap/11-paired-8192-1024.log": "693aaafbf1ce09f8c1ad859ff46b42f7bb9e4e74d8c233de717a22a2675e2e18", + "batch-proof-cpu-overlap/2-current-2048-1024.json": "df587499c06db59e9bfcc7f54391f707f82a2fafa0b108ca947a9c45fc9fa9f8", + "batch-proof-cpu-overlap/2-current-2048-1024.log": "a6e60d7b73270eec97e8d3395c57663cf09d00fd16cead0afb91a331d96e5492", + "batch-proof-cpu-overlap/3-paired-2048-128.json": "bf07642f6511d0209999dbde294bdb92553ad8183ba63f9369047999630cc5e3", + "batch-proof-cpu-overlap/3-paired-2048-128.log": "3abf7120a89d3ff17e97e5d25061c4199236419aba394e4cada6f084877cc954", + "batch-proof-cpu-overlap/4-paired-2048-512.json": "17ff3800e2009858f67ea68f040a22fef1f79b0cbd64a0c3b036b6f0e8cf7014", + "batch-proof-cpu-overlap/4-paired-2048-512.log": "027e330fce708cd3666505e90436b8b18d6d7a977b16d760bc3e34194ec8f325", + "batch-proof-cpu-overlap/5-paired-2048-1024.json": "b49529ef675762a299338d2595824afd6f6ff6804868888422a58b968f4cc9eb", + "batch-proof-cpu-overlap/5-paired-2048-1024.log": "a6e60d7b73270eec97e8d3395c57663cf09d00fd16cead0afb91a331d96e5492", + "batch-proof-cpu-overlap/6-current-8192-128.json": "a75245e722d5849f48b6762d60bec9dd4d5ec642baeb5ed5f48276e75e9e96fc", + "batch-proof-cpu-overlap/6-current-8192-128.log": "5cf360bde5c1a9e0ff68b1cbc385982da901a621793fe54b8ef9fac5763c83ef", + "batch-proof-cpu-overlap/7-current-8192-512.json": "0e423b24b7497aa7b37ccd81e40229a240a28cdf8a36b89088184d588a40e062", + "batch-proof-cpu-overlap/7-current-8192-512.log": "e1b2a45a79d96cdab9d8cc6f875d25100ac33ba777cf7639f6b2ee460514311c", + "batch-proof-cpu-overlap/8-current-8192-1024.json": "eb72b25ac6460aae682d36d395a6e077775db5f104d65209a997ffef819cf85a", + "batch-proof-cpu-overlap/8-current-8192-1024.log": "693aaafbf1ce09f8c1ad859ff46b42f7bb9e4e74d8c233de717a22a2675e2e18", + "batch-proof-cpu-overlap/9-paired-8192-128.json": "9be85bc2d4a7380aa4e4e5bfd2e824bc6a59438d86eddf886e9b35d860c14f81", + "batch-proof-cpu-overlap/9-paired-8192-128.log": "5cf360bde5c1a9e0ff68b1cbc385982da901a621793fe54b8ef9fac5763c83ef", + "batch-proof-cpu-overlap/events.json": "908cb91625c9b6cc47be33695f68b4e885285ae3e9abea2cc4896ba317495e64", + "batch-proof-cpu-overlap/records.json": "7dbfdaf498e0351d843aad4dc24ce5aeb2cb14e8c59b4ecc0d37007ba703d3f6", + "batch-http-results/events.json": "d0c7ee00352eb24cf4e9f62d517103104b89b310414a1803fc30e6979bf3c813", + "batch-http-results/manifest.json": "1621296c8c2c5dd4418ce76df95d8852426d51934c85ebbc596d8600e8596714", + "batch-http-results/records.json": "6720c00b383b1be051ac5b94c94bb75d1b911ce062459c7cc5fe217c106c6fe8", + "batch-http-results/summary.json": "1fd8d85dde73f53b7df030eec3cadf8f645221fd8f70fca24fac54f21ae4c5e1", + "batch-http-results/text-4-0-batch128.after.health": "aafbd1bfafda3d655953e1fbf374a4b538ac6beb86745507d8f0e52ac7bc8da5", + "batch-http-results/text-4-0-batch128.after.metrics": "8186f92fe0b3a978ffb003033065fae6afdc795fa54cdc8cede57b9766d1031d", + "batch-http-results/text-4-0-batch128.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-0-batch128.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "batch-http-results/text-4-0-batch128.before.metrics": "71f2b5590d688aa2e3eb65bfbb767409db9ded4db139eac8b46ef95a0671a604", + "batch-http-results/text-4-0-batch128.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-0-batch128.cold-2048.client.log": "8cca6d4bfaa730306250057111c9d4f7b5c2698d0a9079b0a1a49c5dc079efce", + "batch-http-results/text-4-0-batch128.cold-2048.json": "0833705cb3eebd04c104a4f99b9605e1cbcecb244e53a1a2d582be392975cc78", + "batch-http-results/text-4-0-batch128.cold-8192.client.log": "8be2f7473963c3cfde53c441e2c4fb0a49037c64a8e97d1fe115eae847a6c331", + "batch-http-results/text-4-0-batch128.cold-8192.json": "2f11c23b2183b49dcd9160bfaf63a9030ac219ee8b92e47a4dfa4553685943af", + "batch-http-results/text-4-0-batch128.concurrent.client.log": "807bc346faaf103ddffcc7a763639467fce0c5f52692a0fbe7e4c2a0f021ebef", + "batch-http-results/text-4-0-batch128.concurrent.json": "2dda4a19c1b7e72931e842531ad7dfbfd48cbb1cd9ef15d927d709b8dd04552c", + "batch-http-results/text-4-0-batch128.server.log": "07efb3e16cd0a2c3fac73935e2db2990f06126ca3428101109d19a700ae066b9", + "batch-http-results/text-4-1-batch512.after.health": "490b87e88a6d848b908a53902677ceb3f29e5e9806fe03965664b819148e0c9c", + "batch-http-results/text-4-1-batch512.after.metrics": "5c119b56ce47469448efcd47cf2bc799e85f6dbe16565dc6847ae4241cca850e", + "batch-http-results/text-4-1-batch512.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-1-batch512.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "batch-http-results/text-4-1-batch512.before.metrics": "db6147183ab871ff468ddb40575e4ec477b405be22791d2c03cefcfc329c0655", + "batch-http-results/text-4-1-batch512.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-1-batch512.cold-2048.client.log": "785973d50b59b117f1fc6fc1926bb4b281087f59404c95cb14e1e64e69ebf224", + "batch-http-results/text-4-1-batch512.cold-2048.json": "07d5a9cc103cac3c15336534edb828f4f3d254c69e1b700df45d4249814bb017", + "batch-http-results/text-4-1-batch512.cold-8192.client.log": "7386f9a64b191eded0cfb409840795096d5ed1d99c3bd1ccda096047aa24d727", + "batch-http-results/text-4-1-batch512.cold-8192.json": "22f7ae4d64d6ecf7e12703c0ac5f59bb4e2cc1e72fb9e16633544c5973fd86c4", + "batch-http-results/text-4-1-batch512.concurrent.client.log": "45785f02d0734998e11c9122e872b814108b9ea54e7607a36acb825a3be87fda", + "batch-http-results/text-4-1-batch512.concurrent.json": "34175439bdbe5e8cc015b5ccdd6535a6b345b35ba6f63fdef5aa7f0992dda0d6", + "batch-http-results/text-4-1-batch512.server.log": "ebb4a315385b2dabda8679787cf70d0f0448c3bd87aebc78119649dca5b8c7f9", + "batch-http-results/text-4-2-batch1024.after.health": "dbb1feaf616e1a00d04401c87f0109647587ea10085e790160b396a55684aacb", + "batch-http-results/text-4-2-batch1024.after.metrics": "a4c1965c89d265bf7a92b2304f68638664dad15d6eb1284ff020aff5dfdd3537", + "batch-http-results/text-4-2-batch1024.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-2-batch1024.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "batch-http-results/text-4-2-batch1024.before.metrics": "533736c30e067b86991f68140fbc3e2c46e2a52164c690bb14b8700445131b3d", + "batch-http-results/text-4-2-batch1024.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-2-batch1024.cold-2048.client.log": "0f9b810b58f749e2f6e85969a04b0d445676ac11fa1869bdbe40b6e69d7da6c1", + "batch-http-results/text-4-2-batch1024.cold-2048.json": "67dada418b4ab98f232f61dddcdcce81a8ef1828b286725bc4b089b1a037f66b", + "batch-http-results/text-4-2-batch1024.cold-8192.client.log": "45d0a2919e3318305e647e6f5e0ecac838d19f1f52efe8899ba9a0ec423c39d3", + "batch-http-results/text-4-2-batch1024.cold-8192.json": "c15cfd990c2ddd82da49a8b786b3504855f5ef95b4f52b8d350a0b18d39fe5c7", + "batch-http-results/text-4-2-batch1024.concurrent.client.log": "a719a41f39e5a79fbd064707d72909c83054ab67773edbea1b1147d26fef360e", + "batch-http-results/text-4-2-batch1024.concurrent.json": "de5d7b38b8f8a9572ad3e29a97b54c661daecf1e84fc236adcae41a1ba5de032", + "batch-http-results/text-4-2-batch1024.server.log": "f2390a73ff9affd4977f6d086caa29ffaed789ab7426b43f44ad403f4a0ec27f", + "batch-http-results/text-4-3-batch1024.after.health": "a47e0156c35f771a328162673c73a84f98cdb4fab472190dce8838634b793f06", + "batch-http-results/text-4-3-batch1024.after.metrics": "c872474a26661c0934a8d3a0bcfa2e9b78a631cf88deeba16bece74e7d186094", + "batch-http-results/text-4-3-batch1024.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-3-batch1024.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "batch-http-results/text-4-3-batch1024.before.metrics": "af3c161b85c32f5f64487695e2a5206fd927bf093fe8be10b127ee435dffd592", + "batch-http-results/text-4-3-batch1024.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-3-batch1024.cold-2048.client.log": "85d2effaa59ed328ea3a6294ccc6517ebfd2685241e8a9c563c7e5cef88fc1be", + "batch-http-results/text-4-3-batch1024.cold-2048.json": "51e62bb54cd232cefb58d055bb9935e7fac673edd973244abd0f5a094a72de86", + "batch-http-results/text-4-3-batch1024.cold-8192.client.log": "9b5dad818b31007586ce4de023acb638e8b13c07fb9c4738be7c5b8bb569960f", + "batch-http-results/text-4-3-batch1024.cold-8192.json": "1819ca126b1f917928c296e0b013d52ddb8cef47ebc84b555373409d50b7a0c3", + "batch-http-results/text-4-3-batch1024.concurrent.client.log": "08265839e8715440942bbab88978e8e3291aa68cf9ed9f63f0595573336fadc2", + "batch-http-results/text-4-3-batch1024.concurrent.json": "40cb038f367ab1e5d5cbcbb3e4e267d168d37101044288982cf13b30c9e5b83a", + "batch-http-results/text-4-3-batch1024.server.log": "46864e805da2d76e6f4dc429ce886026305ff3ab147662a02d1f0df05c24e5ef", + "batch-http-results/text-4-4-batch512.after.health": "7089918ee646a3c7c0d15100f552a3636bb7eda622c10390507b0b6ff60c9063", + "batch-http-results/text-4-4-batch512.after.metrics": "13afef5db57a45083269b5261d370af70450af121f8fbf05b76f87203c6f1b8a", + "batch-http-results/text-4-4-batch512.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-4-batch512.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "batch-http-results/text-4-4-batch512.before.metrics": "9b62fcd32fcebfe444483006d14e938fdf7a0225a6b14ffeb07c1f11c98e612d", + "batch-http-results/text-4-4-batch512.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-4-batch512.cold-2048.client.log": "1b421e1e4f2d2b7608d8088f8fde3f9f0e5b3a72b2d66dba57ccd24906ff04ec", + "batch-http-results/text-4-4-batch512.cold-2048.json": "6aeea0380bd51b9396afd86bf2e0f37590d96e32a2b2e1149250d64ddccb1471", + "batch-http-results/text-4-4-batch512.cold-8192.client.log": "813b2e43ad7a54234963032faaff19191bd5a617198b897ea9f3d38f6e8e12f5", + "batch-http-results/text-4-4-batch512.cold-8192.json": "378cf0bb89e218a5f66775632391c8eca1503260195fec877cc15f2949ce0c59", + "batch-http-results/text-4-4-batch512.concurrent.client.log": "5ef46ac876e97622c5db583884cbddad7af9287cf52e506941fb71639a3c411a", + "batch-http-results/text-4-4-batch512.concurrent.json": "5f936794654c24b12f24409f94fc797338e5160080a4bfee719f2bffd9e8a725", + "batch-http-results/text-4-4-batch512.server.log": "4e4bfa2837c027a11ec79429f9e4dc84ca7d4ece2e3b4100397b75283ea9d837", + "batch-http-results/text-4-5-batch128.after.health": "d5a1b186460fef2802d4f6632c6c78486fdc920617b50b2966a40c3e33e30938", + "batch-http-results/text-4-5-batch128.after.metrics": "cbe61937c7b5d3c06fd6784def48ff320aa113c3681e3139cf4406b735deed3a", + "batch-http-results/text-4-5-batch128.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-5-batch128.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "batch-http-results/text-4-5-batch128.before.metrics": "a13c9b31b693edb8615b4906cb566a2321c1367926ab6679144d3d8eba6ad5b7", + "batch-http-results/text-4-5-batch128.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-5-batch128.cold-2048.client.log": "a74aa6c77fc854ccb492ee137bcf7fadb9125385c0923269be9b67139d40480f", + "batch-http-results/text-4-5-batch128.cold-2048.json": "d1861a0ecdaaecc7ba59cd0faaf2f71746d6ab073035634f5f1840364fa25b96", + "batch-http-results/text-4-5-batch128.cold-8192.client.log": "e68eba7024c7dd71d2a71f4720c19f12535ca500b1292ae86f56eb04038c38f1", + "batch-http-results/text-4-5-batch128.cold-8192.json": "34a9115628fde19cfadab69b1b6f83cde92670a54b01a2c8fe3a8848f773e331", + "batch-http-results/text-4-5-batch128.concurrent.client.log": "6b72967dd74c86a261e184d8005e250a10a49891c2f410400c36d7bef39c2483", + "batch-http-results/text-4-5-batch128.concurrent.json": "e3af1986b198812ffdcada6fca074b2eceace600c2740c4cb891764fea15d69a", + "batch-http-results/text-4-5-batch128.server.log": "01172e5369920ed37fff9b52d38241137d2f42228486c6aeccad23155be9c3f0" + } +} diff --git a/docs/receipts/lfm3/m5-private-gate-model-observation.json b/docs/receipts/lfm3/m5-private-gate-model-observation.json new file mode 100644 index 0000000000..88c92346d9 --- /dev/null +++ b/docs/receipts/lfm3/m5-private-gate-model-observation.json @@ -0,0 +1,277 @@ +{ + "scope": "Private M5 TEXT4 layer-0 gate candidate integration observation. This is not full-model FP64 fidelity qualification, public-word identity, or a serving-performance result. No production source or default change.", + "checkpoint": "LiquidAI/LFM3-1.7B-MLX-4bit", + "candidate_scope": "Exact checkpoint/config/header digest allowlist; single segment,128 rows, last head, layer0 gate only; no trace/profile/images; four-split strict-MPP candidate plus repair-failure scan.", + "fixture": "Pinned VL-rendered text-only token arrays executed against TEXT4; no image or repeated activation rows.", + "implementation_checks": [ + "Nonvacuous encoded/completed route counts", + "Short and deferred-head forwards have zero candidate hits", + "Atomic execute exclusion protects shared scratch; no claim of general backend thread safety", + "Actual failure scan tested on clean buffer and nonzero last element", + "Forced unresolved status after GPU work prevents cache length/logit publication and poisons subsequent forward, cache allocation, first-token and read calls" + ], + "cases": [ + { + "prompt_tokens": 128, + "candidate_completed": 1, + "ineligible_hits": 0, + "scan_checks": true, + "poison_checks": true, + "serialization_check": true, + "greedy16_token_ids_equal": true, + "baseline_tokens": [ + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018 + ], + "candidate_tokens": [ + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018 + ], + "downstream_buffers": { + "state": { + "words": 1601536, + "different_words": 1316124, + "rms_difference": 0.011154250803192524, + "max_abs_difference": 0.359375, + "baseline_sha256": "203d561363fca5dd17ff1b1febda88b21a9641620ded5af9b27193c09526a10b", + "candidate_sha256": "9dded8e6500567c77319d04ab492a491b10acec8d6b9c3bf5d67fd9164850543" + }, + "logits": { + "words": 128000, + "different_words": 97551, + "rms_difference": 0.03399319135647476, + "max_abs_difference": 0.21875, + "baseline_sha256": "0a4eb9bdec658d09c27da35983f67b796b9ec488f53e718e1f0ce080ce76497e", + "candidate_sha256": "d51fdbe0ca74e46c03693a8722153ce00e2e5d5c2a9b7e9900eeb144ad73c335" + } + }, + "observation_sha256": "bb37c67a204a6e400ef264ec85af7b4b7d1bd98ad1ce5e670692234e2d7c11f2", + "analysis_sha256": "490f7215404a5f3982e350836215524b4af894c2d599579048ea58cd4ef3fc4f" + }, + { + "prompt_tokens": 2048, + "candidate_completed": 16, + "ineligible_hits": 0, + "scan_checks": true, + "poison_checks": true, + "serialization_check": true, + "greedy16_token_ids_equal": true, + "baseline_tokens": [ + 597, + 8055, + 509, + 4116, + 9543, + 11004, + 302, + 278, + 2936, + 2837, + 496, + 73, + 10, + 10601, + 890, + 309 + ], + "candidate_tokens": [ + 597, + 8055, + 509, + 4116, + 9543, + 11004, + 302, + 278, + 2936, + 2837, + 496, + 73, + 10, + 10601, + 890, + 309 + ], + "downstream_buffers": { + "state": { + "words": 23228416, + "different_words": 19721412, + "rms_difference": 0.0153449730062706, + "max_abs_difference": 0.8515625, + "baseline_sha256": "7f7e23060107d5252391778f46727022f8ee157fa34eb1753c424cac2f3f771f", + "candidate_sha256": "010f1179e2a804b31e5512704a8d1ea7a0e6c3d2f0aec5d7b6159ad43dd8c999" + }, + "logits": { + "words": 128000, + "different_words": 120104, + "rms_difference": 0.09752749988979859, + "max_abs_difference": 0.484375, + "baseline_sha256": "8700e9a12456dce4f744caa329ad2013f44df7f2704e58e245e52aa0e9e9ddc5", + "candidate_sha256": "71baf564b7417fc42db80191a90390fc6294ca87baf233cd8139c55cc8b7a745" + } + }, + "observation_sha256": "6a680ccd17f51cb2ea56ecb87d7ef15a938651e702c672f3dda64e242c41f9e8", + "analysis_sha256": "aff780ef0decf788b34e775e0017c9903f92f2fb29b32d875d5a6219ebad681a" + }, + { + "prompt_tokens": 8192, + "candidate_completed": 64, + "ineligible_hits": 0, + "scan_checks": true, + "poison_checks": true, + "serialization_check": true, + "greedy16_token_ids_equal": true, + "baseline_tokens": [ + 597, + 4695, + 4116, + 267, + 1442, + 1286, + 4160, + 302, + 10601, + 496, + 73, + 10, + 8426, + 20, + 643, + 7204 + ], + "candidate_tokens": [ + 597, + 4695, + 4116, + 267, + 1442, + 1286, + 4160, + 302, + 10601, + 496, + 73, + 10, + 8426, + 20, + 643, + 7204 + ], + "downstream_buffers": { + "state": { + "words": 92434432, + "different_words": 78320197, + "rms_difference": 0.015190919206897453, + "max_abs_difference": 0.8515625, + "baseline_sha256": "5524361e72b466c99d75fbf958f1faa0c715920b3456f01532907c47a76cb27b", + "candidate_sha256": "3a03bef9484008ce8ac3aaabd855287db2532d6a711b7ae27e777634a4c3d2f7" + }, + "logits": { + "words": 128000, + "different_words": 124217, + "rms_difference": 0.13458813193510316, + "max_abs_difference": 0.66796875, + "baseline_sha256": "19a2f9b7b0794ef2f2be0edb7d16564e24d33f02e0d01972780ae28d09010bf5", + "candidate_sha256": "db2dc7710b7413cdd5cdd11cd3ae493e24dbb25bf13d0a2b43f0ddb2d56f9ed1" + } + }, + "observation_sha256": "3f979d1cb5d08f59b7398e481b1ddff4dc8432b22fd86c7ecd505654f52d5276", + "analysis_sha256": "95642ce696cc5cae23e1f42118f7414c0372a7680f2cd404454e9e6053dd44ff" + } + ], + "limitations": [ + "Many downstream BF16 words differ despite matching sixteen greedy tokens; these differences are not shown no-worse against a full-model FP64 reference.", + "Initial128-token smoke generates the same repeated token; longer prompts yield meaningful matching token sequences.", + "First integrated gate now matches the genuine input and standalone primary-qualified output exactly. This excludes an initial adapter binding/input mismatch; downstream differences are consistent with propagation of changed BF16 values but have not been shown no-worse against a full-model reference or for all later candidate-propagated inputs.", + "Extra scan and loss of dual gate/up batching must be included in future serving measurement." + ], + "raw_receipts": { + "smoke": { + "receipt_sha256": "38f67c0f703c5ec5a48ce5392cd7097e78e2200858e73b97af06b8679b9582d2", + "events_sha256": "6073e4081e89467e99119378cfee40ab7b28f20352a9f1277e32cf72dcc495cc", + "children_exit_zero": true, + "log_hashes_independently_verified": true, + "lock_released": true + }, + "positive": { + "receipt_sha256": "dc41777745e5142ecf6812297acf5553793632b0981ff407173b0313ed0e477f", + "events_sha256": "cde249b856b27b09cf22b3407805fbd56060eee5cba097f28729ee201dd7633c", + "children_exit_zero": true, + "log_hashes_independently_verified": true, + "lock_released": true + } + }, + "first_integrated_gate_binding": { + "result": { + "scope": "integrated first gate binding/primary validation only; diagnostic command boundaries; no full-model fidelity or timing claim", + "checks": { + "baseline_norm_matches_genuine128": true, + "baseline_gate_matches_standalone": true, + "candidate_norm_matches_genuine128": true, + "candidate_gate_matches_standalone": true + }, + "baseline": { + "rms": 0.0006554460990912766, + "max_abs": 0.007761517306789756, + "worst_column_rms": 0.007279313909585347 + }, + "candidate": { + "rms": 0.0006554460950282121, + "max_abs": 0.007761517306789756, + "worst_column_rms": 0.007279313909585347 + }, + "primary_per_column_no_worse": true, + "public_words_corrected": 125, + "candidate_matches_frozen_primary_qualified_words": true + }, + "candidate_completed": 1, + "diagnostic_command_boundaries": true, + "receipt_sha256": "72635da8f181a33cf89a5bc620855daca4976d79b841577626e6e72185021518", + "analysis_sha256": "286c1504e8a2b2b9c074e209dbd8fd4c278d53420c1602967cade410e9ad4d5a", + "capture_log_sha256": "e99c327f90777d4f4abd0ba45cc55be59ffe96627297a7f1fddf230fc1831ccc", + "logs_exits_and_lease_release_independently_verified": true + }, + "inline_error_substitution": { + "scope": "Removed separate error scan dispatch; repeat model observation with fusedfinalflagatomic. No arithmetic change.", + "prompt_lengths": [ + 128, + 2048, + 8192 + ], + "prior_baseline_and_candidate_full_state_logits_hashes_identical": true, + "original_scan_poison_publication_and_nonvacuous_hit_checks_repeated": true, + "receipt_sha256": { + "smoke": "c7f500948b37a26f4573cb7c4c55a705826456f17f71a48808cfe6d1cbd7f900", + "positive": "984f406761505af5c5e5698cad9066c58ebbd6b1ed7897e1beee859ab3f47887" + }, + "all_children_exit_zero_logs_hash_verified_and_leases_released": true + } +} diff --git a/docs/receipts/lfm3/m5-private-gate-served-rejected.json b/docs/receipts/lfm3/m5-private-gate-served-rejected.json new file mode 100644 index 0000000000..7d5120ec30 --- /dev/null +++ b/docs/receipts/lfm3/m5-private-gate-served-rejected.json @@ -0,0 +1,185 @@ +{ + "scope": "Rejected private M5 TEXT4 layer0 gate integration. Same-source native ABBA cold2k8k diagnostic with default-off candidate. No public integration or promotion.", + "outcome": "Stable TTFT regression despite prior standalone gate operator gain; complete model path includes failure scan, separate up dispatch, and per-prefill hit logging in both variants. These components are not individually attributed.", + "qualification": "Exact integrated firstgate matches standalone primary-qualified output; full-model FP64 no-worse remains open because downstream logits/state differ. Separate observation matches16 greedy IDs; serving response text hashes match.", + "protocol": { + "order": [ + "public", + "candidate", + "candidate", + "public" + ], + "legs": 4, + "cold_output_tokens": 16, + "prompt_cache": 0, + "baseline_drift_limit": 0.05, + "no_hidden_retries": true, + "both_routes_exact_tokenize_ids": true, + "fixture": "Pinned text-only chat fixtures against exact TEXT4 checkpoint; no image" + }, + "cases": [ + { + "prompt_tokens": 2048, + "baseline_ttft_s": [ + 0.8589870408177376, + 0.8586471672169864 + ], + "candidate_ttft_s": [ + 0.8639319580979645, + 0.8644525003619492 + ], + "baseline_median_s": 0.858817104017362, + "candidate_median_s": 0.8641922292299569, + "baseline_over_candidate": 0.9937801740969316, + "baseline_first_last_drift": -0.00039566790254208506, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "not emitted by cold client; separate observation matches16 IDs but logits differ and full-model fidelity remains open", + "prompt_ids": "both routes /tokenize exact pinned arrays", + "candidate_ttft_increase_fraction": 0.006258754264966582 + }, + { + "prompt_tokens": 8192, + "baseline_ttft_s": [ + 4.619476750027388, + 4.619642375037074 + ], + "candidate_ttft_s": [ + 4.658193541225046, + 4.6532827080227435 + ], + "baseline_median_s": 4.619559562532231, + "candidate_median_s": 4.655738124623895, + "baseline_over_candidate": 0.9922292532089987, + "baseline_first_last_drift": 3.5853629891002825e-05, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "not emitted by cold client; separate observation matches16 IDs but logits differ and full-model fidelity remains open", + "prompt_ids": "both routes /tokenize exact pinned arrays", + "candidate_ttft_increase_fraction": 0.007831604204239673 + } + ], + "route_hit_logs": [ + { + "event": "candidate_route_counts", + "tag": "text-4-0-public", + "counts": [ + [ + 0, + 2048 + ], + [ + 0, + 8192 + ] + ] + }, + { + "event": "candidate_route_counts", + "tag": "text-4-1-candidate", + "counts": [ + [ + 16, + 2048 + ], + [ + 64, + 8192 + ] + ] + }, + { + "event": "candidate_route_counts", + "tag": "text-4-2-candidate", + "counts": [ + [ + 16, + 2048 + ], + [ + 64, + 8192 + ] + ] + }, + { + "event": "candidate_route_counts", + "tag": "text-4-3-public", + "counts": [ + [ + 0, + 2048 + ], + [ + 0, + 8192 + ] + ] + } + ], + "server_prefill_metric_deltas": [ + { + "leg": 0, + "mode": "public", + "server_prefill_seconds_sum_delta": 5.465842, + "request_count_delta": 2.0, + "scope": "combined two requests (2048 and8192); not separate per-length prefill timing" + }, + { + "leg": 1, + "mode": "candidate", + "server_prefill_seconds_sum_delta": 5.50936, + "request_count_delta": 2.0, + "scope": "combined two requests (2048 and8192); not separate per-length prefill timing" + }, + { + "leg": 2, + "mode": "candidate", + "server_prefill_seconds_sum_delta": 5.505171, + "request_count_delta": 2.0, + "scope": "combined two requests (2048 and8192); not separate per-length prefill timing" + }, + { + "leg": 3, + "mode": "public", + "server_prefill_seconds_sum_delta": 5.465736, + "request_count_delta": 2.0, + "scope": "combined two requests (2048 and8192); not separate per-length prefill timing" + } + ], + "cleanup": { + "all_four_owned_servers_exit_zero": true, + "lock_released": true, + "raw_hashes_independently_verified": true, + "frozen_analysis_rerun_matches_remote": true + }, + "memory_observations": [ + { + "leg": "text-4-0-public", + "max_rss_bytes": 1217200128, + "peak_physical_footprint_bytes": 1630881448 + }, + { + "leg": "text-4-1-candidate", + "max_rss_bytes": 1223245824, + "peak_physical_footprint_bytes": 1706362560 + }, + { + "leg": "text-4-2-candidate", + "max_rss_bytes": 1221017600, + "peak_physical_footprint_bytes": 1704199944 + }, + { + "leg": "text-4-3-public", + "max_rss_bytes": 1216495616, + "peak_physical_footprint_bytes": 1630144192 + } + ], + "raw_sha256": { + "summary": "ebe922d66c372fd385cf4f252820bb4edb1546c0a6f9143b6250eff7dda63af1", + "records": "986edd0faea9cb358d191db160dd407a1bff9ea04cbab43703508d92e7ce5238", + "events": "b1c92932edaeef38c04163342a05594c08014b667b5e9e0ce26079f65b0f2fb8", + "manifest": "01ae2203ec9140de442dc1d0cc5e7ca53926174dfcae7b9f19a2e5aeec7939d6" + }, + "next_constraint": "Do not infer a serving win from a projection-only gain. Any changed scan, dual-projection or joint gate/up implementation requires fresh numerical and complete-path measurements." +} diff --git a/docs/receipts/lfm3/m5-prompt-2048-server-prefill.json b/docs/receipts/lfm3/m5-prompt-2048-server-prefill.json new file mode 100644 index 0000000000..5c02bfbf8d --- /dev/null +++ b/docs/receipts/lfm3/m5-prompt-2048-server-prefill.json @@ -0,0 +1,58 @@ +{ + "scope": "Server-reported prompt-pass prefill walltime from existing paired cold128/2048 experiment; separate from clientTTFT and GPU time", + "source_receipt_commit": "b3081add1", + "cases": [ + { + "prompt_tokens": 2048, + "raw_prefill_seconds": [ + 0.551959417, + 0.4683235, + 0.467633292, + 0.5516325 + ], + "baseline_median_seconds": 0.5517959585, + "candidate_median_seconds": 0.467978396, + "baseline_prompt_tokens_per_second": 3711.5168541054113, + "candidate_prompt_tokens_per_second": 4376.2703951829435, + "relative_time_reduction": 0.15189955853944503, + "raw_sha256": { + "text-4-0-public.cold-2048.json": "a104164b27d9d2e5ad7a71c16d95650fccd905b164dd048960ccc22879c6f14c", + "text-4-1-candidate.cold-2048.json": "4227598bcd7b46673406e7c1b73a4eaba6861155f8017fc822b1a5a246376968", + "text-4-2-candidate.cold-2048.json": "102a1e3668892e08908f7bd5e6ccc3fddeb07aa2db0ddc2a88fe997353c77668", + "text-4-3-public.cold-2048.json": "d67e864d18e53f789b1a990a08ec56754d74598588c93e690766ee2f76870576" + } + }, + { + "prompt_tokens": 8192, + "raw_prefill_seconds": [ + 2.43307025, + 2.043556417, + 2.042926916, + 2.431910291 + ], + "baseline_median_seconds": 2.4324902704999998, + "candidate_median_seconds": 2.0432416665, + "baseline_prompt_tokens_per_second": 3367.7421444798338, + "candidate_prompt_tokens_per_second": 4009.3152632466636, + "relative_time_reduction": 0.1600206211390064, + "raw_sha256": { + "text-4-0-public.cold-8192.json": "9b1096973cb60264acb2c5cafd7a8fcd6db2b3b6bc953e838edcafb822b7d144", + "text-4-1-candidate.cold-8192.json": "26ed4735844d414887827df22a5d9cabaf5982d9c77f48ff35020e47dd665777", + "text-4-2-candidate.cold-8192.json": "f183d60421549194a89342b89781d30c9ce60324f834d99b3594bcabfce3f81e", + "text-4-3-public.cold-8192.json": "ccd96a59d52a7567b4ee5929fc2723681194e056006aed34b9ea6271bff82228" + } + } + ], + "order": [ + 128, + 2048, + 2048, + 128 + ], + "samples_per_route": 2, + "no_new_runs": true, + "limitations": [ + "Same boundedprivatebinary andTEXT4scope as referencedreceipt.", + "No claim of default-readiness, broadfidelity, or GPU-only timing." + ] +} diff --git a/docs/receipts/lfm3/m5-prompt-2048-tail-provisional.json b/docs/receipts/lfm3/m5-prompt-2048-tail-provisional.json new file mode 100644 index 0000000000..e71c9ee2ec --- /dev/null +++ b/docs/receipts/lfm3/m5-prompt-2048-tail-provisional.json @@ -0,0 +1,249 @@ +{ + "scope": "Provisional M5 TEXT4 combined-enabled 128-row versus2048-row prompt batching with explicit prompt-only small-tail routes; not a new FP32/model-fidelity qualification.", + "cases": [ + { + "prompt_tokens": 2048, + "baseline_ttft_s": [ + 0.5582766253501177, + 0.5579384174197912 + ], + "candidate_ttft_s": [ + 0.4746196251362562, + 0.4739405419677496 + ], + "baseline_median_s": 0.5581075213849545, + "candidate_median_s": 0.4742800835520029, + "baseline_over_candidate": 1.1767466961824473, + "baseline_first_last_drift": -0.0006058070765803114, + "candidate_relative_spread": 0.0014328446468983014, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "Not emitted; served text digests equal. Separate teacher-forced full-logit identity recorded.", + "prompt_ids": "both routes /tokenize exact pinned arrays" + }, + { + "prompt_tokens": 8192, + "baseline_ttft_s": [ + 2.4393980419263244, + 2.438312124926597 + ], + "candidate_ttft_s": [ + 2.049930708948523, + 2.0491615422070026 + ], + "baseline_median_s": 2.4388550834264606, + "candidate_median_s": 2.0495461255777627, + "baseline_over_candidate": 1.1899488637948816, + "baseline_first_last_drift": -0.0004451577729683853, + "candidate_relative_spread": 0.0003753568109090555, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "Not emitted; served text digests equal. Separate teacher-forced full-logit identity recorded.", + "prompt_ids": "both routes /tokenize exact pinned arrays" + } + ], + "predeclared_max_baseline_drift": 0.05, + "no_timing_retries": true, + "default_off": true, + "raw_sha256": { + "text-4-0-public.cold-2048.json": "a104164b27d9d2e5ad7a71c16d95650fccd905b164dd048960ccc22879c6f14c", + "text-4-1-candidate.cold-8192.client.log": "14593271e6d541a22d372c6be0d9fdb98f7260b40cc807c4e047aa5df16fb989", + "text-4-1-candidate.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-0-public.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "text-4-0-public.after.metrics": "1db24ac66ad401b20a1e1228f1ec19555fa377e1183746ff85cac1f7e662f3af", + "text-4-3-public.cold-8192.client.log": "e0be9d6e69617f2fc5728be9385e5a424acc55001b5b49f11f80d7238e87d755", + "text-4-3-public.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "text-4-1-candidate.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "text-4-1-candidate.after.metrics": "5bb0e92aaa6b9434f460e060d2348c335f5299dedc84229638614de51feaff22", + "text-4-0-public.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "text-4-1-candidate.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "text-4-1-candidate.before.metrics": "43f025ab8ac84832a00b0959bf5d9456768008ee26706d5aec2d0a0a85cf6852", + "text-4-0-public.cold-8192.json": "9b1096973cb60264acb2c5cafd7a8fcd6db2b3b6bc953e838edcafb822b7d144", + "events.json": "b3ce9b494a753e5910df2d0938b44fce0611e8880404e97f2a1dad658e1138e6", + "text-4-3-public.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "text-4-0-public.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "text-4-1-candidate.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "text-4-0-public.cold-2048.client.log": "34c9c4bcfc16ed7804bd41dc9621f121e11c49a006713de838f44f2e06b3a31c", + "text-4-1-candidate.cold-2048.json": "4227598bcd7b46673406e7c1b73a4eaba6861155f8017fc822b1a5a246376968", + "text-4-2-candidate.after.metrics": "147cf942fd31acc92c10afebe413be08e66e61acc8868ad47af5301abe736add", + "text-4-2-candidate.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "text-4-2-candidate.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "text-4-3-public.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-3-public.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "text-4-0-public.before.metrics": "6d9dc7af656235d2fc964d5f97c4e6a6cddb71b694969ffedadb09ba3e1b471b", + "text-4-1-candidate.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "text-4-2-candidate.server.log": "d3b219e218fbcd4dbc5ccaf7b808163d3845a2b76b68e1cc8d58f652734883f7", + "text-4-2-candidate.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-2-candidate.cold-2048.client.log": "4cfa76cd2746e661ff22a88fb913162cefdcd9b6979aebe293ad28eba73bbd58", + "text-4-1-candidate.cold-8192.json": "26ed4735844d414887827df22a5d9cabaf5982d9c77f48ff35020e47dd665777", + "text-4-0-public.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "text-4-0-public.after.health": "1a1b548b32e4b10f92f034af719cfd34450f88f589671e59ca4931557add078b", + "text-4-1-candidate.after.health": "61ce7238803a0dddc5c45f53a75e3acc31b93a4040f73bea155268931ad964b1", + "text-4-0-public.server.log": "5f81c71623bed833af90594f6dbfd3e348b90fa966d3f34ab593d85411a8322c", + "text-4-3-public.cold-8192.json": "ccd96a59d52a7567b4ee5929fc2723681194e056006aed34b9ea6271bff82228", + "text-4-0-public.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-1-candidate.server.log": "8d52f6623f69793d9a3f5b35c4a1a84f4da17ee4bd54c588450e1a2df1ff2b05", + "text-4-3-public.after.health": "27bda87cd574fb51c3027075ec6e3c7ac42471271c53b12af2a7f255abca1258", + "text-4-2-candidate.before.metrics": "a23c30c3e0da576af669496c053cb48fdeac4b45421b5d63296d1a8de54365fd", + "text-4-2-candidate.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "text-4-1-candidate.cold-2048.client.log": "aa8efcd5a34de5d941367ebe01892e8cb2fc5397308f7269f82eceeba2ad29f0", + "manifest.json": "d904dd1ab411e53e9bb18344f982a64d86cb83be1ecc96da97597604c8c305c8", + "text-4-2-candidate.after.health": "71fbdf122b42a60d0c69d8ef91e2c0a1cda87b09c4d95afb389a4381592f7467", + "text-4-3-public.cold-2048.client.log": "d714a40c70faf4edf5f0484b80ec5c4406fa545b469f670056a79ae41e0dba34", + "text-4-2-candidate.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "text-4-3-public.cold-2048.json": "d67e864d18e53f789b1a990a08ec56754d74598588c93e690766ee2f76870576", + "records.json": "7e1bf2858da5e043828d8a536386f7a3f70554fedd3f92de87280044e4c05be2", + "text-4-2-candidate.cold-8192.json": "f183d60421549194a89342b89781d30c9ce60324f834d99b3594bcabfce3f81e", + "text-4-2-candidate.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-1-candidate.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "text-4-2-candidate.cold-8192.client.log": "75285506e321ce1ab084bad02f6ecffbf130d321000ef99c89a22c70fbeebe72", + "text-4-3-public.before.metrics": "be0914cb7b9c74285341629e436f7e3ee97ff5b89bad97ae31595a5999c7be8a", + "text-4-0-public.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "text-4-0-public.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-0-public.cold-8192.client.log": "66080cccaa55379b344ac9fd0d3536d1d3dda8037ab6755390c8d781462d9b00", + "text-4-3-public.server.log": "0ebf2304fa0c26766d6c2ff377eb5fe1749adc3c52ec9e80e5f91c07dc51c3e4", + "text-4-1-candidate.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-3-public.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-3-public.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "text-4-3-public.after.metrics": "c6c86ac4d5961c333a49b2ad55d15b7ddc137030ee0e5997e433b0c8b9808d2b", + "text-4-2-candidate.cold-2048.json": "102a1e3668892e08908f7bd5e6ccc3fddeb07aa2db0ddc2a88fe997353c77668", + "text-4-3-public.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "text-4-2-candidate.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd" + }, + "binary_sha256": "cc3a6b56010b1a501a5e340c149bc2cf9c1a990af2837066b92df34b03669f1d", + "packet_pins_sha256": "0742adf0d42c67437be9903561a3fe61e3e7a2c7bf78bf68296bd3ae8c36b2c0", + "settings": { + "baseline_prompt_rows": 128, + "candidate_prompt_rows": 2048, + "tail_mode_baseline": 0, + "tail_mode_candidate": 3, + "order": [ + "baseline", + "candidate", + "candidate", + "baseline" + ], + "cache_gib": 0, + "output_tokens": 16, + "thinking": false, + "temperature": 0, + "same_binary": true, + "cache_mark_policy_rows": 128 + }, + "numerical_ablation": { + "uniform128": { + "sha256": "1deb25778922c5e40d5febc80f26669b20b4f7bc2de2f42da3a780efd7ce81b8", + "different_bf16_words": 0, + "prompt_state_equal": true, + "final_state_equal": true, + "routes": { + "chunk": 128, + "prompt": 8192, + "cache_len": 8208, + "prompt_state": "423af58169410fdb9206158e613cb4beba40d09274bee3f564f7840e8e5f62fe", + "final_state": "1158d77bcfe9a4f51fa60599fb7d09c7ff61785d775a7aeff619dff640de28d5", + "ffn_calls": 4608, + "ffn_rows": 589824, + "attention_calls": 1408 + } + }, + "seam0": { + "sha256": "9cdf28725190dd55fc386045b6a132ec9a87fededcb7ad3c8cdcf0dfe85a9b3c", + "different_bf16_words": 1875952, + "prompt_state_equal": false, + "final_state_equal": false, + "routes": { + "chunk": 2048, + "prompt": 8192, + "cache_len": 8208, + "prompt_state": "b3f8efab9bea583cade58cf2f4aaf8b179a65c61b17552af5f5af615816753d8", + "final_state": "1a861a4c172305604808793f95daa638e6890fb74bf23a232c7cd9911460a107", + "ffn_calls": 288, + "ffn_rows": 589464, + "attention_calls": 1408 + } + }, + "seam1": { + "sha256": "b911b76ea5e02437761547ed2c394d75ad29e2d90234baeddd556de11b50a12f", + "different_bf16_words": 1887147, + "prompt_state_equal": false, + "final_state_equal": false, + "routes": { + "chunk": 2048, + "prompt": 8192, + "cache_len": 8208, + "prompt_state": "63cd14f44573ae6d6eee405de8278ce1823946ff404f18e52a8eb315d755e3e7", + "final_state": "efe57c1d96595b01350c795dc162ecbfeafcbd2bef6f245d8a5f0488bf1aaeef", + "ffn_calls": 360, + "ffn_rows": 589824, + "attention_calls": 1408 + } + }, + "seam2": { + "sha256": "67dc5125e4d065dfa6164f3656b8865a5bf7685e4af76f3a66a016c561136a76", + "different_bf16_words": 1855282, + "prompt_state_equal": false, + "final_state_equal": false, + "routes": { + "chunk": 2048, + "prompt": 8192, + "cache_len": 8208, + "prompt_state": "07036428b54479bd4984c1608dab8d2bb43c975bb529657fd2ee0d62b7d7cd3a", + "final_state": "4290d9607782a2803197be4d924fa94358835e4f9fba20dbbe6e48a82eaa0828", + "ffn_calls": 288, + "ffn_rows": 589464, + "attention_calls": 1419 + } + }, + "seam3": { + "sha256": "1deb25778922c5e40d5febc80f26669b20b4f7bc2de2f42da3a780efd7ce81b8", + "different_bf16_words": 0, + "prompt_state_equal": true, + "final_state_equal": true, + "routes": { + "chunk": 2048, + "prompt": 8192, + "cache_len": 8208, + "prompt_state": "423af58169410fdb9206158e613cb4beba40d09274bee3f564f7840e8e5f62fe", + "final_state": "1158d77bcfe9a4f51fa60599fb7d09c7ff61785d775a7aeff619dff640de28d5", + "ffn_calls": 360, + "ffn_rows": 589824, + "attention_calls": 1419 + } + }, + "seam3repeat": { + "sha256": "1deb25778922c5e40d5febc80f26669b20b4f7bc2de2f42da3a780efd7ce81b8", + "different_bf16_words": 0, + "prompt_state_equal": true, + "final_state_equal": true, + "routes": { + "chunk": 2048, + "prompt": 8192, + "cache_len": 8208, + "prompt_state": "423af58169410fdb9206158e613cb4beba40d09274bee3f564f7840e8e5f62fe", + "final_state": "1158d77bcfe9a4f51fa60599fb7d09c7ff61785d775a7aeff619dff640de28d5", + "ffn_calls": 360, + "ffn_rows": 589824, + "attention_calls": 1419 + } + } + }, + "numerical_scope": "Same8192prompt+fixed16continuation; all17fullvocabularyBF16outputs and fullactiveKV/convolutionstate. Bothprompttailroutesrestoreuniform128 bytes; eachcomponentalonefails. Repeatbothbytesidentical. Not an independentFP32claim.", + "actual_route_checks": { + "candidate_ffn_tail_calls_per_request": 72, + "candidate_attention_tail_calls_per_request": 11, + "baseline_tail_calls": 0, + "cancel_callback_installed": true, + "eight_clients_exit_zero": true, + "four_servers_exit_zero": true, + "lock_released": true + }, + "limitations": [ + "Private experiment; public artifact validation separate.", + "Broad small-row NF/guard/FP32-envelope qualification remains pending; prior small-query numerical failure is not waived.", + "No warm-cache, concurrency, image, other quantization/backend or long-context qualification.", + "Generated text digests match; served output token arrays were not emitted. Teacher-forced full-logit identity is a separate direct model test.", + "No cancellation-latency claim for this new tail-routing binary.", + "Earlier uncorrected2048 run changed8k text; that failure remains preserved." + ] +} diff --git a/docs/receipts/lfm3/m5-prompt-256-provisional.json b/docs/receipts/lfm3/m5-prompt-256-provisional.json new file mode 100644 index 0000000000..f0d27eecc9 --- /dev/null +++ b/docs/receipts/lfm3/m5-prompt-256-provisional.json @@ -0,0 +1,151 @@ +{ + "scope": "Provisional M5 TEXT4 cold served 128-to-256 prompt batching, both optimized paths enabled; not MLX comparison or broad fidelity qualification.", + "cases": [ + { + "prompt_tokens": 2048, + "baseline_ttft_s": [ + 0.5574995409697294, + 0.5575295421294868 + ], + "candidate_ttft_s": [ + 0.5003845002502203, + 0.5008832919411361 + ], + "baseline_median_s": 0.5575145415496081, + "candidate_median_s": 0.5006338960956782, + "baseline_over_candidate": 1.1136172478482345, + "baseline_first_last_drift": 5.381378378399582e-05, + "candidate_relative_spread": 0.0009968168291911716, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "Not emitted by cold client; output text digests equal, not a token-array equality claim.", + "prompt_ids": "both routes /tokenize exact pinned arrays" + }, + { + "prompt_tokens": 8192, + "baseline_ttft_s": [ + 2.437989375088364, + 2.437773124780506 + ], + "candidate_ttft_s": [ + 2.2083320831879973, + 2.207139417063445 + ], + "baseline_median_s": 2.437881249934435, + "candidate_median_s": 2.207735750125721, + "baseline_over_candidate": 1.1042450391971086, + "baseline_first_last_drift": -8.870026673113429e-05, + "candidate_relative_spread": 0.0005403673711465995, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "Not emitted by cold client; output text digests equal, not a token-array equality claim.", + "prompt_ids": "both routes /tokenize exact pinned arrays" + } + ], + "predeclared_max_baseline_drift": 0.05, + "no_timing_retries": true, + "default_off": true, + "raw_sha256": { + "text-4-0-public.cold-2048.json": "8fc08b416a98298795ca0f09774b451b6d59f36466e1bf70eeda765531172c20", + "text-4-1-candidate.cold-8192.client.log": "f90feec98107bc82d3403ec05540a56aa177e38e201ea9ef109619e5a62b9463", + "text-4-1-candidate.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-0-public.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "text-4-0-public.after.metrics": "ad28e7781d030261092e6483a9e77822f99ddd3a56c813c328b0cb2af75c49a1", + "text-4-3-public.cold-8192.client.log": "e85d84f412078f5fe5af666c23593a09787c7e5bb2b8a8a3c1e45cbc92e2083f", + "text-4-3-public.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "text-4-1-candidate.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "text-4-1-candidate.after.metrics": "65e10b2498e449de1bf4b3de648ad29b4045fda58ea76a5cbad23d7b608b1ed9", + "text-4-0-public.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "text-4-1-candidate.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "text-4-1-candidate.before.metrics": "b50bd3f1dc97c34a706f7878e028f8ddb316fbcce133149de0a62240076a09f2", + "text-4-0-public.cold-8192.json": "d0d3b078a6db6a20ef7b272ae630125320bbb7479c126c16afe141cef4573a0e", + "events.json": "291cac355171d91adf7d6f8e84dbeef0619e3f90a536ca40bae16f5625aaae48", + "text-4-3-public.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "text-4-0-public.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "text-4-1-candidate.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "text-4-0-public.cold-2048.client.log": "67a13a1977e91150d248803cefd91aa2ebb2fa98bc1764feed7e96f28a9fdbd3", + "text-4-1-candidate.cold-2048.json": "4ec78965d1151db5f7a2552339b3fe3058c86c4a0cd0ced257efbee2b5ecf128", + "text-4-2-candidate.after.metrics": "2664181d959b55faf46cdc2f55aa464bb25d0402799bfbd8dc93073a96845f30", + "text-4-2-candidate.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "text-4-2-candidate.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "text-4-3-public.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-3-public.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "text-4-0-public.before.metrics": "a9f1536f44a173e8636aa60856739b526e617ed064b442abbc3f22776448be58", + "text-4-1-candidate.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "text-4-2-candidate.server.log": "7a1dc0c5cdfd5db8ba619141ab5f836c5dee58aff71de8c1023e2d05a9689166", + "text-4-2-candidate.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-2-candidate.cold-2048.client.log": "6cea9b86e81f08bda943d647f7050dc0f2f45435f4b008fb99da073d8576dd5a", + "text-4-1-candidate.cold-8192.json": "55dadf0ceed4659f7c15af692efb5bae217f0f73c6ed85066aa7366b73d44ae3", + "text-4-0-public.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "text-4-0-public.after.health": "98fc2c787308cc611d8b3cf24ca768adc2963d9938558a6b0e31bb4404109424", + "text-4-1-candidate.after.health": "c951270b71a7422141ac003ae89409a3e4e791dca23ab5f8c6e0f2bcb7632652", + "text-4-0-public.server.log": "fe9b91b8a1955166388abac52e69f207b73e26875f35ce23bf3c8c6200152382", + "text-4-3-public.cold-8192.json": "a82b23a2e111353f977fc17ea93c8f76401e4af324e20776476fdbaab8a4d8a3", + "text-4-0-public.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-1-candidate.server.log": "268fba67dd54a8ac38cd2804cd0eeb701e3834c4f11d5a0afac42f67105b08c1", + "text-4-3-public.after.health": "98fc2c787308cc611d8b3cf24ca768adc2963d9938558a6b0e31bb4404109424", + "text-4-2-candidate.before.metrics": "57a2c71cac04c213bb0494f4a25fc537b138e7a069686abbbc1feeded9c8c3da", + "text-4-2-candidate.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "text-4-1-candidate.cold-2048.client.log": "118b18bb7f9a0d54ab73e647eefb394aff839bf7b2c3019b262ee1328adebf50", + "manifest.json": "ddea1872c702ca2d197d33df3c38f1a63b370c79480a2c550151247efc004637", + "text-4-2-candidate.after.health": "4eb9aceedd8ed2dfacc5ef3f6fdd98c5a25fe3cc00c8fba235b7e908a61bbbe0", + "text-4-3-public.cold-2048.client.log": "f3b731c01fef1610697af0bf22bbb42bd5ac4ccac52d415c1f2fee22eabebf8e", + "text-4-2-candidate.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "text-4-3-public.cold-2048.json": "ae36721127d62d7ae97b8d8bdb4c8900099a775d0a3deb5272f0036702bf8981", + "records.json": "bfcea073666e52dee3e3669a5a22faa7faba4a55551723bf54e222cefa4012b8", + "text-4-2-candidate.cold-8192.json": "5a81b32396f3db040e7484a170e4c262404139b47e5c9858b1bb38ca69090d15", + "text-4-2-candidate.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-1-candidate.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "text-4-2-candidate.cold-8192.client.log": "931000836aa1c76e13094494aa5950c099cc06c31cb9fe6a2244f269f5fd6404", + "text-4-3-public.before.metrics": "5f91196d10195498a9131061c36e2b9edf1e667dd03b759c8808ffe42107088e", + "text-4-0-public.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "text-4-0-public.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-0-public.cold-8192.client.log": "3dfbef509552ef5ae31177a107b345d9fb3b250f520b4fd68a15163c00904e9a", + "text-4-3-public.server.log": "bf525814018d02d3ef5b7c68280b0bbdc6eb7c5b5ebe0df8993dfeaef213877f", + "text-4-1-candidate.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-3-public.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-3-public.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "text-4-3-public.after.metrics": "d62f8b7358c60b749eac399549a8fa9bcc78cd155f7c796206eb4e9198d56b3a", + "text-4-2-candidate.cold-2048.json": "bfd83f11ddd05bd0997eb59f3c563c25a233006b7343708481ecf61aac2fd37e", + "text-4-3-public.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "text-4-2-candidate.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd" + }, + "implementation_commit": "13d590bbf", + "measured_private_binary_sha256": "cdcc83c3ebacb47b4b76dd0166c4151052f3abbb92e0a7afa596e94dfa5a0452", + "protocol": { + "order": [ + 128, + 256, + 256, + 128 + ], + "fresh_server_per_leg": true, + "prompt_cache_gib": 0, + "output_tokens": 16, + "temperature": 0, + "thinking": false, + "actual_256_rows_verified": true, + "cancel_callback_installed": true + }, + "public_smoke": { + "binary_sha256": "b4fae76cecbeb35bacfd9bf6508103e514dea8bde227b3bb9b158adbb4715221", + "prompt_tokens": [ + 2048, + 8192 + ], + "ttft_s": [ + 0.5011340416967869, + 2.206330666784197 + ], + "text_equal_private256": true, + "paired_timing": false + }, + "limitations": [ + "Measured private v1 dynamically set cache marks; cold cache was disabled. Public256 preserves original128 cache policy and has separate HTTP smoke only.", + "No new full-model FP32/BF16 envelope claim; previous probability diagnostics remain unmet.", + "512 and2048 private tests changed8k generated text and are not qualified; initial512 route assertion failure preserved and corrected using actual assistant-marker seam.", + "HTTP cancellation latency and warm prefix behavior are not established by this timing receipt.", + "No other quantization, image, concurrency, backend or long-context qualification." + ], + "raw_summary_sha256": "82198b6fb6920e0c910fcf826eb157325cb58998c43f9c975221f88d87cdeee9" +} diff --git a/docs/receipts/lfm3/m5-public-2048-prefill-cancellation.json b/docs/receipts/lfm3/m5-public-2048-prefill-cancellation.json new file mode 100644 index 0000000000..63e561178d --- /dev/null +++ b/docs/receipts/lfm3/m5-public-2048-prefill-cancellation.json @@ -0,0 +1,43 @@ +{ + "scope": "Actual public TEXT4 native HTTP disconnect during unfinished 8192-token prefill and cold recovery, one observation per width.", + "model_revision": "d0ca73dcfa70ce18733754a492cdc1fb0123262d", + "hardware": "Apple M5 Ultra", + "source_revisions": { + "128": "1e43c6722", + "2048": "d9c1ea531" + }, + "binary_sha256": { + "128": "3c987ac2decf021380122a22f2e30c12590045a959e6f160e9ed7570577248bd", + "2048": "46a9b73c763b4c94957332f4d8099098d4883b9a00486b5b4f1fd9121a6faec6" + }, + "prompt_tokens": 8192, + "prompt_cache_tokens": 0, + "output_tokens": 16, + "cases": [ + { + "prompt_rows": 128, + "cancel_to_observed_release_ms": 31.826874999999998, + "completed_tokens_at_disconnect": 128, + "generation_tokens_running": 0, + "recovery_text_equal": true + }, + { + "prompt_rows": 2048, + "cancel_to_observed_release_ms": 479.790375, + "completed_tokens_at_disconnect": 2048, + "generation_tokens_running": 0, + "recovery_text_equal": true + } + ], + "events_sha256": "f41547c5094a00495bf8d7d2b4e6cc240b6cb5c6a71163d1aecee86cda6cd2c3", + "records_sha256": "810240488972be4460409a970d5c3120238b6963ad7e1b43eddf105bc3f516e7", + "packet_pins_sha256": "c2f9945a05da81125055e874af76179384482335d2490f3bcd26c7a6a8f77a2b", + "all_clients_and_servers_exit_zero": true, + "lock_released": true, + "limitations": [ + "One observation per width; no latency percentile or worst-case guarantee.", + "Disconnect is observed after a completed chunk while prefill remains active; it does not prove the signal arrived during a particular GPU command.", + "Different public source revisions: this is a bounded lifecycle check, not an isolated performance comparison.", + "No broad image, width, or full-model FP32/BF16 fidelity qualification." + ] +} diff --git a/docs/receipts/lfm3/m5-public-prompt-2048-smoke.json b/docs/receipts/lfm3/m5-public-prompt-2048-smoke.json new file mode 100644 index 0000000000..cd534a0a37 --- /dev/null +++ b/docs/receipts/lfm3/m5-public-prompt-2048-smoke.json @@ -0,0 +1,60 @@ +{ + "scope": "M5 public developer2048 prompttail cold2k8k basic smoke, not paired timing or broad qualification", + "binary_sha256": "46a9b73c763b4c94957332f4d8099098d4883b9a00486b5b4f1fd9121a6faec6", + "build_options": { + "cpu": "apple_m1", + "lfm3-residual-ffn": true, + "lfm3-tiled-attention": true, + "lfm3-prompt-rows": 2048 + }, + "source_pins_sha256": "3178db24cfda2bb950df4f8c25cfdd2efbd219c31573e4b0bcc860ee6785e95c", + "packet_pins_sha256": "e2b77edcda4b017b75146ed70ee8dd480264d1059d1f1adf00cae74be54a7b8a", + "cases": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.47450420912355185, + "output_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 2.0480394582264125, + "output_sha256": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0" + } + ], + "exact_private_candidate_text_digest_match": true, + "exact_tokenize_prompt_arrays": true, + "clients_exit_zero": 2, + "servers_exit_zero": 1, + "lock_released": true, + "limitations": [ + "Single public observation each; paired private timing is a separate receipt.", + "Served text hashes, not emitted token arrays. Actual route counters belong to the separate instrumented private test.", + "Broad small-row nonfinite/guards/FP32-envelope checks, image/otherwidth/backend and warm/concurrent behavior remain unqualified." + ], + "raw_sha256": { + "text-4-0-candidate.cold-8192.json": "2fdb0281e43bc4eafc1ead58e1d52ac3770ed4d918e7532658fd09f925704d2a", + "text-4-0-candidate.cold-2048.json": "32fc44aae208b7c78005db6f088caeefc8194cbc9121af905d5e7c8688957aa5", + "text-4-0-candidate.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "events.json": "d085a99fae0ca7540aaad6ae5d99da0abc079f08db2589cd5fc8dd5adbded39b", + "text-4-0-candidate.before.metrics": "06316272b58bbb868efb7e9a96a582ede9a79af90cb1dd23749ed0606a1388cd", + "text-4-0-candidate.cold-8192.client.log": "935621011f68b40d88aa6b1fb5a4b951ca6396733192e6cfa7f2f2d3254997fe", + "text-4-0-candidate.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "text-4-0-candidate.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "text-4-0-candidate.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "text-4-0-candidate.after.metrics": "22f1e5284665e6aa70705a5c4f38c35801674bc241e9b150d03b32b38ecabdbf", + "text-4-0-candidate.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "manifest.json": "6ff7c6e74b3c4b4e91c80074cd7b63eb3b9ceedcc751b2170a75b23a81165801", + "text-4-0-candidate.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "records.json": "fa3f3757c105d561d373a1ec756476c211d19f88d5d96102de5bb4030c3e16da", + "text-4-0-candidate.server.log": "07240fa074f9a9440a0081d636e55919f3327bb3cebfbebab122995bed312b78", + "text-4-0-candidate.cold-2048.client.log": "9528d3490ed52cf7ef53829c1e908a54c713b310c11f94982083f74fde1f9f43", + "text-4-0-candidate.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-0-candidate.after.health": "e5ed9dc37687f2fa719289587b3f79491c67bd34a80fb5e19bdd5e3bb8a10b06" + } +} diff --git a/docs/receipts/lfm3/m5-public-prompt-prefill-cancellation.json b/docs/receipts/lfm3/m5-public-prompt-prefill-cancellation.json new file mode 100644 index 0000000000..83ca5375d4 --- /dev/null +++ b/docs/receipts/lfm3/m5-public-prompt-prefill-cancellation.json @@ -0,0 +1,133 @@ +{ + "scope": "M5 public native HTTP cancellation during unfinished 8192-token prefill and subsequent cold recovery, one observation per prompt width.", + "passed": true, + "hardware": "Apple M5", + "os": "macOS 27.0 (26A428)", + "model": { + "repository": "LiquidAI/LFM3-1.7B-MLX-4bit", + "revision": "d0ca73dcfa70ce18733754a492cdc1fb0123262d" + }, + "public_builds": { + "128": { + "source_revision": "1e43c6722", + "binary_sha256": "3c987ac2decf021380122a22f2e30c12590045a959e6f160e9ed7570577248bd" + }, + "256": { + "implementation_revision": "13d590bbf", + "binary_sha256": "b4fae76cecbeb35bacfd9bf6508103e514dea8bde227b3bb9b158adbb4715221" + } + }, + "build_options": { + "cpu": "apple_m1", + "lfm3-residual-ffn": true, + "lfm3-tiled-attention": true, + "lfm3-prompt-rows": [ + 128, + 256 + ] + }, + "protocol": { + "fresh_server_per_width": true, + "request_order": [ + "cold_before", + "disconnect_during_prefill", + "cold_recovery" + ], + "context": 70000, + "parallel": 4, + "active_requests": 1, + "prompt_cache_gib": 0, + "thinking": false, + "temperature": 0, + "output_tokens": 16, + "dashboard_and_metrics_enabled": true, + "exact_prompt_ids_checked_by_tokenize": 8192, + "disconnect_trigger": "Observe at least one configured-width chunk complete, unfinished prefill, and no generated SSE content.", + "release_observation": "Running and waiting both zero, and server disconnection counter increased.", + "poll_sleep_s": 0.01 + }, + "cases": [ + { + "prompt_rows": 128, + "passed": true, + "observations": 1, + "cancel_to_observed_release_ms": 32.11075, + "unfinished_prefill_tokens_at_disconnect": 8064, + "generation_tokens_running_at_disconnect": 0, + "generated_sse_content_before_disconnect": false, + "server_disconnect_counter_delta": 1, + "recovery_completion_tokens": 16, + "before_and_recovery_cached_tokens": 0, + "recovery_text_sha256_equal": true, + "recovery_text_sha256": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349" + }, + { + "prompt_rows": 256, + "passed": true, + "observations": 1, + "cancel_to_observed_release_ms": 63.6395, + "unfinished_prefill_tokens_at_disconnect": 7936, + "generation_tokens_running_at_disconnect": 0, + "generated_sse_content_before_disconnect": false, + "server_disconnect_counter_delta": 1, + "recovery_completion_tokens": 16, + "before_and_recovery_cached_tokens": 0, + "recovery_text_sha256_equal": true, + "recovery_text_sha256": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349" + } + ], + "execution": { + "runner_exit": 0, + "clients_exit_zero": 6, + "servers_exit_zero": 2, + "lock_released": true, + "model_admission_checked": true, + "owned_process_cleanup_verified": true + }, + "packet_pins_sha256": "5949a612b46fe8b5b00920157a9094068471ec60ac1858f4badac650ce2e8166", + "packet_files_sha256": { + "baseline/tensorfold-native": "3c987ac2decf021380122a22f2e30c12590045a959e6f160e9ed7570577248bd", + "cancel_prefill.py": "3a99f29a3834e11b234216ce1eaf2fd3ed1b02a35173c967aaeac100180fb56a", + "candidate/tensorfold-native": "b4fae76cecbeb35bacfd9bf6508103e514dea8bde227b3bb9b158adbb4715221", + "cold_client.py": "7b8ea55aa88834814f95a6f679654e16074b7a8364e34b39a276f009fe0231f2", + "config.json": "af5adbac9613c1f5f82cca8955fe95d2307f34a604ec403ff272db98076b0e72", + "fixtures/text-2048.body.json": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "fixtures/text-2048.ids.json": "f5edac3447405086d7ec361d6c899410ea7cbbb2ef697c27fd7486ef5ff877fe", + "fixtures/text-8192.body.json": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "fixtures/text-8192.ids.json": "a9b3cdf070b9ca5ea523565bcb82c80b6af5aae71f640a8fba0460c9069d22cd", + "freeze.py": "c4f0e5585516781604fa88be7ac930d60477da24813fb0ca83e1c92fadce3193", + "guard.py": "ee70dc087509a6e16e3b82dde8c5eca89be481a59a630a956412b335f7d3b4ce", + "model-revisions.json": "fbe3ba2fea04e926e5c237044f2c5315b31a3d991e5c9c528521dc597a924f88", + "run.py": "37c2293d87eb647150ee7cc26781c7516d7eea2113b9f13b47e56e7bd5a011ed", + "server.py": "dded32e854d813eadf1f128ae184ae69993de5209fa802227c5b6ec175a5ac79" + }, + "raw_sha256": { + "events.json": "56abd35c505d8fdfbbc1a47926ff0bed8dfeff205988613106f72d12000522fe", + "records.json": "5a20e08e936f500cf119a8ce087c3aa99e53b24d6c6e01cfee6128640e901ef8", + "rows128-candidate-after.client.log": "56b8d68f459766b53a10a942f2377c5bce76d433e93f14f5180bc145e242fc9f", + "rows128-candidate-before.client.log": "2c446b6ab8668de14af3c1014e10d27c74ccc434b65ff8f682accc531d3d1c24", + "rows128-candidate-cancel.client.log": "5d18e1b9d95b4fde3707b252a62b64d44c88c5db69cdace0169001538ea3d39f", + "rows128-candidate.after.json": "31cf6f5fdb2ed5dfd2c41a0b3f71a6a2bd2fc22b87f9d9098e799b14f0df3077", + "rows128-candidate.before.json": "c56c6acef61c0a56c0d6795d512f5dc0f5a4ea05f70a3f5e1dba234fad9193a9", + "rows128-candidate.cancel.json": "68ee1c50438b4e5255abc64e8d1e769f03669a37d32a28c895210e39addfa31b", + "rows128-candidate.server.log": "64bdb42cd94613248b3ad3bc8ef290bd687615a53ee1486d6498f2ef0cc74465", + "rows128-candidate.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "rows128-candidate.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "rows256-candidate-after.client.log": "b80cf45e4b70a4281bd2192bd4e1ec4c34fb328b1116d3f35010517e21b215da", + "rows256-candidate-before.client.log": "7f60404dfb3341791666a446e64c74b3025f5fe438db7f121ac9c4b8dff1d773", + "rows256-candidate-cancel.client.log": "e4589c66b29a6d9e0610f8c79a80e8b5bf28e137c86565e9bc123fe96cd086b7", + "rows256-candidate.after.json": "33a2c3a64aad8241ea59b359173f897e471d552f4d177cd4ebf71679826c1789", + "rows256-candidate.before.json": "165e2066a9b460d546ab1dc9cd84d2ddb62af67b6fbe8bfde391a29567294013", + "rows256-candidate.cancel.json": "3af49697625fefe2dbcbb3e5226f12cf80d441f24e9fd160b0244aaca6e53fae", + "rows256-candidate.server.log": "25580fc9050549e39de71eaeffd75c3a1ba980a5996d186e70dd38ebc485302c", + "rows256-candidate.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "rows256-candidate.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee" + }, + "limitations": [ + "Single observation per width; no latency bound, distribution, SLA or isolated throughput comparison. Two unrelated CPU builds were visible on the test host.", + "Latency includes client/server detection, remaining work, cleanup and polling granularity. HTTP evidence does not locate the disconnect signal inside an individual GPU command.", + "Recovery compares emitted text hashes, not output token arrays or internal state. Separate test-only in-flight cancellation/state checks are not this public HTTP run.", + "Only text 4-bit checkpoint, 8192-token prompt and widths128/256. No image, other quantization, multi-request contention, long-context or independent numerical-fidelity claim.", + "Public route selection comes from the pinned binaries/build options; these servers do not expose per-layer route counters." + ] +} diff --git a/docs/receipts/lfm3/m5-public-tiled-prompt-runtime.json b/docs/receipts/lfm3/m5-public-tiled-prompt-runtime.json new file mode 100644 index 0000000000..b5051a0aee --- /dev/null +++ b/docs/receipts/lfm3/m5-public-tiled-prompt-runtime.json @@ -0,0 +1,251 @@ +{ + "scope": "PROVISIONAL actual public source revision default/enabled M5 TEXT4. Runtime and timing evidence, not broad production qualification.", + "source_commit": "8df18abc47568246c0d38b70c2e280521a24b31a", + "build_options": { + "baseline": { + "lfm3-residual-ffn": false, + "lfm3-tiled-attention": false + }, + "candidate": { + "lfm3-residual-ffn": true, + "lfm3-tiled-attention": true + } + }, + "binary_sha256": { + "baseline": "d9d5ee7f8a04f1e8caeaea789a8bd026a15eb87964ccd4262e56bb840ba10069", + "candidate": "44fdcc3d4f95e5af399fe598e8bb063c0bfc7dc752494b9114249af3c41eb553" + }, + "early_checks": { + "enabled_native_and_dev_tools_exit": 0, + "default_native_exit": 0, + "both_strict_Metal_compiles_exit": 0, + "zig_fmt_and_diff_check_exit": 0, + "independent_source_reviews": "pass" + }, + "basic": { + "source": "Same public source with test-only counters and explicit model enable APIs; not the unmodified server binary. Shader/math unchanged.", + "commands": 4, + "all_exit_zero": true, + "prompt_tokens": 128, + "teacher_forced_tokens": 16, + "logit_shape": [ + 17, + 128000 + ], + "finite": true, + "cache_length": 144, + "ffn_dispatch_hits": 72, + "attention_dispatch_hits": 22, + "baseline_zero_routes": "Source-established by null fields; not independently counted in baseline.", + "candidate_matches_prior_private_composed_all_words": true, + "candidate_logits_sha256": "583760763110f09b4557d7d66e4c8801ce2f1996f1052214c9bf99db2cf34cf1", + "baseline_logits_sha256": "ac5b5fd5fb92bc650b9048ea8b5e2233e5b6e4f98c3166f0eebb1fedbc15b8b8", + "instrumentation_diff_sha256": "f1d7cb23550f431a0789304d78f413082d100250e939deca0b66848e6b969edf", + "manifest_sha256": "f1273bde39b6fd9155f6cf475d298da74e888dc10602861b7b55824f304ab579" + }, + "numerical_policy": "Independent propagated FP32 F and actual official MLX-VLM BF16 B; no FP64 acceptance claim. RMS/max logits pass each phase. Extra top20 probability diagnostics are executable proposal, not a source-verbatim maintainer rule, and still breach. No tie waiver.", + "phase_check_exit": 1, + "phases": { + "prompt_boundary": { + "native_logit_errors": { + "mean_absolute_error": 0.03329696540903778, + "rms_error": 0.041121336314284725, + "relative_rms": 0.011827876222482102, + "maximum_absolute_error": 0.20430231094360352, + "maximum_coordinate": [ + 0, + 49445 + ] + }, + "reference_bf16_logit_errors": { + "mean_absolute_error": 0.03884851189472984, + "rms_error": 0.04764064939088356, + "relative_rms": 0.01370304943028551, + "maximum_absolute_error": 0.20851778984069824, + "maximum_coordinate": [ + 0, + 91107 + ] + }, + "logit_rms_and_max_no_worse": true, + "all_proposed_metrics_pass": false, + "native_fp32_top1_misses": 0, + "reference_fp32_top1_misses": 0 + }, + "teacher_forced_continuation": { + "native_logit_errors": { + "mean_absolute_error": 0.02985543358314638, + "rms_error": 0.037950087085717864, + "relative_rms": 0.010836676663429445, + "maximum_absolute_error": 0.20357131958007812, + "maximum_coordinate": [ + 10, + 26150 + ] + }, + "reference_bf16_logit_errors": { + "mean_absolute_error": 0.032687712900759155, + "rms_error": 0.0425864734405558, + "relative_rms": 0.01216060036617698, + "maximum_absolute_error": 0.256467342376709, + "maximum_coordinate": [ + 14, + 4438 + ] + }, + "logit_rms_and_max_no_worse": true, + "all_proposed_metrics_pass": false, + "native_fp32_top1_misses": 0, + "reference_fp32_top1_misses": 0 + } + }, + "phase_receipt_sha256": "1c525ce36259e53193bd6234d03fac99ada3ac8eda37a251b4b490c90287f15d", + "serving": { + "order": [ + "baseline", + "candidate", + "candidate", + "baseline" + ], + "prompt_ids_exact": true, + "prompt_cache_tokens": 0, + "completion_tokens_per_request": 16, + "fresh_servers": 4, + "all_server_exit_zero": true, + "all_client_exit_zero": true, + "same_source_separate_builds": true, + "server_route_counts": "Not logged. Actual route counts belong only to separate test probe.", + "no_timing_retries": true, + "max_baseline_drift": 0.05, + "cases": [ + { + "prompt_tokens": 2048, + "baseline_ttft_s": [ + 0.8587595410645008, + 0.8587297503836453 + ], + "candidate_ttft_s": [ + 0.5555961248464882, + 0.5554991248063743 + ], + "baseline_median_s": 0.858744645724073, + "candidate_median_s": 0.5555476248264313, + "baseline_over_candidate": 1.5457624285449318, + "baseline_first_last_drift": -3.4690363752587494e-05, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "Not emitted by HTTP client; output text SHA equal. Separate teacher-forced check reports top1 picks.", + "prompt_ids": "both routes /tokenize exact pinned arrays", + "ttft_reduction_fraction": 0.35307005686421866, + "candidate_spread": 0.0001746025647112279 + }, + { + "prompt_tokens": 8192, + "baseline_ttft_s": [ + 4.621298334095627, + 4.616932542063296 + ], + "candidate_ttft_s": [ + 2.4133794172666967, + 2.4143210002221167 + ], + "baseline_median_s": 4.6191154380794615, + "candidate_median_s": 2.4138502087444067, + "baseline_over_candidate": 1.9135882671369862, + "baseline_first_last_drift": -0.0009447111432129285, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "Not emitted by HTTP client; output text SHA equal. Separate teacher-forced check reports top1 picks.", + "prompt_ids": "both routes /tokenize exact pinned arrays", + "ttft_reduction_fraction": 0.4774215450765962, + "candidate_spread": 0.00039007513888353204 + } + ], + "new_ITL_or_long_decode_claim": false, + "TTFT_is_not_engine_prefill_time": true + }, + "unrun": [ + "New public kernel-contract registrations/full summaries", + "Broad 6/8-bit, VL images, cancellation, resume and concurrency qualification for these candidates", + "Long-context FP32/BF16 full-model qualification", + "Sustained decode and token-level ITL benchmark" + ], + "raw_sha256": { + "text-4-0-public.cold-2048.json": "91762cbe91fbe1a0e4ff80547016a166385f794f792e8a9e6bfc1f07663ba6eb", + "text-4-1-candidate.cold-8192.client.log": "18f75c2de925022e78161813eeaca41be447ebf22b7d9c1911e4a6528d2167d6", + "text-4-1-candidate.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-0-public.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "text-4-0-public.after.metrics": "12cbb5fb1980f5ae63fbd59b03f96c0d03b7ba8fe1288176819122e51f9eb8ee", + "text-4-3-public.cold-8192.client.log": "bd4153b629678bd46ab8a6569a971aeadb500cc7bd3db34cb3ff49743ecdc5ea", + "text-4-3-public.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "text-4-1-candidate.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "text-4-1-candidate.after.metrics": "a36438bee19b17953929b26c83045f47eb36d39ed63c140f4bc330d3c15e8b47", + "text-4-0-public.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "text-4-1-candidate.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "text-4-1-candidate.before.metrics": "1f67cac48f3a1ecb0870882b7823b73e7ae75c005c523532c629e7fae789f8bb", + "text-4-0-public.cold-8192.json": "675f1f368fd033814f9017dd8f00db1f7c74b9d2457a2e8206baa39bd4d79d45", + "events.json": "fabc69a1819cdb23ab4506bd75d591d24f296701db15119f5434da425303961f", + "text-4-3-public.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "text-4-0-public.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "text-4-1-candidate.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "text-4-0-public.cold-2048.client.log": "f9777000adce6a8d2aee6bf377622105a0eac7ccd00ec67abe1082031a2db0bc", + "text-4-1-candidate.cold-2048.json": "90955fdc97a531da7af16c3cd18e0a93fef3e8685676de10fe6a46b05cf8ccf8", + "text-4-2-candidate.after.metrics": "5e422c69ce85d464540cc4439d3609732621b1e12bdc7f04106e6e311f0d7a61", + "text-4-2-candidate.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "text-4-2-candidate.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "text-4-3-public.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-3-public.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "text-4-0-public.before.metrics": "c8cc1d0dff40410f5d21c9cd9a8ad2c393b9f3a1bb76783e656b5b872cae7dbc", + "text-4-1-candidate.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "text-4-2-candidate.server.log": "7642893f45baf0d055471029ffb8b3b2a040e07ca6987b12e68f4cfa117e9cc2", + "text-4-2-candidate.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-2-candidate.cold-2048.client.log": "83b55e34e609feb589817e8ef87180478ae6155ada9cc810a68068aed8b44cf3", + "text-4-1-candidate.cold-8192.json": "7531eb0319c49413620d8b0dee5d9d4dbef70864ad51a06bc46eb162468af460", + "text-4-0-public.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "text-4-0-public.after.health": "5c202b6bd43cec4e548a4addeba7d85d065f54da93bd1c39ac731db5654cdf6b", + "text-4-1-candidate.after.health": "fcef26e2c0024e0e9829bbe0c2d7200ff0231a0bdfc8b384fbd87ff255b441b3", + "text-4-0-public.server.log": "d9daeb9b19053093b3729ca933f7476452c63b2eb492b8851e45258a537b0753", + "text-4-3-public.cold-8192.json": "1a5b62fc597042dea706cea3a9182f36572d173bdbdbd064ae0fb80c1e20b0d1", + "text-4-0-public.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-1-candidate.server.log": "209758f443d654ddc42650596ba4de8e12abac534338ffe5fde36f2e92ad259e", + "text-4-3-public.after.health": "e99b46b99af09e76fc16af3aed7cd238ef16dfcddde2e34182860b11f2a921d8", + "text-4-2-candidate.before.metrics": "6eeb470af860fbb30df4be5e224c64ba91adc8ef7879adeed0bdbf014a522870", + "text-4-2-candidate.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "text-4-1-candidate.cold-2048.client.log": "cb4619e839c1d6d7bceae803ab56c81de90c0ea3368e6c2e28d2c4d63720065e", + "manifest.json": "c0d08d17dae60d5056d93b161228f1fd67339b6c3eb6db8512849d59bf7ae986", + "text-4-2-candidate.after.health": "c3883cf23f1059d836ec3744b631dd2f2f9d159c7439e57356e8212bee60bc11", + "text-4-3-public.cold-2048.client.log": "988eb699dd8163901433785f01c983595f8981825f65a3ab0928f6051abe8154", + "text-4-2-candidate.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "text-4-3-public.cold-2048.json": "8a1a8a4d40f3ca6560b78ad0edb6e4310855210b951bd6213927aad052e4398b", + "records.json": "d85f122c4c670df812fff9967dbca544c1b6d66b12bfb997ceb20bcfa7db71bd", + "text-4-2-candidate.cold-8192.json": "7ffbd4d45de384ca75de517cf6c7d56276152131747ef62c7bccfa638baa7bae", + "text-4-2-candidate.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-1-candidate.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "text-4-2-candidate.cold-8192.client.log": "7a387b5b7fbe8572e1bfc31c84272e3d6ac3ce292c1efea5c0b654823737d721", + "text-4-3-public.before.metrics": "6dd999e386e8622f51b7f48f688618a503ae80e19b5b35503ad1374001169cb3", + "text-4-0-public.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "text-4-0-public.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-0-public.cold-8192.client.log": "8b06958eb699bce4d58077ec90cdbb85040b66d859ffced3fa7a0c1442b07d79", + "text-4-3-public.server.log": "8fd0a3b9d81b047276147dc41d8cd5dd01e81d9d8806e8486f127832025b2a33", + "text-4-1-candidate.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-3-public.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "text-4-3-public.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "text-4-3-public.after.metrics": "984f726dd55d3e2519685961da487fc011d43d878c05a2345474b4dc4bb1ea69", + "text-4-2-candidate.cold-2048.json": "e5a39a00378970b81cdb748b2efb2439c09ccca8fb5f844e400b78f623511d19", + "text-4-3-public.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "text-4-2-candidate.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd" + }, + "basic_raw_sha256": { + "candidate.phases.aggregate.json": "1d185e278d83d66ea5f1b04009dfa562426e41b0ff53e6b78d4c6ecd01412002", + "phase-check.log": "c666175ceca215ed1c5ef63e934dcc049ce62d060bf50651ecb901dd8b63a3c1", + "candidate.phases.json": "1c525ce36259e53193bd6234d03fac99ada3ac8eda37a251b4b490c90287f15d", + "positive/candidate.bf16": "583760763110f09b4557d7d66e4c8801ce2f1996f1052214c9bf99db2cf34cf1", + "positive/events.json": "764a7b477079f98363fcf55393c1a2390488178b0c7d69e130ae20bdbe3d5cf8", + "positive/baseline.bf16": "ac5b5fd5fb92bc650b9048ea8b5e2233e5b6e4f98c3166f0eebb1fedbc15b8b8", + "positive/002.log": "fc53f966b6cc57e6f6eb7dfdfc2174b7e9f9f86f99109be304c95f273363028b", + "positive/003.log": "b6e4827a57124fd59b886fb26c0d0cb42ef0256a6799f944a7aadfd06ddb36a4", + "positive/001.log": "e379c00ba397b39da362cb18233058f130a8967a56b8b427087da3f22666bed7", + "positive/000.log": "aa542502c37939707c493b1a622cfda5d3fc51ce13a720b9d6e5231a3986b9c2", + "positive/receipt.json": "6190b436654e1cc6580ecacdbd85bff0e555e552dfb701983a295e369e40e76a" + } +} diff --git a/docs/receipts/lfm3/m5-public-tiled-serving-lifecycle.json b/docs/receipts/lfm3/m5-public-tiled-serving-lifecycle.json new file mode 100644 index 0000000000..84983a37c2 --- /dev/null +++ b/docs/receipts/lfm3/m5-public-tiled-serving-lifecycle.json @@ -0,0 +1,635 @@ +{ + "scope": "Bounded M5 TEXT4 enabled public tiled prompt serving/lifecycle checks; no performance or broad numerical qualification.", + "source_commit": "8df18abc47568246c0d38b70c2e280521a24b31a", + "build_options": { + "lfm3-residual-ffn": true, + "lfm3-tiled-attention": true + }, + "binary_sha256": "84b6716b59b19e10592e38f48d159f2c0e440c48cf707bbacf20ba62688537bc", + "instrumentation": "Only aggregate dispatch counters and model-deinit logging; shaders and forward control flow unchanged.", + "instrumentation_sha256": "211810b34c79ad89e9d7b6c22068ff8e3e98dde7fe35d45f96c8467de18a64bf", + "shader_sha256": { + "residual_ffn.metal": "add7140a9ec0589fe674c2066317b9e511430596892cbeb1b831feadd465c86b", + "attention_tiled.metal": "126a69dbfcf1e57d6fb1f9ceabaab840741a7acb54902b622c534648335521b9" + }, + "model": "LFM3-1.7B-MLX-4bit", + "checkpoint_sha256": "1be5bf672013cd91aa62148f77c5eea8d2784a95845d5fe826e08c7b84823815", + "serving": { + "model": "LFM3-1.7B-MLX-4bit", + "binary_sha256": "84b6716b59b19e10592e38f48d159f2c0e440c48cf707bbacf20ba62688537bc", + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 193, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "644594753a98", + "token_sha256": "644594753a98675a02af5cb19ca628a3264567727d10849d9155051c08668310", + "output_sha256": "5d4e262cc905e76142ebc5e775f2f5ffbf2b18e2981e56d5eb8e7da1a7d58d90", + "text": " \nWrite a Python function that returns the smaller of two integers. \nWrite a Python function that returns the sum of two" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 202, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "f6d87f5c720c", + "token_sha256": "f6d87f5c720cc029037a27f699cad20ac08a471e5d1c3c26c78fd0633cd35fdb", + "output_sha256": "d1f5745bade9df792ecbaabfaa0d71992c4ca82ef25a9fb955682ef523d84b61", + "text": "The sky appears blue because molecules in the atmosphere scatter sunlight, and shorter wavelengths of light (like blue) are scattered more" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 193, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "644594753a98", + "token_sha256": "644594753a98675a02af5cb19ca628a3264567727d10849d9155051c08668310", + "output_sha256": "5d4e262cc905e76142ebc5e775f2f5ffbf2b18e2981e56d5eb8e7da1a7d58d90", + "text": " \nWrite a Python function that returns the smaller of two integers. \nWrite a Python function that returns the sum of two" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 193, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "644594753a98", + "token_sha256": "644594753a98675a02af5cb19ca628a3264567727d10849d9155051c08668310", + "output_sha256": "5d4e262cc905e76142ebc5e775f2f5ffbf2b18e2981e56d5eb8e7da1a7d58d90", + "text": " \nWrite a Python function that returns the smaller of two integers. \nWrite a Python function that returns the sum of two" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 202, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "f6d87f5c720c", + "token_sha256": "f6d87f5c720cc029037a27f699cad20ac08a471e5d1c3c26c78fd0633cd35fdb", + "output_sha256": "d1f5745bade9df792ecbaabfaa0d71992c4ca82ef25a9fb955682ef523d84b61", + "text": "The sky appears blue because molecules in the atmosphere scatter sunlight, and shorter wavelengths of light (like blue) are scattered more" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 202, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "f6d87f5c720c", + "token_sha256": "f6d87f5c720cc029037a27f699cad20ac08a471e5d1c3c26c78fd0633cd35fdb", + "output_sha256": "d1f5745bade9df792ecbaabfaa0d71992c4ca82ef25a9fb955682ef523d84b61", + "text": "The sky appears blue because molecules in the atmosphere scatter sunlight, and shorter wavelengths of light (like blue) are scattered more" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 193, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "644594753a98", + "token_sha256": "644594753a98675a02af5cb19ca628a3264567727d10849d9155051c08668310", + "output_sha256": "5d4e262cc905e76142ebc5e775f2f5ffbf2b18e2981e56d5eb8e7da1a7d58d90", + "text": " \nWrite a Python function that returns the smaller of two integers. \nWrite a Python function that returns the sum of two" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 202, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "f6d87f5c720c", + "token_sha256": "f6d87f5c720cc029037a27f699cad20ac08a471e5d1c3c26c78fd0633cd35fdb", + "output_sha256": "d1f5745bade9df792ecbaabfaa0d71992c4ca82ef25a9fb955682ef523d84b61", + "text": "The sky appears blue because molecules in the atmosphere scatter sunlight, and shorter wavelengths of light (like blue) are scattered more" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 193, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "644594753a98", + "token_sha256": "644594753a98675a02af5cb19ca628a3264567727d10849d9155051c08668310", + "output_sha256": "5d4e262cc905e76142ebc5e775f2f5ffbf2b18e2981e56d5eb8e7da1a7d58d90", + "text": " \nWrite a Python function that returns the smaller of two integers. \nWrite a Python function that returns the sum of two" + }, + { + "prompt_tokens": 193, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "644594753a98", + "token_sha256": "644594753a98675a02af5cb19ca628a3264567727d10849d9155051c08668310", + "output_sha256": "5d4e262cc905e76142ebc5e775f2f5ffbf2b18e2981e56d5eb8e7da1a7d58d90", + "text": " \nWrite a Python function that returns the smaller of two integers. \nWrite a Python function that returns the sum of two" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 202, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "f6d87f5c720c", + "token_sha256": "f6d87f5c720cc029037a27f699cad20ac08a471e5d1c3c26c78fd0633cd35fdb", + "output_sha256": "d1f5745bade9df792ecbaabfaa0d71992c4ca82ef25a9fb955682ef523d84b61", + "text": "The sky appears blue because molecules in the atmosphere scatter sunlight, and shorter wavelengths of light (like blue) are scattered more" + }, + { + "prompt_tokens": 202, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "f6d87f5c720c", + "token_sha256": "f6d87f5c720cc029037a27f699cad20ac08a471e5d1c3c26c78fd0633cd35fdb", + "output_sha256": "d1f5745bade9df792ecbaabfaa0d71992c4ca82ef25a9fb955682ef523d84b61", + "text": "The sky appears blue because molecules in the atmosphere scatter sunlight, and shorter wavelengths of light (like blue) are scattered more" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 193, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "644594753a98", + "token_sha256": "644594753a98675a02af5cb19ca628a3264567727d10849d9155051c08668310", + "output_sha256": "5d4e262cc905e76142ebc5e775f2f5ffbf2b18e2981e56d5eb8e7da1a7d58d90", + "text": " \nWrite a Python function that returns the smaller of two integers. \nWrite a Python function that returns the sum of two" + }, + { + "prompt_tokens": 193, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "644594753a98", + "token_sha256": "644594753a98675a02af5cb19ca628a3264567727d10849d9155051c08668310", + "output_sha256": "5d4e262cc905e76142ebc5e775f2f5ffbf2b18e2981e56d5eb8e7da1a7d58d90", + "text": " \nWrite a Python function that returns the smaller of two integers. \nWrite a Python function that returns the sum of two" + }, + { + "prompt_tokens": 193, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "644594753a98", + "token_sha256": "644594753a98675a02af5cb19ca628a3264567727d10849d9155051c08668310", + "output_sha256": "5d4e262cc905e76142ebc5e775f2f5ffbf2b18e2981e56d5eb8e7da1a7d58d90", + "text": " \nWrite a Python function that returns the smaller of two integers. \nWrite a Python function that returns the sum of two" + }, + { + "prompt_tokens": 193, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "644594753a98", + "token_sha256": "644594753a98675a02af5cb19ca628a3264567727d10849d9155051c08668310", + "output_sha256": "5d4e262cc905e76142ebc5e775f2f5ffbf2b18e2981e56d5eb8e7da1a7d58d90", + "text": " \nWrite a Python function that returns the smaller of two integers. \nWrite a Python function that returns the sum of two" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 202, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "f6d87f5c720c", + "token_sha256": "f6d87f5c720cc029037a27f699cad20ac08a471e5d1c3c26c78fd0633cd35fdb", + "output_sha256": "d1f5745bade9df792ecbaabfaa0d71992c4ca82ef25a9fb955682ef523d84b61", + "text": "The sky appears blue because molecules in the atmosphere scatter sunlight, and shorter wavelengths of light (like blue) are scattered more" + }, + { + "prompt_tokens": 202, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "f6d87f5c720c", + "token_sha256": "f6d87f5c720cc029037a27f699cad20ac08a471e5d1c3c26c78fd0633cd35fdb", + "output_sha256": "d1f5745bade9df792ecbaabfaa0d71992c4ca82ef25a9fb955682ef523d84b61", + "text": "The sky appears blue because molecules in the atmosphere scatter sunlight, and shorter wavelengths of light (like blue) are scattered more" + }, + { + "prompt_tokens": 202, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "f6d87f5c720c", + "token_sha256": "f6d87f5c720cc029037a27f699cad20ac08a471e5d1c3c26c78fd0633cd35fdb", + "output_sha256": "d1f5745bade9df792ecbaabfaa0d71992c4ca82ef25a9fb955682ef523d84b61", + "text": "The sky appears blue because molecules in the atmosphere scatter sunlight, and shorter wavelengths of light (like blue) are scattered more" + }, + { + "prompt_tokens": 202, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "f6d87f5c720c", + "token_sha256": "f6d87f5c720cc029037a27f699cad20ac08a471e5d1c3c26c78fd0633cd35fdb", + "output_sha256": "d1f5745bade9df792ecbaabfaa0d71992c4ca82ef25a9fb955682ef523d84b61", + "text": "The sky appears blue because molecules in the atmosphere scatter sunlight, and shorter wavelengths of light (like blue) are scattered more" + } + ] + } + ], + "logs": [ + "server-sj2gki7v.log", + "server-qv4xq_o5.log" + ], + "commands": [ + "instrumented-native serve TEXT4 --host loopback --context 2048 --parallel 4 --temperature 0 --prompt-cache-gib 0 --no-thinking --no-update-check", + "instrumented-native serve TEXT4 --host loopback --context 2048 --parallel 4 --temperature 0 --prompt-cache-gib 0.1 --no-thinking --no-update-check" + ], + "speed_qualified": false, + "counts": { + "passed": 22, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 0 + }, + "cancellation": { + "model": "LFM3-1.7B-MLX-4bit", + "binary_sha256": "84b6716b59b19e10592e38f48d159f2c0e440c48cf707bbacf20ba62688537bc", + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 193, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "644594753a98", + "token_sha256": "644594753a98675a02af5cb19ca628a3264567727d10849d9155051c08668310", + "output_sha256": "5d4e262cc905e76142ebc5e775f2f5ffbf2b18e2981e56d5eb8e7da1a7d58d90", + "text": " \nWrite a Python function that returns the smaller of two integers. \nWrite a Python function that returns the sum of two" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 202, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "f6d87f5c720c", + "token_sha256": "f6d87f5c720cc029037a27f699cad20ac08a471e5d1c3c26c78fd0633cd35fdb", + "output_sha256": "d1f5745bade9df792ecbaabfaa0d71992c4ca82ef25a9fb955682ef523d84b61", + "text": "The sky appears blue because molecules in the atmosphere scatter sunlight, and shorter wavelengths of light (like blue) are scattered more" + } + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "cmpl-42d70d05d00cef9032d26e7a5b7e27ec", + "first_content": " \n", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 193, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "644594753a98", + "token_sha256": "644594753a98675a02af5cb19ca628a3264567727d10849d9155051c08668310", + "output_sha256": "5d4e262cc905e76142ebc5e775f2f5ffbf2b18e2981e56d5eb8e7da1a7d58d90", + "text": " \nWrite a Python function that returns the smaller of two integers. \nWrite a Python function that returns the sum of two" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 193, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "644594753a98", + "token_sha256": "644594753a98675a02af5cb19ca628a3264567727d10849d9155051c08668310", + "output_sha256": "5d4e262cc905e76142ebc5e775f2f5ffbf2b18e2981e56d5eb8e7da1a7d58d90", + "text": " \nWrite a Python function that returns the smaller of two integers. \nWrite a Python function that returns the sum of two" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "chatcmpl-a2eb3266cdc140dad39c2ede23e6294f", + "first_content": "The", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 202, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "f6d87f5c720c", + "token_sha256": "f6d87f5c720cc029037a27f699cad20ac08a471e5d1c3c26c78fd0633cd35fdb", + "output_sha256": "d1f5745bade9df792ecbaabfaa0d71992c4ca82ef25a9fb955682ef523d84b61", + "text": "The sky appears blue because molecules in the atmosphere scatter sunlight, and shorter wavelengths of light (like blue) are scattered more" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 202, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "f6d87f5c720c", + "token_sha256": "f6d87f5c720cc029037a27f699cad20ac08a471e5d1c3c26c78fd0633cd35fdb", + "output_sha256": "d1f5745bade9df792ecbaabfaa0d71992c4ca82ef25a9fb955682ef523d84b61", + "text": "The sky appears blue because molecules in the atmosphere scatter sunlight, and shorter wavelengths of light (like blue) are scattered more" + } + } + ], + "logs": [ + "server-zkwzoo3t.log", + "server-412qwbc7.log" + ], + "commands": [ + "instrumented-native serve TEXT4 --host loopback --context 2048 --parallel 4 --temperature 0 --prompt-cache-gib 0 --no-thinking --no-update-check", + "instrumented-native serve TEXT4 --host loopback --context 2048 --parallel 4 --temperature 0 --prompt-cache-gib 0 --no-thinking --no-update-check" + ], + "speed_qualified": false, + "counts": { + "passed": 8, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6 + }, + "actual_dispatches": { + "serving_fresh": { + "ffn": 1440, + "attention": 704 + }, + "serving_cache_enabled": { + "ffn": 4176, + "attention": 1518 + }, + "cancel_fresh": { + "ffn": 432, + "attention": 385 + }, + "cancel_and_recovery_cache_off": { + "ffn": 1008, + "attention": 550 + } + }, + "cleanup": "Both harnesses exited zero; four servers stopped by owned harness; admission empty after each harness; shared GPU lock held throughout then released.", + "limits": [ + "Cancellation closes HTTP after first content, not during an in-flight prefill kernel.", + "Counters aggregate per server, not per request.", + "Text4 only; image and other quantization widths untested here.", + "Raw token evidence retained; this same-engine check does not substitute for FP32/official-BF16 fidelity." + ], + "raw_sha256": { + "serving.public.json": "7738c75b4da102ccc39c53f35976d9860cdb008ce83c4c28cc4147670c1597df", + "events.json": "dfc762b99f85b37f5f3982c06c919df588d871c7934e54d5d46d268c244782b9", + "serving.json": "ed93b42a565ed7209cb1896a63e3e65b96bd68122076ec685dcec96d8e99d4ed", + "cancellation-server-0.log": "ca726859124aee078faec3047a5149a218f7b8a03b8770633576f9a61230e1c6", + "cancellation-server-1.log": "59d705644712aab9cfe5617b3bb54d505e38100947ec1949ed2c310264ef7253", + "cancellation.json": "7ed65389456fef3121a231b9a0a9f251c4f1c8cab93d5f5acd5df5a6595cb8ae", + "cancellation.log": "a1180c100df550d9ae7315c7c49e6b3325ed2f0b2071d2fe5e9b3fbdb8c4828b", + "instrumentation.diff": "211810b34c79ad89e9d7b6c22068ff8e3e98dde7fe35d45f96c8467de18a64bf", + "serving.log": "5566755682627ad00526d67b5fcb487e69a0ae5e22e6f75c07ce29418b7d1c1f", + "serving-server-0.log": "d340a4602fea390e5918fbddc91b5abbf7acc4a65a9fc7e8b9350cfbbb705cf4", + "serving-server-1.log": "11f4d11bd7cb5f8b5f4751cd52e0fea51e38544817d045784aad64388b3463d7", + "pins.json": "67f9464acdaf91f8882d476181f8e9ac57275a2e89458ca89c054a557204a8ca", + "cancellation.public.json": "4358f00aee369f53c43d4f53e743d2727ded615402736e658b470cac421329b1" + } +} diff --git a/docs/receipts/lfm3/m5-public-tiled-stage-profile.json b/docs/receipts/lfm3/m5-public-tiled-stage-profile.json new file mode 100644 index 0000000000..09d3bcf0b6 --- /dev/null +++ b/docs/receipts/lfm3/m5-public-tiled-stage-profile.json @@ -0,0 +1,901 @@ +{ + "scope": "Calibrated M5 TEXT4 diagnostic attribution for public default-off paths; not a new served performance or fidelity qualification.", + "source_commit": "8df18abc47568246c0d38b70c2e280521a24b31a", + "manifest_sha256": "e86ae5704cee80ca65b9f55409c8cfbc5880a3dd906e33a3f5ac652219ad841b", + "receipt_sha256": "d1e684ecf15408b6333e58d205041cea2430ede0814f3673fd1abaedf1018363", + "events_sha256": "50ad47ca4ebf325521d3bd5f6c1ff4b4bbe46e5f7f170e441f2f1657937a40fa", + "commands": 8, + "all_exit_zero": true, + "final_lock_released": true, + "method": "Two mirrored runs per default/enabled and 2k/8k context. Calibrated timestamp instrumentation; every chunk logit byte-identical to uninstrumented same-route execution, repeated chunk-logit digests equal. Stage sums are diagnostic and not HTTP TTFT.", + "runs": { + "default-2048": { + "stages": { + "ffn_gate_up": { + "gpu_seconds_median": 0.36670456, + "calls": [ + 384, + 384 + ], + "fraction": 0.4304667386158554 + }, + "ffn_down": { + "gpu_seconds_median": 0.1799675685, + "calls": [ + 384, + 384 + ], + "fraction": 0.2112601279591957 + }, + "attention": { + "gpu_seconds_median": 0.12023944650000001, + "calls": [ + 704, + 704 + ], + "fraction": 0.14114654693094253 + }, + "conv_in": { + "gpu_seconds_median": 0.0809096085, + "calls": [ + 208, + 208 + ], + "fraction": 0.09497808070257072 + }, + "conv_out": { + "gpu_seconds_median": 0.028126993500000003, + "calls": [ + 208, + 208 + ], + "fraction": 0.03301768366069504 + }, + "attn_out": { + "gpu_seconds_median": 0.0237998375, + "calls": [ + 176, + 176 + ], + "fraction": 0.027938126616730192 + }, + "attn_q": { + "gpu_seconds_median": 0.023795313499999998, + "calls": [ + 176, + 176 + ], + "fraction": 0.027932815988671734 + }, + "attn_kv": { + "gpu_seconds_median": 0.012722100500000002, + "calls": [ + 176, + 176 + ], + "fraction": 0.014934205101180481 + }, + "activation": { + "gpu_seconds_median": 0.0058169565, + "calls": [ + 768, + 768 + ], + "fraction": 0.006828402388084022 + }, + "norm": { + "gpu_seconds_median": 0.0042347255, + "calls": [ + 784, + 784 + ], + "fraction": 0.004971054832038078 + }, + "output_head": { + "gpu_seconds_median": 0.002278064, + "calls": [ + 16, + 16 + ], + "fraction": 0.002674171219573026 + }, + "attn_gate": { + "gpu_seconds_median": 0.0019773969999999997, + "calls": [ + 176, + 176 + ], + "fraction": 0.0023212245780057286 + }, + "attention_prepare": { + "gpu_seconds_median": 0.0012086955, + "calls": [ + 176, + 176 + ], + "fraction": 0.0014188621212254918 + }, + "embedding": { + "gpu_seconds_median": 9.537550000000001e-05, + "calls": [ + 16, + 16 + ], + "fraction": 0.00011195928523184036 + }, + "ffn_gate": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "ffn_up": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "attn_k": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "attn_v": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "dense": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "xsum": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "copy": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "gdn_prepare": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "gdn_scan": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "gdn_gate": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "state_publish": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + } + }, + "stage_sum": 0.8518766425000001, + "plain_wall": [ + 0.880904, + 0.880832 + ], + "profiled_wall": [ + 0.915819, + 0.916234 + ], + "counter_modes": [ + "stage", + "stage" + ], + "logits_sha256": "62f262beea3c59b4e537cb7d3c5c15383686f31201bf68193048970cdd54208c" + }, + "default-8192": { + "stages": { + "attention": { + "gpu_seconds_median": 1.6782165900000003, + "calls": [ + 2816, + 2816 + ], + "fraction": 0.3644590474611633 + }, + "ffn_gate_up": { + "gpu_seconds_median": 1.4670490225, + "calls": [ + 1536, + 1536 + ], + "fraction": 0.3185996923789084 + }, + "ffn_down": { + "gpu_seconds_median": 0.7196665485, + "calls": [ + 1536, + 1536 + ], + "fraction": 0.15629030622082757 + }, + "conv_in": { + "gpu_seconds_median": 0.32362187500000006, + "calls": [ + 832, + 832 + ], + "fraction": 0.07028110733912818 + }, + "conv_out": { + "gpu_seconds_median": 0.11248509949999999, + "calls": [ + 832, + 832 + ], + "fraction": 0.024428439369285564 + }, + "attn_out": { + "gpu_seconds_median": 0.095198976, + "calls": [ + 704, + 704 + ], + "fraction": 0.02067440419727834 + }, + "attn_q": { + "gpu_seconds_median": 0.09502975549999998, + "calls": [ + 704, + 704 + ], + "fraction": 0.020637654505606593 + }, + "attn_kv": { + "gpu_seconds_median": 0.050905192500000015, + "calls": [ + 704, + 704 + ], + "fraction": 0.011055103423436637 + }, + "activation": { + "gpu_seconds_median": 0.02329471950000001, + "calls": [ + 3072, + 3072 + ], + "fraction": 0.005058924652773805 + }, + "norm": { + "gpu_seconds_median": 0.016906372, + "calls": [ + 3136, + 3136 + ], + "fraction": 0.0036715643689019194 + }, + "output_head": { + "gpu_seconds_median": 0.0091181655, + "calls": [ + 64, + 64 + ], + "fraction": 0.0019801960798893317 + }, + "attn_gate": { + "gpu_seconds_median": 0.0078879645, + "calls": [ + 704, + 704 + ], + "fraction": 0.0017130327784910474 + }, + "attention_prepare": { + "gpu_seconds_median": 0.004899619500000002, + "calls": [ + 704, + 704 + ], + "fraction": 0.0010640525582530094 + }, + "embedding": { + "gpu_seconds_median": 0.00039818800000000006, + "calls": [ + 64, + 64 + ], + "fraction": 8.64746660563436e-05 + }, + "ffn_gate": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "ffn_up": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "attn_k": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "attn_v": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "dense": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "xsum": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "copy": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "gdn_prepare": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "gdn_scan": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "gdn_gate": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "state_publish": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + } + }, + "stage_sum": 4.6046780885, + "plain_wall": [ + 4.640737, + 4.637995 + ], + "profiled_wall": [ + 4.819606, + 4.828097 + ], + "counter_modes": [ + "stage", + "stage" + ], + "logits_sha256": "44061d8b62260e616a7ab4539194c77cca4193d699df4596a289e6050fda26c5" + }, + "enabled-2048": { + "stages": { + "ffn_down": { + "gpu_seconds_median": 0.15348988599999996, + "calls": [ + 384, + 384 + ], + "fraction": 0.2806297718512267 + }, + "ffn_up": { + "gpu_seconds_median": 0.0877102045, + "calls": [ + 384, + 384 + ], + "fraction": 0.16036297452106676 + }, + "ffn_gate": { + "gpu_seconds_median": 0.087660356, + "calls": [ + 384, + 384 + ], + "fraction": 0.1602718351401819 + }, + "conv_in": { + "gpu_seconds_median": 0.08123227600000002, + "calls": [ + 208, + 208 + ], + "fraction": 0.14851920002622118 + }, + "attention": { + "gpu_seconds_median": 0.032313975499999995, + "calls": [ + 352, + 352 + ], + "fraction": 0.05908052842107869 + }, + "conv_out": { + "gpu_seconds_median": 0.028146275, + "calls": [ + 208, + 208 + ], + "fraction": 0.05146060719409151 + }, + "attn_q": { + "gpu_seconds_median": 0.023827164499999998, + "calls": [ + 176, + 176 + ], + "fraction": 0.0435638589079195 + }, + "attn_out": { + "gpu_seconds_median": 0.0238000405, + "calls": [ + 176, + 176 + ], + "fraction": 0.04351426735416923 + }, + "attn_kv": { + "gpu_seconds_median": 0.012818893500000001, + "calls": [ + 176, + 176 + ], + "fraction": 0.023437134863010932 + }, + "activation": { + "gpu_seconds_median": 0.0058541045, + "calls": [ + 768, + 768 + ], + "fraction": 0.010703219951757863 + }, + "norm": { + "gpu_seconds_median": 0.00436006, + "calls": [ + 784, + 784 + ], + "fraction": 0.007971617381080468 + }, + "output_head": { + "gpu_seconds_median": 0.0023417094999999997, + "calls": [ + 16, + 16 + ], + "fraction": 0.004281411758471501 + }, + "attn_gate": { + "gpu_seconds_median": 0.0020024250000000004, + "calls": [ + 176, + 176 + ], + "fraction": 0.0036610885938060623 + }, + "attention_prepare": { + "gpu_seconds_median": 0.0011686905000000002, + "calls": [ + 176, + 176 + ], + "fraction": 0.0021367489215523695 + }, + "embedding": { + "gpu_seconds_median": 0.00022191600000000006, + "calls": [ + 16, + 16 + ], + "fraction": 0.0004057351143653651 + }, + "ffn_gate_up": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "attn_k": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "attn_v": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "dense": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "xsum": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "copy": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "gdn_prepare": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "gdn_scan": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "gdn_gate": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "state_publish": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + } + }, + "stage_sum": 0.546947977, + "plain_wall": [ + 0.577531, + 0.576574 + ], + "profiled_wall": [ + 0.612701, + 0.651447 + ], + "counter_modes": [ + "stage", + "stage" + ], + "logits_sha256": "749937125e1d436fe591e893039c9b1f4f4800a27f354f879ddc8cc32c948be6" + }, + "enabled-8192": { + "stages": { + "ffn_down": { + "gpu_seconds_median": 0.6144221610000001, + "calls": [ + 1536, + 1536 + ], + "fraction": 0.25381376755675106 + }, + "attention": { + "gpu_seconds_median": 0.36267626900000005, + "calls": [ + 1408, + 1408 + ], + "fraction": 0.14981918960816212 + }, + "ffn_gate": { + "gpu_seconds_median": 0.3502354495000001, + "calls": [ + 1536, + 1536 + ], + "fraction": 0.14467996861449017 + }, + "ffn_up": { + "gpu_seconds_median": 0.35005327650000007, + "calls": [ + 1536, + 1536 + ], + "fraction": 0.14460471414222006 + }, + "conv_in": { + "gpu_seconds_median": 0.3250024225000001, + "calls": [ + 832, + 832 + ], + "fraction": 0.13425637054747447 + }, + "conv_out": { + "gpu_seconds_median": 0.11259111749999998, + "calls": [ + 832, + 832 + ], + "fraction": 0.04651065267500961 + }, + "attn_q": { + "gpu_seconds_median": 0.09530000250000001, + "calls": [ + 704, + 704 + ], + "fraction": 0.0393678064009361 + }, + "attn_out": { + "gpu_seconds_median": 0.09522042200000003, + "calls": [ + 704, + 704 + ], + "fraction": 0.03933493221798643 + }, + "attn_kv": { + "gpu_seconds_median": 0.05122985049999999, + "calls": [ + 704, + 704 + ], + "fraction": 0.021162715461973878 + }, + "activation": { + "gpu_seconds_median": 0.023562301, + "calls": [ + 3072, + 3072 + ], + "fraction": 0.009733432106977994 + }, + "norm": { + "gpu_seconds_median": 0.017552623499999996, + "calls": [ + 3136, + 3136 + ], + "fraction": 0.0072508737213991285 + }, + "output_head": { + "gpu_seconds_median": 0.009236876000000002, + "calls": [ + 64, + 64 + ], + "fraction": 0.003815692933664436 + }, + "attn_gate": { + "gpu_seconds_median": 0.0079516615, + "calls": [ + 704, + 704 + ], + "fraction": 0.003284779247490336 + }, + "attention_prepare": { + "gpu_seconds_median": 0.0047380140000000005, + "calls": [ + 704, + 704 + ], + "fraction": 0.0019572425286864484 + }, + "embedding": { + "gpu_seconds_median": 0.0009873365000000003, + "calls": [ + 64, + 64 + ], + "fraction": 0.0004078622367777782 + }, + "ffn_gate_up": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "attn_k": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "attn_v": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "dense": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "xsum": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "copy": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "gdn_prepare": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "gdn_scan": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "gdn_gate": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + }, + "state_publish": { + "gpu_seconds_median": 0.0, + "calls": [ + 0, + 0 + ], + "fraction": 0.0 + } + }, + "stage_sum": 2.4207597835000003, + "plain_wall": [ + 2.435951, + 2.436173 + ], + "profiled_wall": [ + 2.641021, + 2.853679 + ], + "counter_modes": [ + "stage", + "stage" + ], + "logits_sha256": "2ca71e1e89e5cf00d362d9513785ccd2fa710822420f46d149039d7d56728b7d" + } + }, + "raw_json_sha256": { + "default-2048-0.json": "e6252e5dd057188bf8e8b8a82ee33f1010ce3874d7dab5a02375e2fc71dd0663", + "default-2048-1.json": "67661aad13667767651bee344ec7aa756c982373a1f642f2a1aeb66cb7fb3ff0", + "default-8192-0.json": "6ed4268a5f1c82b35ab56836fe5f8399ecf3dca3fe322eccbf0be0fff9a2206f", + "default-8192-1.json": "5af4550d90339a5749838af77c8a14928eaa2d8b04c3ce9375596937bf0cd024", + "enabled-2048-0.json": "83df955a70c18bd0ea472c82f5e45eea4ed4d2b83a344df7c349d723b792de6b", + "enabled-2048-1.json": "76f12fb75915074933f43bd4cede4491aa1b22702f1f767f5bf2df9bceeb856c", + "enabled-8192-0.json": "0b3e50b5b3d17fdd80d2e4bd16207fac98733c14352a9d30b7e151044d9e8555", + "enabled-8192-1.json": "6ff34af901bba1d3ff4844cc03ad786fd52b763d1914f383086f65017c313294" + }, + "limitations": [ + "TEXT4 only; no new width/image/cancellation/kernel-contract coverage.", + "Per-stage instrumentation may alter scheduling; percentages are measured diagnostic shares, not universal bottleneck attribution.", + "No occupancy, compute or bandwidth saturation inference.", + "Only two diagnostic observations per cell; not the five-round operator performance protocol." + ] +} diff --git a/docs/receipts/lfm3/m5-residual-ffn-contract.json b/docs/receipts/lfm3/m5-residual-ffn-contract.json new file mode 100644 index 0000000000..bcfdbac551 --- /dev/null +++ b/docs/receipts/lfm3/m5-residual-ffn-contract.json @@ -0,0 +1,53 @@ +{ + "scope": "Targeted M5 residual FFN mechanical kernel contract diagnostic; legacy FP64 formula checks are not the approved independent FP32/official-BF16 fidelity gate.", + "runtime_source_commit": "8df18abc47568246c0d38b70c2e280521a24b31a", + "shader_sha256": "add7140a9ec0589fe674c2066317b9e511430596892cbeb1b831feadd465c86b", + "command": "tf-kernel-contract --only \"LFM3 residual FFN\" --no-limits", + "result": { + "exit": 0, + "elapsed_s": 60.50941908406094, + "binary_sha256": "7dfcad20f56cb8aebe5458d385e821a645183b998f6a90f1c3152cd3d3128a26", + "scope": "Targeted diagnostic; full runner and independent FP32/BF16 fidelity separate." + }, + "summary": "SUMMARY 1 kernels: 1 pass gates 1-4, 0 fail, 0 known findings; 2 controls: 2 red as planted, 0 missed; 0 limit problems; gates 60138 ms, limits 0 ms", + "log": [ + "tf-kernel-contract on Apple M5 Ultra (applegpu_g17, tensor units true), seed 0x6b63", + "FIXTURE metal LFM3 residual FFN: nonfinite rows 8 expanded to 33 for the enabled route", + "PASS nonfinite metal LFM3 residual FFN: 30720 non-finite and 222720 finite outputs match the float64 classes", + "PASS repeat metal LFM3 residual FFN: five runs byte-identical (983040 bytes), inputs and guards untouched, no overlap", + "PASS shapes metal LFM3 residual FFN: 12 shapes within the float64 bound (worst 0.991 of the bound), 0 edges refused by the kernel", + "FIXTURE metal LFM3 residual FFN: stress rows 4 expanded to 33 for the enabled route", + "PASS sums metal LFM3 residual FFN: exact sums at k 7680, where fp16 sums would miss by 16990x and bf16 by 256x", + "FAIL shapes control: LFM3 residual FFN pack: at 128x7680x2048 output 0 is -1.710938e0 where float64 gives -1.231283e0: 14.02 of its bound", + "CONTROL control: LFM3 residual FFN pack: red at shapes, as planted", + "FIXTURE control: LFM3 residual FFN nonfinite: nonfinite rows 8 expanded to 33 for the enabled route", + "FAIL nonfinite control: LFM3 residual FFN nonfinite: 30720 of 253440 outputs differ in class; first at 0: finite where float64 gives nan", + "CONTROL control: LFM3 residual FFN nonfinite: red at nonfinite, as planted", + "SUMMARY 1 kernels: 1 pass gates 1-4, 0 fail, 0 known findings; 2 controls: 2 red as planted, 0 missed; 0 limit problems; gates 60138 ms, limits 0 ms" + ], + "fixture_note": "Generic single-case gates request fewer rows than the production fast-path minimum. Nonfinite/stress fixtures are explicitly expanded to 33 rows and all references/counts use the actual expanded shape.", + "served_shapes": [ + { + "rows": 128, + "n": 7680, + "k": 2048 + }, + { + "rows": 128, + "n": 2048, + "k": 7680 + }, + { + "rows": 33, + "n": 128, + "k": 64 + } + ], + "initial_invalid_attempt": { + "sha256": "d4da8e4ba9416fdd911d1de5c351df5fea453cf2fc3a9f0d1d537b5242ece8eb", + "reason": "NF/stress fixture refused before launch. The initial nominal red NF control is invalid evidence and is not counted. Corrected attempt above exercises real arithmetic failure." + }, + "limits": "Runtime reflection checked in the probe; global full runner/launch-limit qualification not run by this targeted command. Zero limit problems in its SUMMARY means skipped limits, not a pass.", + "raw_log_sha256": "8f8a0226ee146ff7a7e47aa46c487a804ffe31174089bfe1e3abc6879955d59f", + "qualification": false +} diff --git a/docs/receipts/lfm3/m5-text6-route-isolation-row-sweep.json b/docs/receipts/lfm3/m5-text6-route-isolation-row-sweep.json new file mode 100644 index 0000000000..cd31ada39f --- /dev/null +++ b/docs/receipts/lfm3/m5-text6-route-isolation-row-sweep.json @@ -0,0 +1,747 @@ +{ + "scope": "TEXT6 (LFM3-1.7B-MLX-6bit) prompt128+16 teacher-forced: route isolation and 25-row prompt-boundary sweep. Bounded diagnostic; no width, image, long-context or release qualification.", + "source_commit": "583a8f390 plus an uncommitted-at-run forward-tool flag change (--tiled-attention, --residual-ffn) committed with this receipt", + "host": "M5 Ultra, one guarded model job at a time under the shared lock", + "route_isolation": { + "same_bytes": { + "baseline_equals_residual_ffn_only": true, + "tiled_attention_only_equals_tiled_prompt": true + }, + "output_sha256": { + "baseline": "8e2fce1b1c1a1d6abcb2f80640472d9d8cab571d4492ce63918b442c2ee5662c", + "residual_ffn_only": "8e2fce1b1c1a1d6abcb2f80640472d9d8cab571d4492ce63918b442c2ee5662c", + "tiled_attention_only": "998933420ad173aa3e4809ab4491d6c568b7a4289a8edced245ebed62b36d7ff", + "tiled_prompt": "998933420ad173aa3e4809ab4491d6c568b7a4289a8edced245ebed62b36d7ff" + }, + "finding": "The 6-bit checkpoint does not use the 4-bit residual FFN path; only tiled attention changes its output.", + "phase_scores_prompt128_row_and_16_continuation": { + "baseline": { + "prompt_boundary": { + "native_rms": 0.04901861875813257, + "native_max": 0.2050304412841797, + "bf16_rms": 0.031214103195262688, + "bf16_max": 0.19215011596679688, + "proposed_metric_results": { + "rms_error": false, + "maximum_absolute_error": false, + "top20_probability_mean": false, + "top20_probability_max": false + }, + "top20_probability": { + "native_mean": 1.2156167466048575e-05, + "reference_mean": 5.438477557612127e-06, + "native_max": 0.00014394107004278567, + "reference_max": 3.2107429089545114e-05 + }, + "native_fp32_top1_misses": 0, + "bf16_fp32_top1_misses": 0 + }, + "teacher_forced_continuation": { + "native_rms": 0.04184006094548536, + "native_max": 0.28766918182373047, + "bf16_rms": 0.05064403338575528, + "bf16_max": 0.3306717872619629, + "proposed_metric_results": { + "rms_error": true, + "maximum_absolute_error": true, + "top20_probability_mean": false, + "top20_probability_max": false + }, + "top20_probability": { + "native_mean": 1.3799773429444968e-05, + "reference_mean": 1.3041501782842574e-05, + "native_max": 0.0003969739185127885, + "reference_max": 0.0002879273057369014 + }, + "native_fp32_top1_misses": 0, + "bf16_fp32_top1_misses": 0 + } + }, + "attn": { + "prompt_boundary": { + "native_rms": 0.05813316250926237, + "native_max": 0.22626113891601562, + "bf16_rms": 0.031214103195262688, + "bf16_max": 0.19215011596679688, + "proposed_metric_results": { + "rms_error": false, + "maximum_absolute_error": false, + "top20_probability_mean": false, + "top20_probability_max": false + }, + "top20_probability": { + "native_mean": 2.4365584210391822e-05, + "reference_mean": 5.438477557612127e-06, + "native_max": 0.00028596345265041734, + "reference_max": 3.2107429089545114e-05 + }, + "native_fp32_top1_misses": 0, + "bf16_fp32_top1_misses": 0 + }, + "teacher_forced_continuation": { + "native_rms": 0.04758823452986163, + "native_max": 0.24394440650939941, + "bf16_rms": 0.05064403338575528, + "bf16_max": 0.3306717872619629, + "proposed_metric_results": { + "rms_error": true, + "maximum_absolute_error": true, + "top20_probability_mean": false, + "top20_probability_max": false + }, + "top20_probability": { + "native_mean": 1.3406862109416334e-05, + "reference_mean": 1.3041501782842574e-05, + "native_max": 0.00033607338410834853, + "reference_max": 0.0002879273057369014 + }, + "native_fp32_top1_misses": 0, + "bf16_fp32_top1_misses": 0 + } + } + }, + "finding_phases": "The proposed prompt-phase logit RMS/max and probability diagnostics fail for BOTH baseline and tiled attention on the single prompt-boundary row at N=128; tiled attention is further from FP32 there (RMS 0.0581 vs 0.0490). The proposed continuation probability diagnostics also fail on both routes. Continuation logit RMS/max pass on both. All 17 top picks match FP32 on both routes." + }, + "prompt_boundary_row_sweep": { + "scope": "Per-row boundary error versus independent FP32; descriptive statistics, no pass/fail rule", + "rows": 25, + "summary": { + "official_bf16": { + "rms": 0.04018031004122828, + "max": 0.19565048962831497, + "top20_probability_max": 0.0004705272920054893, + "top1_misses": 0 + }, + "baseline": { + "rms": 0.03904510857050718, + "max": 0.1850430727005005, + "top20_probability_max": 0.00039090498853548635, + "top1_misses": 0 + }, + "tiled_attention": { + "rms": 0.03889914475997641, + "max": 0.18597806632518768, + "top20_probability_max": 0.0004109389619873649, + "top1_misses": 0 + }, + "rows_where_route_rms_exceeds_official_bf16": { + "baseline": 9, + "tiled_attention": 11 + } + }, + "per_row": [ + { + "prompt_length": 32, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.03976233231910736, + "max": 0.2087164893746376, + "top20_probability_max": 0.0032537480217538217, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.026819654572983853, + "max": 0.13836026191711426, + "top20_probability_max": 0.0018293817622364802, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.04227642995861886, + "max": 0.16788291931152344, + "top20_probability_max": 0.0018636441745943089, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 36, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.03080779656339401, + "max": 0.1591489315032959, + "top20_probability_max": 0.0011985592435845849, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.02926782126783764, + "max": 0.14392757415771484, + "top20_probability_max": 0.0011574321735139392, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.03593410608231325, + "max": 0.19315195083618164, + "top20_probability_max": 0.0011514354515780312, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 40, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.03309655847734577, + "max": 0.16278481483459473, + "top20_probability_max": 0.0010421974222822872, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.027848813484300648, + "max": 0.1446385383605957, + "top20_probability_max": 0.001052807914827092, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.031185467182035905, + "max": 0.14122691750526428, + "top20_probability_max": 0.0012826159437662188, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 44, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.03316512204588175, + "max": 0.19300466775894165, + "top20_probability_max": 0.001227914588654122, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.036614246311039565, + "max": 0.17698287963867188, + "top20_probability_max": 0.0005085567628388965, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.03190379590225403, + "max": 0.17340421676635742, + "top20_probability_max": 0.0005185608345426029, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 48, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.030279244688240923, + "max": 0.18629074096679688, + "top20_probability_max": 0.000327467382107368, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.02965907601820752, + "max": 0.15568938851356506, + "top20_probability_max": 0.0009484653666017584, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.0300995724854818, + "max": 0.13663412630558014, + "top20_probability_max": 0.00035088584982545523, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 52, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.034784372629980236, + "max": 0.17693257331848145, + "top20_probability_max": 0.0004477491159463609, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.033041077012022954, + "max": 0.2043132781982422, + "top20_probability_max": 0.0005399274803227216, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.03011213990267148, + "max": 0.15331554412841797, + "top20_probability_max": 0.0008724697088714484, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 56, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.030770128922654735, + "max": 0.15156316757202148, + "top20_probability_max": 0.00011187145229134797, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.045287599634868836, + "max": 0.17670154571533203, + "top20_probability_max": 0.0001921535216700132, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.03500732000641, + "max": 0.19055402278900146, + "top20_probability_max": 7.536459951716363e-05, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 60, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.03269374260168729, + "max": 0.18070220947265625, + "top20_probability_max": 0.0003918909117336389, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.04380859175551235, + "max": 0.17037725448608398, + "top20_probability_max": 0.00010506906210931092, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.042795764416603606, + "max": 0.19571471214294434, + "top20_probability_max": 0.00010526871194356958, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 64, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.043858222394534636, + "max": 0.17912554740905762, + "top20_probability_max": 0.0007770456931190939, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.033912832150465644, + "max": 0.17676019668579102, + "top20_probability_max": 6.177931271444947e-05, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.034923334681959635, + "max": 0.19501233100891113, + "top20_probability_max": 0.0005733931118793167, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 68, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.03327854633896781, + "max": 0.19359242916107178, + "top20_probability_max": 0.00017002240258989776, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.052187463677182445, + "max": 0.22980403900146484, + "top20_probability_max": 0.00020824351949377817, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.04204762855160615, + "max": 0.2206680178642273, + "top20_probability_max": 0.0002991654694334267, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 72, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.05194892418742913, + "max": 0.25265586376190186, + "top20_probability_max": 0.00010786254491271566, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.036663238489017376, + "max": 0.23136691749095917, + "top20_probability_max": 0.00014089190166344245, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.036142674450218217, + "max": 0.17760801315307617, + "top20_probability_max": 0.00033487588220881825, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 76, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.04155062043098694, + "max": 0.2230692207813263, + "top20_probability_max": 0.00013393631954561336, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.03189733378649612, + "max": 0.17526674270629883, + "top20_probability_max": 0.0001938512462887898, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.035089205533822866, + "max": 0.18467950820922852, + "top20_probability_max": 0.00019827994413113004, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 80, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.041043408311579196, + "max": 0.19153368473052979, + "top20_probability_max": 0.00016857156344007862, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.033329789719621236, + "max": 0.1611614227294922, + "top20_probability_max": 0.00015246758939657656, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.04777225846599014, + "max": 0.19099998474121094, + "top20_probability_max": 0.00019945526915421397, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 84, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.03665466164203777, + "max": 0.21520519256591797, + "top20_probability_max": 0.0002861158266269648, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.036601492706390355, + "max": 0.19108664989471436, + "top20_probability_max": 0.00026785863334843896, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.03322934691436011, + "max": 0.1823410987854004, + "top20_probability_max": 0.000115461321217869, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 88, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.039216420446088966, + "max": 0.22095680236816406, + "top20_probability_max": 5.7472444053074945e-05, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.044303234489248014, + "max": 0.24634921550750732, + "top20_probability_max": 0.0004073445375285534, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.03784907138207381, + "max": 0.1904163360595703, + "top20_probability_max": 0.00015790628452738709, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 92, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.02957815231971923, + "max": 0.1651597023010254, + "top20_probability_max": 0.00025952225668135487, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.056984911259793394, + "max": 0.21957707405090332, + "top20_probability_max": 2.1944074520694606e-05, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.04507530186526227, + "max": 0.2137523889541626, + "top20_probability_max": 2.2105992155794878e-05, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 96, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.04215044747420486, + "max": 0.2117934226989746, + "top20_probability_max": 4.5188616786033206e-05, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.040645146474558816, + "max": 0.18277287483215332, + "top20_probability_max": 0.00042819003566152425, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.04367835243002802, + "max": 0.18214869499206543, + "top20_probability_max": 0.0006100215474373893, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 100, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.050008337081004724, + "max": 0.18951702117919922, + "top20_probability_max": 0.00019690380337478164, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.029700930143676234, + "max": 0.15193645656108856, + "top20_probability_max": 0.0001592931691535293, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.04021727253017334, + "max": 0.2136911153793335, + "top20_probability_max": 6.258407985093584e-05, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 104, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.07571420260578858, + "max": 0.27094125747680664, + "top20_probability_max": 0.00044130888416471503, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.04236098431320027, + "max": 0.1941680908203125, + "top20_probability_max": 0.00019166469287901222, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.031211861609196014, + "max": 0.16148138046264648, + "top20_probability_max": 7.588719296025559e-05, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 108, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.043819194999991935, + "max": 0.18938517570495605, + "top20_probability_max": 0.0002837446010606204, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.03155986560035978, + "max": 0.16353273391723633, + "top20_probability_max": 0.00018781796542688411, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.040019718693838136, + "max": 0.18419241905212402, + "top20_probability_max": 0.00013844999007317504, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 112, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.03428625608019581, + "max": 0.17277026176452637, + "top20_probability_max": 0.0001444165063861913, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.06374090815621541, + "max": 0.2557234764099121, + "top20_probability_max": 0.0002899818195412074, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.04678385551085235, + "max": 0.21122074127197266, + "top20_probability_max": 0.00024220497742977187, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 116, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.060949252483137054, + "max": 0.20413434505462646, + "top20_probability_max": 0.00042304447876484463, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.03002384240889594, + "max": 0.16187453269958496, + "top20_probability_max": 2.5467639321756605e-05, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.043439848129088805, + "max": 0.1672534942626953, + "top20_probability_max": 0.0003947588513765199, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 120, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.039883908485483885, + "max": 0.18085312843322754, + "top20_probability_max": 0.00019398905356160512, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.026609757772176185, + "max": 0.1322917938232422, + "top20_probability_max": 0.00013383672083633336, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.04213891972157278, + "max": 0.20961952209472656, + "top20_probability_max": 0.00027606862799056664, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 124, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.04399379430600178, + "max": 0.21927547454833984, + "top20_probability_max": 4.0531737626569046e-05, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.06424048430047624, + "max": 0.23638343811035156, + "top20_probability_max": 0.00042425674144919157, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.03541221008371638, + "max": 0.18622106313705444, + "top20_probability_max": 6.664678056833573e-05, + "top1_matches_fp32": true + } + }, + { + "prompt_length": 128, + "fp32_top1": 23018, + "official_bf16": { + "rms": 0.031214103195262688, + "max": 0.19215011596679688, + "top20_probability_max": 3.2107429089545114e-05, + "top1_matches_fp32": true + }, + "baseline": { + "rms": 0.04901861875813257, + "max": 0.2050304412841797, + "top20_probability_max": 0.00014394107004278567, + "top1_matches_fp32": true + }, + "tiled_attention": { + "rms": 0.05813316250926237, + "max": 0.22626113891601562, + "top20_probability_max": 0.00028596345265041734, + "top1_matches_fp32": true + } + } + ], + "files_sha256": { + "fp32-rows.npy": "76811be2332d5f2f59a8e180884f7234f942443d9a2002d722e87e7b4123db41", + "mlx-rows.npy": "a69dff01c633ba000c01605f231f00ccd5fea0456a797ed7433d72bfb6fdb4c2", + "baseline_32.bf16": "3aada3fc898b2f6f14164596b7facc30233b5af48e450f9e2f183ec71b03eceb", + "attn_32.bf16": "53ba286650d5700eaf0667e24077211369c63d6460f2e09fa805e891a8e30196", + "baseline_36.bf16": "d8e3e5521de14146abcd84327c805826960912292a18db1ab6fae3a3ba0c5465", + "attn_36.bf16": "6a13deb966f666c8b00c7335d48a93197d6e9f7e4878df7dbe33815dc33ddd28", + "baseline_40.bf16": "ed174bb245177f60da5196fb39f287a242bd3f9e4212dc553dc6136b5cb74126", + "attn_40.bf16": "597b7282f6ee1b024810f9d8c0d055fa04e5ecb7099a2a57943b9f65b68a555a", + "baseline_44.bf16": "8fe8ad0ffdd33e21bcbe769af750d1b85c848be70fa4359e4507585c1f3d2478", + "attn_44.bf16": "c6333314b89b6c2179468a618c9c4d41fce516f6c3f95dc8789528926e701e2f", + "baseline_48.bf16": "143888684fe5b8d9fe57489baa69b0be5344892fbcfa7587d2dc6163359088e2", + "attn_48.bf16": "b99d9ada3000a5cec7591e7b877558e0ec5e4f0d02fe5628b9f4d7cbc56d22d8", + "baseline_52.bf16": "4e040bd16748202082515c4ca8db11f71c7606e4df13106408ba8f9e50905a34", + "attn_52.bf16": "458ef524b890591efc2f3be20cd898ca009f99a920ec205017e834a663bb042f", + "baseline_56.bf16": "eeaf9173e8e05dca1baa2998532e9ee40d3623e53483695794e5b54e7a833681", + "attn_56.bf16": "41c525e26265bea13e1ccbe00bd02987e23af4ce0a499d02bfd20d22f3ff63b6", + "baseline_60.bf16": "71c001320cfe02472a3916736a543591bbe3c6340fb13275ffe8b813643c091f", + "attn_60.bf16": "a92bfb8944b39a74b9d5c3671e3bd0a14115deb7b56745178b0131d9930f9ad7", + "baseline_64.bf16": "7cd41af4f49f0f56bb72d533163c3a028464ee74d5cbfeca99a9881700c4fcf0", + "attn_64.bf16": "9844fbb8298d44821bb18884a763a0313dae821bed684b72ba36ac04af2eced0", + "baseline_68.bf16": "4f2dc6f0727357b965f4f84d3755359a92463464f85aaa4c3fd1f026597b5c58", + "attn_68.bf16": "5f1ced4ce5256d13b2c65e10552b134ced164ab5abbe7970657049c28740be31", + "baseline_72.bf16": "e293b3b2b2538985950d79b8c5991b98301ed798aa902635e8c6dc6f1b89a449", + "attn_72.bf16": "7eff39ddb176236ab6b50e0f9ff6570ddbe7ce9fd781b579f9b3e66fef9014c5", + "baseline_76.bf16": "0af4f6961dc48ff59fa654cb001d2ce4e3b44dd8f28db90c8db77005ef22c9a8", + "attn_76.bf16": "21b13555ed6c0add739091d5c2fa11932a9fc0a9dcb0b898e4d68cac3579309c", + "baseline_80.bf16": "20729353bd61699f9969261de63052825a6393917f09fdaf334fabcb45f8deb4", + "attn_80.bf16": "3e784cb15dafe93e21c31038dcc6d9bc83a5fa8571de14562e3a919dca677ac6", + "baseline_84.bf16": "7d554732e26ecb639f186b2d91345d88c175b529b8118693174d21074d9d079c", + "attn_84.bf16": "66d314fb0f086acfe7d9c19c8417e61c8fd580d8c12fc19e7df70dd21a3c2eb0", + "baseline_88.bf16": "c33ab1061be13855b7c3eac60a2f0adc22a28d90be13f71863ea0953510f0cc6", + "attn_88.bf16": "aa349a706d298d2385078a040c7dc021b404c639a09c2b7b45631d585ce81749", + "baseline_92.bf16": "f30362bf695b83dad547abd05f0ddd9143536db1214d6a4146298f6c63874dc0", + "attn_92.bf16": "c2e72d19ea9aab958bce87c6ef425f14901b8160a8f0239cde5bbe2667d42e1c", + "baseline_96.bf16": "3ef5647b2ce8c2e9f7fd251dadf02ee2398671e670ca4cfa31ed623dbabc62fe", + "attn_96.bf16": "0d7d17055ed1a5c13cf86984764396880feff133959b917a8312a0086b4c583f", + "baseline_100.bf16": "2db15cf79128cdb3856c8ffed0900af13cf32952539a8aabf3ec049d61909af8", + "attn_100.bf16": "8584a9a0f596b56a35d31fe03000580b9a5dcefe38c136cde4f10411dbc67284", + "baseline_104.bf16": "c79f124fb79b10e169e89b0325890db78356f3ebf8d644dca0010e98ded910a8", + "attn_104.bf16": "3429d47e11a088ce8c840925a96046b2710ce2276065144fc384011084a1e7cd", + "baseline_108.bf16": "763f8f210019dd525f8a2375f84125a6e5b73e27cb4e2c8957e642d3f855857d", + "attn_108.bf16": "dc268e17c694af27ee58c182026674116b988425f20a120e91eb7ce66cdeb9df", + "baseline_112.bf16": "9e0b5d3c484124e3a02eaf00444b739f1453175eb4c8ff3e1fc0cda5a97b9259", + "attn_112.bf16": "c2094783bf8b600db7dffded18b5f7b35b59e7e3cee51424ac37db50ac82280b", + "baseline_116.bf16": "20de4919ac46688c97d89950722867e1b16d8dbaab30efcb99eeb4a6dc6f2f65", + "attn_116.bf16": "d21d393e07c0b068d612791cc2cd28e25148e7964bba7725838413bdd29ba9e4", + "baseline_120.bf16": "1c26e7ca166c69529d1aea3b0699a91a2245a4363a6e491660841985d8397da8", + "attn_120.bf16": "66528d4375f33df8f2437097132b15503f475d1f1ea43e6272508b2ec76a5c7f", + "baseline_124.bf16": "d530bc7c6f2b063bf0e22e2b9e850f28a8d4f1bb3e917e674cc80dab3c2f4e97", + "attn_124.bf16": "586c8a86def2ab90fcebdfac3a102f6224507bee2d4e4953e14047a6c5b5e93f", + "baseline_128.bf16": "f0067bcabfa213f6f46e87d0e5855ddc22a9560126d72bfd6d451025157d3390", + "attn_128.bf16": "3dd80ef6916a8140bf85e18194fef5a4717e9b4d9f9a36d55afde288e796c8ab" + }, + "method": "FP32 and official MLX BF16 (0.32.4) rows were produced once for the 144-token sequence with --rows 31..127; native used tokens[:N] with prompt:N for N in 32..128 step 4 (25 prompt lengths) for baseline and --tiled-attention. tools/lfm3/prompt_row_sweep.py.", + "finding": "Across 25 boundary rows baseline, tiled attention and official BF16 have indistinguishable mean distance to FP32 (mean RMS 0.0390 / 0.0389 / 0.0402; mean max 0.185 / 0.186 / 0.196; mean top-20 probability max 3.9e-4 / 4.1e-4 / 4.7e-4). Tiled attention RMS exceeded official BF16 in 11/25 rows; baseline in 9/25. No top-1 miss in any route. The N=128 single-row failure is within the per-row spread (official BF16 per-row RMS ranges 0.0296-0.0757). Rows come from one sequence and are correlated, so this is not independent-prompt evidence." + }, + "not_established": [ + "arbitrary-prompt fidelity", + "TEXT8/VL/image-row fidelity", + "a metric that is statistically meaningful on one boundary row", + "that the single-row N=128 diagnostic failure is acceptable under the maintainer criterion" + ] +} diff --git a/docs/receipts/lfm3/m5-tiled-attention-contract.json b/docs/receipts/lfm3/m5-tiled-attention-contract.json new file mode 100644 index 0000000000..33b66cf8fa --- /dev/null +++ b/docs/receipts/lfm3/m5-tiled-attention-contract.json @@ -0,0 +1,254 @@ +{ + "scope": "Targeted M5 mechanical attention contract and finite captured-output regression check. Independent official-BF16/FP32 fidelity remains a separate partial qualification.", + "runtime_source_base": "8df18abc47568246c0d38b70c2e280521a24b31a", + "source_sha256": { + "zig/kernels/metal/lfm3/attention_tiled.metal": "6b134e9062fd55dc2b04bf5124c54c540e9a74d669cdb88336460093abb24c81", + "zig/tests/contract/metal_lfm3_tiled.zig": "7df293e5fcd84581c17d1c0add99176ef2f4de0ba86ff3f1192e7cbe43f28eb9", + "zig/tests/contract/metal_lfm3_residual.zig": "34b283ef26141f964230cccd3aaf332c94cc6c9dbfebc23674a5321ca7837463", + "zig/tests/contract/metal_main.zig": "47b96d3d956a71347899d796f6be823893c05bb3e79c6dbfaff4410623d321f2", + "zig/tests/contract/metal_limits.zig": "96e9bc9fd60083ec29fc5cb352542404817c9799445c2c54f70d59fe01934d82", + "zig/src/core/contract/lfm3_records.zig": "6b0a47c6d23f022ece4dd2fd5a09ccd3ce49dc71e0d6798e10814a2b5ddba431", + "binary": "2f6773f4a7285e1dd9cb15ffd1386c45ddef248ba761187a480d592c23d6eb08" + }, + "command": "tf-kernel-contract --only \"LFM3 tiled structural\" --no-limits", + "result": { + "exit": 0, + "elapsed_s": 8.608491875, + "binary_sha256": "2f6773f4a7285e1dd9cb15ffd1386c45ddef248ba761187a480d592c23d6eb08", + "scope": "Targeted diagnostic; full runner and independent FP32/BF16 fidelity separate." + }, + "log": [ + "tf-kernel-contract on Apple M5 Ultra (applegpu_g17, tensor units true), seed 0x6b63", + "tiled structural nonfinite: expanding fixture rows 8 to eligible 9", + "PASS nonfinite metal LFM3 tiled structural full: the stated rule holds: visible-query/key/value nonfinite classes; masked future values must not contaminate", + "PASS repeat metal LFM3 tiled structural full: five runs byte-identical (36864 bytes), inputs and guards untouched, no overlap", + "PASS shapes metal LFM3 tiled structural full: 13 shapes exact by its own check (worst 0.000 of the bound), 0 edges refused by the kernel", + "tiled structural stress: expanding fixture rows 4 to eligible 9", + "PASS sums metal LFM3 tiled structural full: within its bound at k 8192; fp16 sums would miss 0x, bf16 0.0x (uniform-score structural fixture; no claimed 100x reduced-accumulator witness)", + "tiled class mismatch row=8 head=0 col=0: got=nan want=finite", + "FAIL shapes metal LFM3 tiled structural bad-tail: at 9x2048x0: visible/masked nonfinite class mismatch", + "CONTROL metal LFM3 tiled structural bad-tail: red at shapes, as planted", + "tiled structural nonfinite: expanding fixture rows 8 to eligible 9", + "tiled class mismatch row=0 head=0 col=0: got=finite want=nan", + "FAIL nonfinite metal LFM3 tiled structural bad-nonfinite: visible/masked nonfinite class mismatch", + "CONTROL metal LFM3 tiled structural bad-nonfinite: red at nonfinite, as planted", + "tiled structural nonfinite: expanding fixture rows 8 to eligible 9", + "tiled sliding nonfinite: expanding history to 257 for a masked-past plant", + "PASS nonfinite metal LFM3 tiled structural sliding: the stated rule holds: visible-query/key/value nonfinite classes; masked future values must not contaminate", + "PASS repeat metal LFM3 tiled structural sliding: five runs byte-identical (36864 bytes), inputs and guards untouched, no overlap", + "PASS shapes metal LFM3 tiled structural sliding: 13 shapes exact by its own check (worst 0.000 of the bound), 0 edges refused by the kernel", + "tiled structural stress: expanding fixture rows 4 to eligible 9", + "PASS sums metal LFM3 tiled structural sliding: within its bound at k 8192; fp16 sums would miss 0x, bf16 0.0x (uniform-score structural fixture; no claimed 100x reduced-accumulator witness)", + "SUMMARY 2 kernels: 2 pass gates 1-4, 0 fail, 0 known findings; 2 controls: 2 red as planted, 0 missed; 0 limit problems; gates 7405 ms, limits 0 ms" + ], + "raw_log_sha256": "285f900c1204c44ba8a23654894485491b4a75aa762e83202ed8203e36e55c87", + "repair": { + "trigger": "Nonfinite BF16 result in a SIMD-owned query row.", + "action": "Uniform SIMD device-memory barrier after fragment stores, then existing extended-range causal/window replay overwrites only flagged visible rows. No CPU scan, global flags, allocation, or extra dispatch.", + "finite_arithmetic": "Unchanged; captured byte identity measured below. Instruction-cost impact has not been measured." + }, + "fixtures": { + "reference": "Uniform zero Q/K and bounded integer V: scalar FP32 visible-value sum times FP32 reciprocal, then BF16 rounding. This is not labeled official MLX BF16.", + "nonfinite": "Real Q NaN/+Inf/-Inf; future K NaN; future V NaN; sliding masked-past V NaN; visible V signed infinities and mixed infinities. Eligible 9-row cases, sliding history257.", + "shapes": "Full and sliding each 13 served/tail shapes; declared rows9,17,31,32,33,63,64 and histories0,31,255,256,257,2048,8192; 19 poisoned extra capacity rows.", + "repeat": "Five byte-identical runs, unchanged inputs and guards, no overlap.", + "accumulation": "Uniform-score fixture has no claimed 100x reduced-accumulator witness; sums diagnostic is explicitly limited." + }, + "prior_failure": { + "exit": 1, + "raw_log_sha256": "7013e5580fb0453fa57cbab2ce7820c9764114fc6350742d340b0b9068c44d6a", + "shader_sha256": "126a69dbfcf1e57d6fb1f9ceabaab840741a7acb54902b622c534648335521b9", + "observed": "Both full/sliding NF gates failed: masked future V NaN contaminated an earlier finite query. Valid failing evidence retained, not reclassified." + }, + "finite_regression": { + "cases": 22, + "rows_per_case": 128, + "histories": [ + 0 + ], + "windows": [ + 0, + 256 + ], + "captures": "11 real attention layers times baseline/candidate propagated inputs from the prior text4 128-prompt capture.", + "result": { + "cases": [ + { + "case": "baseline-layer10.normal.pair.out.bf16", + "bytes": 524288, + "sha256": "fe8fa34850f838464182a4e08c71db6ba9addab8559a51c05725e418787fb6fe", + "equal": true + }, + { + "case": "baseline-layer12.normal.pair.out.bf16", + "bytes": 524288, + "sha256": "ecab322a30bf5c4ab160a2ea927185fe04c164067340827ae252b7a8dca4bed6", + "equal": true + }, + { + "case": "baseline-layer14.normal.pair.out.bf16", + "bytes": 524288, + "sha256": "10fb1243c4f529d3d120d2aa3d4317923e2cefa505b2882cf940cc9e2094e304", + "equal": true + }, + { + "case": "baseline-layer16.normal.pair.out.bf16", + "bytes": 524288, + "sha256": "91b49ca7cca79fb86cf55e49a57044a6c369bdb59c7c88e60a3cccb889d72e9e", + "equal": true + }, + { + "case": "baseline-layer18.normal.pair.out.bf16", + "bytes": 524288, + "sha256": "568bc8fb2d0b8f92da55372b96ef8bb8abeb0129e29203a7ce317f55a75e1a23", + "equal": true + }, + { + "case": "baseline-layer2.normal.pair.out.bf16", + "bytes": 524288, + "sha256": "85ff34be2a28de74eb6a022c7ed045fb4c25c53dbc488bbd5edb6bcd9da103c0", + "equal": true + }, + { + "case": "baseline-layer20.normal.pair.out.bf16", + "bytes": 524288, + "sha256": "36d3090814266bf63ff60e4aa6ffe2d079e79f2b9f30f8f74257021fe1e29475", + "equal": true + }, + { + "case": "baseline-layer22.normal.pair.out.bf16", + "bytes": 524288, + "sha256": "a8219b759942fbb06c3e22c940910fb6fb319feb634e183fae692b0a9d59009c", + "equal": true + }, + { + "case": "baseline-layer4.normal.pair.out.bf16", + "bytes": 524288, + "sha256": "7c511a38caf2d2d0a8f21d1e26eab05d778db396082c9c57774e313d198f43f2", + "equal": true + }, + { + "case": "baseline-layer6.normal.pair.out.bf16", + "bytes": 524288, + "sha256": "7744851abf8c8486274cf29dd7bfef11904799fc2a7cbec109387fbb5e2bf362", + "equal": true + }, + { + "case": "baseline-layer8.normal.pair.out.bf16", + "bytes": 524288, + "sha256": "885422dfda4181d6dd7263562389ee9007e6fb06fa933861ec4a87dbac18a59d", + "equal": true + }, + { + "case": "candidate-layer10.normal.pair.out.bf16", + "bytes": 524288, + "sha256": "b72ed2cc8dfb12b434410d9990918998ec1fcf25fa86eaebf30c1744a7dbafad", + "equal": true + }, + { + "case": "candidate-layer12.normal.pair.out.bf16", + "bytes": 524288, + "sha256": "0c7c84f23b842c97133bc02185818d6e752b37bc777fa754c70f701572746e2c", + "equal": true + }, + { + "case": "candidate-layer14.normal.pair.out.bf16", + "bytes": 524288, + "sha256": "e0ec62481bbc69bd5e20a5cc8bec354f55a1fd4880ace5bda1bd547bf7c58321", + "equal": true + }, + { + "case": "candidate-layer16.normal.pair.out.bf16", + "bytes": 524288, + "sha256": "af28ed5395e31c216622261b68468e7ecbee603897c597674b66f45467dca87f", + "equal": true + }, + { + "case": "candidate-layer18.normal.pair.out.bf16", + "bytes": 524288, + "sha256": "0977391be5bdba267351d94fc8ff6f5af303f4b6ee4f64000d3afb3d64eb92a3", + "equal": true + }, + { + "case": "candidate-layer2.normal.pair.out.bf16", + "bytes": 524288, + "sha256": "445ca5e6b8c765b4e99cf5cc638cb944c1c020ef38377a184385de7b3190f66d", + "equal": true + }, + { + "case": "candidate-layer20.normal.pair.out.bf16", + "bytes": 524288, + "sha256": "364b79b37fbe2b57812aaae003ef0f5a1289ae5ff2b3827360ca2caf6a9721ed", + "equal": true + }, + { + "case": "candidate-layer22.normal.pair.out.bf16", + "bytes": 524288, + "sha256": "8a00eabaffe5c486fe2fcffe977fbeddab6417c70e7b3f5b682aad1f6611bbdd", + "equal": true + }, + { + "case": "candidate-layer4.normal.pair.out.bf16", + "bytes": 524288, + "sha256": "5af9ba9c1f72d09de9890cca342fa87122491e6fc19892b051d61a1fad1a117e", + "equal": true + }, + { + "case": "candidate-layer6.normal.pair.out.bf16", + "bytes": 524288, + "sha256": "cc7abd9a9009f7cd181c10c974ffe326baf6ea029c1ab39bead464ae44b1c554", + "equal": true + }, + { + "case": "candidate-layer8.normal.pair.out.bf16", + "bytes": 524288, + "sha256": "c62fd3977be8e9cbaabefc29771cc7806dfbbc064add5b83e6ccd7661346978e", + "equal": true + } + ], + "all_22_byte_equal": true, + "bytes_checked": 11534336 + }, + "all_children_exit_zero": true, + "guarded_controller_released": true, + "manifest_sha256": "aa307c6ff1b0f84d4786f969e787b9cd5dfd2e24106cb678ab3b492bb84e028d", + "raw_files_sha256": { + "000.log": "c7a3c0aaf76c88435c22eb9dba2cfcd9c7289c2cbcd9613578e5ab0b5ff422d5", + "001.log": "a86abe4210b242b65c6833a59bf34f829f502b34f2cdb42d7412cdde1d4a6867", + "002.log": "c7a3c0aaf76c88435c22eb9dba2cfcd9c7289c2cbcd9613578e5ab0b5ff422d5", + "003.log": "a86abe4210b242b65c6833a59bf34f829f502b34f2cdb42d7412cdde1d4a6867", + "004.log": "c7a3c0aaf76c88435c22eb9dba2cfcd9c7289c2cbcd9613578e5ab0b5ff422d5", + "005.log": "a86abe4210b242b65c6833a59bf34f829f502b34f2cdb42d7412cdde1d4a6867", + "006.log": "c7a3c0aaf76c88435c22eb9dba2cfcd9c7289c2cbcd9613578e5ab0b5ff422d5", + "007.log": "a86abe4210b242b65c6833a59bf34f829f502b34f2cdb42d7412cdde1d4a6867", + "008.log": "c7a3c0aaf76c88435c22eb9dba2cfcd9c7289c2cbcd9613578e5ab0b5ff422d5", + "009.log": "a86abe4210b242b65c6833a59bf34f829f502b34f2cdb42d7412cdde1d4a6867", + "010.log": "c7a3c0aaf76c88435c22eb9dba2cfcd9c7289c2cbcd9613578e5ab0b5ff422d5", + "011.log": "c7a3c0aaf76c88435c22eb9dba2cfcd9c7289c2cbcd9613578e5ab0b5ff422d5", + "012.log": "a86abe4210b242b65c6833a59bf34f829f502b34f2cdb42d7412cdde1d4a6867", + "013.log": "c7a3c0aaf76c88435c22eb9dba2cfcd9c7289c2cbcd9613578e5ab0b5ff422d5", + "014.log": "a86abe4210b242b65c6833a59bf34f829f502b34f2cdb42d7412cdde1d4a6867", + "015.log": "c7a3c0aaf76c88435c22eb9dba2cfcd9c7289c2cbcd9613578e5ab0b5ff422d5", + "016.log": "a86abe4210b242b65c6833a59bf34f829f502b34f2cdb42d7412cdde1d4a6867", + "017.log": "c7a3c0aaf76c88435c22eb9dba2cfcd9c7289c2cbcd9613578e5ab0b5ff422d5", + "018.log": "a86abe4210b242b65c6833a59bf34f829f502b34f2cdb42d7412cdde1d4a6867", + "019.log": "c7a3c0aaf76c88435c22eb9dba2cfcd9c7289c2cbcd9613578e5ab0b5ff422d5", + "020.log": "a86abe4210b242b65c6833a59bf34f829f502b34f2cdb42d7412cdde1d4a6867", + "021.log": "c7a3c0aaf76c88435c22eb9dba2cfcd9c7289c2cbcd9613578e5ab0b5ff422d5", + "events.json": "9d576412b02704d0c0735c3e60efc5d163ad88f702bf9f2113cd309d757fd485", + "identity.json": "c560fa12820872e41e9ee99002f79d048f648d8730891e7f141a35df5df82fa0", + "receipt.json": "6ec7769e7ccc9bff776a69be73eeec2f6b97bed29ded007c2e6e801d405c3066" + } + }, + "build_checks": { + "tf-kernel-contract": "exit0, Zig0.17.0, -Dcpu=apple_m1", + "test-kernel-contract": "exit0, host contract tests" + }, + "limitations": [ + "--no-limits skips the global device/offline launch-limit suite; zero limit problems in SUMMARY is not that suite passing. Probe runtime reflection checked SIMD32, threads>=128, TG memory0.", + "Older-GPU capability skip is source-reviewed; this receipt is M5 only.", + "No full-model rerun, width/image promotion, new FP32/BF16 fidelity acceptance, or performance claim.", + "Existing proposed whole-model probability diagnostic failures remain unchanged." + ], + "qualification": false +} diff --git a/docs/receipts/lfm3/m5-tiled-attention-provisional-serving.json b/docs/receipts/lfm3/m5-tiled-attention-provisional-serving.json new file mode 100644 index 0000000000..dccf92215f --- /dev/null +++ b/docs/receipts/lfm3/m5-tiled-attention-provisional-serving.json @@ -0,0 +1,82 @@ +{ + "scope": "PROVISIONAL known fidelity-unqualified M5 TEXT4 attention-only prompt tiled SDPA; no production promotion", + "cases": [ + { + "prompt_tokens": 2048, + "baseline_ttft_s": [ + 0.8462561252526939, + 0.8461228748783469 + ], + "candidate_ttft_s": [ + 0.7695160410366952, + 0.7676864583045244 + ], + "baseline_median_s": 0.8461895000655204, + "candidate_median_s": 0.7686012496706098, + "baseline_over_candidate": 1.1009473383346198, + "baseline_first_last_drift": -0.0001574586822721269, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "not asserted; same output text digest only", + "prompt_ids": "both routes /tokenize exact pinned arrays", + "candidate_spread": 0.002383242158799481, + "ttft_reduction_fraction": 0.09169134146536084 + }, + { + "prompt_tokens": 8192, + "baseline_ttft_s": [ + 4.401642707642168, + 4.400061042048037 + ], + "candidate_ttft_s": [ + 3.2477782079949975, + 3.248213625047356 + ], + "baseline_median_s": 4.400851874845102, + "candidate_median_s": 3.2479959165211767, + "baseline_over_candidate": 1.3549437831679025, + "baseline_first_last_drift": -0.00035933529802056885, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "not asserted; same output text digest only", + "prompt_ids": "both routes /tokenize exact pinned arrays", + "candidate_spread": 0.00013406612904987014, + "ttft_reduction_fraction": 0.2619619998831483 + } + ], + "predeclared_max_baseline_drift": 0.05, + "no_timing_retries": true, + "default_off": true, + "same_binary_sha256": "b5d10c1a642603ce2c50a90c0054cbfbd9020f785de2a26ec7b7b4e3e06cc4ff", + "fidelity": { + "basic_finite_logits": true, + "prompt128_envelope_pass": true, + "continuation_envelope_pass": false, + "continuation_native_max_logit_error": 0.2778849602, + "continuation_reference_max_logit_error": 0.2564673424, + "continuation_native_top20_mean_error": 1.89956e-05, + "continuation_reference_top20_mean_error": 1.65508e-05, + "continuation_native_top20_max_error": 0.0004400848, + "continuation_reference_top20_max_error": 0.0004334887, + "all17_top1_match_fp32": true, + "top1_match_does_not_waive_error_gate": true + }, + "protocol": { + "same_binary_env_order": [ + 0, + 1, + 1, + 0 + ], + "fresh_servers": 4, + "cache_tokens": 0, + "completion_tokens": 16, + "all_server_exit_zero": true, + "all_client_exit_zero": true, + "candidate_route_hits_verified": true, + "same_pinned_token_arrays": true, + "no_retries": true + }, + "records_sha256": "45ab9d5d1e8aab6a81140356c7213a4bd0909e73ef2070ac3847c2ac69340d23", + "events_sha256": "f6c2388250561d16d1832edf3728969e84dfd9eb0c62bd1eae3b0db6c638381e" +} diff --git a/docs/receipts/lfm3/m5-tiled-attention-relaxed-pv-exploratory.json b/docs/receipts/lfm3/m5-tiled-attention-relaxed-pv-exploratory.json new file mode 100644 index 0000000000..90f75e1b4a --- /dev/null +++ b/docs/receipts/lfm3/m5-tiled-attention-relaxed-pv-exploratory.json @@ -0,0 +1,80 @@ +{ + "status": "exploratory operator evidence only; no fullmodel qualification or promotion", + "device": "M5", + "algorithm": "64-query by32-key tiled SDPA; strict QK and relaxed float-by-BF16 PV; actual descriptor mapping verified", + "reference": "independent CPU FP32 plus actual official MLX0.32.4 BF16 on identical heldinputs", + "gates": "zero-added-slack whole-cell RMS and maximum error; per-row and pointwise diagnostics retained", + "scope": "four synthetic R64 timing cells; finite only; not model or MLX performance comparison", + "cases": [ + { + "case": "h2048-r64-w0", + "native_fused_baseline_seconds": 0.005553991211854736, + "tiled_seconds": 0.00020092789600312244, + "throughput_ratio": 27.641712884747605, + "baseline_drift": -0.0003676429450156782, + "max_withinround_spread": 0.0015301493214530115, + "stable": true, + "raw_sha256": "d2ce762a9ea2e81aab097414b43a41d93ce293e88b2ffeabdbba8181f3f731f2" + }, + { + "case": "h2048-r64-w256", + "native_fused_baseline_seconds": 0.00011196785499123507, + "tiled_seconds": 4.293196616345085e-05, + "throughput_ratio": 2.6080299831820035, + "baseline_drift": 0.0002964367730724504, + "max_withinround_spread": 0.002822034885964664, + "stable": true, + "raw_sha256": "7834eaa7a0416fd692b875338240cb3e12693534c8645f3444440d28a0c46573" + }, + { + "case": "h8192-r64-w0", + "native_fused_baseline_seconds": 0.00331929622188909, + "tiled_seconds": 0.0007470677082892507, + "throughput_ratio": 4.443099581281754, + "baseline_drift": 0.0028380385134672004, + "max_withinround_spread": 0.004884279005040737, + "stable": true, + "raw_sha256": "7d1e7f6b3df65fa771f0f61b5740bdfe456ba5418caae97f6c378b66cac58716" + }, + { + "case": "h8192-r64-w256", + "native_fused_baseline_seconds": 0.0001659166869103501, + "tiled_seconds": 4.302250158616516e-05, + "throughput_ratio": 3.856509519281517, + "baseline_drift": 0.00018921527823456685, + "max_withinround_spread": 0.002591717554035977, + "stable": true, + "raw_sha256": "fcb0e787cbabb5e4bddcd82b44d5715e4032348bb7d1343cbdc658289fb8284b" + } + ], + "protocol": { + "balanced_rounds": 5, + "min_calibration_ms": 25, + "min_sample_ms": 20, + "max_batch_seconds": 1, + "stability_fraction": 0.05, + "retries": 0 + }, + "excluded_failure": { + "case": "history0 rows3 fullattention", + "native_rms": 0.00026754235842233083, + "reference_rms": 0.00026530804824486114, + "maximum_equal": true, + "decision": "not qualified; keep original small-query path" + }, + "genuine_heldinput": { + "model": "VL4", + "layer": 4, + "rows": 18, + "history": 0, + "window": 256, + "envelope_pass": true, + "wholemodel_pass": false + }, + "limits": [ + "2k full baseline is potentially replay-sensitive; not representative of universal gain", + "No nonfinite or extreme input qualification", + "Fullmodel propagated FP32/BF16 acceptance and end-to-end timing pending", + "Earlier strict-PV kernel had one diagnostic row-max exceedance despite whole-cell pass; retained separately" + ] +} diff --git a/docs/receipts/lfm3/m5-width-image-route-comparison.json b/docs/receipts/lfm3/m5-width-image-route-comparison.json new file mode 100644 index 0000000000..0f142bcd25 --- /dev/null +++ b/docs/receipts/lfm3/m5-width-image-route-comparison.json @@ -0,0 +1,794 @@ +{ + "scope": "TEXT8 and VL4/6/8 (real projected image rows, teacher-forced text) at the tf-lfm3-forward route, native baseline (guarded default path) vs --tiled-prompt (residual FFN + tiled attention, 128-row chunks) vs official MLX-VLM 0.32.4 BF16, each against independent FP32. Descriptive; no width or image qualification and no pass is claimed.", + "source": "583a8f390 + tf-lfm3-forward flags --tiled-attention/--residual-ffn and prompt:N with image rows. Not the 2048-row server route.", + "caveat": [ + "Boundary rows of one sequence per model are correlated; counts are not independent-prompt evidence.", + "Image rows are the retained VL4/6/8 projected rows from the earlier real-image fixture (70 rows at positions 4..73); teacher-forced tokens after the image are the fixture text, not model output.", + "Zero-slack proposed phase metrics are evaluated on one prompt row and 16 continuation rows; baseline fails them too in several cells." + ], + "canonical_phases_prompt_row_plus_16_teacher_forced": { + "text8": { + "baseline": { + "prompt_boundary": { + "native_rms": 0.05756910058878599, + "bf16_rms": 0.05299046310657806, + "native_max": 0.21836501359939575, + "bf16_max": 0.23966991901397705, + "metrics": { + "rms_error": false, + "maximum_absolute_error": true, + "top20_probability_mean": false, + "top20_probability_max": false + }, + "top20": { + "native_mean": 1.621533910296554e-05, + "reference_mean": 8.516216434420207e-06, + "native_max": 0.0002086804369623163, + "reference_max": 5.4924640725873e-05 + }, + "top1_miss": [ + 0, + 0 + ] + }, + "teacher_forced_continuation": { + "native_rms": 0.041127697666286024, + "bf16_rms": 0.04531972721085561, + "native_max": 0.24574851989746094, + "bf16_max": 0.2324376106262207, + "metrics": { + "rms_error": true, + "maximum_absolute_error": false, + "top20_probability_mean": true, + "top20_probability_max": true + }, + "top20": { + "native_mean": 1.4931907591644119e-05, + "reference_mean": 1.725467685225262e-05, + "native_max": 0.0003219552787731006, + "reference_max": 0.0003729856694399958 + }, + "top1_miss": [ + 0, + 0 + ] + } + }, + "tiled": { + "prompt_boundary": { + "native_rms": 0.028363209962736325, + "bf16_rms": 0.05299046310657806, + "native_max": 0.1697826385498047, + "bf16_max": 0.23966991901397705, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true, + "top20_probability_mean": true, + "top20_probability_max": false + }, + "top20": { + "native_mean": 7.47284263667376e-06, + "reference_mean": 8.516216434420207e-06, + "native_max": 7.602041050491781e-05, + "reference_max": 5.4924640725873e-05 + }, + "top1_miss": [ + 0, + 0 + ] + }, + "teacher_forced_continuation": { + "native_rms": 0.03825201234654284, + "bf16_rms": 0.04531972721085561, + "native_max": 0.20448875427246094, + "bf16_max": 0.2324376106262207, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true, + "top20_probability_mean": true, + "top20_probability_max": false + }, + "top20": { + "native_mean": 1.4025369132687608e-05, + "reference_mean": 1.725467685225262e-05, + "native_max": 0.0004453149229557418, + "reference_max": 0.0003729856694399958 + }, + "top1_miss": [ + 0, + 0 + ] + } + }, + "sweep": { + "official_bf16": { + "rms": 0.04279252546116228, + "max": 0.19843310952186585, + "top20_probability_max": 0.0003955591587628029, + "top1_misses": 0 + }, + "baseline": { + "rms": 0.03895808404525676, + "max": 0.1796414351463318, + "top20_probability_max": 0.0004932884817942959, + "top1_misses": 0 + }, + "tiled_prompt": { + "rms": 0.03619301455482047, + "max": 0.17607753872871398, + "top20_probability_max": 0.00043222382258892154, + "top1_misses": 0 + }, + "rows_where_route_rms_exceeds_official_bf16": { + "baseline": 6, + "tiled_prompt": 4 + } + } + }, + "vl4": { + "baseline": { + "prompt_boundary": { + "native_rms": 0.045021395748451196, + "bf16_rms": 0.05724186334479262, + "native_max": 0.22849583625793457, + "bf16_max": 0.336777925491333, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true, + "top20_probability_mean": true, + "top20_probability_max": true + }, + "top20": { + "native_mean": 0.0005640799256254708, + "reference_mean": 0.00296291354265528, + "native_max": 0.0036106027920228545, + "reference_max": 0.029943697458890695 + }, + "top1_miss": [ + 0, + 0 + ] + }, + "teacher_forced_continuation": { + "native_rms": 0.034404836903271833, + "bf16_rms": 0.03698408052111559, + "native_max": 0.29196321964263916, + "bf16_max": 0.23212289810180664, + "metrics": { + "rms_error": true, + "maximum_absolute_error": false, + "top20_probability_mean": false, + "top20_probability_max": false + }, + "top20": { + "native_mean": 0.001188399198526996, + "reference_mean": 0.000845532662613498, + "native_max": 0.027482537320970835, + "reference_max": 0.013107370719816963 + }, + "top1_miss": [ + 0, + 0 + ] + } + }, + "tiled": { + "prompt_boundary": { + "native_rms": 0.05484708159668184, + "bf16_rms": 0.05724186334479262, + "native_max": 0.26920086145401, + "bf16_max": 0.336777925491333, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true, + "top20_probability_mean": true, + "top20_probability_max": true + }, + "top20": { + "native_mean": 0.0008412521617957613, + "reference_mean": 0.00296291354265528, + "native_max": 0.0076153434266265085, + "reference_max": 0.029943697458890695 + }, + "top1_miss": [ + 0, + 0 + ] + }, + "teacher_forced_continuation": { + "native_rms": 0.03335499915133132, + "bf16_rms": 0.03698408052111559, + "native_max": 0.20049190521240234, + "bf16_max": 0.23212289810180664, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true, + "top20_probability_mean": true, + "top20_probability_max": false + }, + "top20": { + "native_mean": 0.0008280848993724596, + "reference_mean": 0.000845532662613498, + "native_max": 0.015296780348855121, + "reference_max": 0.013107370719816963 + }, + "top1_miss": [ + 0, + 0 + ] + } + }, + "sweep": { + "official_bf16": { + "rms": 0.04035298264210717, + "max": 0.20450172159406874, + "top20_probability_max": 0.005705231316648891, + "top1_misses": 0 + }, + "baseline": { + "rms": 0.05491078416293803, + "max": 0.2325352562798394, + "top20_probability_max": 0.0029790308247212146, + "top1_misses": 0 + }, + "tiled_prompt": { + "rms": 0.05732688706944902, + "max": 0.2529616620805528, + "top20_probability_max": 0.004892909396824816, + "top1_misses": 0 + }, + "rows_where_route_rms_exceeds_official_bf16": { + "baseline": 5, + "tiled_prompt": 5 + } + } + }, + "vl6": { + "baseline": { + "prompt_boundary": { + "native_rms": 0.0525786548534892, + "bf16_rms": 0.1442952219059758, + "native_max": 0.30329906940460205, + "bf16_max": 0.42031431198120117, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true, + "top20_probability_mean": true, + "top20_probability_max": true + }, + "top20": { + "native_mean": 0.0036590676418355233, + "reference_mean": 0.0059602204430437965, + "native_max": 0.0229580133308412, + "reference_max": 0.05314411916546147 + }, + "top1_miss": [ + 0, + 1 + ] + }, + "teacher_forced_continuation": { + "native_rms": 0.043509245553194, + "bf16_rms": 0.05030189013781307, + "native_max": 0.4391179084777832, + "bf16_max": 0.27240753173828125, + "metrics": { + "rms_error": true, + "maximum_absolute_error": false, + "top20_probability_mean": true, + "top20_probability_max": false + }, + "top20": { + "native_mean": 0.0011049439766418962, + "reference_mean": 0.0011381003954446989, + "native_max": 0.022210566288353162, + "reference_max": 0.015440174074140445 + }, + "top1_miss": [ + 0, + 0 + ] + } + }, + "tiled": { + "prompt_boundary": { + "native_rms": 0.05682561171501801, + "bf16_rms": 0.1442952219059758, + "native_max": 0.2940101623535156, + "bf16_max": 0.42031431198120117, + "metrics": { + "rms_error": true, + "maximum_absolute_error": true, + "top20_probability_mean": true, + "top20_probability_max": true + }, + "top20": { + "native_mean": 0.0019917546424927514, + "reference_mean": 0.0059602204430437965, + "native_max": 0.01832072294622833, + "reference_max": 0.05314411916546147 + }, + "top1_miss": [ + 0, + 1 + ] + }, + "teacher_forced_continuation": { + "native_rms": 0.04545417399614853, + "bf16_rms": 0.05030189013781307, + "native_max": 0.3274260461330414, + "bf16_max": 0.27240753173828125, + "metrics": { + "rms_error": true, + "maximum_absolute_error": false, + "top20_probability_mean": true, + "top20_probability_max": false + }, + "top20": { + "native_mean": 0.001117110099955857, + "reference_mean": 0.0011381003954446989, + "native_max": 0.022678870875296853, + "reference_max": 0.015440174074140445 + }, + "top1_miss": [ + 0, + 0 + ] + } + }, + "sweep": { + "official_bf16": { + "rms": 0.06791185993219352, + "max": 0.31771284341812134, + "top20_probability_max": 0.014008899926185113, + "top1_misses": 1 + }, + "baseline": { + "rms": 0.05036021844438576, + "max": 0.19850130875905356, + "top20_probability_max": 0.005867781359692831, + "top1_misses": 0 + }, + "tiled_prompt": { + "rms": 0.044764633684090026, + "max": 0.2005244361029731, + "top20_probability_max": 0.005781203911422671, + "top1_misses": 0 + }, + "rows_where_route_rms_exceeds_official_bf16": { + "baseline": 2, + "tiled_prompt": 2 + } + } + }, + "vl8": { + "baseline": { + "prompt_boundary": { + "native_rms": 0.08996466833747349, + "bf16_rms": 0.06759950572101424, + "native_max": 0.41560298204421997, + "bf16_max": 0.3477163314819336, + "metrics": { + "rms_error": false, + "maximum_absolute_error": false, + "top20_probability_mean": true, + "top20_probability_max": false + }, + "top20": { + "native_mean": 0.0026942492524039763, + "reference_mean": 0.0034629256547694296, + "native_max": 0.026271645693792367, + "reference_max": 0.02376195567602457 + }, + "top1_miss": [ + 0, + 0 + ] + }, + "teacher_forced_continuation": { + "native_rms": 0.043630560381030116, + "bf16_rms": 0.038167574377201724, + "native_max": 0.27512621879577637, + "bf16_max": 0.3014151304960251, + "metrics": { + "rms_error": false, + "maximum_absolute_error": true, + "top20_probability_mean": false, + "top20_probability_max": false + }, + "top20": { + "native_mean": 0.0010667723681750834, + "reference_mean": 0.0010593791082796145, + "native_max": 0.03142906525647948, + "reference_max": 0.02334146286597405 + }, + "top1_miss": [ + 1, + 0 + ] + } + }, + "tiled": { + "prompt_boundary": { + "native_rms": 0.0659144404390306, + "bf16_rms": 0.06759950572101424, + "native_max": 0.5613763630390167, + "bf16_max": 0.3477163314819336, + "metrics": { + "rms_error": true, + "maximum_absolute_error": false, + "top20_probability_mean": true, + "top20_probability_max": true + }, + "top20": { + "native_mean": 0.0020414166521818414, + "reference_mean": 0.0034629256547694296, + "native_max": 0.018191858360262103, + "reference_max": 0.02376195567602457 + }, + "top1_miss": [ + 0, + 0 + ] + }, + "teacher_forced_continuation": { + "native_rms": 0.0470500589571398, + "bf16_rms": 0.038167574377201724, + "native_max": 0.3434346914291382, + "bf16_max": 0.3014151304960251, + "metrics": { + "rms_error": false, + "maximum_absolute_error": false, + "top20_probability_mean": false, + "top20_probability_max": true + }, + "top20": { + "native_mean": 0.0011104073251286019, + "reference_mean": 0.0010593791082796145, + "native_max": 0.021137649965129346, + "reference_max": 0.02334146286597405 + }, + "top1_miss": [ + 1, + 0 + ] + } + }, + "sweep": { + "official_bf16": { + "rms": 0.055668635305161786, + "max": 0.32083530558480156, + "top20_probability_max": 0.01261331459321191, + "top1_misses": 0 + }, + "baseline": { + "rms": 0.060090205909432294, + "max": 0.3198500759899616, + "top20_probability_max": 0.011049715148431711, + "top1_misses": 0 + }, + "tiled_prompt": { + "rms": 0.06984953204430809, + "max": 0.350392265452279, + "top20_probability_max": 0.011304059793175636, + "top1_misses": 0 + }, + "rows_where_route_rms_exceeds_official_bf16": { + "baseline": 6, + "tiled_prompt": 6 + } + } + } + }, + "sweeps_short": { + "text8": { + "rows": 25, + "summary": { + "official_bf16": { + "rms": 0.04279252546116228, + "max": 0.19843310952186585, + "top20_probability_max": 0.0003955591587628029, + "top1_misses": 0 + }, + "baseline": { + "rms": 0.03895808404525676, + "max": 0.1796414351463318, + "top20_probability_max": 0.0004932884817942959, + "top1_misses": 0 + }, + "tiled_prompt": { + "rms": 0.03619301455482047, + "max": 0.17607753872871398, + "top20_probability_max": 0.00043222382258892154, + "top1_misses": 0 + }, + "rows_where_route_rms_exceeds_official_bf16": { + "baseline": 6, + "tiled_prompt": 4 + } + }, + "paired_tiled_minus_baseline_rms": { + "mean": -0.0027650694904362933, + "standard_error": 0.0023544163382112513, + "rows_tiled_worse": 11 + }, + "top1_differences": [], + "sweep_file_sha256": "e1e6f6d85c4b5df44291627b44f4c1ec381838a57a176bf22b6d2860a0af54d0" + }, + "vl4": { + "rows": 9, + "summary": { + "official_bf16": { + "rms": 0.04035298264210717, + "max": 0.20450172159406874, + "top20_probability_max": 0.005705231316648891, + "top1_misses": 0 + }, + "baseline": { + "rms": 0.05491078416293803, + "max": 0.2325352562798394, + "top20_probability_max": 0.0029790308247212146, + "top1_misses": 0 + }, + "tiled_prompt": { + "rms": 0.05732688706944902, + "max": 0.2529616620805528, + "top20_probability_max": 0.004892909396824816, + "top1_misses": 0 + }, + "rows_where_route_rms_exceeds_official_bf16": { + "baseline": 5, + "tiled_prompt": 5 + } + }, + "paired_tiled_minus_baseline_rms": { + "mean": 0.002416102906510988, + "standard_error": 0.004157580530907908, + "rows_tiled_worse": 7 + }, + "top1_differences": [], + "sweep_file_sha256": "7758b635f4df5795b378660b27d6b44ef63d629b0705e34c7ca6ee67fa7ae068" + }, + "vl6": { + "rows": 9, + "summary": { + "official_bf16": { + "rms": 0.06791185993219352, + "max": 0.31771284341812134, + "top20_probability_max": 0.014008899926185113, + "top1_misses": 1 + }, + "baseline": { + "rms": 0.05036021844438576, + "max": 0.19850130875905356, + "top20_probability_max": 0.005867781359692831, + "top1_misses": 0 + }, + "tiled_prompt": { + "rms": 0.044764633684090026, + "max": 0.2005244361029731, + "top20_probability_max": 0.005781203911422671, + "top1_misses": 0 + }, + "rows_where_route_rms_exceeds_official_bf16": { + "baseline": 2, + "tiled_prompt": 2 + } + }, + "paired_tiled_minus_baseline_rms": { + "mean": -0.005595584760295729, + "standard_error": 0.009127715236339602, + "rows_tiled_worse": 5 + }, + "top1_differences": [ + { + "prompt_length": 90, + "route": "official_bf16", + "route_top1": 440, + "fp32_top1": 6066, + "fp32_top1_margin": 0.04569435119628906, + "route_top1_margin": 0.25, + "fp32_gap_of_route_top1": -0.04569435119628906 + } + ], + "sweep_file_sha256": "853f0ef67995ea0279636e9206d62e97795a746fbd1375ab5ceb6f3fca7724d1" + }, + "vl8": { + "rows": 9, + "summary": { + "official_bf16": { + "rms": 0.055668635305161786, + "max": 0.32083530558480156, + "top20_probability_max": 0.01261331459321191, + "top1_misses": 0 + }, + "baseline": { + "rms": 0.060090205909432294, + "max": 0.3198500759899616, + "top20_probability_max": 0.011049715148431711, + "top1_misses": 0 + }, + "tiled_prompt": { + "rms": 0.06984953204430809, + "max": 0.350392265452279, + "top20_probability_max": 0.011304059793175636, + "top1_misses": 0 + }, + "rows_where_route_rms_exceeds_official_bf16": { + "baseline": 6, + "tiled_prompt": 6 + } + }, + "paired_tiled_minus_baseline_rms": { + "mean": 0.0097593261348758, + "standard_error": 0.0055259347365127255, + "rows_tiled_worse": 8 + }, + "top1_differences": [], + "sweep_file_sha256": "19d4fc36b761e865de6a87153ab866019a79de85a2f40eda985366db296f4e6a" + } + }, + "sweeps_long_vl_250_tokens_45_rows": { + "vl4": { + "rows": 45, + "summary": { + "official_bf16": { + "rms": 0.03339460902858266, + "max": 0.15890198623140653, + "top20_probability_max": 0.006149748862220873, + "top1_misses": 0 + }, + "baseline": { + "rms": 0.029617537877033515, + "max": 0.1420389109187656, + "top20_probability_max": 0.006166903703145467, + "top1_misses": 0 + }, + "tiled_prompt": { + "rms": 0.02945682283839453, + "max": 0.14509857760535347, + "top20_probability_max": 0.006421012368764529, + "top1_misses": 0 + }, + "rows_where_route_rms_exceeds_official_bf16": { + "baseline": 19, + "tiled_prompt": 15 + } + }, + "paired_tiled_minus_baseline_rms": { + "mean": -0.0001607150386389859, + "standard_error": 0.0014685568541151174, + "rows_tiled_worse": 22 + }, + "top1_differences": [], + "sweep_file_sha256": "7b026a0db9c596fdd00382c05a6d6776580be885ee3b899b966d2916e5104453" + }, + "vl6": { + "rows": 45, + "summary": { + "official_bf16": { + "rms": 0.04184474775150998, + "max": 0.21715606045391825, + "top20_probability_max": 0.009298263377207374, + "top1_misses": 1 + }, + "baseline": { + "rms": 0.0365732936782363, + "max": 0.1759725825654136, + "top20_probability_max": 0.006888464530860934, + "top1_misses": 0 + }, + "tiled_prompt": { + "rms": 0.03274911813885285, + "max": 0.18221158120367262, + "top20_probability_max": 0.008384905066112513, + "top1_misses": 0 + }, + "rows_where_route_rms_exceeds_official_bf16": { + "baseline": 14, + "tiled_prompt": 16 + } + }, + "paired_tiled_minus_baseline_rms": { + "mean": -0.0038241755393834485, + "standard_error": 0.0026308362392368297, + "rows_tiled_worse": 21 + }, + "top1_differences": [ + { + "prompt_length": 90, + "route": "official_bf16", + "route_top1": 440, + "fp32_top1": 6066, + "fp32_top1_margin": 0.04569435119628906, + "route_top1_margin": 0.125, + "fp32_gap_of_route_top1": -0.04569435119628906 + } + ], + "sweep_file_sha256": "ddd86b5346270a7b7f821ca886614915a8fc74dec3a2ae7442f4948e1a7f0698" + }, + "vl8": { + "rows": 45, + "summary": { + "official_bf16": { + "rms": 0.03768535399633972, + "max": 0.1984032659067048, + "top20_probability_max": 0.007951154274390698, + "top1_misses": 0 + }, + "baseline": { + "rms": 0.037341594878442755, + "max": 0.18874870638052624, + "top20_probability_max": 0.0069587672008757685, + "top1_misses": 0 + }, + "tiled_prompt": { + "rms": 0.04092342794136572, + "max": 0.19802138892312846, + "top20_probability_max": 0.0068949121302810295, + "top1_misses": 0 + }, + "rows_where_route_rms_exceeds_official_bf16": { + "baseline": 17, + "tiled_prompt": 25 + } + }, + "paired_tiled_minus_baseline_rms": { + "mean": 0.0035818330629229624, + "standard_error": 0.002365951341137871, + "rows_tiled_worse": 24 + }, + "top1_differences": [], + "sweep_file_sha256": "3ed77ef66f732362ff260ff7e1a3886ca838eac4f051cc2efda015e168862d06" + } + }, + "native_top1_difference_decode_route": { + "vl8_continuation_row_5": { + "native_top": 5831, + "fp32_and_official_bf16_top": 6840, + "fp32_margin": 0.08093595504760742, + "baseline_native_margin": 0.0, + "baseline_note": "exact tie 5831 vs 6840 in baseline native", + "tiled_native_margin": 0.09375, + "official_bf16_margin": 0.09375, + "note": "Teacher-forced row after the prompt, executed by the single-row decode route (tiled attention only applies to 9..64-row prompt tiles and the 1..8-row prompt tail). Baseline and tiled prompt both miss, so this is not introduced by tiled prefill." + } + }, + "findings": [ + "No defect in tiled attention or residual FFN was identified. Where it differs from baseline, differences are mixed in sign across checkpoints and mostly within about two standard errors.", + "VL8 is the checkpoint to watch: tiled prompt RMS to FP32 exceeds baseline in 8/9 short-sweep rows and 24/45 long-sweep rows (mean +0.0036, SE 0.0024), and exceeds official BF16 RMS in 25/45 rows vs 17/45 for baseline. Not statistically established.", + "TEXT8 and VL6 tiled are closer to FP32 than baseline on average; VL4 is equal.", + "Native top-1 differs from FP32 once: VL8 teacher-forced continuation row 5 (decode route), baseline and tiled alike. Official BF16 misses once in VL6 at the prompt boundary row (position 89, FP32 margin 0.0457; the same row appears in the canonical phase, the short sweep and the long sweep).", + "Baseline (guarded path) also fails several zero-slack proposed diagnostics (e.g. TEXT8 prompt RMS, VL8 prompt RMS/max, VL4 continuation probability), so those diagnostics do not discriminate tiled from baseline on this sample." + ], + "raw_sha256": { + "vl4.long.mlx.npy": "a19540905991a12aa3aeb082cbc1b313dcec2f02ce0c53dbe7170bd877951458", + "vl6.long.mlx.npy": "f58a5c310b5badf99aa2395de85def7fdebba1f097a31b85a78ef819db948879", + "vl8.long.mlx.npy": "bb774b584ceec84fedbe575aaddc5cbc7042729b571e27d0caa410278b8331b5", + "text8.canon.baseline.bf16": "fa24e9dd522a738edd46207b2184973b8c5e56b600fe045d5fd9c60e49099160", + "text8.canon.mlx.npy": "c9220d4ac1a5c291db9102de6cabafa1d2a11846b0f6414386c47ddd51102552", + "text8.canon.tiled.bf16": "0db22b4cedc0e7d3128b5ce77d63927856f100cdd8127fa905c2335709e7a69f", + "vl4.canon.baseline.bf16": "3eeb33aa00b4ff51bae8e1c212158143db72f78b44d7d7af07ead70b431a1dd3", + "vl4.canon.mlx.npy": "c297da35d72a2eb0cd957d7d19c737e0c82d7ff18a35cdf87986c4e998ecef2e", + "vl4.canon.tiled.bf16": "1fa882651a57f393cfaccc97be45e8d7f8be07ac5a4124355e3cc3fad605ad00", + "vl6.canon.baseline.bf16": "61b88a07edbdf7e302df09543d8ee528facae497feed5972fedb8219f20f2bbe", + "vl6.canon.mlx.npy": "b13899392819d49cee416aa2734d84f0b5972277c02748edcd080f2405bba141", + "vl6.canon.tiled.bf16": "7f20dda22c9a149420262bb64c46b4d90431a9b8768f34532909201877d8d6e6", + "vl8.canon.baseline.bf16": "5b1c79875ffb0caedfcb6ae4bd425694550556399b7a83b0c16e8e017c0f3b37", + "vl8.canon.mlx.npy": "5fb898c68792c945629a1759af956ecf0ce9b9bf572488b95f2b95cb589b1807", + "vl8.canon.tiled.bf16": "623eac231faef0926e3fd69e8459137117f4ad02903f51d9f2f3d68a4ca40ff2", + "text8.canon.fp32.npy": "a94992caffeeae10f2eafcc3e7200cf3e5f8ff045de0bd87d36136f20b0f72e5", + "vl4.canon.fp32.npy": "3732a8bb2304c9d1735c899bde5df2b35d04fb885afdccea7fae6cf02164b401", + "vl4.long.fp32.npy": "c7bfc6fa3a95a1e982186312a8c9cff33f07f2bae59925e74153e6ab3ad61ce0", + "vl6.canon.fp32.npy": "dc1d06fc4e3295cc943ef9ba86dc38aedf885550101ed796b7da69ef70887954", + "vl6.long.fp32.npy": "4fbac288d8de4bbe72844cc609f52c24e7bf8672a52cd449bc820330b5a6d2ef", + "vl8.canon.fp32.npy": "9479bc77504b48357cfc871814af24ac408cf2e2a6fdd5c6efbe253e5ed1bda5", + "vl8.long.fp32.npy": "d1c9c2dc5c6d0f0d0a13f204cb99b16bcfebe0cf3fbd201c1f6b090f2b2cddf3" + }, + "not_established": [ + "arbitrary-prompt or multi-image fidelity", + "2048-row server route fidelity at these widths", + "long-context fidelity", + "that the proposed zero-slack probability diagnostics are the maintainer rule" + ] +} diff --git a/docs/receipts/lfm3/model-fp64-comparison.json b/docs/receipts/lfm3/model-fp64-comparison.json new file mode 100644 index 0000000000..c2170d5167 --- /dev/null +++ b/docs/receipts/lfm3/model-fp64-comparison.json @@ -0,0 +1,559 @@ +{ + "cells": 17, + "rows": 18, + "native_in_oracle_maximum": 9, + "mlx_in_oracle_maximum": 16, + "both_in_maximum": 8, + "neither_in_maximum": 1, + "accepted_threshold": null, + "rows_detail": [ + { + "row": 82, + "native_top": 2292, + "mlx_top": 3599, + "oracle_maximum_set": [ + 2292, + 3599 + ], + "oracle_margin": 0.0, + "native_in_oracle_maximum_set": true, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.0, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 82, + "reference_top": 3599, + "native_top": 2292, + "reference_top_set": [ + 3599 + ], + "native_top_set": [ + 2292 + ], + "top_sets_overlap": false, + "reference_margin": 0.125, + "native_margin": 0.5, + "reference_native_contender_gap": 0.125, + "native_reference_contender_gap": 0.5 + }, + "cell": "1.7B-6-prose-257" + }, + { + "row": 82, + "native_top": 2292, + "mlx_top": 3599, + "oracle_maximum_set": [ + 2292, + 3599 + ], + "oracle_margin": 0.0, + "native_in_oracle_maximum_set": true, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.0, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 82, + "reference_top": 3599, + "native_top": 2292, + "reference_top_set": [ + 3599 + ], + "native_top_set": [ + 2292 + ], + "top_sets_overlap": false, + "reference_margin": 0.125, + "native_margin": 0.5, + "reference_native_contender_gap": 0.125, + "native_reference_contender_gap": 0.5 + }, + "cell": "1.7B-6-prose-513" + }, + { + "row": 450, + "native_top": 22497, + "mlx_top": 360, + "oracle_maximum_set": [ + 22497 + ], + "oracle_margin": 0.125, + "native_in_oracle_maximum_set": true, + "mlx_in_oracle_maximum_set": false, + "oracle_gap_to_native_top": 0.0, + "oracle_gap_to_mlx_top": 0.125, + "previous_evidence": { + "row": 450, + "reference_top": 360, + "native_top": 22497, + "reference_top_set": [ + 360 + ], + "native_top_set": [ + 22497 + ], + "top_sets_overlap": false, + "reference_margin": 0.125, + "native_margin": 0.125, + "reference_native_contender_gap": 0.125, + "native_reference_contender_gap": 0.125 + }, + "cell": "1.7B-8-code-513" + }, + { + "row": 264, + "native_top": 207, + "mlx_top": 33237, + "oracle_maximum_set": [ + 2993 + ], + "oracle_margin": 0.375, + "native_in_oracle_maximum_set": false, + "mlx_in_oracle_maximum_set": false, + "oracle_gap_to_native_top": 0.375, + "oracle_gap_to_mlx_top": 0.5625, + "previous_evidence": { + "row": 264, + "reference_top": 33237, + "native_top": 207, + "reference_top_set": [ + 33237 + ], + "native_top_set": [ + 207 + ], + "top_sets_overlap": false, + "reference_margin": 0.0625, + "native_margin": 0.25, + "reference_native_contender_gap": 0.0625, + "native_reference_contender_gap": 0.25 + }, + "cell": "VL-2B-4-code-513" + }, + { + "row": 17, + "native_top": 296, + "mlx_top": 405, + "oracle_maximum_set": [ + 405 + ], + "oracle_margin": 0.125, + "native_in_oracle_maximum_set": false, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.125, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 17, + "reference_top": 405, + "native_top": 296, + "reference_top_set": [ + 405 + ], + "native_top_set": [ + 296 + ], + "top_sets_overlap": false, + "reference_margin": 0.0625, + "native_margin": 0.125, + "reference_native_contender_gap": 0.0625, + "native_reference_contender_gap": 0.1875 + }, + "cell": "VL-2B-4-reasoning-257" + }, + { + "row": 17, + "native_top": 296, + "mlx_top": 405, + "oracle_maximum_set": [ + 405 + ], + "oracle_margin": 0.125, + "native_in_oracle_maximum_set": false, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.125, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 17, + "reference_top": 405, + "native_top": 296, + "reference_top_set": [ + 405 + ], + "native_top_set": [ + 296 + ], + "top_sets_overlap": false, + "reference_margin": 0.0625, + "native_margin": 0.125, + "reference_native_contender_gap": 0.0625, + "native_reference_contender_gap": 0.1875 + }, + "cell": "VL-2B-4-reasoning-513" + }, + { + "row": 22, + "native_top": 16033, + "mlx_top": 91, + "oracle_maximum_set": [ + 91 + ], + "oracle_margin": 0.1875, + "native_in_oracle_maximum_set": false, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.1875, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 22, + "reference_top": 91, + "native_top": 16033, + "reference_top_set": [ + 91 + ], + "native_top_set": [ + 16033 + ], + "top_sets_overlap": false, + "reference_margin": 0.125, + "native_margin": 0.125, + "reference_native_contender_gap": 0.125, + "native_reference_contender_gap": 0.125 + }, + "cell": "VL-2B-6-code-257" + }, + { + "row": 83, + "native_top": 3570, + "mlx_top": 496, + "oracle_maximum_set": [ + 496, + 3570 + ], + "oracle_margin": 0.0, + "native_in_oracle_maximum_set": true, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.0, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 83, + "reference_top": 496, + "native_top": 3570, + "reference_top_set": [ + 496 + ], + "native_top_set": [ + 3570 + ], + "top_sets_overlap": false, + "reference_margin": 0.1875, + "native_margin": 0.125, + "reference_native_contender_gap": 0.1875, + "native_reference_contender_gap": 0.1875 + }, + "cell": "VL-2B-6-code-257" + }, + { + "row": 83, + "native_top": 3570, + "mlx_top": 496, + "oracle_maximum_set": [ + 496, + 3570 + ], + "oracle_margin": 0.0, + "native_in_oracle_maximum_set": true, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.0, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 83, + "reference_top": 496, + "native_top": 3570, + "reference_top_set": [ + 496 + ], + "native_top_set": [ + 3570 + ], + "top_sets_overlap": false, + "reference_margin": 0.1875, + "native_margin": 0.125, + "reference_native_contender_gap": 0.1875, + "native_reference_contender_gap": 0.1875 + }, + "cell": "VL-2B-6-code-513" + }, + { + "row": 22, + "native_top": 16033, + "mlx_top": 91, + "oracle_maximum_set": [ + 91 + ], + "oracle_margin": 0.1875, + "native_in_oracle_maximum_set": false, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.1875, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 22, + "reference_top": 91, + "native_top": 16033, + "reference_top_set": [ + 91 + ], + "native_top_set": [ + 16033 + ], + "top_sets_overlap": false, + "reference_margin": 0.0625, + "native_margin": 0.125, + "reference_native_contender_gap": 0.0625, + "native_reference_contender_gap": 0.125 + }, + "cell": "VL-2B-6-code-64" + }, + { + "row": 38, + "native_top": 22, + "mlx_top": 20, + "oracle_maximum_set": [ + 20, + 22 + ], + "oracle_margin": 0.0, + "native_in_oracle_maximum_set": true, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.0, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 38, + "reference_top": 20, + "native_top": 22, + "reference_top_set": [ + 20 + ], + "native_top_set": [ + 22 + ], + "top_sets_overlap": false, + "reference_margin": 0.125, + "native_margin": 0.125, + "reference_native_contender_gap": 0.125, + "native_reference_contender_gap": 0.125 + }, + "cell": "VL-2B-6-prose-257" + }, + { + "row": 38, + "native_top": 22, + "mlx_top": 20, + "oracle_maximum_set": [ + 20, + 22 + ], + "oracle_margin": 0.0, + "native_in_oracle_maximum_set": true, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.0, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 38, + "reference_top": 20, + "native_top": 22, + "reference_top_set": [ + 20 + ], + "native_top_set": [ + 22 + ], + "top_sets_overlap": false, + "reference_margin": 0.125, + "native_margin": 0.125, + "reference_native_contender_gap": 0.125, + "native_reference_contender_gap": 0.125 + }, + "cell": "VL-2B-6-prose-513" + }, + { + "row": 17, + "native_top": 3547, + "mlx_top": 3983, + "oracle_maximum_set": [ + 3983 + ], + "oracle_margin": 0.0625, + "native_in_oracle_maximum_set": false, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.0625, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 17, + "reference_top": 3983, + "native_top": 3547, + "reference_top_set": [ + 3983 + ], + "native_top_set": [ + 3547 + ], + "top_sets_overlap": false, + "reference_margin": 0.0625, + "native_margin": 0.0625, + "reference_native_contender_gap": 0.0625, + "native_reference_contender_gap": 0.0625 + }, + "cell": "VL-2B-6-prose-64" + }, + { + "row": 24, + "native_top": 27, + "mlx_top": 1325, + "oracle_maximum_set": [ + 1325 + ], + "oracle_margin": 0.15625, + "native_in_oracle_maximum_set": false, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.15625, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 24, + "reference_top": 1325, + "native_top": 27, + "reference_top_set": [ + 1325 + ], + "native_top_set": [ + 27 + ], + "top_sets_overlap": false, + "reference_margin": 0.0625, + "native_margin": 0.09375, + "reference_native_contender_gap": 0.0625, + "native_reference_contender_gap": 0.09375 + }, + "cell": "VL-2B-6-reasoning-257" + }, + { + "row": 24, + "native_top": 27, + "mlx_top": 1325, + "oracle_maximum_set": [ + 1325 + ], + "oracle_margin": 0.15625, + "native_in_oracle_maximum_set": false, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.15625, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 24, + "reference_top": 1325, + "native_top": 27, + "reference_top_set": [ + 1325 + ], + "native_top_set": [ + 27 + ], + "top_sets_overlap": false, + "reference_margin": 0.125, + "native_margin": 0.09375, + "reference_native_contender_gap": 0.125, + "native_reference_contender_gap": 0.09375 + }, + "cell": "VL-2B-6-reasoning-513" + }, + { + "row": 24, + "native_top": 27, + "mlx_top": 1325, + "oracle_maximum_set": [ + 1325 + ], + "oracle_margin": 0.15625, + "native_in_oracle_maximum_set": false, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.15625, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 24, + "reference_top": 1325, + "native_top": 27, + "reference_top_set": [ + 1325 + ], + "native_top_set": [ + 27 + ], + "top_sets_overlap": false, + "reference_margin": 0.25, + "native_margin": 0.09375, + "reference_native_contender_gap": 0.25, + "native_reference_contender_gap": 0.09375 + }, + "cell": "VL-2B-6-reasoning-64" + }, + { + "row": 12, + "native_top": 20, + "mlx_top": 296, + "oracle_maximum_set": [ + 20, + 296 + ], + "oracle_margin": 0.0, + "native_in_oracle_maximum_set": true, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.0, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 12, + "reference_top": 296, + "native_top": 20, + "reference_top_set": [ + 296 + ], + "native_top_set": [ + 20 + ], + "top_sets_overlap": false, + "reference_margin": 0.0625, + "native_margin": 0.0625, + "reference_native_contender_gap": 0.0625, + "native_reference_contender_gap": 0.0625 + }, + "cell": "VL-2B-8-prose-64" + }, + { + "row": 16, + "native_top": 20, + "mlx_top": 22, + "oracle_maximum_set": [ + 20, + 22 + ], + "oracle_margin": 0.0, + "native_in_oracle_maximum_set": true, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.0, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 16, + "reference_top": 22, + "native_top": 20, + "reference_top_set": [ + 22 + ], + "native_top_set": [ + 20 + ], + "top_sets_overlap": false, + "reference_margin": 0.0625, + "native_margin": 0.0625, + "reference_native_contender_gap": 0.0625, + "native_reference_contender_gap": 0.0625 + }, + "cell": "VL-2B-8-reasoning-513" + } + ] +} diff --git a/docs/receipts/lfm3/pixel-normalization.json b/docs/receipts/lfm3/pixel-normalization.json new file mode 100644 index 0000000000..bf8b6fc33e --- /dev/null +++ b/docs/receipts/lfm3/pixel-normalization.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "scope": "LFM3-VL CPU image preparation normalization correction; downstream GPU image outputs are not rerun by this receipt", + "review_comment": "https://github.com/ashhart/TensorFold/pull/1502#issuecomment-6494523293", + "reference_fixture": { + "source": "https://github.com/jschmied/TensorFold/blob/1325da3bfab19d2ada499e613f1f0c90b7d95dd3/zig/src/core/picture/patches_test.zig", + "processor": "transformers 5.19.0 / torchvision 0.29.0 as recorded by fixture author", + "input": "RGB217x300; (7*x+13*y+50*c+(x*y)%31)%256", + "grid": [ + 14, + 20 + ], + "pixel_values_sha256": "53153c22eb4e065031bbdf4de0f038db37ef736a880e3438ac8e0d73f82dcea8" + }, + "change": { + "before": "(f32(pixel)*f32(1/255)-0.5)/0.5", + "after": "(f32(pixel)-127.5)/127.5", + "changed_lut_entries": 207, + "lut_entries": 256, + "max_absolute_difference": 1.1920928955078125e-07 + }, + "regression": { + "before_exit": 1, + "before_pixel_values_sha256": "6e7d91ab24b4b18bc383dc5fcf441e2a973904f5a9b6a2af0272f34c48cf0229", + "after_exit": 0, + "after_pixel_values_sha256": "53153c22eb4e065031bbdf4de0f038db37ef736a880e3438ac8e0d73f82dcea8", + "command": "zig build test-lfm3-host -Dcpu=apple_m1 --summary all", + "after_build_steps": [ + 12, + 12 + ], + "image_tests": [ + 26, + 26 + ], + "optional_JPEG_cases": "three unavailable JPEG fixture cases; unrelated to RGB preparation regression", + "logs_sha256": { + "lfm3-pixel-parity-before.log": "ac5bdb487acc143649eb9db314387148d2deeb4c0d2f49efd17580e67ef70941", + "lfm3-pixel-parity-after.log": "028c13fb4b3378574ce6f140297c4352ecf7b72f2c8053916e82c37be349a57d" + } + }, + "validation": { + "zig_fmt_check_exit": 0, + "git_diff_check_exit": 0 + }, + "limits": [ + "One external processor fixture hash checked, not all five cited pictures.", + "Earlier image GPU/model receipts retain their original pixel-input scope; this correction does not imply their byte identity.", + "No performance claim." + ] +} diff --git a/docs/receipts/lfm3/postnorm-fusion.json b/docs/receipts/lfm3/postnorm-fusion.json new file mode 100644 index 0000000000..a6697c3959 --- /dev/null +++ b/docs/receipts/lfm3/postnorm-fusion.json @@ -0,0 +1,383 @@ +{ + "scope": "Qualified private exact post-add plus next RMSNorm fusion; public application requires rebuilding and checking the applied source. Performance measured only for the two 4-bit checkpoints.", + "base_commit": "6b318cedb9d9520ee53ee4a56abeb5d6ced5d461", + "algorithm": "Fuse 48 adjacent dispatch pairs per forward; retain both unchanged 256-thread FP32 reduction trees, all BF16 round sites and the residual h store. No carry crosses a forward call. Norm-class dispatches decrease from 97 to 49.", + "source_sha256": { + "zig/kernels/metal/lfm3/ops.metal": "1659bb9185d966dd30276a6668465d25c06753a03c4cb38a02c3684514a657ae", + "zig/src/families/lfm3/forward.zig": "1ab4e29b47056c3232d3e8278c40c6b7dcf295d52d1f172e71ea47d95e92ff0f", + "zig/src/families/lfm3/kernels.zig": "7f04b485a2e9d1dc5ad6d2c7e49a4278ad61128fb946daee878a665e7ba21fed", + "zig/tests/contract/metal_lfm3_postnorm.zig": "8c362ca20a90fe093f1f08d65ace30affabe63879456ba0394ba84532d688594", + "zig/tests/contract/metal_main.zig": "3eff9d0bb4cfdbd0c990ed4201a094bcf91107e0e2ad5c4792551eda85db6922", + "zig/tests/contract/metal_limits.zig": "5d74368ae9b00d65373421719423ba98b6627d02645909d114e418ab9cc2a0e9", + "zig/src/core/contract/lfm3_records.zig": "33cf7203d5e371a016cceecb165923355a72dbcc4eadd1a3ceb6a92dbad0bd54" + }, + "native_tested_source_sha256": { + "zig/kernels/metal/lfm3/ops.metal": "1659bb9185d966dd30276a6668465d25c06753a03c4cb38a02c3684514a657ae", + "zig/src/families/lfm3/forward.zig": "1ab4e29b47056c3232d3e8278c40c6b7dcf295d52d1f172e71ea47d95e92ff0f", + "zig/src/families/lfm3/kernels.zig": "b5f5406605f665cd6ab3bacbeca4ccf53c39245d13e827ab0819247a981b6dfa" + }, + "source_provenance": "After native binaries were frozen, kernels.zig changed only Zig formatting. Later changes affect contract metadata/reference/simulation only, not runtime arithmetic.", + "artifact_sha256": { + "zig-out/native/bin/tensorfold-native": "a20ea31aa466b7e20a85022a6932ad069ae4f9100d899d9a70743daef9132bae", + "zig-out/bin/tf-lfm3-forward": "4e350b7d02a793d375ba72db48b468b14e1255bfd47e51f9913429befaeea186", + "zig-out/bin/tf-lfm3-exact": "bd899ef674c8668fb8dc7491f45540d4435c070bb4f5bc500af4573f21312286", + "zig-out/bin/tf-kernel-contract": "c29de86253f66822af91b73f753b65247e42a33c41aeaa171e79223b1607542c" + }, + "hosts": [ + { + "chip": "Apple M4 Pro", + "macos": "26.5.1", + "memory_gib": 48 + }, + { + "chip": "Apple M5 Ultra", + "macos": "27.0", + "memory_gib": 256 + } + ], + "primitive": { + "hosts": [ + "M4 Pro", + "M5 Ultra" + ], + "cases_per_host": 320, + "bytes_equal_per_host": 124354560, + "rows": [ + 1, + 2, + 3, + 4, + 5, + 7, + 8, + 9, + 15, + 16, + 17, + 31, + 32, + 33, + 63, + 64, + 65, + 127, + 128, + 129 + ], + "distributions": [ + "random finite", + "signed zero", + "BF16 subnormal", + "maximum finite", + "NaN", + "positive infinity", + "negative infinity", + "random raw BF16" + ], + "outputs": [ + "residual BF16", + "next normalized BF16", + "next normalized pre-store FP32" + ], + "guards": true, + "independent_post_and_next_gain_inputs": true, + "embedded_source_sha256": "e09a71920a04f1e6f864ff9c7a71db3b57f3b23b6fde4efdeb31a372374c240c", + "driver_sha256": "fc67f80fae27155c327c8b161c9585202b7358bee201b55867bf158525a5545e", + "binary_sha256": "65764baa7acf6b5cb76e8400f222a7e07d06169c84c9ee442ddf2de8e7380755" + }, + "canonical_model_logits": { + "hosts": [ + "M4 Pro", + "M5 Ultra" + ], + "quantizations_per_host": 6, + "rows_per_checkpoint": 257, + "bytes_equal_per_host": 394752000 + }, + "trace": { + "files": 620, + "bytes_equal": 68474208, + "rows": 33, + "chunk": 32 + }, + "image_conditioned": { + "family": "VL8", + "rows": 90, + "bytes_equal": 23040000, + "sha256": "db730509b6d7dcb3d6d134b59b6434dfb43ebc85cdd3fe351bc8cc4148b47642" + }, + "state": { + "host": "M4 Pro", + "models": 6, + "passed": 408, + "failed": 0, + "skipped": 0, + "per_checkpoint_passed": 68, + "binary_sha256": "bd899ef674c8668fb8dc7491f45540d4435c070bb4f5bc500af4573f21312286", + "log_sha256": { + "LFM3-1.7B-MLX-4bit": "3f460abee50d57fe6fbde0290b441415f9a7e748f6a79e69088d77ca87cd8ec2", + "LFM3-1.7B-MLX-6bit": "16075f3068f51a7baa8edff1c73a504275b3798d917d33a3596f6bff85baf000", + "LFM3-1.7B-MLX-8bit": "db4169b0e9803ef90f230c3896ba51f0a664e7f29a3ae65af8cd321319c77a92", + "LFM3-VL-2B-MLX-4bit": "005371e44f10619eaa1aeac4b1438334bf34cdd70abcf603a325ebc0a2cc72b8", + "LFM3-VL-2B-MLX-6bit": "5723cdb4ceebcb5ab9668d77d2a8f4d7a21acbf4be99c104b3359d734ba7a75c", + "LFM3-VL-2B-MLX-8bit": "ffca558c50d192a2e296ee82d66a8b527dc1cb3279f080b37f7c1ef6cc25b978" + } + }, + "serving": { + "host": "M5 Ultra", + "models": 6, + "passed": 303, + "failed": 0, + "skipped": 0, + "modes": [ + "greedy shared versus solo", + "seeded sampling", + "cancellation", + "vision inputs" + ], + "receipt_sha256": { + "LFM3-1.7B-MLX-4bit": "d0ba2a8adc090d2e40259456e65e04f60fe2157f3221369b3b961336ff17f408", + "LFM3-1.7B-MLX-6bit": "d9634eeb767ba195531d35e1694dce4df7bbcc38b6782a6825ce456fc96c6a4c", + "LFM3-1.7B-MLX-8bit": "6e1108d7f5158fc27c8560a06a595614bd1d3bf746005c24b9a36bf2ec61abce", + "LFM3-VL-2B-MLX-4bit": "7337b3300a191858cef3858fc813739e99fbe0704f803b64f911783ab04a2642", + "LFM3-VL-2B-MLX-6bit": "fbcc9fec889ff54adcd61e2371d20bf8682a663829bc76b0194c08fcf456d33d", + "LFM3-VL-2B-MLX-8bit": "e9ca9565030968665abb8585c922d768494b095862b2eecf4680c174f4f19640" + } + }, + "contract": { + "hosts": [ + "M4 Pro", + "M5 Ultra" + ], + "kernels": 1, + "gates_1_4_pass": true, + "rounding_controls_red": 1, + "normal_launch_records": 355, + "limit_problems": 0, + "threads": 256, + "static_threadgroup_bytes": 144, + "declared_intermediate_bf16_rounds": 4, + "round_sites": [ + "first normalization before gain", + "first normalization after gain", + "residual addition", + "second normalization before gain" + ], + "prior_stricter_mids2_probe": { + "host": "Apple M4 Pro", + "result": "all four gates pass and removed-rounding control red", + "source_sha256": "27e59b782478d210f80ee5fbee52fd7f156ce94b5b69922dd555dfe74232362a", + "binary_sha256": "5b62de7efd7280a78b7ddeee98dae58ae26015b9ef4dcff4bd6b78d23309571d", + "log_sha256": "b68f40af159eac40419ca1a73878c400e78f24210334dd1cf395ea963617594f" + }, + "metadata_reason": "Four intermediate BF16 nodes describe the actual two-stage graph. The earlier mids2 probe already passed. Exact replacement and raw-byte requirements remain mandatory.", + "log_sha256": { + "m4": "288fdd9dd60928f489a68a779db164399c203045f9e4efade9954f6fc6c5a1ff", + "m5": "acac3a048ebd0a35c7ec29f046cd48ae59deb5e85bcf29c4fb8050b60ef34d0b" + } + }, + "performance": { + "scope": "Same-machine old/new native HTTP ABBA. No new comparison against the standard runtime and no claim that the remaining concurrency target is met.", + "host": "M5 Ultra", + "checkpoint_bits": 4, + "context": 70000, + "parallel": 4, + "prompt_cache_gib": 0, + "temperature": 0, + "drafting": "default enabled", + "thinking": false, + "output_tokens_per_request": 256, + "repetitions_per_leg": 3, + "order": [ + "old", + "new", + "new", + "old" + ], + "measured_requests": 240, + "all_output_lengths_and_token_hashes_equal": true, + "decode": [ + { + "family": "text", + "prompt": "long-code-chat", + "streams": 1, + "old_tps": 323.37859207406774, + "new_tps": 340.56213946137177, + "throughput_gain_percent": 5.313755396451314, + "all_same_token_hash": true + }, + { + "family": "text", + "prompt": "long-code-chat", + "streams": 4, + "old_tps": 877.3614816835146, + "new_tps": 909.6937766053586, + "throughput_gain_percent": 3.6851737393124973, + "all_same_token_hash": true + }, + { + "family": "text", + "prompt": "long-prose-chat", + "streams": 1, + "old_tps": 338.05606841419115, + "new_tps": 355.5690832078852, + "throughput_gain_percent": 5.180505966317073, + "all_same_token_hash": true + }, + { + "family": "text", + "prompt": "long-prose-chat", + "streams": 4, + "old_tps": 897.5693493427952, + "new_tps": 928.0139771021795, + "throughput_gain_percent": 3.3918969917673847, + "all_same_token_hash": true + }, + { + "family": "vl", + "prompt": "long-code-chat", + "streams": 1, + "old_tps": 307.4211480014529, + "new_tps": 323.0724303307775, + "throughput_gain_percent": 5.091153432700946, + "all_same_token_hash": true + }, + { + "family": "vl", + "prompt": "long-code-chat", + "streams": 4, + "old_tps": 852.5923561448815, + "new_tps": 883.2251823336844, + "throughput_gain_percent": 3.5929041549602525, + "all_same_token_hash": true + }, + { + "family": "vl", + "prompt": "long-prose-chat", + "streams": 1, + "old_tps": 305.46397849556536, + "new_tps": 320.49747408261396, + "throughput_gain_percent": 4.921528116372276, + "all_same_token_hash": true + }, + { + "family": "vl", + "prompt": "long-prose-chat", + "streams": 4, + "old_tps": 848.7382388763339, + "new_tps": 879.6497949219245, + "throughput_gain_percent": 3.642060016821591, + "all_same_token_hash": true + } + ], + "cold_prefill": "Exact 2k/8k cached_tokens=0 exercised; tiny TTFT changes are not claimed as an improvement.", + "status_sha256": "83410ab791e78510de0e4a1980ea9b205bac36ae74f7b57503ea40f166e290d2" + }, + "applied_public_validation": { + "parent_commit": "6b318cedb9d9520ee53ee4a56abeb5d6ced5d461", + "scope": "Actual public checkout with only the reviewed fusion patch applied. The frozen six-quantization model/state/serving proofs above were not repeated because the runtime source hashes match.", + "source_sha256": { + "zig/kernels/metal/lfm3/ops.metal": "1659bb9185d966dd30276a6668465d25c06753a03c4cb38a02c3684514a657ae", + "zig/src/families/lfm3/forward.zig": "1ab4e29b47056c3232d3e8278c40c6b7dcf295d52d1f172e71ea47d95e92ff0f", + "zig/src/families/lfm3/kernels.zig": "7f04b485a2e9d1dc5ad6d2c7e49a4278ad61128fb946daee878a665e7ba21fed", + "zig/tests/contract/metal_lfm3_postnorm.zig": "8c362ca20a90fe093f1f08d65ace30affabe63879456ba0394ba84532d688594", + "zig/tests/contract/metal_main.zig": "3eff9d0bb4cfdbd0c990ed4201a094bcf91107e0e2ad5c4792551eda85db6922", + "zig/tests/contract/metal_limits.zig": "5d74368ae9b00d65373421719423ba98b6627d02645909d114e418ab9cc2a0e9", + "zig/src/core/contract/lfm3_records.zig": "33cf7203d5e371a016cceecb165923355a72dbcc4eadd1a3ceb6a92dbad0bd54" + }, + "build_command": "zig build native tf-kernel-contract dev-tools test-cuda-host -Dcpu=apple_m1 -Dversion=1.0.0 --summary all", + "build_exit_code": 0, + "build_summary": "Build Summary: 494/494 steps succeeded; 3429/3444 tests passed (15 skipped)", + "build_log_sha256": "1e4579b3d905dc834b6709210418e2a37f68769473ef402accd76b2d8b03d129", + "artifact_sha256": { + "zig-out/native/bin/tensorfold-native": "ad3ca5a0d39daa399b5bde058514291ec52f87a50799528127c654d30a139672", + "zig-out/bin/tf-kernel-contract": "0769ba295330784fffce2f7fd98f31ecf71edb44a3dbc55070863c3854d9b82b", + "zig-out/bin/tf-lfm3-forward": "8ef7afff7df80dda6d573c1c8c9b0079ecdfc0752fe35c827c4355d6cde549e2" + }, + "gpu_contract": { + "m4": { + "binary_sha256": "0769ba295330784fffce2f7fd98f31ecf71edb44a3dbc55070863c3854d9b82b", + "summary": "SUMMARY 1 kernels: 1 pass gates 1-4, 0 fail, 0 known findings; 1 controls: 1 red as planted, 0 missed; 0 limit problems; gates 27 ms, limits 388 ms", + "full_normal_launch_reflection": true, + "log_sha256": "98e12c1a886974cf94b52b4be60ed9f087b9c59f4570801715df1c70fa2a8467" + }, + "m5": { + "binary_sha256": "0769ba295330784fffce2f7fd98f31ecf71edb44a3dbc55070863c3854d9b82b", + "summary": "SUMMARY 1 kernels: 1 pass gates 1-4, 0 fail, 0 known findings; 1 controls: 1 red as planted, 0 missed; 0 limit problems; gates 15 ms, limits 120 ms", + "full_normal_launch_reflection": true, + "log_sha256": "993f191b92e0f69cd78e019260b4b1bf62d0c7881f938a77a84134ccd6244b1b" + } + }, + "format": "zig fmt --check passes for all six changed Zig files", + "lean": { + "base_findings": 428, + "applied_findings": 428, + "new_findings": 0 + }, + "not_claimed": "This combined host build is not a rerun of zig build test test-golden or the complete backend kernel numerical suite." + }, + "limits_source_selection_correction": { + "reason": "The original Gate 5 postnorm-bad record selected normal source because its fault macro was placed in the vision branch. The numerical removed-rounding control already used its fault macro correctly. Move the macro selection to the generic LFM3 branch and remove it from the vision branch. Runtime arithmetic is unchanged.", + "supersedes": "Only the earlier fault-variant launch-reflection source selection; all exact runtime/model proofs and normal/control numerical checks remain valid.", + "record": "lfm3 postnorm-bad lfm3_post_norm", + "macro": "L3_POSTNORM_SKIP_ROUND=1", + "variants": { + "main": { + "parent_commit": "1cdcd2c1d580f8a4b9bf3f9dec1984548097328e", + "source_selector_sha256": "3cefea93fa4d813d0b2e79fa781d2df4a07d91e150f9ecf4d3e6fb8b7216e684", + "previously_misselected_normal_source_sha256": "2a9edd589a50dea6c3289770d386459e14aa60c707e797e8e23d8f9fc3d73678", + "corrected_fault_source_sha256": "00068f4c021a65ef139e07ba9991f17a15ee3bc3a36a28d68cf88ae6fefa01f6", + "source_construction": "#define L3_POSTNORM_SKIP_ROUND 1 plus newline, ops.metal, newline, attention.metal", + "contract_binary_sha256": "dc23cf60560fd688f873bafb3fe2b31339b13af214d9489c65d9ff42aac1672b", + "build_exit_code": 0, + "build_summary": "Build Summary: 43/43 steps succeeded", + "checks": { + "m4": { + "summary": "SUMMARY 1 kernels: 1 pass gates 1-4, 0 fail, 0 known findings; 1 controls: 1 red as planted, 0 missed; 0 limit problems; gates 9 ms, limits 165 ms", + "full_normal_launch_reflection": true, + "normal_launch_records": 355, + "fault_reflection": { + "threads": 256, + "static_threadgroup_bytes": 144 + }, + "log_sha256": "20d7a6766a63fe4bd4d1262491947436e6c01a6a997c60703debd566df7963c0" + }, + "m5": { + "summary": "SUMMARY 1 kernels: 1 pass gates 1-4, 0 fail, 0 known findings; 1 controls: 1 red as planted, 0 missed; 0 limit problems; gates 9 ms, limits 9644 ms", + "full_normal_launch_reflection": true, + "normal_launch_records": 355, + "fault_reflection": { + "threads": 256, + "static_threadgroup_bytes": 144 + }, + "log_sha256": "da6c1ad478b9b91cd5bae7d44f9e176e86d7d6aeaea97db50f3b569899abd6d4" + } + } + }, + "text_foundation": { + "parent_commit": "fea79f5fad0c8e951ae1157fa245f244c68c492c", + "source_selector_sha256": "b23efe8d782cd45ba452375bab0d72ad297b5bd35e299ddb5ce4a959cea89b9a", + "previously_misselected_normal_source_sha256": "2a9edd589a50dea6c3289770d386459e14aa60c707e797e8e23d8f9fc3d73678", + "corrected_fault_source_sha256": "00068f4c021a65ef139e07ba9991f17a15ee3bc3a36a28d68cf88ae6fefa01f6", + "source_construction": "#define L3_POSTNORM_SKIP_ROUND 1 plus newline, ops.metal, newline, attention.metal", + "contract_binary_sha256": "69153461afe40f5f45d14fd838306b70b85e86d2c937a304a7e9b4d9adf9e249", + "build_exit_code": 0, + "build_summary": "Build Summary: 43/43 steps succeeded", + "checks": { + "m4": { + "summary": "SUMMARY 1 kernels: 1 pass gates 1-4, 0 fail, 0 known findings; 1 controls: 1 red as planted, 0 missed; 0 limit problems; gates 8 ms, limits 1137 ms", + "full_normal_launch_reflection": true, + "normal_launch_records": 337, + "fault_reflection": { + "threads": 256, + "static_threadgroup_bytes": 144 + }, + "log_sha256": "e39f929d7cc6eebd5b92e186cd8177d91b0a34d1ce8720e1fb6e915fb299f95d" + } + } + } + }, + "formatting": "zig fmt --check passes in main and text", + "model_repetition": "None: this correction changes contract source selection only." + } +} diff --git a/docs/receipts/lfm3/prefill-lifecycle.json b/docs/receipts/lfm3/prefill-lifecycle.json new file mode 100644 index 0000000000..02f9eedc82 --- /dev/null +++ b/docs/receipts/lfm3/prefill-lifecycle.json @@ -0,0 +1,87 @@ +{ + "scope": "Local M4, text and VL 4-bit, prefill cancellation/yield/progress ownership correctness; no performance or precision change", + "source_sha256": { + "zig/build/lfm3_host.zig": "2a8e1aec41ed4e43835b6bee7958fdd95f17ddf108c509eab0a88a9a3ec63089", + "zig/src/families/lfm3/backend.zig": "4ed5d33b869dc4be0d82d406cff4abc0f65a63687c45d8b3aaa61e26361db233", + "zig/src/families/lfm3/prefill_control.zig": "f34af732f2fbf77beb2608a1c220ab84579942ea4b0a4a663f16ba2ab9fcad68", + "zig/src/lfm3_prefill_tests.zig": "0946c1207c410b4ff767ac3de8b270e0ccfca5c62a228cecb1e2e147cb9eb4c6", + "zig/tests/lfm3_row_tune.zig": "443174a7ba48a569db49d709296c8ec9f00288f064f4f912a2ed27546693af55" + }, + "patch_sha256": "41eba72d2438b54c1a23a2f0ab0be1c4241a25eb5ed99504af870210949668bb", + "cpu_tests": { + "passed": 10, + "failed": 0, + "command": "zig build test-lfm3-prefill-host --summary all", + "scope": "Actual focused registered target reported 10 tests passing; five source-defined focused tests and their imported real Stream coverage.", + "build_log_sha256": "7c7206129986598a9279ea1ba9046ae2a555fac06a3708ff5e8292ef7014c83e" + }, + "actual_device": { + "untouched_old": { + "exit": 1, + "failure": "ExpectedInterruption", + "callback_checks": 0 + }, + "fixed": { + "exit": 0, + "cases_per_model": 4, + "models": 2, + "resumed_state_logits_draws": "exact", + "ownership": "clean" + } + }, + "diagnostic_excluded_from_patch": "Private actual-model callback diagnostic; immutable binary hashes below", + "evidence_sha256": { + "1.7B-after.log": "6d046fce8b7a213090872cdcc5d5a634358fb9254673dca0ba7369980921546b", + "events.json": "d29a50383a6ac6c86a1cecf58170d058b257f7ad9baca7b1c9f38199966e87fc", + "VL-2B-before.log": "70f184adaad6608a5c4f2ad2a4c528bb4ca43447a6162511329c6c0221c64a32", + "complete.json": "04b28fac6e93b10e83ff2ee9409e029aee6d20fb89ad69d35d6b0f2fd10b11e2", + "1.7B-before.log": "ddcfaf50992be7fdbce8b1e28e56553e294bb6de757c152d21839737e5f3ff92", + "VL-2B-after.log": "7e335597de283db894e2d1cfa22e835783d1abac1b11ef8ba692c0be27dec55d" + }, + "timings": "discarded; correctness only", + "binaries": { + "before": "54ee1e049e9daad368386795c0815c6ad40bb3b9dabaa88b23634be7003ef28f", + "after": "6d58b931d840c73eb6f6a037d8b5868d869584f7829a4c84044e4ec045f4e940" + }, + "host": { + "hardware": "Apple M4 Pro, 20 GPU cores", + "memory_bytes": 51539607552, + "os": "macOS 26.5.1, build 25F80" + }, + "lease": { + "path": "/tmp/tensorfold-model.lock", + "exclusive": true, + "same_alias_inode": true, + "single_model_admission": true, + "owned_children_exited": 4, + "released_after_children": true + }, + "cases": { + "text4": { + "after": [ + "CASE yield=false immediate=true observed=Cancelled cancel_checks=1 yield_checks=0 fill_at=0 caches=0 marks=0", + "CASE yield=false immediate=false observed=Cancelled cancel_checks=4 yield_checks=3 fill_at=256 caches=0 marks=2", + "CASE yield=true immediate=true observed=Yielded cancel_checks=1 yield_checks=1 fill_at=0 caches=0 marks=0", + "CASE yield=true immediate=false observed=Yielded cancel_checks=4 yield_checks=4 fill_at=256 caches=0 marks=2" + ], + "before": [ + "CASE yield=false immediate=true observed=success cancel_checks=0 yield_checks=0 fill_at=0 caches=1 marks=1" + ] + }, + "vl4": { + "after": [ + "CASE yield=false immediate=true observed=Cancelled cancel_checks=1 yield_checks=0 fill_at=0 caches=0 marks=0", + "CASE yield=false immediate=false observed=Cancelled cancel_checks=4 yield_checks=3 fill_at=256 caches=0 marks=2", + "CASE yield=true immediate=true observed=Yielded cancel_checks=1 yield_checks=1 fill_at=0 caches=0 marks=0", + "CASE yield=true immediate=false observed=Yielded cancel_checks=4 yield_checks=4 fill_at=256 caches=0 marks=2" + ], + "before": [ + "CASE yield=false immediate=true observed=success cancel_checks=0 yield_checks=0 fill_at=0 caches=1 marks=1" + ] + } + }, + "scope_limits": [ + "Final-chunk cancellation is covered by the host helper regression; device cases exercise immediate and partial cancellation/yield plus restored-prefix cancellation.", + "No image-input, numerical-fidelity, or performance gate is claimed." + ] +} diff --git a/docs/receipts/lfm3/projection-fp64-bounds.json b/docs/receipts/lfm3/projection-fp64-bounds.json new file mode 100644 index 0000000000..0206a54747 --- /dev/null +++ b/docs/receipts/lfm3/projection-fp64-bounds.json @@ -0,0 +1,13 @@ +{ + "cells": 66, + "outputs": 60954624, + "raw_tap_matches_all_native": true, + "one_row_tap_matches_all_native_first_rows": true, + "raw_FP32_bound_exceedances": 0, + "BF16_bound_exceedances": 0, + "max_raw_FP32_bound_fraction": 0.01465208540032716, + "max_BF16_bound_fraction": 0.9807553251550621, + "max_raw_abs_error": 2.555781975388527e-06, + "max_worst_column_raw_rms": 7.7385387436973e-07, + "model_acceptance_threshold": null +} diff --git a/docs/receipts/lfm3/projection-rounding-replay.json b/docs/receipts/lfm3/projection-rounding-replay.json new file mode 100644 index 0000000000..6427f176ab --- /dev/null +++ b/docs/receipts/lfm3/projection-rounding-replay.json @@ -0,0 +1,43 @@ +{ + "shape": [ + 257, + 6144 + ], + "projection": "language_model.model.layers.0.conv.in_proj", + "scope": "Projection rounding replay; not a whole-model fidelity pass", + "affine_fusion": "FP64 product-plus-add then FP32 round; checkpoint replay, not arbitrary IEEE fmaf proof", + "accepted_threshold": null, + "raw_equal": 1579008, + "values": 1579008, + "bf16_equal": 1579008, + "maximum_absolute_error": 0.0, + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "fixture": "reasoning-257", + "reference_module": "tools/lfm3/row_projection_reference.py", + "input_sha256": "c8be512c9fb15ede6f46052c3e6c467a61bfd333d0eb3f70fdb2d597a7a4fa40", + "native_raw_sha256": "0480f2bd0ad424db14bd7e197317909019ac0fbc413ff728382900b364febd0a", + "rounding_sites": [ + "Sequential FP32 additions of 16 BF16 inputs per lane", + "Four left-associated four-product sums, accumulated in FP32", + "Rounded bias-times-input-sum, fused scale-times-dot plus that product", + "FP32 accumulation across 512-input lane passes", + "FP32 XOR butterfly sum distances 1, 2, 4, 8, 16", + "BF16 RNE output storage" + ], + "explicit_fmaf_accelerator": { + "shape": [ + 257, + 6144 + ], + "projection": "language_model.model.layers.0.conv.in_proj", + "scope": "Projection rounding replay; not a whole-model fidelity pass", + "affine_fusion": "Explicit CPU fmaf", + "accepted_threshold": null, + "raw_equal": 1579008, + "values": 1579008, + "bf16_equal": 1579008, + "maximum_absolute_error": 0.0, + "source_sha256": "ff336f75fcf0a4f6c0ac92ed627bc1a6abb0b36b8f4cc95d876f2a8bb2f5cc42", + "library_sha256": "fd3bdcc07800ca3e4bda472228c5fc86afa1f8eab7c2ba4669b0eae1e07df82b" + } +} diff --git a/docs/receipts/lfm3/public-tiled-status.md b/docs/receipts/lfm3/public-tiled-status.md new file mode 100644 index 0000000000..5d54e9a10a --- /dev/null +++ b/docs/receipts/lfm3/public-tiled-status.md @@ -0,0 +1,23 @@ +# Public tiled LFM3 status + + +The [maintainer response](https://github.com/ashhart/TensorFold/issues/1491#issuecomment-6537143418) +ratifies “no further from an fp32 run than the reference’s own bf16,” with every top-1 difference and margin +retained. Partial comparisons stay partial. It accepts approximately 0.92x standard decode for the opt-in text +family and records the cold-prefill gap as separate follow-up work rather than a blocker for the text slice. +The release context tier is 32k until long-context reference fidelity is measured; longer memory/time runs +must not be labelled fidelity qualification. This policy does not automatically pass the tiled candidates. + +The [public-revision runtime receipt](m5-public-tiled-prompt-runtime.json) binds default and +enabled binaries to `8df18abc4`. M5 TEXT4 cold TTFT fell from 0.858745 to 0.555548 seconds at 2k and +4.619115 to 2.413850 seconds at 8k (35.31% and 47.74% lower). Exact rendered input IDs, zero prompt-cache +use, 16 output tokens, matching output text hashes and stable ABBA legs are retained. These are TTFT +measurements, not sustained decode or token-level ITL results. The source-bound test probe exercised 72 +FFN and 22 attention dispatches and reproduced the private candidate logits exactly. The bounded +prompt/continuation logit RMS/max checks pass; the proposed probability diagnostics still breach. + +The [post-change stage profile](m5-public-tiled-stage-profile.json) verifies every chunk's +logits against uninstrumented same-route execution in eight mirrored diagnostic runs. At 8k, the enabled +stage sum is 2.421 seconds: FFN down accounts for 25.38%, attention 14.98%, gate 14.47%, up 14.46%, and +convolution input 13.43%. This identifies the next measured optimization targets; it is not evidence of +hardware saturation or a separate served speed qualification. diff --git a/docs/receipts/lfm3/real-image-fp32-comparison.json b/docs/receipts/lfm3/real-image-fp32-comparison.json new file mode 100644 index 0000000000..dcb096a478 --- /dev/null +++ b/docs/receipts/lfm3/real-image-fp32-comparison.json @@ -0,0 +1,507 @@ +{ + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-8bit", + "phase": "prompt", + "scope": "Independent CPU FP32 graph diagnostics alongside retained required FP64; no Metal calls in FP32 graph", + "image": { + "url": "https://placecats.com/300/200", + "jpeg_sha256": "d1b324f529a96d8d8a16d973d88e1e974905d9c2c0e3e8b21a50b1c0e3fe2379", + "canonical_png_sha256": "005500792ee8ea6f35ac369d900fc788cd77667f145e7100c2cff57fae5355d9", + "conversion": "macOS sips JPEG to PNG; canonical PNG is the exact native/reference input" + }, + "grid": [ + 14, + 20 + ], + "image_start": 4, + "image_rows": 70, + "suffix_start": 74, + "suffix_rows": 16, + "all90_scope": "Contains70 image-placeholder rows,4 prefix rows and16 suffix rows; never conflate all90 with suffix16", + "modes": { + "fp32": "No internal BF16 activation boundaries; changes model arithmetic and is diagnostic only", + "fp32-bf16": "FP32 graph with the same explicit BF16 boundary sites as ideal FP64 graph; independent CPU reductions, not native lane replay", + "metal_assisted_replay": "Retained separately in real-image-rounding-reproduction.json; explicitly delegates scalar elementary functions to Metal" + }, + "fp32_semantics": { + "compute": "NumPy CPU float32 matmul/reductions/elementary functions; no Metal calls", + "boundaries": "fp32-bf16", + "dequantization": "FP32 q*scale+bias; separate multiply and add, no BF16 dequantized-weight cast", + "matmul_order": "NumPy BLAS implementation, not native GPU lane order", + "erf": "Binary64 libm erf on FP32 arguments then FP32 RNE", + "shape": [ + 90, + 128000 + ], + "numpy": "2.5.4", + "host": "arm64", + "blas": { + "Compilers": { + "c": { + "name": "clang", + "linker": "ld64", + "version": "15.0.0", + "commands": "cc" + }, + "cython": { + "name": "cython", + "linker": "cython", + "version": "3.3.0", + "commands": "cython" + }, + "c++": { + "name": "clang", + "linker": "ld64", + "version": "15.0.0", + "commands": "c++" + } + }, + "Machine Information": { + "host": { + "cpu": "aarch64", + "family": "aarch64", + "endian": "little", + "system": "darwin" + }, + "build": { + "cpu": "aarch64", + "family": "aarch64", + "endian": "little", + "system": "darwin" + } + }, + "Build Dependencies": { + "blas": { + "name": "accelerate", + "found": true, + "version": "unknown", + "detection method": "system", + "include directory": "unknown", + "lib directory": "unknown", + "openblas configuration": "unknown", + "pc file directory": "unknown" + }, + "lapack": { + "name": "accelerate", + "found": true, + "version": "unknown", + "detection method": "system", + "include directory": "unknown", + "lib directory": "unknown", + "openblas configuration": "unknown", + "pc file directory": "unknown" + } + }, + "Python Information": { + "path": "/private/var/folders/s6/5hzmn6lx4dz5nxs7k_0slzph0000gn/T/build-env-q3ulyt3x/bin/python", + "version": "3.13" + }, + "SIMD Extensions": { + "baseline": [ + "NEON", + "NEON_FP16", + "NEON_VFPV4", + "ASIMD" + ], + "found": [ + "ASIMDHP", + "ASIMDDP" + ], + "not found": [ + "ASIMDFHM" + ] + } + }, + "thread_environment": { + "OPENBLAS_NUM_THREADS": "2", + "VECLIB_MAXIMUM_THREADS": "2", + "OMP_NUM_THREADS": null + }, + "source_sha256": "6a8a7dc6434361f22a4da4e24821d14ee45ff2d31d57a9d4cbd5f8d1569f7f4e", + "accepted_threshold": null, + "fidelity_pass": null + }, + "results": { + "ideal_fp64_own_vision": { + "logits_sha256": "c34a303517fe4ea85a1886f9bc9928febebd96b677c1c9412462c5967c8fbc42", + "all90": { + "top1_differences": 7, + "disjoint_maximum_sets": 7, + "relative_rms": 0.022216577726492024, + "maximum_absolute_error": 1.59375 + }, + "suffix16": { + "top1_differences": 2, + "disjoint_maximum_sets": 2, + "relative_rms": 0.013412548767931996, + "maximum_absolute_error": 0.5 + }, + "suffix_differences": [ + { + "row": 81, + "native_top": 78, + "reference_top": 851, + "native_maximum_set": [ + 78 + ], + "reference_maximum_set": [ + 851 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.40625, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.40625 + }, + { + "row": 88, + "native_top": 309, + "reference_top": 22, + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22 + ], + "maximum_sets_overlap": false, + "native_margin": 0.0625, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0625 + } + ], + "disputed_rows": [ + { + "row": 81, + "contender_ids": [ + 78, + 851 + ], + "native_values": [ + 7.28125, + 7.15625 + ], + "reference_values": [ + 7.21875, + 7.625 + ], + "native_maximum_set": [ + 78 + ], + "reference_maximum_set": [ + 851 + ], + "reference_margin": 0.40625 + }, + { + "row": 88, + "contender_ids": [ + 22, + 309 + ], + "native_values": [ + 13.4375, + 13.5 + ], + "reference_values": [ + 13.4375, + 13.375 + ], + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22 + ], + "reference_margin": 0.0625 + } + ] + }, + "independent_fp32_own_vision": { + "logits_sha256": "13a0b9aa18f643ebba3b29bc71784fbe2244ca13e47a9632b0ac573df3e8353e", + "all90": { + "top1_differences": 6, + "disjoint_maximum_sets": 5, + "relative_rms": 0.029089294594932084, + "maximum_absolute_error": 1.75390625 + }, + "suffix16": { + "top1_differences": 2, + "disjoint_maximum_sets": 2, + "relative_rms": 0.01862716329656785, + "maximum_absolute_error": 0.859375 + }, + "suffix_differences": [ + { + "row": 81, + "native_top": 78, + "reference_top": 851, + "native_maximum_set": [ + 78 + ], + "reference_maximum_set": [ + 851 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.21875, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.21875 + }, + { + "row": 88, + "native_top": 309, + "reference_top": 22, + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22 + ], + "maximum_sets_overlap": false, + "native_margin": 0.0625, + "reference_margin": 0.125, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.125 + } + ], + "disputed_rows": [ + { + "row": 81, + "contender_ids": [ + 78, + 851 + ], + "native_values": [ + 7.28125, + 7.15625 + ], + "reference_values": [ + 7.21875, + 7.4375 + ], + "native_maximum_set": [ + 78 + ], + "reference_maximum_set": [ + 851 + ], + "reference_margin": 0.21875 + }, + { + "row": 88, + "contender_ids": [ + 22, + 309 + ], + "native_values": [ + 13.4375, + 13.5 + ], + "reference_values": [ + 13.4375, + 13.3125 + ], + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22 + ], + "reference_margin": 0.125 + } + ] + }, + "independent_fp32_bf16_own_vision": { + "logits_sha256": "4e37c32710720c019db62d3d8fe574b202afc50ad6bdf19164c777e0d368e2e7", + "all90": { + "top1_differences": 5, + "disjoint_maximum_sets": 3, + "relative_rms": 0.018905629285201224, + "maximum_absolute_error": 0.8125 + }, + "suffix16": { + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.013024445100711624, + "maximum_absolute_error": 0.53125 + }, + "suffix_differences": [ + { + "row": 88, + "native_top": 309, + "reference_top": 22, + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22, + 309 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + } + ], + "disputed_rows": [ + { + "row": 81, + "contender_ids": [ + 78, + 851 + ], + "native_values": [ + 7.28125, + 7.15625 + ], + "reference_values": [ + 7.28125, + 7.15625 + ], + "native_maximum_set": [ + 78 + ], + "reference_maximum_set": [ + 78 + ], + "reference_margin": 0.125 + }, + { + "row": 88, + "contender_ids": [ + 22, + 309 + ], + "native_values": [ + 13.4375, + 13.5 + ], + "reference_values": [ + 13.5, + 13.5 + ], + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22, + 309 + ], + "reference_margin": 0.0 + } + ] + }, + "independent_fp32_bf16_native_vision": { + "logits_sha256": "c604db2c1e236eca9ba6197d5f464625e0c9bcc19a303aae0e12adc605977298", + "all90": { + "top1_differences": 3, + "disjoint_maximum_sets": 1, + "relative_rms": 0.015441009381036912, + "maximum_absolute_error": 0.8125 + }, + "suffix16": { + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.012171142512004731, + "maximum_absolute_error": 0.8125 + }, + "suffix_differences": [ + { + "row": 88, + "native_top": 309, + "reference_top": 22, + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22, + 309 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + } + ], + "disputed_rows": [ + { + "row": 81, + "contender_ids": [ + 78, + 851 + ], + "native_values": [ + 7.28125, + 7.15625 + ], + "reference_values": [ + 7.28125, + 7.25 + ], + "native_maximum_set": [ + 78 + ], + "reference_maximum_set": [ + 78 + ], + "reference_margin": 0.03125 + }, + { + "row": 88, + "contender_ids": [ + 22, + 309 + ], + "native_values": [ + 13.4375, + 13.5 + ], + "reference_values": [ + 13.5, + 13.5 + ], + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22, + 309 + ], + "reference_margin": 0.0 + } + ] + } + }, + "conclusion": "Pure FP32 retains both disputed suffix differences. Independent FP32 with model BF16 boundaries removes row81 difference and makes row88 a tied maximum overlapping native309. This demonstrates sensitivity to arithmetic/rounding; it does not establish a new acceptance threshold or attribute every difference to FP64 alone.", + "accepted_threshold": null, + "fidelity_pass": null, + "tests": { + "command": "python -m unittest discover -s tools/lfm3 -p test_fp32_graph_reference.py", + "passed": 6, + "failed": 0 + }, + "sources": { + "tools/lfm3/fp32_graph_reference.py": "6a8a7dc6434361f22a4da4e24821d14ee45ff2d31d57a9d4cbd5f8d1569f7f4e", + "tools/lfm3/test_fp32_graph_reference.py": "dd2c4fc06680dbab904c5d9d54ee0834fd2793f8d32c8c3d5cc9c231aad5468c", + "tools/lfm3/fp64_reference.py": "0fb1128ade095db87b6126792ab14930792ab1f7b41b998570356bc586f16b2e", + "tools/lfm3/compare_logits.py": "b1bf273c8f3ce89591bccb8f098c82763cddff11373dbe850302450fcee1502c" + }, + "input_sha256": { + "lfm3-real-cat-patches.npy": "f1c55a0dd28b91d9929f074995adadea79d7c0862c8cdcd0be6dbfda57383dc1", + "lfm3-real-cat-tokens.npy": "54c970db8c6ba82895cb2f18a48d397ea568ae885241129e9c5ee7dbbc91a24d", + "lfm3-real-cat-native-8.bf16": "aceb2b880177abfae718bdd660398b3cd7ee990d2dfcdce2a110d08dc4e33135" + }, + "raw_fp32_logits_sha256": { + "fp32-ownvision": "28d155c91aa531a7d832429ccbad4a397f0f8cc2a37795dffc3f534e62cc650c", + "fp32-bf16-ownvision-final": "3301aa73c1ff171de9b35d1bc5a64ff143d173f3d72edae4a387916b0e0c8d82", + "fp32-bf16-nativevision": "e0bcba9ec3e724bbd44e9e5d8b63510fa2d5d0dea0f8b24be8a7cdd3ad38b0d5" + }, + "commands": [ + "OPENBLAS_NUM_THREADS=2 VECLIB_MAXIMUM_THREADS=2 python tools/lfm3/fp32_graph_reference.py --model-dir MODEL --tokens TOKENS.npy --patches PATCHES.npy --grid-h 14 --grid-w 20 --image-start 4 --boundaries fp32 --output FP32.npy", + "Repeat with --boundaries fp32-bf16; fixed-native-image ablation replaces patch/grid arguments with --image-rows NATIVE_ROWS.npy" + ], + "native_identity_receipt": "real-image-rounding-reproduction.json", + "ideal_fp64_receipt": "real-image-fp64.json" +} diff --git a/docs/receipts/lfm3/real-image-fp64.json b/docs/receipts/lfm3/real-image-fp64.json new file mode 100644 index 0000000000..67621384ed --- /dev/null +++ b/docs/receipts/lfm3/real-image-fp64.json @@ -0,0 +1,704 @@ +{ + "image": { + "url": "https://placecats.com/300/200", + "jpeg_sha256": "d1b324f529a96d8d8a16d973d88e1e974905d9c2c0e3e8b21a50b1c0e3fe2379", + "canonical_png_sha256": "005500792ee8ea6f35ac369d900fc788cd77667f145e7100c2cff57fae5355d9", + "conversion": "macOS sips JPEG to PNG; canonical PNG is the exact native/reference input" + }, + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "grid": [ + 14, + 20 + ], + "image_start": 4, + "image_rows": 70, + "suffix_start": 74, + "tokens": { + "_trimmed_list": { + "count": 90, + "sha256": "d50acaa064fe912daee0a6ecae10990e6b57f78e0c54846a77160be8d5f0402e", + "first": [ + 124899, + 5922 + ], + "last": [ + 22 + ] + } + }, + "teacher_forced_suffix": "<|vision_end|>Describe this image.<|im_end|>\n<|im_start|>assistant\nThe image contains a cat.", + "patches_sha256": "f1c55a0dd28b91d9929f074995adadea79d7c0862c8cdcd0be6dbfda57383dc1", + "native_projected_rows_sha256": "ab2ac7db759e5c4a32fe7d60680ec47f7d1cbc5938fa7d8091bf4756bfeea4fc", + "fp64_projected_rows_sha256": "48e5056f330f9ce22e6f8a713f169c364a884d976f5d110abe9eeb2db3053719", + "token_fixture_sha256": "54c970db8c6ba82895cb2f18a48d397ea568ae885241129e9c5ee7dbbc91a24d", + "native_forward_diagnostic_sha256": "e80a2a14c0d8ed20b62088d0d8b87be4beb5fed3eb1a7336b3b00f8ac5f74bee", + "vision": { + "relative_rms": 0.01813030697564529, + "maximum_absolute_error": 0.009981155395507812, + "equal_values": 16235, + "values": 143360, + "worst_columns": { + "_trimmed_list": { + "count": 20, + "sha256": "734394e4cecccfbcece3c3c1fa55678af1367e9c260bef80e1c87573219ba5b3", + "first": [ + { + "column": 1990, + "rms": 0.0015688593706404128, + "max_abs": 0.009981155395507812 + }, + { + "column": 397, + "rms": 0.0015344000353975263, + "max_abs": 0.005859375 + } + ], + "last": [ + { + "column": 607, + "rms": 0.0011042805714237814, + "max_abs": 0.004638671875 + } + ] + } + }, + "rows": { + "_trimmed_list": { + "count": 70, + "sha256": "68d92dfddd87b51d130401ce3a4e712d94f9d9b5bc801311b96f0548bb0cc92f", + "first": [ + { + "row": 0, + "rms": 0.00038509225692759163, + "max_abs": 0.00146484375 + }, + { + "row": 1, + "rms": 0.0004149904938254954, + "max_abs": 0.001953125 + } + ], + "last": [ + { + "row": 69, + "rms": 0.00045007914776163624, + "max_abs": 0.00146484375 + } + ] + } + } + }, + "decoder": { + "same_native_projected_rows": { + "rows": 90, + "vocabulary": 128000, + "top1_differences": 3, + "disjoint_maximum_sets": 0, + "relative_rms": 0.012972441485740975, + "maximum_absolute_error": 0.59228515625, + "differences": [ + { + "row": 15, + "native_top": 11876, + "reference_top": 3553, + "native_maximum_set": [ + 11876 + ], + "reference_maximum_set": [ + 3553, + 11876 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + }, + { + "row": 54, + "native_top": 84, + "reference_top": 266, + "native_maximum_set": [ + 84, + 266 + ], + "reference_maximum_set": [ + 266 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.03125, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.03125 + }, + { + "row": 88, + "native_top": 22, + "reference_top": 309, + "native_maximum_set": [ + 22, + 309 + ], + "reference_maximum_set": [ + 309 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + } + ], + "worst_rows": { + "_trimmed_list": { + "count": 20, + "sha256": "5fd0290a4d47405d115695d1baaf2e78c8fb03d41fe8f4f54d5cf207a015090c", + "first": [ + { + "row": 0, + "rms_error": 0.14167088715067103, + "maximum_absolute_error": 0.59228515625 + }, + { + "row": 78, + "rms_error": 0.12248561069361408, + "maximum_absolute_error": 0.357421875 + } + ], + "last": [ + { + "row": 31, + "rms_error": 0.05736327192908228, + "maximum_absolute_error": 0.21875 + } + ] + } + }, + "worst_columns": { + "_trimmed_list": { + "count": 20, + "sha256": "0b0c6eda29c147dec384ea4a56a235db8623f5e2ddfa7a59dc8e8ea9ecbca48d", + "first": [ + { + "column": 283, + "rms_error": 0.09527168511786321, + "maximum_absolute_error": 0.421875 + }, + { + "column": 280, + "rms_error": 0.0923071313133541, + "maximum_absolute_error": 0.421875 + } + ], + "last": [ + { + "column": 29061, + "rms_error": 0.08291168131609514, + "maximum_absolute_error": 0.34375 + } + ] + } + }, + "accepted_threshold": null, + "fidelity_pass": null, + "text_suffix": { + "rows": 16, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.011224499341193007, + "maximum_absolute_error": 0.357421875, + "differences": [ + { + "row": 88, + "native_top": 22, + "reference_top": 309, + "native_maximum_set": [ + 22, + 309 + ], + "reference_maximum_set": [ + 309 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + } + ], + "worst_rows": { + "_trimmed_list": { + "count": 16, + "sha256": "134a97c4dc72f69fe6e4f109bf2eef1efa863fc7200596c80c2973c834cf8435", + "first": [ + { + "row": 78, + "rms_error": 0.12248561069361408, + "maximum_absolute_error": 0.357421875 + }, + { + "row": 86, + "rms_error": 0.07341299374633652, + "maximum_absolute_error": 0.296875 + } + ], + "last": [ + { + "row": 75, + "rms_error": 0.029775309489710713, + "maximum_absolute_error": 0.125 + } + ] + } + }, + "worst_columns": { + "_trimmed_list": { + "count": 20, + "sha256": "3acf30986b80542d323cb0bd069568735f235565a6f8c0e91c0e7774b1e49353", + "first": [ + { + "column": 2411, + "rms_error": 0.11533577547124645, + "maximum_absolute_error": 0.328125 + }, + { + "column": 1447, + "rms_error": 0.11317840379956114, + "maximum_absolute_error": 0.2724609375 + } + ], + "last": [ + { + "column": 37134, + "rms_error": 0.10699182605717175, + "maximum_absolute_error": 0.3125 + } + ] + } + }, + "accepted_threshold": null, + "fidelity_pass": null + }, + "native_logits_sha256": "d2dc2c1014716cb9ead7543ee00562a1816bf50d4a18b927cd8d8151f0074d54", + "reference_logits_sha256": "ea2bb4dc9794ea8d5a6d714eaf2a5bf776713cd0a482f9fc754c22107b95f4a1" + }, + "same_fp64_projected_rows": { + "rows": 90, + "vocabulary": 128000, + "top1_differences": 3, + "disjoint_maximum_sets": 2, + "relative_rms": 0.014158856079061779, + "maximum_absolute_error": 0.59375, + "differences": [ + { + "row": 34, + "native_top": 5594, + "reference_top": 2911, + "native_maximum_set": [ + 5594 + ], + "reference_maximum_set": [ + 2911 + ], + "maximum_sets_overlap": false, + "native_margin": 0.15625, + "reference_margin": 0.03125, + "native_reference_contender_gap": 0.15625, + "reference_native_contender_gap": 0.03125 + }, + { + "row": 54, + "native_top": 266, + "reference_top": 84, + "native_maximum_set": [ + 266 + ], + "reference_maximum_set": [ + 84 + ], + "maximum_sets_overlap": false, + "native_margin": 0.046875, + "reference_margin": 0.03125, + "native_reference_contender_gap": 0.046875, + "reference_native_contender_gap": 0.03125 + }, + { + "row": 88, + "native_top": 309, + "reference_top": 22, + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22, + 309 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": { + "_trimmed_list": { + "count": 20, + "sha256": "945a03adbdd2853db8fa08b5f590ffdb26e3ebe74df8f412b65f2e78406debbc", + "first": [ + { + "row": 64, + "rms_error": 0.14449757029683585, + "maximum_absolute_error": 0.4375 + }, + { + "row": 0, + "rms_error": 0.14167088715067103, + "maximum_absolute_error": 0.59228515625 + } + ], + "last": [ + { + "row": 87, + "rms_error": 0.060788257552294694, + "maximum_absolute_error": 0.2578125 + } + ] + } + }, + "worst_columns": { + "_trimmed_list": { + "count": 20, + "sha256": "4f03ebdd5e980298b8ea9bc17d07ec0bb732f7f32aba183fd732acd17245b79a", + "first": [ + { + "column": 283, + "rms_error": 0.09870564171344597, + "maximum_absolute_error": 0.390625 + }, + { + "column": 518, + "rms_error": 0.09813089945580669, + "maximum_absolute_error": 0.3671875 + } + ], + "last": [ + { + "column": 1894, + "rms_error": 0.08637283820473957, + "maximum_absolute_error": 0.3333740234375 + } + ] + } + }, + "accepted_threshold": null, + "fidelity_pass": null, + "text_suffix": { + "rows": 16, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.010381343967797625, + "maximum_absolute_error": 0.59375, + "differences": [ + { + "row": 88, + "native_top": 309, + "reference_top": 22, + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22, + 309 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": { + "_trimmed_list": { + "count": 16, + "sha256": "d60e294f737fa52aa357f1fb5810883201c5875ea1d888a57a36024959b3bea4", + "first": [ + { + "row": 79, + "rms_error": 0.0780645262711307, + "maximum_absolute_error": 0.1875 + }, + { + "row": 76, + "rms_error": 0.06821821862630469, + "maximum_absolute_error": 0.59375 + } + ], + "last": [ + { + "row": 78, + "rms_error": 0.0288727224827803, + "maximum_absolute_error": 0.15625 + } + ] + } + }, + "worst_columns": { + "_trimmed_list": { + "count": 20, + "sha256": "ade4a3385c53089e6445e56a58b72f250f6b2f8ce7e4d0582616ae4a52373acc", + "first": [ + { + "column": 4752, + "rms_error": 0.17160446376418992, + "maximum_absolute_error": 0.59375 + }, + { + "column": 6440, + "rms_error": 0.12906706056088119, + "maximum_absolute_error": 0.462890625 + } + ], + "last": [ + { + "column": 851, + "rms_error": 0.1091898304385418, + "maximum_absolute_error": 0.3125 + } + ] + } + }, + "accepted_threshold": null, + "fidelity_pass": null + }, + "native_logits_sha256": "7fcb749db23cf2e090441a5c6a97e925add9b81f34b79d6b15b9951099ddd358", + "reference_logits_sha256": "c16af1c1ff0095b2ced285886f4c54e21c48f244cb74b4fa615c8954163e620f" + }, + "end_to_end_own_vision_rows": { + "rows": 90, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 2, + "relative_rms": 0.02812434494869154, + "maximum_absolute_error": 2.5390625, + "differences": [ + { + "row": 34, + "native_top": 5594, + "reference_top": 2911, + "native_maximum_set": [ + 5594 + ], + "reference_maximum_set": [ + 2911 + ], + "maximum_sets_overlap": false, + "native_margin": 0.0625, + "reference_margin": 0.03125, + "native_reference_contender_gap": 0.25, + "reference_native_contender_gap": 0.03125 + }, + { + "row": 40, + "native_top": 4899, + "reference_top": 34541, + "native_maximum_set": [ + 4899 + ], + "reference_maximum_set": [ + 34541 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0625 + } + ], + "worst_rows": { + "_trimmed_list": { + "count": 20, + "sha256": "6e156c6392cf7f38789a948066c8c8d45c2d2150f5b2d1ff368a158f89e9ec47", + "first": [ + { + "row": 31, + "rms_error": 0.6295615473865768, + "maximum_absolute_error": 2.5390625 + }, + { + "row": 14, + "rms_error": 0.4583900105693971, + "maximum_absolute_error": 1.375 + } + ], + "last": [ + { + "row": 33, + "rms_error": 0.09087490275382837, + "maximum_absolute_error": 0.59375 + } + ] + } + }, + "worst_columns": { + "_trimmed_list": { + "count": 20, + "sha256": "7a833252d3e533ace4490a0a8245ba238dbcbf0e87f61425a8d09fba13d62ed9", + "first": [ + { + "column": 42385, + "rms_error": 0.29791967617195436, + "maximum_absolute_error": 2.4921875 + }, + { + "column": 39091, + "rms_error": 0.28829632207176553, + "maximum_absolute_error": 2.5 + } + ], + "last": [ + { + "column": 117785, + "rms_error": 0.2519815358285387, + "maximum_absolute_error": 2.046875 + } + ] + } + }, + "accepted_threshold": null, + "fidelity_pass": null, + "text_suffix": { + "rows": 16, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.012445652348647095, + "maximum_absolute_error": 0.46875, + "differences": [], + "worst_rows": { + "_trimmed_list": { + "count": 16, + "sha256": "2b924c71e248ef49e02b99a55e9233c8bf5033cee5a19f75c24ca81d79411c2c", + "first": [ + { + "row": 89, + "rms_error": 0.09796858084665877, + "maximum_absolute_error": 0.35546875 + }, + { + "row": 82, + "rms_error": 0.09612450835564357, + "maximum_absolute_error": 0.28125 + } + ], + "last": [ + { + "row": 77, + "rms_error": 0.0324625974120783, + "maximum_absolute_error": 0.1875 + } + ] + } + }, + "worst_columns": { + "_trimmed_list": { + "count": 20, + "sha256": "18be457892de9af935c98a87ef742b559aa3890c2f3d6a72514e10eb62fa1dab", + "first": [ + { + "column": 37452, + "rms_error": 0.1330994143546939, + "maximum_absolute_error": 0.34375 + }, + { + "column": 59415, + "rms_error": 0.13228006787653943, + "maximum_absolute_error": 0.46875 + } + ], + "last": [ + { + "column": 38691, + "rms_error": 0.11936098965844703, + "maximum_absolute_error": 0.25 + } + ] + } + }, + "accepted_threshold": null, + "fidelity_pass": null + }, + "native_logits_sha256": "d2dc2c1014716cb9ead7543ee00562a1816bf50d4a18b927cd8d8151f0074d54", + "reference_logits_sha256": "c16af1c1ff0095b2ced285886f4c54e21c48f244cb74b4fa615c8954163e620f" + } + }, + "accepted_threshold": null, + "fidelity_pass": null, + "vision_weight_identity": { + "4": { + "tensors": 138, + "tensor_bytes": 378760704, + "named_tensor_sha256": "4f4d0769ead2dc99fce516e6f43fb10be1070bbad4b708db8bb41df8bcb4c628" + }, + "6": { + "tensors": 138, + "tensor_bytes": 378760704, + "named_tensor_sha256": "40135b664ebd32cc67d662a6b4da767074fa4919668f1c711700f406340cc031" + }, + "8": { + "tensors": 138, + "tensor_bytes": 378760704, + "named_tensor_sha256": "40135b664ebd32cc67d662a6b4da767074fa4919668f1c711700f406340cc031" + } + }, + "other_widths": [ + { + "_trimmed_value": { + "bytes": 21601, + "sha256": "d447e33d9276ea2c6acd9f50d647dafdcd22d8377ca9d7d627a518f950ce2ad0", + "keys": [ + "checkpoint", + "vision", + "decoder", + "text_suffix", + "native_projected_rows_sha256", + "fp64_projected_rows_sha256", + "native_logits_sha256", + "fp64_logits_sha256", + "native_forward_diagnostic_sha256", + "native_vision_diagnostic_sha256", + "fp64_vision_scope", + "same_fp64_projected_rows_decoder" + ], + "count": 15 + } + }, + { + "_trimmed_value": { + "bytes": 23224, + "sha256": "58035d96544323038d9ab2012062a0379d9d4157ffc223e1885f430a2f684d77", + "keys": [ + "checkpoint", + "vision", + "decoder", + "text_suffix", + "native_projected_rows_sha256", + "fp64_projected_rows_sha256", + "native_logits_sha256", + "fp64_logits_sha256", + "native_forward_diagnostic_sha256", + "native_vision_diagnostic_sha256", + "fp64_vision_scope", + "same_fp64_projected_rows_decoder" + ], + "count": 15 + } + } + ], + "_trim": { + "note": "Raw capture replaced by counts, first/last items and SHA-256 of the canonical JSON of each trimmed list or string.", + "original_sha256": "8a8ba82daec4ac1def463860f2be8b071e80b2f3094d87180bd74bbd45a30191", + "original_bytes": 133224 + } +} diff --git a/docs/receipts/lfm3/real-image-mlx-secondary.json b/docs/receipts/lfm3/real-image-mlx-secondary.json new file mode 100644 index 0000000000..fb88484482 --- /dev/null +++ b/docs/receipts/lfm3/real-image-mlx-secondary.json @@ -0,0 +1,68 @@ +{ + "scope": "Secondary MLX developer vision reference comparison on identical frozen native image patches, not official MLX-VLM end-to-end serving or a fidelity pass. 6/8 vision weights are identical by prior full-tensor hash verification; only distinct 4/8 sets are run here.", + "grid": [ + 14, + 20 + ], + "patches_sha256": "f1c55a0dd28b91d9929f074995adadea79d7c0862c8cdcd0be6dbfda57383dc1", + "host": "Apple M4 Pro, macOS 26.5.1, MLX 0.32.4", + "records": [ + { + "vision_weight_set": "4bit export", + "target": "Ideal FP64 vision oracle with its recorded BF16 stores; no acceptance threshold", + "comparators": { + "native": { + "output_sha256": "ab2ac7db759e5c4a32fe7d60680ec47f7d1cbc5938fa7d8091bf4756bfeea4fc", + "rms": 0.000641842947115359, + "relative_rms": 0.01813030697564529, + "max_abs": 0.009981155395507812, + "worst_column": 1990, + "worst_column_rms": 0.0015688593706404128, + "equal_bf16_values": 16235, + "values": 143360 + }, + "mlx_developer_reference": { + "output_sha256": "34a2b8adfb6b17949d16a7f6d0762d1196c8151c5fde19a2bcb93d83c4f84f83", + "rms": 0.0005991838235835683, + "relative_rms": 0.016925303464397964, + "max_abs": 0.00634765625, + "worst_column": 397, + "worst_column_rms": 0.0012908817021458044, + "equal_bf16_values": 16157, + "values": 143360 + } + } + }, + { + "vision_weight_set": "8bit export", + "target": "Ideal FP64 vision oracle with its recorded BF16 stores; no acceptance threshold", + "comparators": { + "native": { + "output_sha256": "aceb2b880177abfae718bdd660398b3cd7ee990d2dfcdce2a110d08dc4e33135", + "rms": 0.0005656584034634817, + "relative_rms": 0.016063874280294436, + "max_abs": 0.0087890625, + "worst_column": 498, + "worst_column_rms": 0.0013091940942879653, + "equal_bf16_values": 16945, + "values": 143360 + }, + "mlx_developer_reference": { + "output_sha256": "4eb3954a4b7c0d5d8e31d01ca2393ef754dbcc1605ad5c71b6679f3b7bbe6039", + "rms": 0.0005638638806455016, + "relative_rms": 0.016012912447561735, + "max_abs": 0.0078125, + "worst_column": 128, + "worst_column_rms": 0.0012127937651889588, + "equal_bf16_values": 17083, + "values": 143360 + } + } + } + ], + "source_sha256": { + "tools/lfm3/vision_reference.py": "99ec81e9ff78e58f9e75c5562466b87aaf520a9a2c2da9a45a0593e587d7b711", + "tools/lfm3/reference.py": "fd4622a6fc35ed61ba4bf7a9d48c06a43e7cc390bc6505e7e6445300386b9f39" + }, + "command": "python tools/lfm3/vision_reference.py --model-dir MODEL --patches PATCHES.npy --grid-h 14 --grid-w 20 --output OUTPUT.bf16" +} diff --git a/docs/receipts/lfm3/real-image-rounding-reproduction.json b/docs/receipts/lfm3/real-image-rounding-reproduction.json new file mode 100644 index 0000000000..25c8ab6980 --- /dev/null +++ b/docs/receipts/lfm3/real-image-rounding-reproduction.json @@ -0,0 +1,349 @@ +{ + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-8bit", + "phase": "prompt", + "image": { + "url": "https://placecats.com/300/200", + "jpeg_sha256": "d1b324f529a96d8d8a16d973d88e1e974905d9c2c0e3e8b21a50b1c0e3fe2379", + "canonical_png_sha256": "005500792ee8ea6f35ac369d900fc788cd77667f145e7100c2cff57fae5355d9", + "conversion": "macOS sips JPEG to PNG; canonical PNG is the exact native/reference input" + }, + "grid": [ + 14, + 20 + ], + "image_start": 4, + "image_rows": 70, + "suffix_start": 74, + "suffix_rows": 16, + "scope": "Independent CPU FP32 arithmetic replay with explicitly delegated Metal scalar elementary functions; not an independent FP64 pass", + "ideal_fp64_retained": { + "receipt": "real-image-fp64.json", + "suffix_differences": [ + { + "row": 81, + "native_top": 78, + "reference_top": 851, + "native_maximum_set": [ + 78 + ], + "reference_maximum_set": [ + 851 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.40625, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.40625 + }, + { + "row": 88, + "native_top": 309, + "reference_top": 22, + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22 + ], + "maximum_sets_overlap": false, + "native_margin": 0.0625, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0625 + } + ] + }, + "vision_stages": [ + { + "stage": "patchifier", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "embedding_norm", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_00", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_01", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_02", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_03", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_04", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_05", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_06", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_07", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_08", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_09", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_10", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_11", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_12", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_13", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "projector_norm", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "projector_gelu", + "values": 286720, + "different_bf16": 0 + } + ], + "text_stages": [ + { + "stage": "layer_00_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_01_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_02_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_03_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_04_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_05_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_06_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_07_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_08_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_09_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_10_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_11_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_12_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_13_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_14_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_15_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_16_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_17_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_18_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_19_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_20_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_21_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_22_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_23_output", + "values": 184320, + "different_bf16": 0 + } + ], + "projected_image": { + "values": 143360, + "different_bf16": 0, + "sha256": "aceb2b880177abfae718bdd660398b3cd7ee990d2dfcdce2a110d08dc4e33135" + }, + "decoder_logits": { + "values": 11520000, + "different_bf16": 0, + "sha256": "db730509b6d7dcb3d6d134b59b6434dfb43ebc85cdd3fe351bc8cc4148b47642", + "suffix_top1_differences": 0 + }, + "first_attention_elementary_diagnostic": { + "scope": "Fixed native Q/K/V, independently replayed FP32 arithmetic with captured elementary functions", + "rows": 280, + "score_values": 1254400, + "score_differences": 0, + "raw_values": 291200, + "raw_differences": 0, + "helper_context_differences": 0, + "exp_values": 2508800, + "exp_differences_vs_fp64_rounded": 463691, + "maximum_exp_ulp_vs_fp64_rounded": 1, + "accepted_threshold": null, + "whole_model_fidelity_pass": null + }, + "rounding_causes": [ + "FP32 lane reduction/FMA sites", + "FP32 normalization reduction and rsqrt sites", + "Metal exp/exp2/cos/sin elementary rounding", + "FP32 subnormal normalization products flushed before BF16 store" + ], + "rotary_fixed_primitive_replay": { + "query_values": 286720, + "key_values": 143360, + "different_bf16": 0 + }, + "accepted_threshold": null, + "independent_fp64_fidelity_pass": null, + "runtime_changes": "Trace-only snapshots; no arithmetic changes", + "reproduction": "fidelity-replay.md#real-image-rounding-causality", + "artifact_identity": { + "tested_host": { + "chip": "Apple M4 Pro", + "model_identifier": "Mac16,7", + "memory_gib": 48, + "macos": "26.5.1", + "macos_build": "25F80", + "architecture": "arm64" + }, + "native_decoder_baseline": { + "diagnostic_sha256": "e80a2a14c0d8ed20b62088d0d8b87be4beb5fed3eb1a7336b3b00f8ac5f74bee", + "logits_sha256": "db730509b6d7dcb3d6d134b59b6434dfb43ebc85cdd3fe351bc8cc4148b47642", + "scope": "Previously frozen native baseline from real-image-fp64.json; not a claim that later runtime revisions were requalified" + }, + "native_vision_canonical_recapture": { + "diagnostic_sha256": "7a635c8439842ae2bf8e7993cacf1a8cea0a386f57033654d74c08e65292efea", + "projected_image_sha256": "aceb2b880177abfae718bdd660398b3cd7ee990d2dfcdce2a110d08dc4e33135", + "source_sha256": { + "zig/src/families/lfm3/vision.zig": "c80661799c9e9fd7d71d6cbb2838c9701388cd3f292496f346956b3edd8a21e0", + "zig/kernels/metal/lfm3/vision.metal": "7f6338698948bebe80fe21b60b2c68c0a55a08208e8a768715016a02976e846a" + } + }, + "reference_source_sha256": { + "tools/lfm3/fp64_reference.py": "0fb1128ade095db87b6126792ab14930792ab1f7b41b998570356bc586f16b2e", + "tools/lfm3/row_projection_reference.py": "d03f224a037a920ba66b9828059d1489e48729cd168077e76789d0ec5c199c0c", + "tools/lfm3/row_projection_replay.c": "ff336f75fcf0a4f6c0ac92ed627bc1a6abb0b36b8f4cc95d876f2a8bb2f5cc42", + "tools/lfm3/elementary_reference.py": "3e4db276278867b7683536044ed7bcbe4c7643867d722fdda5b8ad895b3963e7", + "tools/lfm3/text_rounding_reference.py": "52665eef48570d45a2554861fe8f22b31874f8f464366adf882f40c3d311db27", + "tools/lfm3/text_attention_rounding.py": "ae67f2b2c26f89f75b1f36822a4b30a9f710df2b42f9f7ec42679373028b89e8", + "tools/lfm3/vision_fp64_reference.py": "742b8b4f44dde824c3a5c2d72a632401a0f9df7b7f23a8d3729d15f4f8c6a4d2", + "tools/lfm3/vision_rounding_reference.py": "8b2f5679bbd3f05b3e86a5748c3803f744502cffd0159bee2a218da38bd6f1c4", + "tools/lfm3/vision_attention_rounding.py": "37d99908f7b324740d098276f3d014cc9cc740587a8b798cde8be83d6dcef60d" + }, + "optional_c_accelerator_sha256": "fd3bdcc07800ca3e4bda472228c5fc86afa1f8eab7c2ba4669b0eae1e07df82b", + "elementary_capture": { + "shader_sha256": "7f6338698948bebe80fe21b60b2c68c0a55a08208e8a768715016a02976e846a", + "scope": "Actual production helper diagnostic, not independent FP64 reference" + } + } +} diff --git a/docs/receipts/lfm3/risk-fast-experiment.json b/docs/receipts/lfm3/risk-fast-experiment.json new file mode 100644 index 0000000000..2ae89f48f4 --- /dev/null +++ b/docs/receipts/lfm3/risk-fast-experiment.json @@ -0,0 +1,129 @@ +{ + "status": "default-off development experiment; not a performance improvement", + "date": "2026-10-29", + "backend": "Metal", + "device": "Apple M5", + "scope": "Synthetic attention operator, 64 query rows, full history 2048 and8192; not actual-model prefill or serving", + "candidate": "Exponent predicate early exits and absorbing per-lane risk OR", + "build_option": "-Dlfm3-risk-fast=true", + "default_enabled": false, + "cpu_equivalence": { + "pair_classes": 4194304, + "sum_classes": 20480000, + "random_words": 1000000, + "sticky_sequences": 512, + "mismatches": 0 + }, + "device_numerical": { + "devices": [ + "Apple M4", + "Apple M5" + ], + "normal_cases_per_device": 35, + "repeats": 5, + "negative_controls_per_device": 2, + "checks": [ + "pm/pl/po byte identity", + "range flags byte identity", + "final BF16 byte identity", + "input immutability", + "guard regions" + ] + }, + "cost_protocol": { + "balanced_rounds": 5, + "legs": 20, + "minimum_leg_seconds": 0.02, + "maximum_leg_seconds": 1, + "baseline_drift_limit": 0.05, + "within_round_spread_limit": 0.05 + }, + "cost": [ + { + "case": "h2048-r64-w0", + "stable": true, + "baseline_first_last_drift": -0.0011672461352403296, + "baseline_within_round_spreads": [ + 0.00028488954797434296, + 0.004006824676436649, + 0.0023022151292024784, + 0.008804105837239673, + 0.0009227079128673044 + ], + "median_candidate_over_public_throughput": 0.8900864888254163, + "public_seconds_per_op": 0.005706671858206391, + "pair_seconds_per_op": 0.00641652345075272, + "geometry": { + "rows": 64, + "history": 2048, + "window": 0 + }, + "full_output_matches_frozen_canonical": true, + "analysis_sha256": "50241ab2cbf0f8330c972efc3be5939156ae1cbbd213b5fe0ca63414e4a08775" + }, + { + "case": "h8192-r64-w0", + "stable": true, + "baseline_first_last_drift": -0.00960170710028363, + "baseline_within_round_spreads": [ + 0.012505571895359058, + 0.009459609353000609, + 0.007941873258114018, + 0.027025251958851193, + 0.00567250180932688 + ], + "median_candidate_over_public_throughput": 0.5774839407016837, + "public_seconds_per_op": 0.0037078906316310167, + "pair_seconds_per_op": 0.0063853828178253025, + "geometry": { + "rows": 64, + "history": 8192, + "window": 0 + }, + "full_output_matches_frozen_canonical": true, + "analysis_sha256": "0a9b9cb856dcf6f13eb422ea5659966858e17db16fc9a7df029aaa8ef6194cce" + } + ], + "remaining": [ + "Full public kernel contract suite", + "Real-checkpoint prompt/decode state and token equality", + "Cold 2k/8k prefill, TTFT, ITL and serving comparison to MLX-VLM" + ], + "promotion": "Not recommended: both measured operator cases regressed", + "build_validation": { + "native": "pass", + "kernel_contract_binary": "pass", + "all_developer_tools": "pass", + "cpu_target": "apple_m1", + "lean_baseline_problems": 428, + "lean_current_problems": 428, + "lean_new_problems": 0, + "diff_check": "pass", + "targeted_public_device_contract": "pass on Apple M5" + }, + "reproduce": [ + "clang++ -O2 -std=c++17 tools/check_lfm3_risk.cpp -o risk-proof && ./risk-proof", + "zig build native tf-kernel-contract dev-tools -Dcpu=apple_m1 -Dlfm3-risk-fast=true", + "zig-out/bin/tf-kernel-contract --only risk-fast" + ], + "targeted_public_device_contract": { + "device": "Apple M5", + "exit_code": 0, + "kernels": 2, + "passed": 2, + "failed": 0, + "known": 0, + "controls": 0, + "missed_controls": 0, + "limit_problems": 0, + "gates": "1-4 plus launch limits", + "full_shapes": 7, + "sliding_shapes": 10, + "nonfinite_values_each": 512, + "repeats": 5, + "log_sha256": "31c86c8db07677f4714aa6c136d70ea802ca22cefa1d4720f06f1b3117dea3c8", + "scope": "targeted risk-fast full/sliding contracts; no full-suite or negative-control claim" + }, + "archived_candidate_commit": "1543227fc657c88d8300015a1e1590af7633f1ee", + "superseded_by": "risk-fused-experiment.json" +} diff --git a/docs/receipts/lfm3/risk-fused-experiment.json b/docs/receipts/lfm3/risk-fused-experiment.json new file mode 100644 index 0000000000..6a9813069b --- /dev/null +++ b/docs/receipts/lfm3/risk-fused-experiment.json @@ -0,0 +1,563 @@ +{ + "status": "default-off measured M5 operator/served improvement; all six checkpoint widths passed baseline equivalence; M4 and broader qualification pending", + "date": "2026-10-30", + "backend": "Metal", + "device": "Apple M5 Ultra", + "baseline_commit": "1543227fc657c88d8300015a1e1590af7633f1ee", + "baseline": "Original attention with L3_RISK_FAST=0; the slower early-return experiment is not the baseline", + "candidate": "Branch-free fused product/sum range predicates with shared alpha/probability classification", + "build_option": "-Dlfm3-risk-fast=true", + "default_enabled": false, + "scope": "Synthetic complete attention partial plus merge, 64 query rows; not model prefill or serving", + "cpu_equivalence": { + "pair_class_checks": 4194304, + "fused_operand_result_classes": 20480000, + "random_fused_words": 10000000, + "boolean_truth_table_rows": 256, + "control_witnesses": 2, + "mismatches": 0, + "scope": "Exhaustive predicate category identity plus deterministic random sample; not exhaustive floating-point arithmetic or GPU qualification" + }, + "air": { + "scope": "Optimized AIR static counts, not native instructions or measured registers", + "partial_instructions_before": 423, + "partial_instructions_after": 409, + "conditional_branches_before": 16, + "conditional_branches_after": 16, + "floating_arithmetic_and_exp_call_counts_unchanged": true, + "compile_options": [ + "metal4.0", + "-fmetal-math-mode=safe", + "-fmetal-math-fp32-functions=fast", + "-O2" + ] + }, + "device_numerical": { + "normal_cases": 35, + "repeats": 5, + "guard_regions": 26, + "negative_controls": 2, + "controls_detected": 2, + "checks": [ + "pm/pl/po byte identity", + "range flags byte identity", + "final BF16 byte identity", + "input immutability", + "guard regions" + ], + "metadata": { + "partial_max_threads": 128, + "partial_static_threadgroup_bytes": 2096, + "merge_max_threads": 32, + "merge_static_threadgroup_bytes": 0, + "simd_width": 32 + } + }, + "cost_protocol": { + "dispatches_per_operation": 2, + "barriers_per_operation": 2, + "balanced_rounds": 5, + "legs": 20, + "minimum_leg_seconds": 0.02, + "maximum_leg_seconds": 1, + "baseline_drift_limit": 0.05, + "within_round_spread_limit": 0.05 + }, + "cost": [ + { + "case": "h2048-r64-w0", + "stable": true, + "baseline_first_last_drift": 0.0002169915226417629, + "baseline_within_round_spreads": [ + 0.0015848936687099524, + 0.0003504886096892257, + 0.0026541543167410975, + 0.0007725650054322131, + 0.00026146651287706213 + ], + "round_candidate_over_public_throughput": [ + 1.0186291411709285, + 1.023372901498002, + 1.0216203211937869, + 1.023195084673455, + 1.021426741046012 + ], + "median_candidate_over_public_throughput": 1.0216203211937869, + "public_seconds_per_op": 0.005678570305462927, + "pair_seconds_per_op": 0.005563093756791204, + "geometry": { + "rows": 64, + "history": 2048, + "window": 0 + }, + "full_output_matches_frozen_canonical": true, + "raw_records": [ + { + "round": 0, + "leg": 0, + "variant": 1, + "repeats": 8, + "gpu_seconds": 0.04459441639482975, + "wall_seconds": 0.044812, + "warm_gpu_seconds": 0.04455345869064331 + }, + { + "round": 0, + "leg": 1, + "variant": 0, + "repeats": 8, + "gpu_seconds": 0.04537637485191226, + "wall_seconds": 0.045591, + "warm_gpu_seconds": 0.04538462497293949 + }, + { + "round": 0, + "leg": 2, + "variant": 0, + "repeats": 8, + "gpu_seconds": 0.04544829158112407, + "wall_seconds": 0.045658, + "warm_gpu_seconds": 0.04558699997141957 + }, + { + "round": 0, + "leg": 3, + "variant": 1, + "repeats": 8, + "gpu_seconds": 0.04456920828670263, + "wall_seconds": 0.044781, + "warm_gpu_seconds": 0.04432191653177142 + }, + { + "round": 1, + "leg": 4, + "variant": 1, + "repeats": 8, + "gpu_seconds": 0.0443487917073071, + "wall_seconds": 0.044552, + "warm_gpu_seconds": 0.04441529139876366 + }, + { + "round": 1, + "leg": 5, + "variant": 0, + "repeats": 8, + "gpu_seconds": 0.045413083396852016, + "wall_seconds": 0.045623, + "warm_gpu_seconds": 0.04539479175582528 + }, + { + "round": 1, + "leg": 6, + "variant": 0, + "repeats": 8, + "gpu_seconds": 0.04542900016531348, + "wall_seconds": 0.045647, + "warm_gpu_seconds": 0.04542329162359238 + }, + { + "round": 1, + "leg": 7, + "variant": 1, + "repeats": 8, + "gpu_seconds": 0.044418541714549065, + "wall_seconds": 0.044629, + "warm_gpu_seconds": 0.04462050041183829 + }, + { + "round": 2, + "leg": 8, + "variant": 1, + "repeats": 8, + "gpu_seconds": 0.04459858359768987, + "wall_seconds": 0.044811, + "warm_gpu_seconds": 0.04444362502545118 + }, + { + "round": 2, + "leg": 9, + "variant": 0, + "repeats": 8, + "gpu_seconds": 0.04538512462750077, + "wall_seconds": 0.045597, + "warm_gpu_seconds": 0.045675250235944986 + }, + { + "round": 2, + "leg": 10, + "variant": 0, + "repeats": 8, + "gpu_seconds": 0.04550558375194669, + "wall_seconds": 0.045723, + "warm_gpu_seconds": 0.045425708405673504 + }, + { + "round": 2, + "leg": 11, + "variant": 1, + "repeats": 8, + "gpu_seconds": 0.04436862515285611, + "wall_seconds": 0.044585, + "warm_gpu_seconds": 0.044558833353221416 + }, + { + "round": 3, + "leg": 12, + "variant": 1, + "repeats": 8, + "gpu_seconds": 0.04449574975296855, + "wall_seconds": 0.044707, + "warm_gpu_seconds": 0.0445182085968554 + }, + { + "round": 3, + "leg": 13, + "variant": 0, + "repeats": 8, + "gpu_seconds": 0.0455546248704195, + "wall_seconds": 0.04577, + "warm_gpu_seconds": 0.04532674979418516 + }, + { + "round": 3, + "leg": 14, + "variant": 0, + "repeats": 8, + "gpu_seconds": 0.04551945813000202, + "wall_seconds": 0.04573, + "warm_gpu_seconds": 0.04546125000342727 + }, + { + "round": 3, + "leg": 15, + "variant": 1, + "repeats": 8, + "gpu_seconds": 0.04451375035569072, + "wall_seconds": 0.044724, + "warm_gpu_seconds": 0.044589916709810495 + }, + { + "round": 4, + "leg": 16, + "variant": 1, + "repeats": 8, + "gpu_seconds": 0.04451524978503585, + "wall_seconds": 0.044733, + "warm_gpu_seconds": 0.044402250088751316 + }, + { + "round": 4, + "leg": 17, + "variant": 0, + "repeats": 8, + "gpu_seconds": 0.045428124722093344, + "wall_seconds": 0.045637, + "warm_gpu_seconds": 0.04541187500581145 + }, + { + "round": 4, + "leg": 18, + "variant": 0, + "repeats": 8, + "gpu_seconds": 0.04541624989360571, + "wall_seconds": 0.045642, + "warm_gpu_seconds": 0.0455421251244843 + }, + { + "round": 4, + "leg": 19, + "variant": 1, + "repeats": 8, + "gpu_seconds": 0.04442345816642046, + "wall_seconds": 0.044643, + "warm_gpu_seconds": 0.04442120809108019 + } + ], + "analysis_sha256": "385ebccaacdc63ccd455f87fcdb1f50a6f9cf69609487ee51de4fa6fd7def83e" + }, + { + "case": "h8192-r64-w0", + "stable": true, + "baseline_first_last_drift": 0.0027091003804535774, + "baseline_within_round_spreads": [ + 0.0009845777467132333, + 0.0057089602158091335, + 0.0045751819192068055, + 0.014541111915817195, + 0.013068006608485394 + ], + "round_candidate_over_public_throughput": [ + 1.1252255285291899, + 1.1189308038566097, + 1.122841759973785, + 1.1278408129252813, + 1.1364275184827335 + ], + "median_candidate_over_public_throughput": 1.1252255285291899, + "public_seconds_per_op": 0.003703492198837921, + "pair_seconds_per_op": 0.0032891718728933483, + "geometry": { + "rows": 64, + "history": 8192, + "window": 0 + }, + "full_output_matches_frozen_canonical": true, + "raw_records": [ + { + "round": 0, + "leg": 0, + "variant": 1, + "repeats": 8, + "gpu_seconds": 0.026430791709572077, + "wall_seconds": 0.026641, + "warm_gpu_seconds": 0.05235574999824166 + }, + { + "round": 0, + "leg": 1, + "variant": 0, + "repeats": 8, + "gpu_seconds": 0.029623083770275116, + "wall_seconds": 0.029842, + "warm_gpu_seconds": 0.05940175009891391 + }, + { + "round": 0, + "leg": 2, + "variant": 0, + "repeats": 8, + "gpu_seconds": 0.02965224999934435, + "wall_seconds": 0.029875, + "warm_gpu_seconds": 0.059093999676406384 + }, + { + "round": 0, + "leg": 3, + "variant": 1, + "repeats": 8, + "gpu_seconds": 0.026247833389788866, + "wall_seconds": 0.026456, + "warm_gpu_seconds": 0.052706167101860046 + }, + { + "round": 1, + "leg": 4, + "variant": 1, + "repeats": 8, + "gpu_seconds": 0.02627137489616871, + "wall_seconds": 0.026489, + "warm_gpu_seconds": 0.053468083031475544 + }, + { + "round": 1, + "leg": 5, + "variant": 0, + "repeats": 8, + "gpu_seconds": 0.02971287490800023, + "wall_seconds": 0.029926, + "warm_gpu_seconds": 0.05965212546288967 + }, + { + "round": 1, + "leg": 6, + "variant": 0, + "repeats": 8, + "gpu_seconds": 0.029544208198785782, + "wall_seconds": 0.029758, + "warm_gpu_seconds": 0.05920074973255396 + }, + { + "round": 1, + "leg": 7, + "variant": 1, + "repeats": 8, + "gpu_seconds": 0.02668729145079851, + "wall_seconds": 0.026907, + "warm_gpu_seconds": 0.05246133357286453 + }, + { + "round": 2, + "leg": 8, + "variant": 1, + "repeats": 8, + "gpu_seconds": 0.026162500027567148, + "wall_seconds": 0.026381, + "warm_gpu_seconds": 0.05277195805683732 + }, + { + "round": 2, + "leg": 9, + "variant": 0, + "repeats": 8, + "gpu_seconds": 0.02949783345684409, + "wall_seconds": 0.029718, + "warm_gpu_seconds": 0.059847208205610514 + }, + { + "round": 2, + "leg": 10, + "variant": 0, + "repeats": 8, + "gpu_seconds": 0.02963279141113162, + "wall_seconds": 0.029845, + "warm_gpu_seconds": 0.05959666660055518 + }, + { + "round": 2, + "leg": 11, + "variant": 1, + "repeats": 8, + "gpu_seconds": 0.026499083265662193, + "wall_seconds": 0.026714, + "warm_gpu_seconds": 0.05233533354476094 + }, + { + "round": 3, + "leg": 12, + "variant": 1, + "repeats": 8, + "gpu_seconds": 0.026355375070124865, + "wall_seconds": 0.026566, + "warm_gpu_seconds": 0.05321375001221895 + }, + { + "round": 3, + "leg": 13, + "variant": 0, + "repeats": 8, + "gpu_seconds": 0.029652458149939775, + "wall_seconds": 0.02986, + "warm_gpu_seconds": 0.059119750279933214 + }, + { + "round": 3, + "leg": 14, + "variant": 0, + "repeats": 8, + "gpu_seconds": 0.029227458406239748, + "wall_seconds": 0.029442, + "warm_gpu_seconds": 0.05924620805308223 + }, + { + "round": 3, + "leg": 15, + "variant": 1, + "repeats": 8, + "gpu_seconds": 0.025850499980151653, + "wall_seconds": 0.026063, + "warm_gpu_seconds": 0.05283687496557832 + }, + { + "round": 4, + "leg": 16, + "variant": 1, + "repeats": 8, + "gpu_seconds": 0.026503500062972307, + "wall_seconds": 0.026718, + "warm_gpu_seconds": 0.05225620837882161 + }, + { + "round": 4, + "leg": 17, + "variant": 0, + "repeats": 8, + "gpu_seconds": 0.029525041580200195, + "wall_seconds": 0.029738, + "warm_gpu_seconds": 0.05905879195779562 + }, + { + "round": 4, + "leg": 18, + "variant": 0, + "repeats": 8, + "gpu_seconds": 0.029910875018686056, + "wall_seconds": 0.030126, + "warm_gpu_seconds": 0.05888850009068847 + }, + { + "round": 4, + "leg": 19, + "variant": 1, + "repeats": 8, + "gpu_seconds": 0.025797166395932436, + "wall_seconds": 0.026001, + "warm_gpu_seconds": 0.052467874716967344 + } + ], + "analysis_sha256": "fe857afa6c5349fb9b6b09dc1611ea1cbd4f356025a386e0e1ec5ea7a7480151" + } + ], + "evidence": { + "baseline_source_sha256": "087e7bd23450117b626fb467d6750c46ae74c85ed06f41d7bee025de7617fba0", + "private_candidate_source_sha256": "8014cc1904ac440b4445e0444b46b445cc65662e9de73af2dc76719606ca1f3e", + "remote_manifest_sha256": "217152ba10beeca10bf199faed254eb962346ecbea63ff3a7350dc29e61c4114", + "stage_receipt_sha256": { + "metadata": "deb335a1b3d447e7ea5426040fc68a7d4e75df99194ffb1004d5b646fa17b059", + "smoke": "3a5618a6ff7b83bfd6f6c36bcbe354668ca6eed4bb0df18ae9c42126a8952aaa", + "positive": "d7cc88eaec51a261ac4cf92074d111d18396fe466f9025406f874f269d34183f", + "controls": "7226594720b4938df851b473ed50bda9959af52e0d831842238880d7b3cb4965", + "cost": "54b426df3b7e0c1c2cd1828ee730858f4ffa975f2c0b4f8b91285d7ac7d34223" + } + }, + "remaining": [ + "Full public kernel contract suite", + "M1-M4 and other backend qualification not established by this M5 receipt", + "Broader image, checkpoint-width, cancellation and concurrent-stream baseline/candidate validation", + "Broader served contexts, image requests,6/8-bit checkpoints, decode/ITL and MLX comparisons" + ], + "reproduce": [ + "python3 tools/check_lfm3_risk.py", + "zig build native tf-kernel-contract dev-tools -Dcpu=apple_m1 -Dlfm3-risk-fast=true", + "zig-out/bin/tf-kernel-contract --only risk-fast" + ], + "os": { + "version": "macOS 27.0", + "build": "26A428" + }, + "runtime_metal_options": { + "language": "4.1", + "math_mode": "safe", + "fp32_functions": "fast", + "exp": "metal::precise::exp" + }, + "build_validation": { + "zig": "0.17.0", + "cpu_target": "apple_m1", + "native": "pass", + "kernel_contract_binary": "pass", + "all_developer_tools": "pass", + "exit_code": 0, + "source_linked_cpu_proof": "pass", + "integrated_flag_off_on_air_static_counts_match_measured_sources": true, + "lean_baseline_problems": 428, + "lean_current_problems": 428, + "lean_new_problems": 0, + "diff_check": "pass", + "full_host_suite": "not run", + "full_kernel_contract_suite": "not run", + "targeted_integrated_contract": "pass on Apple M5 Ultra", + "host_contract_golden_fuzz_subset": "pass; details in risk-fused-fullmodel.json" + }, + "targeted_integrated_contract": { + "source_commit": "de7cf96ed", + "device": "Apple M5 Ultra", + "architecture": "applegpu_g17", + "exit_code": 0, + "full_shapes": 7, + "sliding_shapes": 10, + "nonfinite_outputs_each": 512, + "finite_outputs_each": 15872, + "repeats": 5, + "summary": "SUMMARY 2 kernels: 2 pass gates 1-4, 0 fail, 0 known findings; 0 controls: 0 red as planted, 0 missed; 0 limit problems; gates 1033 ms, limits 6323 ms", + "accumulation_caveat": "Nonlinear BF16 nodes do not guarantee a 100x reduced-accumulator witness; full case reports FP16 0x/BF16 0.4x, sliding FP16 0x/BF16 0.1x", + "launch_records_checked": 361, + "scope": "Targeted full/sliding replacements and launch-limit checks; no complete kernel-suite claim; the two mutation controls are recorded separately in private operator qualification", + "log_sha256": "35e5a0c0e68c2f0a0c72b66f72f29c2636fabb3ee6027b6d060b7f1d7f218c2e", + "manifest_sha256": "5b6223276399163e9c626634639a1e13a162de3edf7e31cd296702ebb7e6d508" + }, + "fullmodel_receipt": "risk-fused-fullmodel.json", + "serving_receipt": "risk-fused-serving.json", + "width_image_receipt": "risk-fused-width-image.json", + "text_widths_receipt": "risk-fused-text-widths.json", + "m4_receipt": "risk-fused-m4.json" +} diff --git a/docs/receipts/lfm3/risk-fused-fullmodel.json b/docs/receipts/lfm3/risk-fused-fullmodel.json new file mode 100644 index 0000000000..e7ce59121c --- /dev/null +++ b/docs/receipts/lfm3/risk-fused-fullmodel.json @@ -0,0 +1,136 @@ +{ + "date": "2026-10-30", + "status": "2k/8k checkpoint equivalence and isolated served TTFT improvement passed; remains default-off pending broader qualification", + "device": "Apple M5 Ultra", + "os": "macOS27.0 build26A428", + "backend": "Metal", + "source_commit": "de7cf96ed", + "checkpoint": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "revision": "c823fade98e35a3d6942ad4d38b1ea15f675cd12", + "files_sha256": { + "model.safetensors": "58268dfdf441ae4020d2a52fc97a814645e8a9624946e2d490403d4dcd71a255", + "config.json": "0efb7708e0771e3e104ac628954cdc71070dd436e0b7974615bd1b9eb836f9d9", + "tokenizer.json": "a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7" + } + }, + "scope": "Isolated fused attention with original prompt-head and projection behavior; text prompts through the VL4 checkpoint; no image or served timing claim", + "comparison": "One loaded model with only four attention pipelines changed between baseline macro0 and candidate macro1; all other model operations and weights identical", + "checks": [ + "Full active KV bytes per head and convolution state after every prompt chunk", + "Complete last-row logits after every prompt chunk", + "Snapshot bytes at129 and257 tokens", + "Restore257-token snapshot and finish prompt with full state/logit comparison", + "16greedy decode tokens with full state/logit comparison after each token" + ], + "cases": [ + { + "prompt_tokens": 2048, + "compared_prompt_chunks": 17, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 597, + 8055, + 509, + 5901, + 4116, + 355, + 267, + 1286, + 4160, + 302, + 278, + 7126, + 496, + 73, + 10, + 10601 + ], + "vision_rows": false, + "prompt_u32_npy_sha256": "82c7dac5126bf3f7bf789f2a7385214044ad7ace0d151c52935e139ad99bcf2b", + "result_sha256": "c478ff8b01fc09bd8ff940e838a3e60cab72fcaae008a21f2fd1527d655b4e9a", + "log_sha256": "52fc171b440729618ce26a4f89e0c0c9366f3bb6ae3b1f9e408e46f73e6b120a" + }, + { + "prompt_tokens": 8192, + "compared_prompt_chunks": 65, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 597, + 4695, + 677, + 4116, + 267, + 1442, + 1286, + 4160, + 302, + 496, + 3862, + 10, + 8426, + 20, + 643, + 7204 + ], + "vision_rows": false, + "prompt_u32_npy_sha256": "b371a7a09adfed00451d2aeba2dc64f5851e7ee62136dddcd8f77e1e07673581", + "result_sha256": "fdb1dec6589d628cb277e464776fa417d9cf1b4987389a82cbf0b19c9e1b7813", + "log_sha256": "fbfbc76691049c0e915ffffb39d1217062b355959ac07a0e72b6cc1d707e36f5" + } + ], + "negative_control": { + "mutation": "Actual candidate Metal final attention stores write zero", + "exit_code": 1, + "required_error": "AttentionFusedByteMismatch", + "log_sha256": "ad9d7a7a025bfccf06104d2e48d5323f7d0ba2aa86039cc47cb71925b2311dd5" + }, + "evidence": { + "manifest_sha256": "f23d7b9803b722f2456aff2c30ebc2840eb6c32b58e83652ada2cf0f55e0d4e2", + "probe_sha256": "1e4053893a30190b24e27c9dffba24f32ab98179c56059b27ee263a914e803ae", + "probe_source_sha256": "7d571c502e44eb75915b66b32018d7aac7ab0a231bad546dd0ac0b3888677cc2", + "baseline_shader_sha256": "760ea921822580f7973070fa915808570cbb27214017e5ebe14a5e844faedbfe", + "candidate_shader_sha256": "193a16770a6da1085fbde3247d26da7fa22a3ac4f0f937d9233ea0ffedfb11b3", + "stage_receipts_sha256": { + "smoke": "9e65e09d37c2e194afd7d03a56fa5a231ff65534b3a75a3916a461926a5049a5", + "positive": "145a3b965a80c578b73993fcce15758161bf301b7a85b5f991794abaa08dca82", + "controls": "6fa09e258a9436276ef7c764f56703deda9fd901445ccad9a74f4da8f77a1e0e" + } + }, + "host_subset": { + "command": "zig build test-kernel-contract test-golden fuzz-cortheon -Dcpu=apple_m1 -Dlfm3-risk-fast=true", + "source_snapshot": "de7cf96ed plus a private qualifier target registration", + "exit_code": 0, + "golden": { + "equal": 322, + "total": 335, + "known_differences": 13, + "unexpected_differences": 0 + }, + "security_golden": { + "equal": 26, + "total": 26, + "unexpected_differences": 0 + }, + "log_sha256": "3e154251cc4ab661aa9cc158f2e87a5cccd389d469d68d3529741150febe5f07", + "scope": "Host contract, golden and Cortheon fuzz targets; not the full zig build test suite" + }, + "remaining": [ + "Fresh image-to-decoder inputs", + "Live cancellation and concurrent-stream baseline/candidate comparisons", + "Full host and device test suites", + "Other GPU generations and backends" + ], + "serving_receipt": "risk-fused-serving.json", + "width_image_receipt": "risk-fused-width-image.json", + "text_widths_receipt": "risk-fused-text-widths.json", + "m4_receipt": "risk-fused-m4.json" +} diff --git a/docs/receipts/lfm3/risk-fused-m4.json b/docs/receipts/lfm3/risk-fused-m4.json new file mode 100644 index 0000000000..35a34cf2c5 --- /dev/null +++ b/docs/receipts/lfm3/risk-fused-m4.json @@ -0,0 +1,294 @@ +{ + "date": "2026-10-30", + "status": "Measured isolated M4 served TTFT improvement and full-model baseline equivalence; candidate remains default-off", + "device": "Apple M4 Pro", + "physical_memory_bytes": 51539607552, + "os": "macOS26.5.1 build25F80", + "source_commit": "de7cf96ed", + "checkpoint": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "revision": "c823fade98e35a3d6942ad4d38b1ea15f675cd12" + }, + "scope": "VL4 text prompts, cold prefix2k/8k,16 generated tokens; native baseline versus fused attention only; M4 non-NAX path", + "build": { + "version_string": "tensorfold-native1.1.6", + "compiler": "Zig0.17.0", + "cpu_target": "apple_m1", + "baseline_option": "-Dlfm3-risk-fast=false", + "candidate_option": "-Dlfm3-risk-fast=true", + "original_prompt_head_behavior_both": true, + "projection_candidate_both": false, + "baseline_sha256": "c01f10e0872f80b67ae2125a852d8179c058e9c95f6fbda8ad36e38f594484d8", + "candidate_sha256": "165c31f5ae6d085f8c6ec57fa6c0b2ad2d9737cf8256fb41148e58a203d7e351" + }, + "protocol": { + "order": [ + "baseline", + "candidate", + "candidate", + "baseline" + ], + "measurements_per_variant_and_prompt": 2, + "request_order_each_leg": [ + 2048, + 8192 + ], + "fresh_server_per_leg": true, + "fresh_KV_per_request": true, + "cached_tokens_every_request": 0, + "prompt_cache_gib": 0, + "context": 70000, + "configured_parallel": 4, + "active_measured_requests": 1, + "temperature": 0, + "thinking": false, + "output_tokens_every_request": 16, + "prompt_ids_exactly_verified_by_tokenize": true, + "output_text_digests_equal": true, + "output_token_ids": "Not emitted by this cold client; same exact histories separately passed state/logit/16-token qualification", + "predeclared_baseline_drift_limit": 0.05, + "hidden_warmups": 0, + "timing_retries": 0, + "client_python": "Python 3.9.6", + "client_sha256": "7b8ea55aa88834814f95a6f679654e16074b7a8364e34b39a276f009fe0231f2", + "all_owned_servers_exit": 0 + }, + "cases": [ + { + "prompt_tokens": 2048, + "baseline_ttft_s": [ + 3.385673459, + 3.384928958 + ], + "candidate_ttft_s": [ + 3.3500699579999997, + 3.347492209 + ], + "baseline_median_s": 3.3853012085, + "candidate_median_s": 3.3487810834999996, + "baseline_over_candidate": 1.0109054978780014, + "baseline_first_last_drift": -0.00021989746176520608, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "not emitted by cold client; separate model qualification covers16 IDs/logits", + "prompt_ids": "both routes /tokenize exact pinned arrays", + "ttft_saved_ms": 36.52012500000046, + "ttft_fraction_lower": 0.010787850991901049 + }, + { + "prompt_tokens": 8192, + "baseline_ttft_s": [ + 17.139636916, + 17.139102082999997 + ], + "candidate_ttft_s": [ + 16.666613792, + 16.663699875 + ], + "baseline_median_s": 17.1393694995, + "candidate_median_s": 16.6651568335, + "baseline_over_candidate": 1.0284553377287602, + "baseline_first_last_drift": -3.12044533162803e-05, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "not emitted by cold client; separate model qualification covers16 IDs/logits", + "prompt_ids": "both routes /tokenize exact pinned arrays", + "ttft_saved_ms": 474.2126659999997, + "ttft_fraction_lower": 0.02766803446380184 + } + ], + "memory_legs": [ + { + "leg": 0, + "variant": "baseline", + "max_rss_bytes": 1836318720, + "peak_physical_footprint_bytes": 2172422208 + }, + { + "leg": 1, + "variant": "candidate", + "max_rss_bytes": 1836302336, + "peak_physical_footprint_bytes": 2172831880 + }, + { + "leg": 2, + "variant": "candidate", + "max_rss_bytes": 1836138496, + "peak_physical_footprint_bytes": 2172569760 + }, + { + "leg": 3, + "variant": "baseline", + "max_rss_bytes": 1836236800, + "peak_physical_footprint_bytes": 2172405896 + } + ], + "evidence": { + "raw_records_sha256": "c6dbbd8c16bb0d7387ec13f8ad2683827830fa56e3e16645defc88289be909c5", + "run_manifest_sha256": "6cd8366d1c1b4f17dd9a0d19e4dd8abe83c3476673b6c5aa6a9563c5a5380795", + "events_sha256": "2950cde41d0e51a7ce9b0b41d1bb1e203d2eafc2b9094ed8b24fcedc365362d4", + "config_sha256": "75c52200e57ab68b97912dd9b2150f76e8acd95d2511a48611c4d0c8d6af7b70", + "pins_sha256": "263f956db60d9443f5e528e5e4fcbf51b63997bf98ad37b99156fe694568e6c3", + "raw_files_sha256": { + "vl-4-2-candidate.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "vl-4-0-public.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "vl-4-0-public.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "vl-4-2-candidate.cold-2048.json": "79c2e59e901f9c797dd0bdf69e951e3fa71f18071402447dfe4a82305f26f3ea", + "vl-4-1-candidate.after.metrics": "819edaa78fdc2c58b2afcb52c1dc2129eba9585834dc90f63e481ea32695380d", + "vl-4-1-candidate.before.health": "3ecca835aef6168623e5fd09af1a553676471c6c3d76176c12322725ac4e2881", + "vl-4-3-public.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "vl-4-2-candidate.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "vl-4-0-public.before.metrics": "cef6cd5d59d65c9b027f88fac90fd28584ba69f3301b20fd7566298c1ca9d9c8", + "vl-4-2-candidate.cold-8192.json": "21b474da076f3f27d60fdf3ae93753593c21dec44a4be7e802396fcd2a7ed40d", + "vl-4-1-candidate.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "vl-4-1-candidate.server.log": "fd566f89a1f93b5531686b6bd3aabeb5f66cd9574aec91346f0230edb058658d", + "vl-4-2-candidate.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "vl-4-2-candidate.cold-2048.client.log": "e8784c8a5e3c7cb5a1d858e655a4e3a1f761ada16f0ae941eaae63374c5289b2", + "vl-4-3-public.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "events.json": "2950cde41d0e51a7ce9b0b41d1bb1e203d2eafc2b9094ed8b24fcedc365362d4", + "vl-4-2-candidate.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "vl-4-0-public.cold-2048.client.log": "98a281a571b665381c1538d0d6d03c373c36099f8eab13849bec22d4c8b7ed01", + "vl-4-0-public.after.health": "3c5e807cf2e0ae94385da5880ca71a15ef41499b1926bb71f08ee83760dca585", + "vl-4-3-public.after.metrics": "64f4e6cc771c66ccc9d5523093b5f83dbb265aec42227c037bbdc70415b05f90", + "vl-4-3-public.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "vl-4-0-public.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "vl-4-3-public.before.health": "3ecca835aef6168623e5fd09af1a553676471c6c3d76176c12322725ac4e2881", + "vl-4-0-public.cold-8192.json": "562f972349f9e880b6c229991f9c2f1d1e58468e08b5870da06bf5c390d51fcb", + "vl-4-0-public.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "vl-4-1-candidate.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "vl-4-3-public.after.health": "3c5e807cf2e0ae94385da5880ca71a15ef41499b1926bb71f08ee83760dca585", + "vl-4-1-candidate.cold-8192.client.log": "8f3cca94a803a16f684517edd7bf049d8155cabaa464a6660edb1754d0753d23", + "vl-4-2-candidate.before.health": "3ecca835aef6168623e5fd09af1a553676471c6c3d76176c12322725ac4e2881", + "vl-4-2-candidate.after.metrics": "22372c28c4f9842e588b386ad79e6bf8e1bc38bc806dc01875d9770d4c3d5384", + "vl-4-1-candidate.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "vl-4-3-public.cold-8192.client.log": "431757bbd4a301146bd98beb2a649d808c0839b38599a8ef4da7f78b8126fc0f", + "vl-4-2-candidate.before.metrics": "97957533feedf4919e21f881260237524d38a3e0e480c89661b0179180a4c8c8", + "vl-4-2-candidate.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "vl-4-0-public.cold-2048.json": "47bac048e687b828425db97350b5d810f1b667a5b0d3789cc3f378aa96ea175a", + "vl-4-1-candidate.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "vl-4-3-public.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "vl-4-2-candidate.cold-8192.client.log": "525046f838962b9902da5e1b7ec5256bdf92547c550c864c125ff147efc44ef8", + "vl-4-1-candidate.after.health": "c2470ce8a87fa19e5fd7e74fd83156d15382abeb1fb85542e8e8545109266730", + "vl-4-1-candidate.cold-8192.json": "c637028f47f1f58282e569048719d18b7b5204649ce59041f70040f6eccc4f1a", + "vl-4-3-public.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "vl-4-0-public.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "vl-4-0-public.cold-8192.client.log": "860f41a6242ad40867e114261f93f074061f3d6d847cbd9bcefa0ec0e4c1f970", + "vl-4-1-candidate.cold-2048.json": "c1d992dc58448bf305f9068d4461ff27f4507283631ccb9050bbb0ba6b31e3ad", + "vl-4-3-public.before.metrics": "74ac71d53e103ec893ab9237dfea2b25668cfadc1796c6dfef9c38bd13af2a13", + "vl-4-0-public.server.log": "0a18ba222629efa9d9800e4a53919bf2a246b33cbcb6dce30e519405269522b7", + "vl-4-0-public.after.metrics": "1ff28b9f2f859901bceb3b132114c00f5a321c466ea2c91dccf844b8138e9911", + "manifest.json": "6cd8366d1c1b4f17dd9a0d19e4dd8abe83c3476673b6c5aa6a9563c5a5380795", + "vl-4-0-public.before.health": "3ecca835aef6168623e5fd09af1a553676471c6c3d76176c12322725ac4e2881", + "vl-4-1-candidate.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "vl-4-2-candidate.after.health": "c2470ce8a87fa19e5fd7e74fd83156d15382abeb1fb85542e8e8545109266730", + "records.json": "c6dbbd8c16bb0d7387ec13f8ad2683827830fa56e3e16645defc88289be909c5", + "vl-4-3-public.cold-2048.json": "5873b501f0ae665272e0ac2a5ea21a8200b9d6bfce7749ab1e422b2debe75bfe", + "vl-4-1-candidate.before.metrics": "754ccde7360a9bc95b10d1e48e6f5ce72a87e79ca38c6eeac352dd784b6c840d", + "vl-4-3-public.cold-2048.client.log": "2110be127144ab06abb48d97a46a80964fa04c28a309190131a58453f6b6eaff", + "vl-4-3-public.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "vl-4-3-public.server.log": "c18d98af8362d0f53e496f721a83cc38da911682d5e9e5ed70af503473a7b33f", + "vl-4-2-candidate.server.log": "6f1ad71c23929d68d6acdd62e8534511b3b7b3bea9c51ffb7a3e2ea4a4848f53", + "vl-4-1-candidate.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "vl-4-2-candidate.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "vl-4-0-public.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "vl-4-1-candidate.cold-2048.client.log": "c817f59c86cc3301269b237c09136de3c3b657532cf00e8ff8578a91ddff067c", + "vl-4-3-public.cold-8192.json": "b0a8730631e347cc085279f6c6373ea2415f61a0a37e72ee62ed5f753c561a8d" + } + }, + "limitations": [ + "One M4 Pro with48GiB RAM and VL4 checkpoint; two ABBA legs per variant and prompt", + "No new decode-rate or ITL measurement", + "No MLX performance comparison in this experiment", + "No M4 image-request or6/8-bit qualification in this experiment", + "No full-suite or all-platform qualification claim" + ], + "numerical_qualification": { + "comparison": "One loaded model with only four attention pipelines changed between baseline macro0 and candidate macro1; all other model operations and weights identical", + "checks": [ + "Full active KV bytes per head and convolution state after every prompt chunk", + "Complete last-row logits after every prompt chunk", + "Snapshot bytes at129 and257 tokens", + "Restore257-token snapshot and finish prompt with full state/logit comparison", + "16greedy decode tokens with full state/logit comparison after each token" + ], + "cases": [ + { + "prompt_tokens": 2048, + "compared_prompt_chunks": 17, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 597, + 8055, + 509, + 5901, + 4116, + 355, + 267, + 1286, + 4160, + 302, + 278, + 7126, + 496, + 73, + 10, + 10601 + ], + "vision_rows": false, + "scope": "real checkpoint differential correctness, not served timing or cancellation callback execution", + "result_sha256": "c478ff8b01fc09bd8ff940e838a3e60cab72fcaae008a21f2fd1527d655b4e9a", + "log_sha256": "52fc171b440729618ce26a4f89e0c0c9366f3bb6ae3b1f9e408e46f73e6b120a", + "receipt_sha256": "867f4c062f0721c20a02180850ba0bc2058ad25a67d758af962fe75a2eac83f0" + }, + { + "prompt_tokens": 8192, + "compared_prompt_chunks": 65, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 597, + 4695, + 677, + 4116, + 267, + 1442, + 1286, + 4160, + 302, + 496, + 3862, + 10, + 8426, + 20, + 643, + 7204 + ], + "vision_rows": false, + "scope": "real checkpoint differential correctness, not served timing or cancellation callback execution", + "result_sha256": "fdb1dec6589d628cb277e464776fa417d9cf1b4987389a82cbf0b19c9e1b7813", + "log_sha256": "fbfbc76691049c0e915ffffb39d1217062b355959ac07a0e72b6cc1d707e36f5", + "receipt_sha256": "afc4af4472f35ead2884b768395a67e33891b2d21ba8a313a451145a01730a6e" + } + ], + "negative_control": { + "mutation": "Actual candidate Metal final attention stores write zero", + "exit_code": 1, + "required_error": "AttentionFusedByteMismatch", + "log_sha256": "ad9d7a7a025bfccf06104d2e48d5323f7d0ba2aa86039cc47cb71925b2311dd5", + "receipt_sha256": "4c722fa32661cf6be986d54055c83b5f9c3c19e71ab8c080aa55ff331e5c9700" + }, + "manifest_sha256": "d610bf79f0c02428a12ba8fd2ea3110610a2d7738e5ca0d8381c0d487263bbf0", + "probe_sha256": "1e4053893a30190b24e27c9dffba24f32ab98179c56059b27ee263a914e803ae", + "probe_source_sha256": "7d571c502e44eb75915b66b32018d7aac7ab0a231bad546dd0ac0b3888677cc2", + "baseline_shader_sha256": "760ea921822580f7973070fa915808570cbb27214017e5ebe14a5e844faedbfe", + "candidate_shader_sha256": "193a16770a6da1085fbde3247d26da7fa22a3ac4f0f937d9233ea0ffedfb11b3", + "scope": "Exact baseline equivalence on this checkpoint; no new independent FP64 fidelity qualification" + } +} diff --git a/docs/receipts/lfm3/risk-fused-serving.json b/docs/receipts/lfm3/risk-fused-serving.json new file mode 100644 index 0000000000..d48a5ea0f3 --- /dev/null +++ b/docs/receipts/lfm3/risk-fused-serving.json @@ -0,0 +1,206 @@ +{ + "date": "2026-10-30", + "status": "Measured isolated native served TTFT improvement; candidate remains default-off", + "device": "Apple M5 Ultra", + "physical_memory_bytes": 274877906944, + "os": "macOS27.0 build26A428", + "source_commit": "de7cf96ed", + "checkpoint": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "revision": "c823fade98e35a3d6942ad4d38b1ea15f675cd12" + }, + "scope": "VL4 text prompts, cold prefix2k/8k,16 generated tokens; native baseline versus fused attention only", + "build": { + "version_string": "tensorfold-native1.1.6", + "compiler": "Zig0.17.0", + "cpu_target": "apple_m1", + "baseline_option": "-Dlfm3-risk-fast=false", + "candidate_option": "-Dlfm3-risk-fast=true", + "original_prompt_head_behavior_both": true, + "projection_candidate_both": false, + "baseline_sha256": "c01f10e0872f80b67ae2125a852d8179c058e9c95f6fbda8ad36e38f594484d8", + "candidate_sha256": "165c31f5ae6d085f8c6ec57fa6c0b2ad2d9737cf8256fb41148e58a203d7e351" + }, + "protocol": { + "order": [ + "baseline", + "candidate", + "candidate", + "baseline" + ], + "measurements_per_variant_and_prompt": 2, + "request_order_each_leg": [ + 2048, + 8192 + ], + "fresh_server_per_leg": true, + "fresh_KV_per_request": true, + "cached_tokens_every_request": 0, + "prompt_cache_gib": 0, + "context": 70000, + "configured_parallel": 4, + "active_measured_requests": 1, + "temperature": 0, + "thinking": false, + "output_tokens_every_request": 16, + "prompt_ids_exactly_verified_by_tokenize": true, + "output_text_digests_equal": true, + "output_token_ids": "Not emitted by this cold client; same exact histories separately passed state/logit/16-token qualification", + "predeclared_baseline_drift_limit": 0.05, + "hidden_warmups": 0, + "timing_retries": 0, + "client_python": "Python 3.9.6", + "client_sha256": "7b8ea55aa88834814f95a6f679654e16074b7a8364e34b39a276f009fe0231f2", + "all_owned_servers_exit": 0 + }, + "cases": [ + { + "prompt_tokens": 2048, + "baseline_ttft_s": [ + 0.859373208, + 0.858855417 + ], + "candidate_ttft_s": [ + 0.8477165, + 0.8471230000000001 + ], + "baseline_median_s": 0.8591143125, + "candidate_median_s": 0.84741975, + "baseline_over_candidate": 1.0138002005499636, + "baseline_first_last_drift": -0.0006025216927638688, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "not emitted by cold client; separate model qualification covers16 IDs/logits", + "prompt_ids": "both routes /tokenize exact pinned arrays", + "ttft_saved_ms": 11.694562500000005, + "ttft_fraction_lower": 0.01361234742553541 + }, + { + "prompt_tokens": 8192, + "baseline_ttft_s": [ + 4.64294925, + 4.651438291 + ], + "candidate_ttft_s": [ + 4.429717459, + 4.4373175 + ], + "baseline_median_s": 4.6471937704999995, + "candidate_median_s": 4.4335174795, + "baseline_over_candidate": 1.0481956577340703, + "baseline_first_last_drift": 0.0018283725586705213, + "stable_at_5percent": true, + "output_text_sha256_equal": true, + "output_token_id_equality": "not emitted by cold client; separate model qualification covers16 IDs/logits", + "prompt_ids": "both routes /tokenize exact pinned arrays", + "ttft_saved_ms": 213.67629099999962, + "ttft_fraction_lower": 0.04597963879974165 + } + ], + "memory_legs": [ + { + "leg": 0, + "variant": "baseline", + "max_rss_bytes": 1835204608, + "peak_physical_footprint_bytes": 2290419872 + }, + { + "leg": 1, + "variant": "candidate", + "max_rss_bytes": 1834844160, + "peak_physical_footprint_bytes": 2290010200 + }, + { + "leg": 2, + "variant": "candidate", + "max_rss_bytes": 1834172416, + "peak_physical_footprint_bytes": 2289272992 + }, + { + "leg": 3, + "variant": "baseline", + "max_rss_bytes": 1834188800, + "peak_physical_footprint_bytes": 2289240224 + } + ], + "evidence": { + "raw_records_sha256": "e53f81fe6f66a553aee90c4ae5665e296fc7cbcca7ff2f85c0a961cfd7432bdf", + "run_manifest_sha256": "a1b8d919deb72f85fdaad1999b9cd2827710ede921d97252e371506f68fd2e9d", + "events_sha256": "4084128466f1c6bb6acbda2bff84305e6df75e426143c8c1446afda8992c3d61", + "config_sha256": "80080d9e087f91cacc1399406e5de168eed094dd8d7e94946bb5308438469bb1", + "pins_sha256": "0962da3cfceef95cdce20f624dbdc9c5e10b37a80b32f21d75741a1ea2e955f8", + "raw_files_sha256": { + "vl-4-2-candidate.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "vl-4-0-public.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "vl-4-0-public.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "vl-4-2-candidate.cold-2048.json": "720954418d49adfbb9acf1a776cbc6baa885924063931dcf344f739da7000e5e", + "vl-4-1-candidate.after.metrics": "f01a6bda582a1604bc512e27e0f6a380838c96f0077b93ed6bdda44655f202cd", + "vl-4-1-candidate.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "vl-4-3-public.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "vl-4-2-candidate.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "vl-4-0-public.before.metrics": "ac28361cba697650f05014545c8a633e520dc3f6a2eb17d737268d1f7b11b30f", + "vl-4-2-candidate.cold-8192.json": "4cbf61f0e861a7e1d85287d94da5b58809497a672e73b9872bef21b738f7e406", + "vl-4-1-candidate.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "vl-4-1-candidate.server.log": "7d411bbb95a004f4866ede7041abdf7e632390fb418a9e18c1fe861d076f3202", + "vl-4-2-candidate.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "vl-4-2-candidate.cold-2048.client.log": "129bd17ec574c422bfe55dc47701277bd7aa01cc89716940bdfa4e1e2ed2b610", + "vl-4-3-public.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "events.json": "4084128466f1c6bb6acbda2bff84305e6df75e426143c8c1446afda8992c3d61", + "vl-4-2-candidate.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "vl-4-0-public.cold-2048.client.log": "a2d4b52e0e549c65ee98f42d01179ab0c38253177230045611f660dac84707c9", + "vl-4-0-public.after.health": "65fb8a4a434e9aa79532c77b77fda785189afb47bb0294b9e032b47a7122a348", + "vl-4-3-public.after.metrics": "d484760218ed144c17b316b2b768ce118b3032ab965ea900c8f2a8993b0c6c03", + "vl-4-3-public.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "vl-4-0-public.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "vl-4-3-public.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "vl-4-0-public.cold-8192.json": "fd70fa19f5868d0cbc632c1a7ff0df66bbce88a3e7a776fa7207fab02b0fafb5", + "vl-4-0-public.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "vl-4-1-candidate.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "vl-4-3-public.after.health": "15f6c2e761e03054eaacf6f3f96dd6dcf92dc017bf69c64992ecc799654fde96", + "vl-4-1-candidate.cold-8192.client.log": "ce37e19db16c13943f051a1feabf38beaf1397ae0d6f2b8d25ec3ec9d24702cf", + "vl-4-2-candidate.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "vl-4-2-candidate.after.metrics": "e3d44b7caee031537101aa4d4cdfe711ffa265a0f089fe613b5ba16e3a768cb0", + "vl-4-1-candidate.cold-8192.tokenize.response.json": "1e924ba32cf41b050291eaf02707b996b9f656171f816c15c24c7045a92600ee", + "vl-4-3-public.cold-8192.client.log": "695745c9e4fe677ca5f73d38188eda01133742ceee03feccc8ad84bdd4a1f5c0", + "vl-4-2-candidate.before.metrics": "270dace984bc1c0d9c5cb22be0035387cfef71838f417ccde6498c44e0346a60", + "vl-4-2-candidate.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "vl-4-0-public.cold-2048.json": "04b347278fad621cda9c7c2ea76ef551f02eda82b9d57d0ddbd6fdcc570ea82e", + "vl-4-1-candidate.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "vl-4-3-public.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "vl-4-2-candidate.cold-8192.client.log": "e64806903d1ae7e913b4180d34c2eb90c8fedac3e900a9caaddae6a887e9b992", + "vl-4-1-candidate.after.health": "a56f21d7db6293b2888cdab808f66d2374eafb371cf7c0dbf03360893cfe908d", + "vl-4-1-candidate.cold-8192.json": "c379378a462eaaf065b5dfbe04fa412d42a92943d1c7145bf87491750d2ea16b", + "vl-4-3-public.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "vl-4-0-public.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "vl-4-0-public.cold-8192.client.log": "73d45f64359fbb729d8ca48cdf1f6394e90c03e486d47f95b38bc601491bfe57", + "vl-4-1-candidate.cold-2048.json": "2c78c0119ed630475040fc3b5be66307b6d12d4d7d1de10a596782267cb43ab7", + "vl-4-3-public.before.metrics": "2eadf0d05e2cd61b6145e306310ae428db02a8a1c5f00da6695850c6d445d5d9", + "vl-4-0-public.server.log": "adc4bf07279c1be7de25b69e5a9375dc054fb2249e52d1f52740b0d354771c78", + "vl-4-0-public.after.metrics": "7c31794b9d0d5727d94266546d9813517c800685e0b889e1c3b1ed7ded6b27fe", + "manifest.json": "a1b8d919deb72f85fdaad1999b9cd2827710ede921d97252e371506f68fd2e9d", + "vl-4-0-public.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "vl-4-1-candidate.cold-8192.tokenize.request.json": "c196be253a6a7f7928e3f427e0faad10300dc3216279a3ae8a80a9cd46f77dab", + "vl-4-2-candidate.after.health": "a2bd00c0acf0aa418f4a4946e811171ea36f6e1509d17b84c4eab4ac2da82b22", + "records.json": "e53f81fe6f66a553aee90c4ae5665e296fc7cbcca7ff2f85c0a961cfd7432bdf", + "vl-4-3-public.cold-2048.json": "b2431f60e1d4d57259f5fd34ac430eff16a8d01a2d65d8b94528881a6b0a65ea", + "vl-4-1-candidate.before.metrics": "48cb06efdf74ae5b84f3f2b7cf9dfbb03e6b4230c9aec27d0dcdf59fec53943b", + "vl-4-3-public.cold-2048.client.log": "8bff0ba7f5b944a87935f5a31c731e2851fcb2fbcd8528655400b655dedb205e", + "vl-4-3-public.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "vl-4-3-public.server.log": "af7ee13b3228606211afe2b200b1bb68b1a10a1430bc5a37f573242bb2c286ef", + "vl-4-2-candidate.server.log": "ae044ccce5c82ea2a5c4bf751394f4a561d99680f587222d513d62757764f3cf", + "vl-4-1-candidate.cold-2048.tokenize.request.json": "f69a5dcb97950c19f12fd93b89b288ec0dd214a89ed83fbe5fb1e60270f6dabd", + "vl-4-2-candidate.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "vl-4-0-public.cold-2048.tokenize.response.json": "7f59448440e4d3f003e40ef5773b12b7ef17c6808bc21ea16cb1bf3807ad06f6", + "vl-4-1-candidate.cold-2048.client.log": "5ffa60f6b57d3ea1aec484258d48af8ca0ac86b486de9fd06fafd794f5c29d22", + "vl-4-3-public.cold-8192.json": "b0be53215dbfb55676555774c88fbff1bd74ecada8ed9168e3fdae1a0ceec7b4" + } + }, + "limitations": [ + "One M5 Ultra and VL4 checkpoint; two ABBA legs per variant and prompt", + "No new decode-rate or ITL measurement", + "No MLX performance comparison in this experiment", + "No image-request serving timing or6/8-bit serving timing", + "No full-suite or all-platform qualification claim" + ], + "m4_receipt": "risk-fused-m4.json" +} diff --git a/docs/receipts/lfm3/risk-fused-text-widths.json b/docs/receipts/lfm3/risk-fused-text-widths.json new file mode 100644 index 0000000000..6e0d164ea8 --- /dev/null +++ b/docs/receipts/lfm3/risk-fused-text-widths.json @@ -0,0 +1,281 @@ +{ + "date": "2026-10-30", + "status": "All six LFM3-1.7B text4/6/8 baseline-equivalence cases passed; three kernel faults rejected", + "device": "Apple M5 Ultra", + "os": "macOS27.0 build26A428", + "source_commit": "de7cf96ed", + "scope": "2k/8k exact text histories through LFM3-1.7B checkpoints at4/6/8 bits", + "comparison": "Fused attention versus original attention within each checkpoint; original prompt-head and projection behavior retained", + "fidelity_scope": "Baseline equivalence only; no new independent FP64 fidelity result, tie policy or ledger change", + "checkpoints": { + "LFM3-1.7B-MLX-4bit": { + "repository": "LiquidAI/LFM3-1.7B-MLX-4bit", + "revision": "d0ca73dcfa70ce18733754a492cdc1fb0123262d", + "files": { + "model.safetensors": "1be5bf672013cd91aa62148f77c5eea8d2784a95845d5fe826e08c7b84823815", + "config.json": "7df2ea819f9a31b770192a0a9dc1b7953a7dd8c41b816cc3905bbec99ef332bd", + "tokenizer.json": "41927bd314d17d7d7fadb44e10cd0919f7b71528d7741596d8cbd15c6945ec44" + } + }, + "LFM3-1.7B-MLX-6bit": { + "repository": "LiquidAI/LFM3-1.7B-MLX-6bit", + "revision": "26be898c36b545d625d84ac0761c6c3eb664aef3", + "files": { + "model.safetensors": "503d68675793fba64907842e176d066c8c3adca22e14c8b3538f9ffb76bd5e0e", + "config.json": "2b5879bf6d1d99b43e17726c7ee3fb4868e4c2e29c4a49784803df90f494fc2e", + "tokenizer.json": "41927bd314d17d7d7fadb44e10cd0919f7b71528d7741596d8cbd15c6945ec44" + } + }, + "LFM3-1.7B-MLX-8bit": { + "repository": "LiquidAI/LFM3-1.7B-MLX-8bit", + "revision": "ee7667f3cfccb2a41725f91820531823fcabc6b7", + "files": { + "model.safetensors": "2b06cea98d75527c248a0b04383f7cf0a1c72fb4ed7b45096618fc4e59d015ad", + "config.json": "4ef95e289e7105a09bf010b3ceaa015d3f01163104c039a4e7948739d5b9035b", + "tokenizer.json": "41927bd314d17d7d7fadb44e10cd0919f7b71528d7741596d8cbd15c6945ec44" + } + } + }, + "checks": [ + "Full active KV and convolution state after each prompt chunk", + "Full last-row logits after each prompt chunk", + "Snapshot bytes at129 and257 tokens", + "Restore257-token snapshot and finish with full state/logit equality", + "16greedy decode tokens plus state/logits each step" + ], + "cases": [ + { + "prompt_tokens": 2048, + "compared_prompt_chunks": 17, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 597, + 8055, + 509, + 4116, + 9543, + 11004, + 302, + 278, + 2936, + 2837, + 496, + 73, + 10, + 10601, + 890, + 309 + ], + "vision_rows": false, + "case": "text4-2048", + "result_sha256": "06044f9099be682e34015cd3978c891b48817b4c20846e12bf94bb3d63991ae1" + }, + { + "prompt_tokens": 8192, + "compared_prompt_chunks": 65, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 597, + 4695, + 4116, + 267, + 1442, + 1286, + 4160, + 302, + 10601, + 496, + 73, + 10, + 8426, + 20, + 643, + 7204 + ], + "vision_rows": false, + "case": "text4-8192", + "result_sha256": "56e25597aa103d53dfea5ca3c718b4a38fc27dd48d42dc71ea907932ffeb2262" + }, + { + "prompt_tokens": 2048, + "compared_prompt_chunks": 17, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 597, + 8055, + 509, + 18304, + 355, + 267, + 1286, + 4160, + 302, + 278, + 2936, + 7126, + 7575, + 73, + 962, + 10601 + ], + "vision_rows": false, + "case": "text6-2048", + "result_sha256": "06d91573f9ddccb9a29e5d784ceaaa33989557bcbd067edaddaa813cbedfff6e" + }, + { + "prompt_tokens": 8192, + "compared_prompt_chunks": 65, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 597, + 8055, + 509, + 5901, + 4116, + 9543, + 11004, + 302, + 278, + 2936, + 2837, + 496, + 73, + 10, + 10601, + 890 + ], + "vision_rows": false, + "case": "text6-8192", + "result_sha256": "e3ef945f323aef02de3933bc0d1810a68b2e92dd95ad0dbe705af85ab43faebb" + }, + { + "prompt_tokens": 2048, + "compared_prompt_chunks": 17, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 597, + 8055, + 509, + 5901, + 4116, + 9543, + 11004, + 302, + 278, + 2936, + 2837, + 496, + 73, + 10, + 10601, + 890 + ], + "vision_rows": false, + "case": "text8-2048", + "result_sha256": "01fe6c8cad183c70775e243224485fe0fdd62baf3280c195f1697e6f1e086ba0" + }, + { + "prompt_tokens": 8192, + "compared_prompt_chunks": 65, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 597, + 8055, + 509, + 5901, + 4116, + 355, + 267, + 1286, + 4160, + 302, + 278, + 7126, + 89889, + 73, + 90645, + 10601 + ], + "vision_rows": false, + "case": "text8-8192", + "result_sha256": "bc70501214e17ea78c0a34398a4be073ae962669745a30d2bef301f0e0359bd0" + } + ], + "controls": [ + { + "checkpoint_bits": 4, + "prompt_tokens": 2048, + "mutation": "Candidate final attention stores write zero", + "exit_code": 1, + "required_error": "AttentionFusedByteMismatch", + "log_sha256": "ad9d7a7a025bfccf06104d2e48d5323f7d0ba2aa86039cc47cb71925b2311dd5" + }, + { + "checkpoint_bits": 6, + "prompt_tokens": 2048, + "mutation": "Candidate final attention stores write zero", + "exit_code": 1, + "required_error": "AttentionFusedByteMismatch", + "log_sha256": "ad9d7a7a025bfccf06104d2e48d5323f7d0ba2aa86039cc47cb71925b2311dd5" + }, + { + "checkpoint_bits": 8, + "prompt_tokens": 2048, + "mutation": "Candidate final attention stores write zero", + "exit_code": 1, + "required_error": "AttentionFusedByteMismatch", + "log_sha256": "ad9d7a7a025bfccf06104d2e48d5323f7d0ba2aa86039cc47cb71925b2311dd5" + } + ], + "evidence": { + "remote_manifest_sha256": "a7f4fd40241922b7f873a8b961c2c05ed71f6a1ae2d64da4dba5b76beeec5cb2", + "probe_sha256": "1e4053893a30190b24e27c9dffba24f32ab98179c56059b27ee263a914e803ae", + "probe_source_sha256": "7d571c502e44eb75915b66b32018d7aac7ab0a231bad546dd0ac0b3888677cc2", + "baseline_shader_sha256": "760ea921822580f7973070fa915808570cbb27214017e5ebe14a5e844faedbfe", + "candidate_shader_sha256": "193a16770a6da1085fbde3247d26da7fa22a3ac4f0f937d9233ea0ffedfb11b3", + "stage_receipts_sha256": { + "smoke": "4d71db1fe5fdb7b20e2a4d51f2595f13b0f38450dde2d2139756cd6da81a8ff7", + "positive": "b181649c76b0b30e52059b00812609d3443c0f59b07f2c42619ad76c9a4d74ec", + "controls": "205b76de0b3bf69217ec586b255791a127383c823eb8b951ef060adbe5625cf1" + } + }, + "combined_coverage": { + "text_and_VL_checkpoints": 6, + "text_history_cases_at_2k_and_8k": 12, + "corrected_projected_image_cases": 3, + "other_receipts": [ + "risk-fused-fullmodel.json", + "risk-fused-width-image.json" + ] + }, + "remaining": [ + "M4 fused-attention numerical/performance qualification", + "Longer contexts, full suites and other GPU generations", + "Fresh image-to-decoder and HTTP image requests", + "Independent FP64 difference ledger remains unchanged" + ] +} diff --git a/docs/receipts/lfm3/risk-fused-width-image.json b/docs/receipts/lfm3/risk-fused-width-image.json new file mode 100644 index 0000000000..02f0c3eb4a --- /dev/null +++ b/docs/receipts/lfm3/risk-fused-width-image.json @@ -0,0 +1,324 @@ +{ + "date": "2026-10-30", + "status": "Seven VL baseline-equivalence cases passed; three actual-kernel faults rejected", + "device": "Apple M5 Ultra", + "os": "macOS27.0 build26A428", + "source_commit": "de7cf96ed", + "scope": "VL6/8 text histories at2k/8k and VL4/6/8 decoder inputs containing70 corrected real-image projected rows in512-token prompts", + "comparison": "Fused attention versus original attention within each checkpoint; original prompt-head/projection behavior retained", + "fidelity_scope": "Baseline equivalence only. This does not add an independent FP64 decoder-fidelity result or change the existing FP64 difference ledger. Saved image-row provenance qualifies the inputs, not a new vision-tower execution.", + "checkpoints": { + "LFM3-VL-2B-MLX-4bit": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "revision": "c823fade98e35a3d6942ad4d38b1ea15f675cd12", + "files": { + "model.safetensors": "58268dfdf441ae4020d2a52fc97a814645e8a9624946e2d490403d4dcd71a255", + "config.json": "0efb7708e0771e3e104ac628954cdc71070dd436e0b7974615bd1b9eb836f9d9", + "tokenizer.json": "a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7" + } + }, + "LFM3-VL-2B-MLX-6bit": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-6bit", + "revision": "6d71bfae121abedc75f6d531fc7f6ceb4f5e3501", + "files": { + "model.safetensors": "838c39147541b12e86a4f1b43d6a7814d7fb4efe9d5ff8a151c85ac06f321eef", + "config.json": "4c03aa298372cbcb478eb08be0a4a59be917b4949756066ce4eac06033d200bc", + "tokenizer.json": "a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7" + } + }, + "LFM3-VL-2B-MLX-8bit": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-8bit", + "revision": "83336d5be3874a2bdd3297bde77b2114dbd3a816", + "files": { + "model.safetensors": "50c9476a7f00f5349e81b42ab88b47635b9eba3c7ab5851c598d270cda4c6857", + "config.json": "9f7dc06d04d0610c7e27fb6a110c42a3a09b686a6ec53b0eb383f30a8be5a457", + "tokenizer.json": "a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7" + } + } + }, + "image_inputs": { + "input_png_sha256": "005500792ee8ea6f35ac369d900fc788cd77667f145e7100c2cff57fae5355d9", + "corrected_CPU_patches_sha256": "9d39c11551f105b8982e3a9acceffcdf687d360c17a7931b7fff21757a6e5473", + "prompt_tokens": 512, + "image_rows": 70, + "image_width": 2048, + "image_start": 4, + "prompt_sha256": "57111edcf1dd6adb846c9dedc3f398be5c8e19223cc4e75846d28296dc55e40f", + "projected_rows_sha256": { + "4": "ffa0401f452f39a07833d6692ac9eff0d5485cc93a14fb3433bdad74c42ab05e", + "6": "0bfbdb4d6e8f3485609f0b6e8cf4b84bd8daba4c2ae62f56a4861aad3688b4df", + "8": "0bfbdb4d6e8f3485609f0b6e8cf4b84bd8daba4c2ae62f56a4861aad3688b4df" + }, + "derivation": "Original90-token real cat prompt plus first422 publicVL2k text IDs;70 placeholders at4..73;512total tokens", + "provenance_sha256": { + "independent-vision6.json": "cd65b824e33a1e20bd635c4f7500819b36e4c6702135d3a75e409943cdb3fc13", + "vision68-identity.json": "6e28b34d17c1b5ff57811e1375bf8292fdbe18b2ddeeebe78d12191c64ebab98", + "corrected-capture.json": "967400283a8ceeadd4d45f91c5dc7f85a18ec58af79020b7391621d069d7c6e0", + "independent-vision4.json": "98556f6663ca894b3f185e7873f145fb199f76d32a24bf62b6a5db6023b35bd3" + } + }, + "checks": [ + "Active KV per head and every convolution state after each prompt chunk", + "Full last-row logits after each prompt chunk", + "Snapshot bytes at129 and257 tokens", + "Restore257-token snapshot and compare final state/logits", + "16greedy decode tokens and full state/logits after each token" + ], + "cases": [ + { + "prompt_tokens": 512, + "compared_prompt_chunks": 5, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018 + ], + "vision_rows": true, + "case": "vl4-image512", + "result_sha256": "7614841c497e11aadd1832316fbc086eff5c182501884a5245ce4761d52cf852" + }, + { + "prompt_tokens": 512, + "compared_prompt_chunks": 5, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018 + ], + "vision_rows": true, + "case": "vl6-image512", + "result_sha256": "7614841c497e11aadd1832316fbc086eff5c182501884a5245ce4761d52cf852" + }, + { + "prompt_tokens": 2048, + "compared_prompt_chunks": 17, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 597, + 8055, + 509, + 5901, + 4116, + 355, + 267, + 1286, + 4160, + 302, + 278, + 7126, + 496, + 73, + 10, + 10601 + ], + "vision_rows": false, + "case": "vl6-text2048", + "result_sha256": "c478ff8b01fc09bd8ff940e838a3e60cab72fcaae008a21f2fd1527d655b4e9a" + }, + { + "prompt_tokens": 8192, + "compared_prompt_chunks": 65, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 597, + 4695, + 677, + 4116, + 267, + 1442, + 1286, + 4160, + 302, + 496, + 3862, + 10, + 8426, + 20, + 643, + 7204 + ], + "vision_rows": false, + "case": "vl6-text8192", + "result_sha256": "fdb1dec6589d628cb277e464776fa417d9cf1b4987389a82cbf0b19c9e1b7813" + }, + { + "prompt_tokens": 512, + "compared_prompt_chunks": 5, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018 + ], + "vision_rows": true, + "case": "vl8-image512", + "result_sha256": "7614841c497e11aadd1832316fbc086eff5c182501884a5245ce4761d52cf852" + }, + { + "prompt_tokens": 2048, + "compared_prompt_chunks": 17, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 597, + 8055, + 509, + 5901, + 4116, + 355, + 267, + 1286, + 4160, + 302, + 278, + 7126, + 496, + 73, + 10, + 10601 + ], + "vision_rows": false, + "case": "vl8-text2048", + "result_sha256": "c478ff8b01fc09bd8ff940e838a3e60cab72fcaae008a21f2fd1527d655b4e9a" + }, + { + "prompt_tokens": 8192, + "compared_prompt_chunks": 65, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 597, + 4695, + 677, + 4116, + 267, + 1442, + 1286, + 4160, + 302, + 496, + 3862, + 10, + 8426, + 20, + 643, + 355 + ], + "vision_rows": false, + "case": "vl8-text8192", + "result_sha256": "b091c42d1fec9422d3aade3c2951678002585775e326ad843618fdcdb9cc0b6b" + } + ], + "controls": [ + { + "checkpoint_bits": 4, + "prompt": "image512", + "mutation": "Candidate final attention stores write zero", + "exit_code": 1, + "required_error": "AttentionFusedByteMismatch", + "log_sha256": "ad9d7a7a025bfccf06104d2e48d5323f7d0ba2aa86039cc47cb71925b2311dd5" + }, + { + "checkpoint_bits": 6, + "prompt": "image512", + "mutation": "Candidate final attention stores write zero", + "exit_code": 1, + "required_error": "AttentionFusedByteMismatch", + "log_sha256": "ad9d7a7a025bfccf06104d2e48d5323f7d0ba2aa86039cc47cb71925b2311dd5" + }, + { + "checkpoint_bits": 8, + "prompt": "image512", + "mutation": "Candidate final attention stores write zero", + "exit_code": 1, + "required_error": "AttentionFusedByteMismatch", + "log_sha256": "ad9d7a7a025bfccf06104d2e48d5323f7d0ba2aa86039cc47cb71925b2311dd5" + } + ], + "evidence": { + "remote_manifest_sha256": "8a1657e6f700c97b347e92409b70a63c289ea057d20cc2bdf5a8bd324b83e54d", + "probe_sha256": "1e4053893a30190b24e27c9dffba24f32ab98179c56059b27ee263a914e803ae", + "probe_source_sha256": "7d571c502e44eb75915b66b32018d7aac7ab0a231bad546dd0ac0b3888677cc2", + "baseline_shader_sha256": "760ea921822580f7973070fa915808570cbb27214017e5ebe14a5e844faedbfe", + "candidate_shader_sha256": "193a16770a6da1085fbde3247d26da7fa22a3ac4f0f937d9233ea0ffedfb11b3", + "fault_shader_sha256": "c37fac732d17c22408f627d1b8b66f55415751275a15c28079414e0a726ec593", + "stage_receipts_sha256": { + "smoke": "2177e51514cdb786b6831f8d99c64f10f6722d21d82354c394535cdc830edef6", + "positive": "87da1f092392477e224640ba4f37567b9b89cffa96b8031cdc4fe48645486a57", + "controls": "ff1299861e602c5f464447ef409729e92444f409b45ce29d02f894e0711909c5" + } + }, + "remaining": [ + "Fresh image-to-decoder and HTTP image-request checks", + "Full test suites, other GPUs and longer contexts", + "Existing independent FP64 fidelity ledger remains unchanged" + ], + "text_widths_receipt": "risk-fused-text-widths.json" +} diff --git a/docs/receipts/lfm3/rounded-fp64-prompt.json b/docs/receipts/lfm3/rounded-fp64-prompt.json new file mode 100644 index 0000000000..621c9ba07b --- /dev/null +++ b/docs/receipts/lfm3/rounded-fp64-prompt.json @@ -0,0 +1,366 @@ +{ + "scope": "Actual FP32 affine projection reduction replay in independent FP64 model; remaining nonprojection operations retain FP64 arithmetic", + "accepted_threshold": null, + "fidelity_pass": null, + "completed_cells": 17, + "requested_cells": 17, + "cells": { + "_trimmed_list": { + "count": 17, + "sha256": "8bb3ec2b8b1c39cf4d6a8cdb5db46a11727a80eecccf9a3ce65c1bad90c2a35b", + "first": [ + { + "rows": 257, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.017635787525519895, + "maximum_absolute_error": 1.125, + "differences": [ + { + "row": 58, + "native_top": 3599, + "reference_top": 2292, + "native_maximum_set": [ + 3599 + ], + "reference_maximum_set": [ + 2292, + 3599 + ], + "maximum_sets_overlap": true, + "native_margin": 0.375, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.375, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": { + "_trimmed_list": { + "count": 20, + "sha256": "5a86cf6604a7c4820b258549a1fed6a2a230a4d275b12837871e9b3e2e15be5a", + "first": [ + { + "row": 202, + "rms_error": 0.257782799033294, + "maximum_absolute_error": 0.9453125 + }, + { + "row": 250, + "rms_error": 0.18972153068496678, + "maximum_absolute_error": 1.0 + } + ], + "last": [ + { + "row": 77, + "rms_error": 0.06715550854939922, + "maximum_absolute_error": 0.3125 + } + ] + } + }, + "worst_columns": { + "_trimmed_list": { + "count": 20, + "sha256": "95935644dfc56383705db80daf05ef42c27c6221c884d4feed101b7c78c2277f", + "first": [ + { + "column": 38520, + "rms_error": 0.11248981312110265, + "maximum_absolute_error": 1.0 + }, + { + "column": 90344, + "rms_error": 0.11157782703431807, + "maximum_absolute_error": 1.0 + } + ], + "last": [ + { + "column": 118692, + "rms_error": 0.09592147692708017, + "maximum_absolute_error": 0.921875 + } + ] + } + }, + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "1.7B-6-prose-257", + "phase": "prompt", + "seconds": 127.02277824981138, + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "2fb12160d203b843337c07b5a3e5b7a1f35de658857c3e06d3827c77dac1d001", + "rounded_reference_logits_sha256": "7b94f202ddc70106c25dcc260f29466e6bfea00ba29b61273603d26b87663c15", + "token_fixture_sha256": "522ec0f7a7ef8481d2f5533e3962ddad1f418d4337b095282e40c096e2befb1d", + "original_ideal_fp64_differences": [ + { + "row": 58, + "native_top": 3599, + "reference_top": 2292, + "native_maximum_set": [ + 3599 + ], + "reference_maximum_set": [ + 2292, + 3599 + ], + "maximum_sets_overlap": true, + "native_margin": 0.375, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.375, + "reference_native_contender_gap": 0.0, + "rounded_reference_top": 2292, + "rounded_reference_margin": 0.0, + "rounded_reference_native_contender_gap": 0.0, + "rounded_reference_maximum_set": [ + 2292, + 3599 + ] + } + ] + }, + { + "rows": 481, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.0210872789742184, + "maximum_absolute_error": 2.53125, + "differences": [ + { + "row": 58, + "native_top": 3599, + "reference_top": 2292, + "native_maximum_set": [ + 3599 + ], + "reference_maximum_set": [ + 2292, + 3599 + ], + "maximum_sets_overlap": true, + "native_margin": 0.375, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.375, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": { + "_trimmed_list": { + "count": 20, + "sha256": "b452508813d8630bb7cd078eb9a613f10117251f344f888cc7498f7912457eb1", + "first": [ + { + "row": 353, + "rms_error": 0.4063506040364455, + "maximum_absolute_error": 2.125 + }, + { + "row": 329, + "rms_error": 0.3718739029875104, + "maximum_absolute_error": 1.46875 + } + ], + "last": [ + { + "row": 161, + "rms_error": 0.08692003525411107, + "maximum_absolute_error": 0.40625 + } + ] + } + }, + "worst_columns": { + "_trimmed_list": { + "count": 20, + "sha256": "f52ebd1320933a9733b771ac2220ed505fd3f2154006ed62ef52170d3925e1a3", + "first": [ + { + "column": 42654, + "rms_error": 0.22104044060462807, + "maximum_absolute_error": 2.53125 + }, + { + "column": 17944, + "rms_error": 0.2194537635352515, + "maximum_absolute_error": 2.5 + } + ], + "last": [ + { + "column": 19982, + "rms_error": 0.15688888747079888, + "maximum_absolute_error": 1.728515625 + } + ] + } + }, + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "1.7B-6-prose-513", + "phase": "prompt", + "seconds": 233.82947916723788, + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "b6c07fed3bdccf311a1a46b1b2dc3a5f3de069d0b0a433d679a25b41cfa15968", + "rounded_reference_logits_sha256": "2a0d494528a8b0aee3249fe63f2281d9d3758f07dc24083144f7bbe7d5fd5620", + "token_fixture_sha256": "4b9e533cc1948ee72c4231178ec3af21808d7542c97f35997bb9538161d9a4cb", + "original_ideal_fp64_differences": [ + { + "row": 58, + "native_top": 3599, + "reference_top": 2292, + "native_maximum_set": [ + 3599 + ], + "reference_maximum_set": [ + 2292, + 3599 + ], + "maximum_sets_overlap": true, + "native_margin": 0.375, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.375, + "reference_native_contender_gap": 0.0, + "rounded_reference_top": 2292, + "rounded_reference_margin": 0.0, + "rounded_reference_native_contender_gap": 0.0, + "rounded_reference_maximum_set": [ + 2292, + 3599 + ] + } + ] + } + ], + "last": [ + { + "rows": 513, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.008981459929561184, + "maximum_absolute_error": 0.625, + "differences": [], + "worst_rows": { + "_trimmed_list": { + "count": 20, + "sha256": "a7a40d034a242ca883206eaecc1f63da34794bad17fecc4aed0a70e5bbfb4a54", + "first": [ + { + "row": 453, + "rms_error": 0.12919133095517948, + "maximum_absolute_error": 0.28125 + }, + { + "row": 336, + "rms_error": 0.11239795215540833, + "maximum_absolute_error": 0.28125 + } + ], + "last": [ + { + "row": 51, + "rms_error": 0.07024464365859565, + "maximum_absolute_error": 0.21875 + } + ] + } + }, + "worst_columns": { + "_trimmed_list": { + "count": 20, + "sha256": "51846d93d89ed07491cd331b241041f05c152499b921567be30331cecad91c65", + "first": [ + { + "column": 334, + "rms_error": 0.08306270849169323, + "maximum_absolute_error": 0.5 + }, + { + "column": 1840, + "rms_error": 0.07130993129891984, + "maximum_absolute_error": 0.34375 + } + ], + "last": [ + { + "column": 5435, + "rms_error": 0.06190424021952325, + "maximum_absolute_error": 0.375 + } + ] + } + }, + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-8-reasoning-513", + "phase": "prompt", + "seconds": 193.77772895805538, + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "0fbe7f9ff161bf9b0961bd4d64b081f775dc51bb8eb72e7c1bcef7ac9bcbd582", + "rounded_reference_logits_sha256": "499abbaa05178bc8aa6375bd17848fb1e472f8267a6cc62e74bb826b7f47afe6", + "token_fixture_sha256": "c8ce58a8d512375ea4635bc85d0ea8a235506aa56026087bda216ea61357a863", + "original_ideal_fp64_differences": [ + { + "row": 5, + "native_top": 440, + "reference_top": 334, + "native_maximum_set": [ + 440 + ], + "reference_maximum_set": [ + 334, + 440 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0, + "rounded_reference_top": 440, + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": 0.0625, + "rounded_reference_maximum_set": [ + 440 + ] + }, + { + "row": 33, + "native_top": 1579, + "reference_top": 2925, + "native_maximum_set": [ + 1579, + 2925 + ], + "reference_maximum_set": [ + 2925 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625, + "rounded_reference_top": 1579, + "rounded_reference_margin": 0.0, + "rounded_reference_native_contender_gap": 0.0, + "rounded_reference_maximum_set": [ + 1579, + 2925 + ] + } + ] + } + ] + } + }, + "rows": 5581, + "top1_differences": 22, + "disjoint_maximum_sets": 3, + "_trim": { + "note": "Raw capture replaced by counts, first/last items and SHA-256 of the canonical JSON of each trimmed list or string.", + "original_sha256": "91511ba60ebe10b69b8c65a09a109b65d8c174b3cd7cd87a8551f9f7cb96a7b5", + "original_bytes": 141593 + } +} diff --git a/docs/receipts/lfm3/rounded-fp64-resolution.json b/docs/receipts/lfm3/rounded-fp64-resolution.json new file mode 100644 index 0000000000..365963fb89 --- /dev/null +++ b/docs/receipts/lfm3/rounded-fp64-resolution.json @@ -0,0 +1,847 @@ +{ + "scope": "Recorded ideal-FP64 differences are retained separately; independent reference applies only named production rounding sites, with no tolerance or automatic acceptance", + "affine_only": { + "cells": 17, + "rows": 5581, + "top1_differences": 22, + "disjoint_maximum_sets": 3 + }, + "guided_norm_conv_cases": [ + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.008043711960330414, + "maximum_absolute_error": 0.34375, + "differences": [ + { + "row": 6, + "native_top": 9798, + "reference_top": 9743, + "native_maximum_set": [ + 9798 + ], + "reference_maximum_set": [ + 9743, + 9798 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 51, + "rms_error": 0.11861646575515228, + "maximum_absolute_error": 0.34375 + }, + { + "row": 10, + "rms_error": 0.10028614776804698, + "maximum_absolute_error": 0.21875 + }, + { + "row": 57, + "rms_error": 0.0861529837832831, + "maximum_absolute_error": 0.21875 + }, + { + "row": 24, + "rms_error": 0.05687998224278724, + "maximum_absolute_error": 0.1875 + }, + { + "row": 48, + "rms_error": 0.05600190458937229, + "maximum_absolute_error": 0.1875 + }, + { + "row": 60, + "rms_error": 0.05498064187219183, + "maximum_absolute_error": 0.3125 + }, + { + "row": 32, + "rms_error": 0.05294731569054374, + "maximum_absolute_error": 0.21875 + }, + { + "row": 17, + "rms_error": 0.04935522540415026, + "maximum_absolute_error": 0.15625 + }, + { + "row": 50, + "rms_error": 0.0478751943152622, + "maximum_absolute_error": 0.171875 + }, + { + "row": 30, + "rms_error": 0.047753962581513175, + "maximum_absolute_error": 0.1953125 + }, + { + "row": 9, + "rms_error": 0.04555611920115846, + "maximum_absolute_error": 0.1875 + }, + { + "row": 31, + "rms_error": 0.045409813623385815, + "maximum_absolute_error": 0.1640625 + }, + { + "row": 56, + "rms_error": 0.04311155120797251, + "maximum_absolute_error": 0.203125 + }, + { + "row": 62, + "rms_error": 0.04123161695385987, + "maximum_absolute_error": 0.171875 + }, + { + "row": 45, + "rms_error": 0.04050624266366054, + "maximum_absolute_error": 0.1875 + }, + { + "row": 42, + "rms_error": 0.038526539946546666, + "maximum_absolute_error": 0.125 + }, + { + "row": 53, + "rms_error": 0.037666491313599185, + "maximum_absolute_error": 0.15625 + }, + { + "row": 63, + "rms_error": 0.03744615993758534, + "maximum_absolute_error": 0.125 + }, + { + "row": 44, + "rms_error": 0.037078881417759536, + "maximum_absolute_error": 0.203125 + }, + { + "row": 28, + "rms_error": 0.03621204994136106, + "maximum_absolute_error": 0.15625 + } + ], + "worst_columns": [ + { + "column": 7161, + "rms_error": 0.0739487202983055, + "maximum_absolute_error": 0.203125 + }, + { + "column": 229, + "rms_error": 0.07305314144244157, + "maximum_absolute_error": 0.1875 + }, + { + "column": 1840, + "rms_error": 0.07213394953111382, + "maximum_absolute_error": 0.3125 + }, + { + "column": 10494, + "rms_error": 0.07134923616253058, + "maximum_absolute_error": 0.28125 + }, + { + "column": 10625, + "rms_error": 0.07134766978366326, + "maximum_absolute_error": 0.2578125 + }, + { + "column": 3431, + "rms_error": 0.07027679916665594, + "maximum_absolute_error": 0.1875 + }, + { + "column": 15114, + "rms_error": 0.06613980864073528, + "maximum_absolute_error": 0.33984375 + }, + { + "column": 117113, + "rms_error": 0.06395546022850564, + "maximum_absolute_error": 0.1875 + }, + { + "column": 5037, + "rms_error": 0.0637027232793923, + "maximum_absolute_error": 0.296875 + }, + { + "column": 1703, + "rms_error": 0.06359664123111303, + "maximum_absolute_error": 0.1875 + }, + { + "column": 4827, + "rms_error": 0.06355588798115991, + "maximum_absolute_error": 0.234375 + }, + { + "column": 8187, + "rms_error": 0.06341566551278807, + "maximum_absolute_error": 0.3203125 + }, + { + "column": 42985, + "rms_error": 0.06277423398584353, + "maximum_absolute_error": 0.2109375 + }, + { + "column": 36483, + "rms_error": 0.06250661250613916, + "maximum_absolute_error": 0.15625 + }, + { + "column": 62333, + "rms_error": 0.06225591465536169, + "maximum_absolute_error": 0.1875 + }, + { + "column": 398, + "rms_error": 0.0621247416424273, + "maximum_absolute_error": 0.2734375 + }, + { + "column": 677, + "rms_error": 0.061515686515717274, + "maximum_absolute_error": 0.1875 + }, + { + "column": 117761, + "rms_error": 0.061515625957307814, + "maximum_absolute_error": 0.1875 + }, + { + "column": 24129, + "rms_error": 0.06147691693569196, + "maximum_absolute_error": 0.3125 + }, + { + "column": 43345, + "rms_error": 0.06129237335627735, + "maximum_absolute_error": 0.15625 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-reasoning-64", + "reference_sha256": "8f7df6375fb0a0f3a96fabe16ae7f798be48785534fcb5cc3218b8225279378e" + }, + { + "rows": 257, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.00890914680506212, + "maximum_absolute_error": 0.375, + "differences": [ + { + "row": 6, + "native_top": 9798, + "reference_top": 9743, + "native_maximum_set": [ + 9798 + ], + "reference_maximum_set": [ + 9743, + 9798 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 102, + "rms_error": 0.12569535229309456, + "maximum_absolute_error": 0.375 + }, + { + "row": 141, + "rms_error": 0.12195062473144336, + "maximum_absolute_error": 0.25 + }, + { + "row": 51, + "rms_error": 0.11861646575515228, + "maximum_absolute_error": 0.34375 + }, + { + "row": 10, + "rms_error": 0.10028614776804698, + "maximum_absolute_error": 0.21875 + }, + { + "row": 121, + "rms_error": 0.09380736244683188, + "maximum_absolute_error": 0.25 + }, + { + "row": 219, + "rms_error": 0.09166578124562276, + "maximum_absolute_error": 0.25 + }, + { + "row": 57, + "rms_error": 0.0861529837832831, + "maximum_absolute_error": 0.21875 + }, + { + "row": 96, + "rms_error": 0.07888112333375753, + "maximum_absolute_error": 0.287109375 + }, + { + "row": 133, + "rms_error": 0.07786581656174694, + "maximum_absolute_error": 0.19140625 + }, + { + "row": 122, + "rms_error": 0.07066079887865517, + "maximum_absolute_error": 0.20703125 + }, + { + "row": 87, + "rms_error": 0.0700584951337384, + "maximum_absolute_error": 0.1875 + }, + { + "row": 77, + "rms_error": 0.06858409301583214, + "maximum_absolute_error": 0.22265625 + }, + { + "row": 174, + "rms_error": 0.06369936770618398, + "maximum_absolute_error": 0.2109375 + }, + { + "row": 109, + "rms_error": 0.06126578238264121, + "maximum_absolute_error": 0.1875 + }, + { + "row": 148, + "rms_error": 0.06105379414796407, + "maximum_absolute_error": 0.25 + }, + { + "row": 107, + "rms_error": 0.06088625375806041, + "maximum_absolute_error": 0.232421875 + }, + { + "row": 159, + "rms_error": 0.060035975951836636, + "maximum_absolute_error": 0.34375 + }, + { + "row": 228, + "rms_error": 0.05946765702581071, + "maximum_absolute_error": 0.171875 + }, + { + "row": 180, + "rms_error": 0.05838313205777614, + "maximum_absolute_error": 0.1875 + }, + { + "row": 146, + "rms_error": 0.05802808873794389, + "maximum_absolute_error": 0.3125 + } + ], + "worst_columns": [ + { + "column": 1008, + "rms_error": 0.06943595438926299, + "maximum_absolute_error": 0.3125 + }, + { + "column": 1703, + "rms_error": 0.0689546629516174, + "maximum_absolute_error": 0.34375 + }, + { + "column": 677, + "rms_error": 0.06758341579884398, + "maximum_absolute_error": 0.25 + }, + { + "column": 10625, + "rms_error": 0.06617273514319813, + "maximum_absolute_error": 0.2578125 + }, + { + "column": 334, + "rms_error": 0.06563620020264926, + "maximum_absolute_error": 0.28125 + }, + { + "column": 1840, + "rms_error": 0.06536974854013013, + "maximum_absolute_error": 0.3125 + }, + { + "column": 229, + "rms_error": 0.06493757056972531, + "maximum_absolute_error": 0.1875 + }, + { + "column": 10494, + "rms_error": 0.0645074011981935, + "maximum_absolute_error": 0.28125 + }, + { + "column": 3498, + "rms_error": 0.0636772495717454, + "maximum_absolute_error": 0.25 + }, + { + "column": 9798, + "rms_error": 0.06304420037383449, + "maximum_absolute_error": 0.234375 + }, + { + "column": 50474, + "rms_error": 0.06263534483934208, + "maximum_absolute_error": 0.21875 + }, + { + "column": 3431, + "rms_error": 0.06230194018215942, + "maximum_absolute_error": 0.25 + }, + { + "column": 3464, + "rms_error": 0.06201285570379185, + "maximum_absolute_error": 0.21875 + }, + { + "column": 4475, + "rms_error": 0.06195830330667422, + "maximum_absolute_error": 0.21875 + }, + { + "column": 405, + "rms_error": 0.06172813829250489, + "maximum_absolute_error": 0.28125 + }, + { + "column": 278, + "rms_error": 0.06139696424144821, + "maximum_absolute_error": 0.205810546875 + }, + { + "column": 25248, + "rms_error": 0.061211164521044456, + "maximum_absolute_error": 0.25 + }, + { + "column": 20, + "rms_error": 0.06067311917866256, + "maximum_absolute_error": 0.1875 + }, + { + "column": 417, + "rms_error": 0.059608407993605596, + "maximum_absolute_error": 0.375 + }, + { + "column": 25234, + "rms_error": 0.05953700598964491, + "maximum_absolute_error": 0.21875 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-reasoning-257", + "reference_sha256": "76598d5e25b26caa3a49d87ab324926e3acf6714bebc1d049b215aabf0b3edd7", + "native_logits_sha256": "9be9b95af0ff0c00697f8737ce9271ccac0e6dcd519b56d49f3666623ae6fd2c" + }, + { + "rows": 513, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.009213169639915648, + "maximum_absolute_error": 0.383544921875, + "differences": [ + { + "row": 6, + "native_top": 9798, + "reference_top": 9743, + "native_maximum_set": [ + 9798 + ], + "reference_maximum_set": [ + 9743, + 9798 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 336, + "rms_error": 0.1881454590456434, + "maximum_absolute_error": 0.383544921875 + }, + { + "row": 102, + "rms_error": 0.12569535229309456, + "maximum_absolute_error": 0.375 + }, + { + "row": 141, + "rms_error": 0.12195062473144336, + "maximum_absolute_error": 0.25 + }, + { + "row": 51, + "rms_error": 0.11861646575515228, + "maximum_absolute_error": 0.34375 + }, + { + "row": 258, + "rms_error": 0.11009589234272762, + "maximum_absolute_error": 0.25 + }, + { + "row": 472, + "rms_error": 0.10046899842055106, + "maximum_absolute_error": 0.234375 + }, + { + "row": 10, + "rms_error": 0.10028614776804698, + "maximum_absolute_error": 0.21875 + }, + { + "row": 297, + "rms_error": 0.09552272607184104, + "maximum_absolute_error": 0.25 + }, + { + "row": 121, + "rms_error": 0.09380736244683188, + "maximum_absolute_error": 0.25 + }, + { + "row": 219, + "rms_error": 0.09166578124562276, + "maximum_absolute_error": 0.25 + }, + { + "row": 282, + "rms_error": 0.09102058524587438, + "maximum_absolute_error": 0.25 + }, + { + "row": 409, + "rms_error": 0.08696751249564189, + "maximum_absolute_error": 0.25 + }, + { + "row": 453, + "rms_error": 0.08674642545604302, + "maximum_absolute_error": 0.3125 + }, + { + "row": 57, + "rms_error": 0.0861529837832831, + "maximum_absolute_error": 0.21875 + }, + { + "row": 96, + "rms_error": 0.07888112333375753, + "maximum_absolute_error": 0.287109375 + }, + { + "row": 341, + "rms_error": 0.07793201637777376, + "maximum_absolute_error": 0.2265625 + }, + { + "row": 133, + "rms_error": 0.07786581656174694, + "maximum_absolute_error": 0.19140625 + }, + { + "row": 398, + "rms_error": 0.0764737367001577, + "maximum_absolute_error": 0.1875 + }, + { + "row": 360, + "rms_error": 0.07402143924854386, + "maximum_absolute_error": 0.25 + }, + { + "row": 473, + "rms_error": 0.07355884307503846, + "maximum_absolute_error": 0.1875 + } + ], + "worst_columns": [ + { + "column": 10625, + "rms_error": 0.06695082098979481, + "maximum_absolute_error": 0.2578125 + }, + { + "column": 50474, + "rms_error": 0.06628851063006382, + "maximum_absolute_error": 0.34375 + }, + { + "column": 1008, + "rms_error": 0.06555307895250304, + "maximum_absolute_error": 0.3125 + }, + { + "column": 229, + "rms_error": 0.06511421706049732, + "maximum_absolute_error": 0.3125 + }, + { + "column": 334, + "rms_error": 0.06446316244476662, + "maximum_absolute_error": 0.28125 + }, + { + "column": 3498, + "rms_error": 0.06416930226123667, + "maximum_absolute_error": 0.3125 + }, + { + "column": 1394, + "rms_error": 0.06413130019571951, + "maximum_absolute_error": 0.3125 + }, + { + "column": 677, + "rms_error": 0.0639696526546457, + "maximum_absolute_error": 0.25 + }, + { + "column": 278, + "rms_error": 0.06311357950539488, + "maximum_absolute_error": 0.2734375 + }, + { + "column": 10494, + "rms_error": 0.06296311952457531, + "maximum_absolute_error": 0.28125 + }, + { + "column": 32564, + "rms_error": 0.06291107980700647, + "maximum_absolute_error": 0.34375 + }, + { + "column": 3464, + "rms_error": 0.06151705018559607, + "maximum_absolute_error": 0.3125 + }, + { + "column": 25234, + "rms_error": 0.061327690045660375, + "maximum_absolute_error": 0.296875 + }, + { + "column": 1703, + "rms_error": 0.061288737510933354, + "maximum_absolute_error": 0.34375 + }, + { + "column": 1840, + "rms_error": 0.06109651463987475, + "maximum_absolute_error": 0.3125 + }, + { + "column": 405, + "rms_error": 0.06102842083328508, + "maximum_absolute_error": 0.28125 + }, + { + "column": 7161, + "rms_error": 0.060976322209715904, + "maximum_absolute_error": 0.2421875 + }, + { + "column": 4475, + "rms_error": 0.06074095619198881, + "maximum_absolute_error": 0.25 + }, + { + "column": 3431, + "rms_error": 0.06038757000041191, + "maximum_absolute_error": 0.25 + }, + { + "column": 22, + "rms_error": 0.06027237557813311, + "maximum_absolute_error": 0.19921875 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-reasoning-513", + "reference_sha256": "bcd07462fa0dcdf7ff8eeb62e056046efa5f1a53c759fba11637eb80700dc2bd", + "native_logits_sha256": "049ebf779f8b00545a73253fb30fcd55cb1343321cb2f035fd5ad725153ddb66" + } + ], + "resolved_case_view": { + "cells": 17, + "rows": 5581, + "top1_differences": 19, + "disjoint_maximum_sets": 0, + "per_cell": [ + { + "cell": "1.7B-6-prose-257", + "rows": 257, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "1.7B-6-prose-513", + "rows": 481, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "1.7B-8-code-513", + "rows": 513, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-4-code-513", + "rows": 513, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-4-reasoning-257", + "rows": 257, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-4-reasoning-513", + "rows": 513, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-6-code-257", + "rows": 257, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-6-code-513", + "rows": 513, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-6-code-64", + "rows": 64, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-6-prose-257", + "rows": 257, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-6-prose-513", + "rows": 481, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-6-prose-64", + "rows": 64, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-6-reasoning-257", + "rows": 257, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane, RMS norm and seven-tap convolution reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-6-reasoning-513", + "rows": 513, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane, RMS norm and seven-tap convolution reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-6-reasoning-64", + "rows": 64, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane, RMS norm and seven-tap convolution reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-8-prose-64", + "rows": 64, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-8-reasoning-513", + "rows": 513, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + } + ] + }, + "accepted_threshold": null, + "fidelity_pass": null +} diff --git a/docs/receipts/lfm3/rounding-reproduction.json b/docs/receipts/lfm3/rounding-reproduction.json new file mode 100644 index 0000000000..4774101681 --- /dev/null +++ b/docs/receipts/lfm3/rounding-reproduction.json @@ -0,0 +1,604 @@ +{ + "scope": "Independent whole-model FP64 calculation with production FP32 affine projection round sites; remaining operations retain FP64 arithmetic", + "accepted_threshold": null, + "fidelity_pass": null, + "cases": [ + { + "rows": 257, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.008228574718582158, + "maximum_absolute_error": 0.40234375, + "differences": [], + "worst_rows": [ + { + "row": 9, + "rms_error": 0.11675100600170359, + "maximum_absolute_error": 0.40234375 + }, + { + "row": 72, + "rms_error": 0.09182025948456946, + "maximum_absolute_error": 0.34375 + }, + { + "row": 30, + "rms_error": 0.08301323679998208, + "maximum_absolute_error": 0.2890625 + }, + { + "row": 34, + "rms_error": 0.08161881181226428, + "maximum_absolute_error": 0.234375 + }, + { + "row": 142, + "rms_error": 0.07404224594098688, + "maximum_absolute_error": 0.1875 + }, + { + "row": 250, + "rms_error": 0.06245770977221806, + "maximum_absolute_error": 0.171875 + }, + { + "row": 196, + "rms_error": 0.05974744907730566, + "maximum_absolute_error": 0.15625 + }, + { + "row": 181, + "rms_error": 0.05968768378833964, + "maximum_absolute_error": 0.15625 + }, + { + "row": 21, + "rms_error": 0.059383517969852855, + "maximum_absolute_error": 0.21875 + }, + { + "row": 141, + "rms_error": 0.05867767050374448, + "maximum_absolute_error": 0.203125 + }, + { + "row": 189, + "rms_error": 0.05833769401635122, + "maximum_absolute_error": 0.1875 + }, + { + "row": 29, + "rms_error": 0.05833556144228005, + "maximum_absolute_error": 0.21875 + }, + { + "row": 115, + "rms_error": 0.057935115778285945, + "maximum_absolute_error": 0.1640625 + }, + { + "row": 129, + "rms_error": 0.057801159107845855, + "maximum_absolute_error": 0.171875 + }, + { + "row": 106, + "rms_error": 0.05778176406071792, + "maximum_absolute_error": 0.1953125 + }, + { + "row": 107, + "rms_error": 0.05745385391010654, + "maximum_absolute_error": 0.25 + }, + { + "row": 63, + "rms_error": 0.05726298677642242, + "maximum_absolute_error": 0.25 + }, + { + "row": 85, + "rms_error": 0.05675869907432556, + "maximum_absolute_error": 0.203125 + }, + { + "row": 150, + "rms_error": 0.05647164936702668, + "maximum_absolute_error": 0.21875 + }, + { + "row": 125, + "rms_error": 0.056325135119553775, + "maximum_absolute_error": 0.15625 + } + ], + "worst_columns": [ + { + "column": 10625, + "rms_error": 0.0719168286234429, + "maximum_absolute_error": 0.3125 + }, + { + "column": 16420, + "rms_error": 0.06935220883801685, + "maximum_absolute_error": 0.3125 + }, + { + "column": 7161, + "rms_error": 0.06872535279635689, + "maximum_absolute_error": 0.28125 + }, + { + "column": 334, + "rms_error": 0.06852811266791806, + "maximum_absolute_error": 0.29296875 + }, + { + "column": 1840, + "rms_error": 0.06808063619059351, + "maximum_absolute_error": 0.34375 + }, + { + "column": 50474, + "rms_error": 0.06801655566196134, + "maximum_absolute_error": 0.34375 + }, + { + "column": 229, + "rms_error": 0.06790218938914094, + "maximum_absolute_error": 0.28125 + }, + { + "column": 10494, + "rms_error": 0.06760618030686806, + "maximum_absolute_error": 0.296875 + }, + { + "column": 3464, + "rms_error": 0.06581870418435679, + "maximum_absolute_error": 0.28125 + }, + { + "column": 278, + "rms_error": 0.06574005733457104, + "maximum_absolute_error": 0.359375 + }, + { + "column": 677, + "rms_error": 0.0644634874054111, + "maximum_absolute_error": 0.21875 + }, + { + "column": 2925, + "rms_error": 0.06352312981336071, + "maximum_absolute_error": 0.28125 + }, + { + "column": 90208, + "rms_error": 0.06342204452206861, + "maximum_absolute_error": 0.34375 + }, + { + "column": 3431, + "rms_error": 0.0632776334042306, + "maximum_absolute_error": 0.265625 + }, + { + "column": 3300, + "rms_error": 0.0631922709910923, + "maximum_absolute_error": 0.203125 + }, + { + "column": 3498, + "rms_error": 0.06288495654896703, + "maximum_absolute_error": 0.21875 + }, + { + "column": 32564, + "rms_error": 0.062457497914349076, + "maximum_absolute_error": 0.28125 + }, + { + "column": 405, + "rms_error": 0.06221173376451575, + "maximum_absolute_error": 0.296875 + }, + { + "column": 20, + "rms_error": 0.06212506474275177, + "maximum_absolute_error": 0.21875 + }, + { + "column": 91312, + "rms_error": 0.061823161313847616, + "maximum_absolute_error": 0.21875 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-4-reasoning-257", + "original_ideal_fp64_differences": [ + { + "row": 17, + "native_top": 296, + "reference_top": 405, + "native_maximum_set": [ + 296 + ], + "reference_maximum_set": [ + 405 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.125, + "native_reference_contender_gap": 0.1875, + "reference_native_contender_gap": 0.125, + "rounded_reference_top": 296, + "rounded_reference_maximum_set": [ + 296 + ], + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": 0.0625 + }, + { + "row": 18, + "native_top": 22, + "reference_top": 1018, + "native_maximum_set": [ + 22, + 1018 + ], + "reference_maximum_set": [ + 1018 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625, + "rounded_reference_top": 22, + "rounded_reference_maximum_set": [ + 22 + ], + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": 0.0625 + } + ], + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "118440ccd7fc07a502485a3266a2878ec815f316bcfa9dddd859e5d324fb8654", + "ideal_reference_logits_sha256": "eaad83f4ec389f5c731c0d3422f2af47f218f8cdb220a6d0b752641c8f8f9bfe", + "rounded_reference_logits_sha256": "b59c43083bfc6008233e7e249ced2f589ace92e1461e95c65300141aea14ab2b" + }, + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "relative_rms": 0.013509190417066086, + "maximum_absolute_error": 0.625, + "differences": [ + { + "row": 13, + "native_top": 22, + "reference_top": 63908, + "native_maximum_set": [ + 22, + 63908 + ], + "reference_maximum_set": [ + 63908 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + }, + { + "row": 22, + "native_top": 16033, + "reference_top": 91, + "native_maximum_set": [ + 16033 + ], + "reference_maximum_set": [ + 91, + 16033 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 57, + "rms_error": 0.09000787683806573, + "maximum_absolute_error": 0.625 + }, + { + "row": 10, + "rms_error": 0.08800637679348153, + "maximum_absolute_error": 0.234375 + }, + { + "row": 54, + "rms_error": 0.07760607841137634, + "maximum_absolute_error": 0.375 + }, + { + "row": 36, + "rms_error": 0.058355442854574543, + "maximum_absolute_error": 0.1953125 + }, + { + "row": 53, + "rms_error": 0.057159352869539715, + "maximum_absolute_error": 0.21875 + }, + { + "row": 20, + "rms_error": 0.055633565618538675, + "maximum_absolute_error": 0.296875 + }, + { + "row": 39, + "rms_error": 0.053347736683817384, + "maximum_absolute_error": 0.1953125 + }, + { + "row": 13, + "rms_error": 0.052123672609206625, + "maximum_absolute_error": 0.21875 + }, + { + "row": 8, + "rms_error": 0.05116690334326474, + "maximum_absolute_error": 0.25 + }, + { + "row": 2, + "rms_error": 0.05008304981339593, + "maximum_absolute_error": 0.125 + }, + { + "row": 52, + "rms_error": 0.04974962183100654, + "maximum_absolute_error": 0.1875 + }, + { + "row": 62, + "rms_error": 0.04902776384179979, + "maximum_absolute_error": 0.21875 + }, + { + "row": 55, + "rms_error": 0.047242131140363866, + "maximum_absolute_error": 0.21875 + }, + { + "row": 23, + "rms_error": 0.044142037283058606, + "maximum_absolute_error": 0.1875 + }, + { + "row": 11, + "rms_error": 0.043190213412404824, + "maximum_absolute_error": 0.25 + }, + { + "row": 22, + "rms_error": 0.04095067360254933, + "maximum_absolute_error": 0.3125 + }, + { + "row": 38, + "rms_error": 0.04044440498995601, + "maximum_absolute_error": 0.234375 + }, + { + "row": 30, + "rms_error": 0.04023781052786102, + "maximum_absolute_error": 0.1953125 + }, + { + "row": 50, + "rms_error": 0.0400162730201504, + "maximum_absolute_error": 0.171875 + }, + { + "row": 26, + "rms_error": 0.03986160470382173, + "maximum_absolute_error": 0.25 + } + ], + "worst_columns": [ + { + "column": 538, + "rms_error": 0.09985948465502727, + "maximum_absolute_error": 0.625 + }, + { + "column": 33237, + "rms_error": 0.09714630618708237, + "maximum_absolute_error": 0.375 + }, + { + "column": 723, + "rms_error": 0.09163830636319745, + "maximum_absolute_error": 0.4375 + }, + { + "column": 16033, + "rms_error": 0.0907320804033186, + "maximum_absolute_error": 0.34375 + }, + { + "column": 6986, + "rms_error": 0.0893078048926329, + "maximum_absolute_error": 0.28125 + }, + { + "column": 30270, + "rms_error": 0.08854197407562756, + "maximum_absolute_error": 0.34375 + }, + { + "column": 9700, + "rms_error": 0.0861797386586336, + "maximum_absolute_error": 0.3125 + }, + { + "column": 61089, + "rms_error": 0.08516518525177089, + "maximum_absolute_error": 0.25 + }, + { + "column": 116016, + "rms_error": 0.08419055912772351, + "maximum_absolute_error": 0.3125 + }, + { + "column": 29956, + "rms_error": 0.08386143846355991, + "maximum_absolute_error": 0.28125 + }, + { + "column": 39382, + "rms_error": 0.0833193654285085, + "maximum_absolute_error": 0.328125 + }, + { + "column": 274, + "rms_error": 0.08328050464397146, + "maximum_absolute_error": 0.46875 + }, + { + "column": 6907, + "rms_error": 0.08258207761026035, + "maximum_absolute_error": 0.552734375 + }, + { + "column": 42847, + "rms_error": 0.08252694023323655, + "maximum_absolute_error": 0.296875 + }, + { + "column": 1583, + "rms_error": 0.08156546550782054, + "maximum_absolute_error": 0.25 + }, + { + "column": 295, + "rms_error": 0.08148593392714178, + "maximum_absolute_error": 0.4375 + }, + { + "column": 717, + "rms_error": 0.08125915112683267, + "maximum_absolute_error": 0.515625 + }, + { + "column": 93538, + "rms_error": 0.08125813356366732, + "maximum_absolute_error": 0.46875 + }, + { + "column": 2741, + "rms_error": 0.08083424732039579, + "maximum_absolute_error": 0.45703125 + }, + { + "column": 346, + "rms_error": 0.08050453816084938, + "maximum_absolute_error": 0.5 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-code-64", + "original_ideal_fp64_differences": [ + { + "row": 13, + "native_top": 22, + "reference_top": 63908, + "native_maximum_set": [ + 22, + 63908 + ], + "reference_maximum_set": [ + 63908 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625, + "rounded_reference_top": 63908, + "rounded_reference_maximum_set": [ + 63908 + ], + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": -0.0625 + }, + { + "row": 22, + "native_top": 16033, + "reference_top": 91, + "native_maximum_set": [ + 16033 + ], + "reference_maximum_set": [ + 91 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.1875, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.1875, + "rounded_reference_top": 91, + "rounded_reference_maximum_set": [ + 91, + 16033 + ], + "rounded_reference_margin": 0.0, + "rounded_reference_native_contender_gap": 0.0 + } + ], + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "02225f1c8c7f5de05ebd44bd49948ad000aac040f3d6025d6655a9772ed08f9b", + "ideal_reference_logits_sha256": "d16f06c7813c231e4f3b7172456959da94d32adf024381666104fa28c0d3a859", + "rounded_reference_logits_sha256": "3f061bbfb2d31d235f0c4c5681e170f1bb2fe5f928ff805bf768d7727787e614" + } + ], + "first_projection_raw_replay": { + "shape": [ + 257, + 6144 + ], + "projection": "language_model.model.layers.0.conv.in_proj", + "scope": "Projection rounding replay; not a whole-model fidelity pass", + "affine_fusion": "Explicit CPU fmaf", + "accepted_threshold": null, + "raw_equal": 1579008, + "values": 1579008, + "bf16_equal": 1579008, + "maximum_absolute_error": 0.0 + }, + "first_convolution_same_input_rounding_replay": { + "conv": { + "raw_fp32_sum": true, + "bf16_equal": 526336, + "values": 526336, + "max_error": 0.0 + } + }, + "optional_accelerator": { + "source": "tools/lfm3/row_projection_replay.c", + "compile": "cc -std=c11 -O3 -ffp-contract=off -shared -fPIC tools/lfm3/row_projection_replay.c -o row-replay.so", + "equivalence": "All 1579008 captured pre-store FP32 outputs equal both NumPy and explicit C fmaf replays" + } +} diff --git a/docs/receipts/lfm3/scratch-alias.json b/docs/receipts/lfm3/scratch-alias.json new file mode 100644 index 0000000000..3d71217cdb --- /dev/null +++ b/docs/receipts/lfm3/scratch-alias.json @@ -0,0 +1,406 @@ +{ + "date": "2026-10-30", + "status": "Scratch allocation reuse; exact native baseline equivalence on the recorded fixtures", + "base_commit": "e9aeab27ce21b66fc9652fe8ce1163dc0f9d48cd", + "change": "Reuse attention partial-output storage for vocabulary logits after all attention layers complete. Allocate max(partial_bytes, logits_bytes) once.", + "scope": "Allocation and lifetime change only; no shader, arithmetic, dispatch, or performance change claimed.", + "checkpoint": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "revision": "c823fade98e35a3d6942ad4d38b1ea15f675cd12" + }, + "ownership": { + "storage_owner": "One entry in Scratch.memory.buffers owns the shared allocation; po and logits are borrowed handles.", + "lifetime": "Attention partials expire before the final head. Command completion precedes cache-logit copy. Cache.logits remains separate.", + "checked_scratch_rows": [ + 1, + 32, + 64, + 65, + 128 + ], + "checks": [ + "Shared handle identity and required buffer lengths", + "One fewer owned allocation and no duplicate owner IDs", + "Sum of actual allocated buffer lengths decreases by min(partial_bytes, logits_bytes)", + "Two 16-row streams match solo state and every logit row; partial keep lengths 3 and 1", + "Text callback cancellation, yield, reuse and unchanged deferred cache logits" + ] + }, + "allocation": { + "normal_partial_bytes": 33554432, + "normal_logits_bytes": 8192000, + "normal_saved_bytes": 8192000, + "single_row_saved_bytes": 256000, + "process_footprint_or_speed_measured": false + }, + "build": { + "compiler": "Zig 0.17.0", + "cpu_target": "apple_m1", + "options": [ + "-Dversion=1.1.6", + "-Dlfm3-risk-fast=true", + "-Dlfm3-defer-prompt-head=false" + ], + "native_dev_tools_exit": 0, + "qualifier_build_exit": 0, + "private_differential_probe_sha256": "798da300ec67bba39a6852774b808058988fe4f5f0781569123793391c43cdd8", + "public_state_sha256": "617208965442fa41fc341c470bf022ae1ac8b17a682a0384cd429273a846219d", + "public_qualifier_sha256": "52373585f3ab4b9627156decc2aa543b4c81ee1c0777403d9e8f6a49a802da49" + }, + "fixtures_sha256": { + "vl-2048.npy": "82c7dac5126bf3f7bf789f2a7385214044ad7ace0d151c52935e139ad99bcf2b", + "vl-8192.npy": "b371a7a09adfed00451d2aeba2dc64f5851e7ee62136dddcd8f77e1e07673581", + "image512.npy": "57111edcf1dd6adb846c9dedc3f398be5c8e19223cc4e75846d28296dc55e40f", + "image-vl4.bf16": "ffa0401f452f39a07833d6692ac9eff0d5485cc93a14fb3433bdad74c42ab05e" + }, + "devices": [ + { + "device": "Apple M5 Ultra", + "physical_memory_bytes": 274877906944, + "os": "macOS27.0 build26A428", + "manifest_sha256": "febe8bc72090b84fe74c03b1d1cf63c572bfd8a4e626988ec5bbbad7e94a02c1", + "stages": [ + { + "stage": "smoke", + "receipt_sha256": "8b14e0b21263f7b4596c950cf225684936cde99ea383b16e3d2fbf72f125cd67", + "events_sha256": "6c6ded5e7b43e457efe914fe64bb851a202b972955cca8a8e557d5d39ae459b6", + "runs": [ + { + "index": 0, + "exit": 0, + "log_sha256": "455aee97ad2530dab52975947dd8606a0f6c539f3704e7075cf8ebf69bcfb249" + } + ], + "results": [ + { + "case": "ordinary2k", + "result_sha256": "ca3115ee28cf383fae20c74a7360f6a7d9e34beccc7765d95f60e8460ce63423", + "scratch_alias_bytes_saved": 8192000, + "scratch_alias_owners_valid": true, + "scratch_alias_partial_keep_qualified": true, + "final_state_sha256": "8b3760aa217e7ee625b1af7f6e884ae32f8d86ff853bd55befa3619c7d4a2f10", + "final_logits_sha256": "8e67a0869f49e095ea7d6a6c134c0ee4e2722da949a96431e20b20c2df804a76", + "prompt_tokens": 2048, + "skipped_heads": 0, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 597, + 8055, + 509, + 5901, + 4116, + 355, + 267, + 1286, + 4160, + 302, + 278, + 7126, + 496, + 73, + 10, + 10601 + ], + "vision_rows": false, + "scope": "real checkpoint differential correctness; backend callbacks exercised separately for text fixtures; not served timing" + } + ] + }, + { + "stage": "positive", + "receipt_sha256": "5027e799cb4c0bf492cb338fd9c2475129c34f91835218b8eb125d47b4086c3c", + "events_sha256": "0ee12d9b3ac2ccc95655ab57ec9a5aa8b6018b5aaa574fe366b168fd48e43bc9", + "runs": [ + { + "index": 0, + "exit": 0, + "log_sha256": "9b6aac2ec7c4deb563780bede833b6705d5e95aa08929bae7347a459d5a34a08" + }, + { + "index": 1, + "exit": 0, + "log_sha256": "7753ff8f0e14ba14005ab5bc0de8a9d606495dc3e926cfbb1e39a055fc167ea8" + } + ], + "results": [ + { + "case": "deferred8k", + "result_sha256": "22beb4f4e6d594a338a239e31d88034e22b055330a048f7d2a4ead393296017f", + "scratch_alias_bytes_saved": 8192000, + "scratch_alias_owners_valid": true, + "scratch_alias_partial_keep_qualified": true, + "final_state_sha256": "8b9175c2c0062d43e54db026e161988e1e156bbf6a140a89ae29d671bf0d4a49", + "final_logits_sha256": "f7ed4ed4d826a6bebf5d95b79278c3f2f9079dca80ba11918ac0a2aae77504fa", + "prompt_tokens": 8192, + "skipped_heads": 62, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 597, + 4695, + 677, + 4116, + 267, + 1442, + 1286, + 4160, + 302, + 496, + 3862, + 10, + 8426, + 20, + 643, + 7204 + ], + "vision_rows": false, + "scope": "real checkpoint differential correctness; backend callbacks exercised separately for text fixtures; not served timing" + }, + { + "case": "image512", + "result_sha256": "ed043108162aac16e9cd2faf459f8f384e0c1d8a858cfdddba799e7b2830fd51", + "scratch_alias_bytes_saved": 8192000, + "scratch_alias_owners_valid": true, + "scratch_alias_partial_keep_qualified": true, + "final_state_sha256": "610ac8da7d7b099a0ed329257aaa6c67c8985cfb863595ae271fb321ec3463a9", + "final_logits_sha256": "9ab8840f27a8c003547a7b204f6d9fd489323cabe16098a419c2c44e47a5bef3", + "prompt_tokens": 512, + "skipped_heads": 0, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018 + ], + "vision_rows": true, + "scope": "real checkpoint differential correctness; backend callbacks exercised separately for text fixtures; not served timing" + } + ] + }, + { + "stage": "controls", + "receipt_sha256": "b57fb072fbdee02defe0681a76bcae328e5b04b59972296b7d0815039fd569d5", + "events_sha256": "3c4d77bd6725fbd456739e2f1438d746a66ab255e42bdcd013a83d5a18130388", + "runs": [ + { + "index": 0, + "exit": 1, + "log_sha256": "fd4515cea36bf1663d137ad5f5733db628535a9f909791bd8a9b1a306a2ad820" + } + ], + "results": [], + "intended_failure": "Actual final-head suppression causes DeferredHeadByteMismatch" + } + ] + }, + { + "device": "Apple M4 Pro", + "physical_memory_bytes": 51539607552, + "os": "macOS26.5.1 build25F80", + "manifest_sha256": "9d1decfd297fbfd22ffae19039e67e100374da350085f09e0399048ac25ae7df", + "stages": [ + { + "stage": "smoke", + "receipt_sha256": "647efc0d9e83a436bf7cb4e729cf4fefc0f58987107c89abad5646e7a520fb7c", + "events_sha256": "3820aea44e97e595fdabf1d1576dbeee62cebb8e627ad318ed466282853ff8db", + "runs": [ + { + "index": 0, + "exit": 0, + "log_sha256": "455aee97ad2530dab52975947dd8606a0f6c539f3704e7075cf8ebf69bcfb249" + } + ], + "results": [ + { + "case": "ordinary2k", + "result_sha256": "ca3115ee28cf383fae20c74a7360f6a7d9e34beccc7765d95f60e8460ce63423", + "scratch_alias_bytes_saved": 8192000, + "scratch_alias_owners_valid": true, + "scratch_alias_partial_keep_qualified": true, + "final_state_sha256": "8b3760aa217e7ee625b1af7f6e884ae32f8d86ff853bd55befa3619c7d4a2f10", + "final_logits_sha256": "8e67a0869f49e095ea7d6a6c134c0ee4e2722da949a96431e20b20c2df804a76", + "prompt_tokens": 2048, + "skipped_heads": 0, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 597, + 8055, + 509, + 5901, + 4116, + 355, + 267, + 1286, + 4160, + 302, + 278, + 7126, + 496, + 73, + 10, + 10601 + ], + "vision_rows": false, + "scope": "real checkpoint differential correctness; backend callbacks exercised separately for text fixtures; not served timing" + } + ] + }, + { + "stage": "positive", + "receipt_sha256": "0cceb7dd25376e3576f8c055f83d8dc9ac51df84900cf60fed5b066144aeb0de", + "events_sha256": "63d0f91da4fc96aaf4238c6ef47d63815e6822d68ffef4ac24d34bd996487489", + "runs": [ + { + "index": 0, + "exit": 0, + "log_sha256": "7753ff8f0e14ba14005ab5bc0de8a9d606495dc3e926cfbb1e39a055fc167ea8" + }, + { + "index": 1, + "exit": 0, + "log_sha256": "7fcfc39468e24fc314e1f918c1adadc3d1954c75093859b7c0aea636bf29742a" + } + ], + "results": [ + { + "case": "image512", + "result_sha256": "ed043108162aac16e9cd2faf459f8f384e0c1d8a858cfdddba799e7b2830fd51", + "scratch_alias_bytes_saved": 8192000, + "scratch_alias_owners_valid": true, + "scratch_alias_partial_keep_qualified": true, + "final_state_sha256": "610ac8da7d7b099a0ed329257aaa6c67c8985cfb863595ae271fb321ec3463a9", + "final_logits_sha256": "9ab8840f27a8c003547a7b204f6d9fd489323cabe16098a419c2c44e47a5bef3", + "prompt_tokens": 512, + "skipped_heads": 0, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018, + 23018 + ], + "vision_rows": true, + "scope": "real checkpoint differential correctness; backend callbacks exercised separately for text fixtures; not served timing" + }, + { + "case": "public2k", + "result_sha256": "73ffbc25c72ca4b7c6f39620cbf098744a8f993b80f4fdf12b71752c9ed1b0bb", + "final_state_sha256": "8b3760aa217e7ee625b1af7f6e884ae32f8d86ff853bd55befa3619c7d4a2f10", + "final_logits_sha256": "8e67a0869f49e095ea7d6a6c134c0ee4e2722da949a96431e20b20c2df804a76", + "prompt_tokens": 2048, + "skipped_heads": 14, + "mark_snapshots_equal": 2, + "resumed_prefix": 257, + "resume_state_logits_equal": true, + "full_active_kv_conv_equal": true, + "full_logits_equal": true, + "decode_tokens": [ + 597, + 8055, + 509, + 5901, + 4116, + 355, + 267, + 1286, + 4160, + 302, + 278, + 7126, + 496, + 73, + 10, + 10601 + ], + "vision_rows": false, + "scope": "real checkpoint differential correctness; backend callbacks exercised separately for text fixtures; not served timing" + } + ] + }, + { + "stage": "controls", + "receipt_sha256": "739b41fd43b2906558b136bbce165687f3537df82cac4bbf4ac6bade2e5bd192", + "events_sha256": "13c1ee8a65409eec7df0dfee4ef94717985ad6a7a53cca05b0ce0ffee9df3c6a", + "runs": [ + { + "index": 0, + "exit": 1, + "log_sha256": "fd4515cea36bf1663d137ad5f5733db628535a9f909791bd8a9b1a306a2ad820" + } + ], + "results": [], + "intended_failure": "Actual final-head suppression causes DeferredHeadByteMismatch" + } + ] + } + ], + "limitations": [ + "Only the pinned VL4 checkpoint was exercised for this allocation change; no new text-family or 6/8-bit qualification.", + "M4 did not run the 8k alias fixture.", + "Image case injects corrected native projected image rows; it does not rerun the vision tower.", + "Baseline byte equivalence is not independent FP64 model-fidelity qualification.", + "No new served timing, decode-rate, peak process footprint, or all-platform claim." + ], + "integrated_public_check": "M4 public qualifier final state/logit SHA and all 16 decode tokens equal the private original-layout 2k baseline; both public qualifier branches use the integrated alias.", + "host_checks": { + "command": "zig build test test-kernel-contract test-golden fuzz-cortheon -Dcpu=apple_m1", + "exit": 0, + "golden_equal": 322, + "golden_total": 335, + "golden_known_differences": 13, + "golden_unexpected_differences": 0, + "log_sha256": "aaf56811f1742f67753fded085a230b0bffd6944a5f5a8731015a064f52ca984", + "plain_build_command": "zig build -Dcpu=apple_m1 -Dversion=1.1.6", + "plain_build_exit": 0, + "plain_build_log_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "lean_merge_base": "b30d5b9d36786ff778b4bb7a7e1c4b5d26de126e", + "lean_inherited_problems": 428, + "lean_new_problems": 0, + "git_diff_check_exit": 0, + "changed_zig_fmt_check_exit": 0 + } +} diff --git a/docs/receipts/lfm3/scratch-reuse.json b/docs/receipts/lfm3/scratch-reuse.json new file mode 100644 index 0000000000..3dba7cf53c --- /dev/null +++ b/docs/receipts/lfm3/scratch-reuse.json @@ -0,0 +1,173 @@ +{ + "scope": "Private one-allocation bcx/act alias; local M4 Pro text and VL 4-bit correctness only", + "runtime_change": { + "file": "zig/src/families/lfm3/state.zig", + "baseline_sha256": "1161aea9174fd6a6dbecc09bfd26565459c9d7b8d1674ca06c7c78c1b0265517", + "candidate_sha256": "8a7083b1c43c75803596d441f20c5cfefb00820738adbcc65ffc6b043e647e10", + "patch_sha256": "3748e191f85ff8871df3fe108a836764ba6b269276f937169a565aed65aabba6", + "source_base": "Private GPU proof predates lifecycle92c64e96f; final applied public-source host/build check below combines lifecycle plus the exact same state alias. GPU binaries and final public build binaries are distinct artifacts." + }, + "allocation": { + "method": "Same instrumentedMetal mock invokes actual baseline/candidateScratch.init withactualconfig.schedule,rows128;testingallocator checkslifetime. No mirroredallocationimplementation.", + "baseline": { + "owned_allocations": 34, + "requested_buffer_bytes": 63677056, + "joins": 34, + "leaves": 34, + "frees": 34, + "actual_exit": 0 + }, + "candidate": { + "owned_allocations": 33, + "requested_buffer_bytes": 62104192, + "joins": 33, + "leaves": 33, + "frees": 33, + "actual_exit": 0 + }, + "saved_bytes": 1572864, + "saved_allocations": 1, + "test_sources_sha256": { + "lfm3-scratch-alias-ownership-test.zig": "5bbd1fc7ae26ebcc3f72dfbdd1f243e84795f038c915abdd50bcc19f75022d2c", + "lfm3-scratch-alias-metal-mock.zig": "e4f52bcadfaf21d56116fe98d71f15d9033e718f0ecc28e82329f2cc4ba139c6" + }, + "limits": "Allocation counts are requested-buffer bytes from the instrumented host allocator; actual 4-bit GPU correctness is recorded below. No physical-footprint or speed claim." + }, + "saved_requested_buffer_bytes_at128rows": 1572864, + "build": { + "native_forward_exact": "65/65 steps succeeded", + "profile": "Build succeeded", + "clock_fields_discarded": true + }, + "binaries": { + "zig-out/bin/tf-lfm3-forward": "160b350b19fc5c8f9273cb23d4cb2de649a8cdb4eca22f8318686ef7beeab2d5", + "zig-out/bin/tf-lfm3-exact": "addf6e1bd5e1a27e58f2a98104dd083ed9debe01efd85f3c34f650cb07df0463", + "zig-out/bin/tf-lfm3-profile": "d06ef0507b33b1ae3900e67eccd648bc2e4553c87b3de32d4189e370ab8db1fb", + "zig-out/native/bin/tensorfold-native": "da3157a08b0b7899d4c37ed74b0251b112f6f09bd3108032baa4c9d5f9a67507" + }, + "models": [ + { + "model": "LFM3-1.7B-MLX-4bit", + "state_passed": 68, + "state_failed": 0, + "logit_bytes": 65792000, + "logits_current_public_exact": true, + "logits_sha256": "f1dbd42c2556fcb05031a00b209c61fadee45e74ee1ff94628d4802479fcb518" + }, + { + "model": "LFM3-VL-2B-MLX-4bit", + "state_passed": 68, + "state_failed": 0, + "logit_bytes": 65792000, + "logits_current_public_exact": true, + "logits_sha256": "a4dde0a00ee9acb294c2b5a791e7e73cd05c5d6904ffe171e439ad1b01e25c5d" + } + ], + "traces": [ + { + "event": "trace_complete", + "model": "LFM3-1.7B", + "files": 620, + "bytes": 68474208, + "exact": true + }, + { + "event": "trace_complete", + "model": "LFM3-VL-2B", + "files": 620, + "bytes": 68474208, + "exact": true + }, + { + "event": "image_complete", + "rows": 90, + "image_rows": 70, + "trace_files": 930, + "exact": true, + "decoder_logit_bytes": 23040000, + "trace_bytes": 186747840 + } + ], + "profile_identity": [ + { + "event": "profile_identity", + "model": "LFM3-1.7B", + "mode": "stage", + "plain_profile_current_public_final_logits_equal": true, + "timing_fields_discarded": true + }, + { + "event": "profile_identity", + "model": "LFM3-VL-2B", + "mode": "stage", + "plain_profile_current_public_final_logits_equal": true, + "timing_fields_discarded": true + } + ], + "lease": { + "path": "/tmp/tensorfold-model.lock", + "exclusive": true, + "same_alias_inode": true, + "single_model_admission_before_and_after_child": true, + "owned_children_exited_before_release": true, + "local_only": true + }, + "scope_limits": [ + "No 6/8-bit runs.", + "No speed, independent FP64 fidelity, or physical-footprint reduction claim.", + "Image test injects retained real-cat projected rows; vision tower itself is unchanged and was not rerun.", + "All timing fields from correctness tools are discarded." + ], + "evidence_sha256": { + "lfm3-scratch-alias-proof/LFM3-1.7B-MLX-4bit-state.log": "fca04ca0d1f4afdde25a037178a11177a62909f668eca86ed8923d47bc00c1f0", + "lfm3-scratch-alias-proof/events.json": "e6b6e0682e6badd3070ada9243bce39a7e3d24ed0eac87f949eb935a38476231", + "lfm3-scratch-alias-proof/LFM3-1.7B-MLX-4bit-combined-logits.log": "da6d8a43e75cd674009591a7a830d2907f6b8f558c0c494f40d1cc4178bd82dc", + "lfm3-scratch-alias-proof/complete.json": "ceb54aca0f65edf2a3ad310e66a956075b4c9881d5557bfca5f09ed5f29b7405", + "lfm3-scratch-alias-proof/LFM3-VL-2B-MLX-4bit-state.log": "e7ccee7df28d5c8188e225b2064bffb275ca94e835da1548dc912b0456ba16d5", + "lfm3-scratch-alias-proof/LFM3-VL-2B-MLX-4bit-public-logits.log": "da6d8a43e75cd674009591a7a830d2907f6b8f558c0c494f40d1cc4178bd82dc", + "lfm3-scratch-alias-proof/records.json": "38b3e5edcfa2b6db327578ab4a19d1064ab105d126d15c4526b1f44201185309", + "lfm3-scratch-alias-proof/LFM3-1.7B-old-trace.log": "790d8930bff73e6e081f6e535f57733e9ea8b951dc3cb473c9a309bba80cb2ba", + "lfm3-scratch-alias-proof/LFM3-1.7B-MLX-4bit-public-logits.log": "da6d8a43e75cd674009591a7a830d2907f6b8f558c0c494f40d1cc4178bd82dc", + "lfm3-scratch-alias-proof/LFM3-1.7B-new-trace.log": "790d8930bff73e6e081f6e535f57733e9ea8b951dc3cb473c9a309bba80cb2ba", + "lfm3-scratch-alias-proof/LFM3-VL-2B-new-trace.log": "790d8930bff73e6e081f6e535f57733e9ea8b951dc3cb473c9a309bba80cb2ba", + "lfm3-scratch-alias-proof/LFM3-VL-2B-MLX-4bit-combined-logits.log": "da6d8a43e75cd674009591a7a830d2907f6b8f558c0c494f40d1cc4178bd82dc", + "lfm3-scratch-alias-proof/LFM3-VL-2B-old-trace.log": "790d8930bff73e6e081f6e535f57733e9ea8b951dc3cb473c9a309bba80cb2ba", + "lfm3-scratch-alias-image-proof/LFM3-1.7B-profile.log": "d930bc329a5754114b8de4a5ef47a0674fcdfacc593b37007d7a073de2b6e1f4", + "lfm3-scratch-alias-image-proof/image-old.log": "aafffce9158d8f03ef1a53cabcbc801a169cf1f973d8a2faaf5cee07d5787444", + "lfm3-scratch-alias-image-proof/events.json": "4f708b7320dd227d9f17495f1f97d68ef7bab04592f1e58ad309d4b7acb28d57", + "lfm3-scratch-alias-image-proof/LFM3-VL-2B-profile.json": "382ada3c09848ebc0eb8c2d2db6c53eda087863ff9281b8daf1f7dd69c11b121", + "lfm3-scratch-alias-image-proof/LFM3-1.7B-profile.json": "2b29b7e287e1b5f917509f7a6e3bcf038c2c8de2e6fb634afe5446e63bdc6380", + "lfm3-scratch-alias-image-proof/LFM3-VL-2B-profile.log": "d930bc329a5754114b8de4a5ef47a0674fcdfacc593b37007d7a073de2b6e1f4", + "lfm3-scratch-alias-image-proof/image-new.log": "aafffce9158d8f03ef1a53cabcbc801a169cf1f973d8a2faaf5cee07d5787444" + }, + "inputs": { + "code-257.npy": "96859365056d513a414e6eeef9d4a85bf72c3f9630d3e13a9ea0f57dd81b8bea", + "tokens.npy": "ab18390677209ae355cbc0d64e33f94856dc70264f01c8b94a18df310a7712e8", + "lfm3-real-cat-tokens.npy": "54c970db8c6ba82895cb2f18a48d397ea568ae885241129e9c5ee7dbbc91a24d", + "lfm3-real-cat-native.bf16": "ab2ac7db759e5c4a32fe7d60680ec47f7d1cbc5938fa7d8091bf4756bfeea4fc" + }, + "combined_public_build": { + "base_commit": "92c64e96f56d1a7a68f8476362696b4419b973b9", + "command": "ZIG_GLOBAL_CACHE_DIR=$WRITABLE_CACHE zig build test-lfm3-host native tf-lfm3-forward tf-lfm3-exact --cache-dir $PUBLIC_CACHE --summary all", + "actual_exit": 0, + "steps_passed": 77, + "steps_total": 77, + "host_tests_passed": 44, + "host_tests_total": 44, + "jpeg_limitation": "Pinned libjpeg-turbo3.1.4.1 unavailable; JPEG pixel cases skipped, warning retained.", + "fmt_check": true, + "diff_check": true, + "lean_problems": 428, + "lean_new_problems": 0, + "state_source_sha256": "8a7083b1c43c75803596d441f20c5cfefb00820738adbcc65ffc6b043e647e10", + "backend_source_sha256": "4ed5d33b869dc4be0d82d406cff4abc0f65a63687c45d8b3aaa61e26361db233", + "binaries_sha256": { + "zig-out/native/bin/tensorfold-native": "aae2de20cec9698afa9c108a4f7de27b83e5695b61f45cbb5e3dc21c34f6fbb3", + "zig-out/bin/tf-lfm3-forward": "7b81ba7487d3c07625ea3c6be45e0e4ae475373b6797af3ac6aae0717aa78b14", + "zig-out/bin/tf-lfm3-exact": "6dda9934d296db6bc76147fba113848fdf22df0ae45d1ead56f827cc00eb290b" + }, + "build_log_sha256": "2bacc236d342187ff20fdc4e1ce7a809e969af40da610c544ccfd36b4d2b4ddd", + "lean_log_sha256": "2aceae81fc338ffce3e33e5ec0a51b733e33eee717e102384f750bf41e808a01", + "gpu_rerun": false + } +} diff --git a/docs/receipts/lfm3/served-checks.json b/docs/receipts/lfm3/served-checks.json new file mode 100644 index 0000000000..2f33fee594 --- /dev/null +++ b/docs/receipts/lfm3/served-checks.json @@ -0,0 +1,207 @@ +{ + "scope": "M4 Pro frozen benchmark candidate and M5 Ultra final-source candidate; generated text, transient request IDs, log paths and local commands omitted", + "cells": { + "_trimmed_list": { + "count": 12, + "sha256": "890cd8137d0d1d0f1770aa3954e0559e9f00be227e731a4605da66931ad15d9a", + "first": [ + { + "checkpoint": "LFM3-1.7B-MLX-4bit", + "native_binary_sha256": "5c8a04d4af599879797f4f5bed8e00ab7d9d70732e698bd2a1ec2a01ec762e09", + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6, + "tests": { + "_trimmed_list": { + "count": 40, + "sha256": "ec93990f3e5951b65e9273ee8a6948400e438b6aead1aefa5d4167eacac6e7ea", + "first": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + } + ], + "last": [ + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + } + ] + } + }, + "platform": "M4 Pro, macOS 26.5.1" + }, + { + "checkpoint": "LFM3-1.7B-MLX-6bit", + "native_binary_sha256": "5c8a04d4af599879797f4f5bed8e00ab7d9d70732e698bd2a1ec2a01ec762e09", + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6, + "tests": { + "_trimmed_list": { + "count": 40, + "sha256": "a896485b9a7998f3c7d1c958fed66fa1333b95158968a66e137a79017ebfe7a7", + "first": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + } + ], + "last": [ + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + } + ] + } + }, + "platform": "M4 Pro, macOS 26.5.1" + } + ], + "last": [ + { + "platform": "M5 Ultra, macOS 27.0", + "checkpoint": "LFM3-VL-2B-MLX-8bit", + "native_binary_sha256": "251079309919aa9b21ca8cc6a89c58fc3fc9a9baf800fa5ce6a6af313a5b5f65", + "counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 9, + "tests": { + "_trimmed_list": { + "count": 61, + "sha256": "6085976c1edcf2c60f95e8f23ed21556e9342d92065ee07b21f064af0ca83b50", + "first": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + } + ], + "last": [ + { + "name": "red/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + } + } + ] + } + } + } + ] + } + }, + "_trim": { + "note": "Raw capture replaced by counts, first/last items and SHA-256 of the canonical JSON of each trimmed list or string.", + "original_sha256": "ed6a432c1b51c63f9ba27e52adf438ec609948ba5fe9561f5daa056a6549b3f7", + "original_bytes": 404305 + } +} diff --git a/docs/receipts/lfm3/shared-attention.json b/docs/receipts/lfm3/shared-attention.json new file mode 100644 index 0000000000..88d99c5924 --- /dev/null +++ b/docs/receipts/lfm3/shared-attention.json @@ -0,0 +1,54 @@ +{ + "scope": "Shared-stream native Metal attention preserves serial attention bytes; this is not a tensor-unit prompt-attention qualification.", + "binary_sha256": "d1656249bdb581b147faacbd3b3c557c00b6c45b710eeddce517a67178b6ba2b", + "source_manifest": { + "zig/kernels/metal/lfm3/attention.metal": "cfd691baa70a30cd7629b1ac4c795d58ae2bad95aff07a2e6fb2347e1bf9b514", + "zig/src/families/lfm3/kernels.zig": "73406a56a0ad15700b9da797cab84fea75d0b94c0e58cd4d8abf7b84638048ec", + "zig/tests/contract/metal_lfm3_segments.zig": "036899bf4a5323ccce661386a35a9b3cd638f7a8c6a9c004b78094a9e0be4c24", + "tools/lfm3/serve_check.py": "87327328ef1a1fcc35536efa0c18829779ee278a4d325f2af69356bc68a71f5c" + }, + "source_manifest_scope": "Hashes identify reviewed shared-attention source and checker files; they are not a whole-tree build manifest.", + "operator_contract": { + "hardware": "M4 Pro, 48 GB, macOS 26.5.1", + "command": "tf-kernel-contract --only LFM3", + "exit_status": 0, + "summary": "SUMMARY 52 kernels: 52 pass gates 1-4, 0 fail, 0 known findings; 14 controls: 14 red as planted, 0 missed; 0 limit problems; gates 28006 ms, limits 107 ms", + "shared_operator_scope": "Full and sliding serial-to-shared byte replacement; real shader missing-tail control; existing FP64 and accumulation gates retained." + }, + "exactness": { + "hardware": "M4 Pro, 48 GB, macOS 26.5.1", + "checkpoint": "LFM3-1.7B-MLX-4bit", + "command": "tf-lfm3-exact MODEL_DIR", + "exit_status": 0, + "summary": "LFM3_EXACT mode=default passed=68 failed=0 skipped=0 prompt=5 windows=60 shared=2 snapshots=1 long=0 gpu_seconds=148.781", + "scope": "Includes two unequal streams and four unequal streams with 32/16 aggregate rows, partial keeps, idle stream, distinct cache capacities and state/logit equality against one-row execution. This run did not include --long." + }, + "served": { + "hardware": "M5 Ultra, 256 GiB, macOS 27.0 (26A428)", + "command": "python3 tools/lfm3/serve_check.py --binary NATIVE --model-dir MODEL_DIR --sampled --cancel --output RESULT.json", + "counts": { + "passed": 303, + "failed": 0, + "skipped": 0 + }, + "cells": { + "_trimmed_value": { + "bytes": 131212, + "sha256": "3fde4f7bbb8d3e1543aa6be6b28cdfc873bda525b0909532bb81ae7270404f0b", + "keys": null, + "count": 6 + } + }, + "lifecycle": "One owned model process at a time; every test server exited after its suite." + }, + "limits": [ + "No new tolerance or reduced-precision probability storage.", + "This receipt does not establish a whole-model FP64 fidelity gate or a performance gate.", + "Named sessions and private machine addresses are omitted." + ], + "_trim": { + "note": "Raw capture replaced by counts, first/last items and SHA-256 of the canonical JSON of each trimmed list or string.", + "original_sha256": "7f9d5ab2d11dcca6f9f876cc6b6af956447bfdd7603150cf7f9455be832d279d", + "original_bytes": 216720 + } +} diff --git a/docs/receipts/lfm3/shared-convolution-text.json b/docs/receipts/lfm3/shared-convolution-text.json new file mode 100644 index 0000000000..e25530e883 --- /dev/null +++ b/docs/receipts/lfm3/shared-convolution-text.json @@ -0,0 +1,43 @@ +{ + "commit": "65051196df3f5d996aae9b4be132240969703dfe", + "scope": "Text-only shared-convolution cherry-pick, actual M4 validation only; no M5 claim", + "sources": { + "zig/kernels/metal/lfm3/ops.metal": "02746819be142ae0788823ffb6963336adc9344fe948d7a9dc94406373e61074", + "zig/src/families/lfm3/forward.zig": "082577207be3a85b3e54011eb7dd2d218e96c28a2f66e969431e4bf3a3f7f8e9", + "zig/src/families/lfm3/kernels.zig": "ad21db3cb7ba951df07b066f6ec244b61a77edb9c8ba8419cf7c786c75de4071", + "zig/src/families/lfm3/state.zig": "13950e1bb0313ae788c43432113d31f27b2b4fa97712d404a3f23ffe2d01ce80", + "zig/tests/contract/metal_main.zig": "e38cedc1d7c8fe134cf14bf422920da0f4612981afda177cc85feedfee4ad2c4", + "zig/tests/contract/metal_lfm3_conv_segments.zig": "b824c468f403e8384942829ace1b6a7c589a4b091a8f2a8f8af6b81244751857", + "zig/tests/contract/metal_limits.zig": "6863bb52b3c48ecd02c0e72a611f9094b94b7e1af9b1642d6c68904ecccb8216", + "zig/src/core/contract/lfm3_records.zig": "ce6f0e8cf8ac7d08e27e825ce9e41d0a0c694875811ef11de5f8f876eaba4c0d" + }, + "build_status": "67/67 steps succeeded; 1/1 newly executed host test passed, others cached", + "build_command": "zig build native tf-kernel-contract test-lfm3-host -Dcpu=apple_m1 --summary all", + "m4_gates": { + "passed": 1, + "failed": 0, + "wrong_state_control_red": 1, + "controls_missed": 0 + }, + "m4_normal_limits": { + "registered_launches": 335, + "problems": 0, + "new_pipeline_max_threads": 256, + "new_pipeline_static_tg_bytes": 0 + }, + "m5_executed": false, + "tree_clean": true, + "binary_sha256": { + "zig-out/native/bin/tensorfold-native": "56b9c8a43342f0d93a73ccefe1e0c8813a9bf19bbe2ef9765380e1ee79c0c21e", + "zig-out/bin/tf-kernel-contract": "86546abb63ddc5ac0de44192db1dff0fd005d3ac9857929075c3519f10400ee5" + }, + "log_sha256": { + "lfm3-shared-conv-text-build.log": "07798b1176d5f6ea8cf3048ef755e55f6b222eb2f33ad42820a0857f5930994f", + "lfm3-shared-conv-text-gates-m4.log": "9b6909521ec6b8665f436367424100eb196a715c4fa3cf1f57353c10d60b0c7e", + "lfm3-shared-conv-text-limits-m4.log": "a7d20c8682be9b90a31adc564ade14832f146a41d43061873d702acbfb59dd27" + }, + "commands": [ + "zig-out/bin/tf-kernel-contract --only \"shared conv\" --no-limits", + "zig-out/bin/tf-kernel-contract --only \"shared conv\" --no-gates" + ] +} diff --git a/docs/receipts/lfm3/shared-convolution.json b/docs/receipts/lfm3/shared-convolution.json new file mode 100644 index 0000000000..fb5d3055c0 --- /dev/null +++ b/docs/receipts/lfm3/shared-convolution.json @@ -0,0 +1,447 @@ +{ + "scope": "Shared-convolution evidence with private qualification/performance and applied-public validation identified separately", + "base_commit": "d05f599a3f341b9bb9bcfbebb9b47b6d63e618ca", + "algorithm": "One descriptor-driven convolution launch across recorded stream segments; both entries invoke the unchanged seven-tap helper and snapshot equations", + "hosts": [ + { + "chip": "Apple M4 Pro", + "macos": "26.5.1", + "build": "25F80", + "memory_gib": 48 + }, + { + "chip": "Apple M5 Ultra", + "macos": "27.0", + "build": "26A428", + "memory_gib": 256 + } + ], + "layout": { + "descriptor_bytes": 16, + "descriptor_alignment": 8, + "offsets": { + "state": 0, + "base": 8, + "rows": 12 + }, + "max_rows": 32, + "scratch_payload_bytes": 12416, + "lifetime": "Separate per-layer descriptor slices and independent per-call row map remain immutable through command completion", + "resources": "Every indirect per-cache convolution-state buffer declared read; input/output/snapshot row spans remain cache-local", + "admission": "Backend init qualification exercises multiple streams and allocates the scratch before native metal_admit.room samples device allocation/footprint; Storage owns and tracks both buffers" + }, + "guards": [ + "Compile-time layout assertions", + "record=false multi-segment calls rejected before encoding, with unchanged cache/history/logits/inputIDs witness on allsix widths", + "Descriptor count and row sums bounded<=32", + "Shared convolution explicitly requires recorded total<=32" + ], + "frozen_private_artifacts": { + "base_commit": "d05f599a3f341b9bb9bcfbebb9b47b6d63e618ca", + "scope": "Private shared-convolution candidate only; defaultOFF unless explicit private environment/testflag", + "source_sha256": { + "zig/kernels/metal/lfm3/ops.metal": "02746819be142ae0788823ffb6963336adc9344fe948d7a9dc94406373e61074", + "zig/src/families/lfm3/model.zig": "7d26a3afe310208ec3b45378890836dac845d262c9186d394951a42906504d60", + "zig/src/families/lfm3/state.zig": "13950e1bb0313ae788c43432113d31f27b2b4fa97712d404a3f23ffe2d01ce80", + "zig/src/families/lfm3/kernels.zig": "4fc6f329a322b8025351b7f097a8dff6757bc6fc4f31b2db32be5960685ce853", + "zig/src/families/lfm3/forward.zig": "e0d2c61773b1e7b6ce5c7ae8d622f65664a3590cddeb750edd99c3f70cc3d32e", + "zig/tests/lfm3_exact.zig": "e3b8902be3b375d8103e9f799051c680219567e871d17c4565bb5a7fb78e2f63" + }, + "binary_sha256": { + "zig-out/bin/tf-lfm3-exact": "98632db29ed6f16eb8b64da8db8a077f8bb7be73beaa7150988a940288844515", + "zig-out/bin/tf-lfm3-forward": "4c0ad68dc32dfa367beb5755be6b1166a23dfc62f0a2203813d6cbdf34e4a548", + "zig-out/native/bin/tensorfold-native": "b06de5679c58f63cffca95c34d0d78ee53678f60e04dff894aded547213ec1a5" + }, + "scratch_bytes": 12416, + "guards": [ + "ConvSegment size16/alignment8/offsets0,8,12 asserted at compile time", + "Unrecorded multi-segment calls rejected before encoding", + "Descriptor row sums validated<=32 before allocation/population", + "Shared launch explicitly requires recorded rows<=32" + ] + }, + "contract_artifact": { + "binary_sha256": "408887b56dc2f899a64f2101341637f01ea0bb8ffdd74a0c3adbc29b664754c7", + "source_sha256": { + "zig/tests/contract/metal_lfm3_conv_segments.zig": "0b1361bfeb044d2033e695d1d5eac3fa2488140b5677ef7a63b8cb172f27f6b2", + "zig/tests/contract/metal_main.zig": "d4b6d164fe1f8c61d5e55c63e8d74ab8a77021b41d5fc2c21c1fea32e0741aab", + "zig/tests/contract/metal_limits.zig": "51c3294d185d69bf56f2d20cadfff50c891bcc2ee4df40e22b2f2e7895694c56", + "zig/src/core/contract/lfm3_records.zig": "a5fa76f575767256030e034a74bc6582d9052180ed6be838bb5736bdc808913b" + }, + "build_steps": [ + 43, + 43 + ] + }, + "serving_scope": "Allsix widths; seeded sampling, actual HTTP cancellation/recovery, unequal/shared streams, prompt-cache reuse and VL image/order cases", + "performance": { + "scope": "Four-stream steady greedy decode on4bit text/VL, two public code/prose chat prompts,256tokens,3reps each leg, ABBA; no speed claim for6/8bit or cold-prefill", + "baseline_sha256": "e9a1488abb130af39c21ee08068fe490d5926271cebcd2e9d0f815711e3730b4", + "candidate_sha256": "b06de5679c58f63cffca95c34d0d78ee53678f60e04dff894aded547213ec1a5", + "order": [ + "old", + "new", + "new", + "old" + ], + "server_options": { + "context": 70000, + "parallel": 4, + "prompt_cache_gib": 0, + "temperature": 0, + "thinking": false + }, + "private_enablement": "TF_LFM3_SHARED_CONV=1 on candidate servers only; no quad/NAX/publication candidate", + "measured_requests": 192, + "counts": { + "alone_equal": 192, + "alone_unequal": 0, + "alone_failed": 0, + "serial_equal": 16, + "serial_unequal": 0, + "serial_failed": 0 + }, + "all_measured_full_token_hashes_match_by_family_prompt": true, + "cells": [ + { + "family": "text", + "prompt": "long-code-chat", + "baseline_leg_rates": [ + 848.2, + 849.5 + ], + "candidate_leg_rates": [ + 877.4, + 876.7 + ], + "baseline_median_tps": 848.85, + "candidate_median_tps": 877.05, + "rate_ratio": 1.0332214172115215, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7" + }, + { + "family": "text", + "prompt": "long-prose-chat", + "baseline_leg_rates": [ + 868.6, + 869.4 + ], + "candidate_leg_rates": [ + 896.7, + 896.2 + ], + "baseline_median_tps": 869.0, + "candidate_median_tps": 896.45, + "rate_ratio": 1.0315880322209436, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a" + }, + { + "family": "vl", + "prompt": "long-code-chat", + "baseline_leg_rates": [ + 825.2, + 824.9 + ], + "candidate_leg_rates": [ + 851.3, + 847.1 + ], + "baseline_median_tps": 825.05, + "candidate_median_tps": 849.2, + "rate_ratio": 1.029270953275559, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee" + }, + { + "family": "vl", + "prompt": "long-prose-chat", + "baseline_leg_rates": [ + 822.2, + 821.1 + ], + "candidate_leg_rates": [ + 846.8, + 843.7 + ], + "baseline_median_tps": 821.6500000000001, + "candidate_median_tps": 845.25, + "rate_ratio": 1.0287226921438568, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa" + } + ], + "runner_sha256": "a50e6058f59fdd5cbca1e937b8bee3c719ae4c201336832425d1aa7787770a64", + "artifact_scope": "Measured on the frozen private opt-in binary; the applied public binary has a distinct SHA and was not this benchmark artifact" + }, + "evidence_sha256": { + "lfm3-shared-conv-contract-m4.log": "d747ee0a98514126816594491d6c2262be22c35c73db61390cb6dda9f38f1e4f", + "lfm3-shared-conv-contract-m5.log": "71c7c162fc0dfd982046f617529a119cd6f568b206110281c126a91e7bffd684", + "lfm3-shared-conv-guard-quick-m4.log": "6b305e0707b3d3867faf380c82f6d5bdbd60326cac431b4813b90ac3308a32fa", + "lfm3-shared-conv-guard-build.log": "bba528bbaa79057884e661f5b832bada7b107dcfdc69c2fd4d19755c6eb666de", + "lfm3-shared-conv-contract-final-build.log": "4eb06fdeef03301fed52b14d74b32b887b6c7fa888f640b6af7f5bb74ebeca0f" + }, + "private_state_suites": [ + { + "model": "LFM3-1.7B-MLX-4bit", + "passed": 68, + "failed": 0, + "skipped": 0, + "unrecorded_rejected_unchanged": true, + "seconds": 69.34337883302942, + "log_sha256": "89f8b71a016ce3eaa2996bc73c6d1dea3f7cb90007f09109b82d346ccdec478e" + }, + { + "model": "LFM3-1.7B-MLX-6bit", + "passed": 68, + "failed": 0, + "skipped": 0, + "unrecorded_rejected_unchanged": true, + "seconds": 81.18592800013721, + "log_sha256": "f0ee8d743c8c22a3e34801e70d0ed2e6e88e23f87f5f7b2380cee48e06b529f8" + }, + { + "model": "LFM3-1.7B-MLX-8bit", + "passed": 68, + "failed": 0, + "skipped": 0, + "unrecorded_rejected_unchanged": true, + "seconds": 83.94439495913684, + "log_sha256": "394710a17d58e50092dcf3d7773946e90a0e5a4018c54da541c0384d67158060" + }, + { + "model": "LFM3-VL-2B-MLX-4bit", + "passed": 68, + "failed": 0, + "skipped": 0, + "unrecorded_rejected_unchanged": true, + "seconds": 73.88264883402735, + "log_sha256": "4b9be778ab98d731a9c0882355ddf670d94e5345e8777121dbf8858fc0a32c79" + }, + { + "model": "LFM3-VL-2B-MLX-6bit", + "passed": 68, + "failed": 0, + "skipped": 0, + "unrecorded_rejected_unchanged": true, + "seconds": 81.54222154105082, + "log_sha256": "69c0c38931a19db1617229aff21d7b352038593f3e364f7e936c980c32d44023" + }, + { + "model": "LFM3-VL-2B-MLX-8bit", + "passed": 68, + "failed": 0, + "skipped": 0, + "unrecorded_rejected_unchanged": true, + "seconds": 83.95681104203686, + "log_sha256": "02d16d94104abfa400ca98f3c6e1ec2edcb1937dd3699ac016b0b80fb87edf1c" + } + ], + "private_state_total": 408, + "private_serving_suites": [ + { + "model": "LFM3-1.7B-MLX-4bit", + "passed": 40, + "failed": 0, + "skipped": 0, + "receipt_sha256": "85e7081cc498d19626f1dbcc0f16a310455d2762f8b202da2f0c5a3606538282" + }, + { + "model": "LFM3-1.7B-MLX-6bit", + "passed": 40, + "failed": 0, + "skipped": 0, + "receipt_sha256": "74f26e831a7b76582f7c8b6ca288f2f31b2bfad4c9836f249a4802af2850920b" + }, + { + "model": "LFM3-1.7B-MLX-8bit", + "passed": 40, + "failed": 0, + "skipped": 0, + "receipt_sha256": "f0f90275348eed175783b3cdcdefb830da3de7984c5751acd0b748b38f2bf92b" + }, + { + "model": "LFM3-VL-2B-MLX-4bit", + "passed": 61, + "failed": 0, + "skipped": 0, + "receipt_sha256": "70115d39e759a7a169c3f300cded1790f445301624fa7f1cbf14d26b5d102075" + }, + { + "model": "LFM3-VL-2B-MLX-6bit", + "passed": 61, + "failed": 0, + "skipped": 0, + "receipt_sha256": "08f4af3efa91ce27b652e3c407bb625cd627af05eda9b85e5b1d80bc74da1ac1" + }, + { + "model": "LFM3-VL-2B-MLX-8bit", + "passed": 61, + "failed": 0, + "skipped": 0, + "receipt_sha256": "582ed6d544d14e058781f1cfa9ce64db36dd3b28520e488583d12b2ff62de8ec" + } + ], + "private_serving_total": 303, + "private_operator_gates": { + "M4": { + "kernels_passed": 1, + "failed": 0, + "controls_red": 1, + "controls_missed": 0, + "nonfinite_classes": 16, + "finite_classes": 114672, + "replacement_shapes": 5, + "repeat_runs": 5, + "repeat_bytes": 229376, + "reduced_precision_witness": "fp16 infx; bf16 963x relative miss", + "limits": { + "launched_threads": 256, + "max_threads": 256, + "static_threadgroup_bytes": 0 + } + }, + "M5": { + "kernels_passed": 1, + "failed": 0, + "controls_red": 1, + "controls_missed": 0, + "limits": { + "launched_threads": 256, + "max_threads": 256, + "static_threadgroup_bytes": 0 + } + }, + "bound": "Existing scalar conv FP64 equation, depth and intermediate-round allowances reused unchanged per independent cache", + "control": "Wrong stream-state address gives raw replacement mismatch and is red as planted", + "limits_scope": "Targeted runs used --no-limits; each new pipeline was explicitly reflected/checked at probe init. Normal gate5 declarations and bad-source compilation are registered for the final full limits run." + }, + "private_immutable_solo_captures": [ + { + "model": "LFM3-1.7B-MLX-4bit", + "rows": 257, + "bytes": 65792000, + "canonical_equal": true, + "sha256": "f1dbd42c2556fcb05031a00b209c61fadee45e74ee1ff94628d4802479fcb518" + }, + { + "model": "LFM3-1.7B-MLX-6bit", + "rows": 257, + "bytes": 65792000, + "canonical_equal": true, + "sha256": "08bf833a94fa0403bc65c70677d20372b5ab0616fb0a58d26119c486069ef59c" + }, + { + "model": "LFM3-1.7B-MLX-8bit", + "rows": 257, + "bytes": 65792000, + "canonical_equal": true, + "sha256": "1345bd80acaa16fe08a00a14d839e3e37cf9d0d8cd7e98c01b68cd1b8d2f51d4" + }, + { + "model": "LFM3-VL-2B-MLX-4bit", + "rows": 257, + "bytes": 65792000, + "canonical_equal": true, + "sha256": "a4dde0a00ee9acb294c2b5a791e7e73cd05c5d6904ffe171e439ad1b01e25c5d" + }, + { + "model": "LFM3-VL-2B-MLX-6bit", + "rows": 257, + "bytes": 65792000, + "canonical_equal": true, + "sha256": "93ea83c90c5623f4e17502a526e2358f9e1ba3f2d03999295745257dc95bbca6" + }, + { + "model": "LFM3-VL-2B-MLX-8bit", + "rows": 257, + "bytes": 65792000, + "canonical_equal": true, + "sha256": "5a013f3eed97f143429f09a81a829b07dc32360734cd54855e18e514160e0a48" + } + ], + "private_immutable_solo_bytes": 394752000, + "applied_public": { + "base_commit": "d05f599a3f341b9bb9bcfbebb9b47b6d63e618ca", + "source_sha256": { + "zig/kernels/metal/lfm3/ops.metal": "02746819be142ae0788823ffb6963336adc9344fe948d7a9dc94406373e61074", + "zig/src/families/lfm3/forward.zig": "082577207be3a85b3e54011eb7dd2d218e96c28a2f66e969431e4bf3a3f7f8e9", + "zig/src/families/lfm3/state.zig": "13950e1bb0313ae788c43432113d31f27b2b4fa97712d404a3f23ffe2d01ce80", + "zig/src/families/lfm3/kernels.zig": "ad21db3cb7ba951df07b066f6ec244b61a77edb9c8ba8419cf7c786c75de4071", + "zig/tests/contract/metal_lfm3_conv_segments.zig": "b824c468f403e8384942829ace1b6a7c589a4b091a8f2a8f8af6b81244751857", + "zig/tests/contract/metal_main.zig": "d4b6d164fe1f8c61d5e55c63e8d74ab8a77021b41d5fc2c21c1fea32e0741aab", + "zig/tests/contract/metal_limits.zig": "51c3294d185d69bf56f2d20cadfff50c891bcc2ee4df40e22b2f2e7895694c56", + "zig/src/core/contract/lfm3_records.zig": "a5fa76f575767256030e034a74bc6582d9052180ed6be838bb5736bdc808913b" + }, + "binary_sha256": { + "zig-out/native/bin/tensorfold-native": "195a8aceefd625f76049ed7cb48459b43f6ebbeedde5e82a07b0726f29010a9d", + "zig-out/bin/tf-lfm3-exact": "7c04d0da0bed247f7bbb08077b74aa190b5a07322c8d54dadfb2898ffbb650b1", + "zig-out/bin/tf-kernel-contract": "31557ec36605bcaf5c40f04218f10ea5374d3cdfb87b6ad644c2ae8cf1139437" + }, + "default": "Shared convolution for recorded multi-segment rounds; unchanged single-stream entry; no private getenv/host-fault flags", + "private_shader_byte_identity": true, + "build": "64/64 steps succeeded", + "operator_and_normal_limits": { + "m4": { + "kernel_gates_passed": 1, + "failed": 0, + "wrong_state_controls_red": 1, + "controls_missed": 0, + "launched_threads": 256, + "pipeline_max_threads": 256, + "static_threadgroup_bytes": 0, + "normal_limits_registered_launches": 353, + "normal_limits_problems": 0, + "scope": "All normal launch declarations against M1-M5 and actual device reflection; no offline M1/M2 compiler sweep" + }, + "m5": { + "kernel_gates_passed": 1, + "failed": 0, + "wrong_state_controls_red": 1, + "controls_missed": 0, + "launched_threads": 256, + "pipeline_max_threads": 256, + "static_threadgroup_bytes": 0, + "normal_limits_registered_launches": 353, + "normal_limits_problems": 0, + "scope": "All normal launch declarations against M1-M5 and actual device reflection; no offline M1/M2 compiler sweep" + } + }, + "state_suite": { + "host": "Apple M4 Pro", + "model": "LFM3-1.7B-MLX-4bit", + "passed": 68, + "failed": 0, + "skipped": 0, + "mode": "default, no enablement flags" + }, + "host_validation": "zig build test-lfm3-host test-lfm3-image-host -Dcpu=apple_m1 --summary all: 10/10 steps; 1/1 newly executed test passed, remaining executions cached", + "fmt": "zig fmt --check on all seven changed Zig sources: passed", + "source_maximum_lines": 468, + "lean": { + "baseline_problems": 428, + "head_problems": 428, + "new_problems": 0, + "note": "Nonzero lean-check findings are inherited; lean_new comparison exits zero", + "baseline_log_sha256": "3d3fc6c70d708ea107e4cd1f87f8674de115b367131a267e68f639006cd0165c" + }, + "commands": [ + "zig build native tf-lfm3-exact tf-kernel-contract -Dcpu=apple_m1 --summary all", + "zig-out/bin/tf-kernel-contract --only \"shared conv\" --no-limits", + "zig-out/bin/tf-kernel-contract --only \"shared conv\" --no-gates", + "zig-out/bin/tf-lfm3-exact MODEL_DIR", + "zig build test-lfm3-host test-lfm3-image-host -Dcpu=apple_m1 --summary all", + "python3 tools/zig/lean_check.py", + "python3 tools/zig/lean_new.py PINNED_BASE_LEAN.txt HEAD_LEAN.txt" + ], + "log_sha256": { + "lfm3-shared-conv-public-build.log": "bfbb9951b04b1fd507c424a9e3d4373f5b7c075b49b26d0b186a18895bb6af07", + "lfm3-shared-conv-public-gates-m4.log": "f351162af6678cf688fabd1a53b6d79bb54229e95ee899d422a55da39cdab541", + "lfm3-shared-conv-public-gates-m5.log": "4d07a2f99af41772509ec501d502bf07e350440074dc4ec675cbbda6b3d3f90d", + "lfm3-shared-conv-public-limits-m4.log": "e7f338d0699f9c2be4aa97ef5995b5bb244bf8cd6046ddd3f91652bf555e6718", + "lfm3-shared-conv-public-limits-m5.log": "89d77a1125cfad02ba0ac94a811989c907a352feeab6a7474b9cae838607ef70", + "lfm3-shared-conv-public-state68-m4.log": "47c399fac3257c8c00d62ab84645cfce8ea44a183af88a1a74d3765bf08e78fd", + "lfm3-shared-conv-public-host-tests.log": "b0b75fac398c6200f54aabc87233d3d4cc1da79b271a98f3b15868a7b52902e4", + "lfm3-shared-conv-public-lean-head.txt": "e570a5cbfae6f29b03a2e746fa5f8141b203ee51412e7b76f9333f05cc658d6f", + "lfm3-shared-conv-public-lean-new.txt": "7c56e1d13027048ee9b6d376556965cbfd2aff0b2b2a67f03612c87643893209", + "lfm3-shared-conv-public-fmt.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + } +} diff --git a/docs/receipts/lfm3/six-width-fp64-decode.json b/docs/receipts/lfm3/six-width-fp64-decode.json new file mode 100644 index 0000000000..fc0f75da81 --- /dev/null +++ b/docs/receipts/lfm3/six-width-fp64-decode.json @@ -0,0 +1,769 @@ +{ + "scope": "Actual one-row decode after 63-row teacher-forced history, all six quantizations; reference re-evaluates identical full history in FP64", + "binary_sha256": "e80a2a14c0d8ed20b62088d0d8b87be4beb5fed3eb1a7336b3b00f8ac5f74bee", + "fidelity_pass": null, + "cells": [ + { + "rows": 1, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.01665075641707762, + "maximum_absolute_error": 0.21875, + "differences": [], + "worst_rows": [ + { + "row": 63, + "rms_error": 0.04319844149784029, + "maximum_absolute_error": 0.21875 + } + ], + "worst_columns": [ + { + "column": 48006, + "rms_error": 0.21875, + "maximum_absolute_error": 0.21875 + }, + { + "column": 30122, + "rms_error": 0.21197509765625, + "maximum_absolute_error": 0.21197509765625 + }, + { + "column": 3568, + "rms_error": 0.20703125, + "maximum_absolute_error": 0.20703125 + }, + { + "column": 42890, + "rms_error": 0.203125, + "maximum_absolute_error": 0.203125 + }, + { + "column": 6723, + "rms_error": 0.203125, + "maximum_absolute_error": 0.203125 + }, + { + "column": 3641, + "rms_error": 0.203125, + "maximum_absolute_error": 0.203125 + }, + { + "column": 7068, + "rms_error": 0.203125, + "maximum_absolute_error": 0.203125 + }, + { + "column": 38765, + "rms_error": 0.203125, + "maximum_absolute_error": 0.203125 + }, + { + "column": 30663, + "rms_error": 0.2001953125, + "maximum_absolute_error": 0.2001953125 + }, + { + "column": 116632, + "rms_error": 0.1953125, + "maximum_absolute_error": 0.1953125 + }, + { + "column": 20560, + "rms_error": 0.1953125, + "maximum_absolute_error": 0.1953125 + }, + { + "column": 20326, + "rms_error": 0.1953125, + "maximum_absolute_error": 0.1953125 + }, + { + "column": 12263, + "rms_error": 0.1953125, + "maximum_absolute_error": 0.1953125 + }, + { + "column": 469, + "rms_error": 0.1875, + "maximum_absolute_error": 0.1875 + }, + { + "column": 11460, + "rms_error": 0.1875, + "maximum_absolute_error": 0.1875 + }, + { + "column": 23796, + "rms_error": 0.1875, + "maximum_absolute_error": 0.1875 + }, + { + "column": 31233, + "rms_error": 0.1875, + "maximum_absolute_error": 0.1875 + }, + { + "column": 36584, + "rms_error": 0.1875, + "maximum_absolute_error": 0.1875 + }, + { + "column": 9178, + "rms_error": 0.1875, + "maximum_absolute_error": 0.1875 + }, + { + "column": 93833, + "rms_error": 0.1875, + "maximum_absolute_error": 0.1875 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "checkpoint": "LiquidAI/LFM3-1.7B-MLX-4bit", + "phase": "decode", + "prefilled_history": 63, + "current_row": 63, + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "6d6828e56b463b0a09f9f559ec1251a21ff1445d53e67811dda0318185ce39ff", + "reference_full_logits_sha256": "6286092ef4d800dd2fefe310628c1a5270cc489f33c72a373010b02d38164023" + }, + { + "rows": 1, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.021845787266094638, + "maximum_absolute_error": 0.328125, + "differences": [], + "worst_rows": [ + { + "row": 63, + "rms_error": 0.06023315568323548, + "maximum_absolute_error": 0.328125 + } + ], + "worst_columns": [ + { + "column": 20341, + "rms_error": 0.328125, + "maximum_absolute_error": 0.328125 + }, + { + "column": 35931, + "rms_error": 0.3203125, + "maximum_absolute_error": 0.3203125 + }, + { + "column": 43041, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + }, + { + "column": 8553, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + }, + { + "column": 35725, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + }, + { + "column": 6953, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + }, + { + "column": 15136, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + }, + { + "column": 12603, + "rms_error": 0.310546875, + "maximum_absolute_error": 0.310546875 + }, + { + "column": 9323, + "rms_error": 0.3046875, + "maximum_absolute_error": 0.3046875 + }, + { + "column": 7032, + "rms_error": 0.3046875, + "maximum_absolute_error": 0.3046875 + }, + { + "column": 94782, + "rms_error": 0.296875, + "maximum_absolute_error": 0.296875 + }, + { + "column": 23812, + "rms_error": 0.2939453125, + "maximum_absolute_error": 0.2939453125 + }, + { + "column": 36145, + "rms_error": 0.28125, + "maximum_absolute_error": 0.28125 + }, + { + "column": 93950, + "rms_error": 0.28125, + "maximum_absolute_error": 0.28125 + }, + { + "column": 117795, + "rms_error": 0.28125, + "maximum_absolute_error": 0.28125 + }, + { + "column": 52318, + "rms_error": 0.28125, + "maximum_absolute_error": 0.28125 + }, + { + "column": 94667, + "rms_error": 0.28125, + "maximum_absolute_error": 0.28125 + }, + { + "column": 53589, + "rms_error": 0.28125, + "maximum_absolute_error": 0.28125 + }, + { + "column": 58716, + "rms_error": 0.28125, + "maximum_absolute_error": 0.28125 + }, + { + "column": 32140, + "rms_error": 0.28125, + "maximum_absolute_error": 0.28125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "checkpoint": "LiquidAI/LFM3-1.7B-MLX-6bit", + "phase": "decode", + "prefilled_history": 63, + "current_row": 63, + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "e86739c46d55e1f137ac55c9e43119fda97914902334f12df9069907c1a0cab9", + "reference_full_logits_sha256": "badfeed6f4b81e8f448eb0bd1f523d8dc3f8891e56f0067fe170f566e87eb9e5" + }, + { + "rows": 1, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.03398969064458055, + "maximum_absolute_error": 0.35546875, + "differences": [], + "worst_rows": [ + { + "row": 63, + "rms_error": 0.09125710667798623, + "maximum_absolute_error": 0.35546875 + } + ], + "worst_columns": [ + { + "column": 19987, + "rms_error": 0.35546875, + "maximum_absolute_error": 0.35546875 + }, + { + "column": 21929, + "rms_error": 0.34716796875, + "maximum_absolute_error": 0.34716796875 + }, + { + "column": 7600, + "rms_error": 0.34375, + "maximum_absolute_error": 0.34375 + }, + { + "column": 34767, + "rms_error": 0.34375, + "maximum_absolute_error": 0.34375 + }, + { + "column": 54720, + "rms_error": 0.34375, + "maximum_absolute_error": 0.34375 + }, + { + "column": 29199, + "rms_error": 0.34375, + "maximum_absolute_error": 0.34375 + }, + { + "column": 6143, + "rms_error": 0.33984375, + "maximum_absolute_error": 0.33984375 + }, + { + "column": 60109, + "rms_error": 0.3359375, + "maximum_absolute_error": 0.3359375 + }, + { + "column": 29486, + "rms_error": 0.3330078125, + "maximum_absolute_error": 0.3330078125 + }, + { + "column": 33211, + "rms_error": 0.328125, + "maximum_absolute_error": 0.328125 + }, + { + "column": 54050, + "rms_error": 0.328125, + "maximum_absolute_error": 0.328125 + }, + { + "column": 10936, + "rms_error": 0.328125, + "maximum_absolute_error": 0.328125 + }, + { + "column": 14054, + "rms_error": 0.322540283203125, + "maximum_absolute_error": 0.322540283203125 + }, + { + "column": 46076, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + }, + { + "column": 23955, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + }, + { + "column": 1985, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + }, + { + "column": 63853, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + }, + { + "column": 123494, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + }, + { + "column": 49094, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + }, + { + "column": 10534, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "checkpoint": "LiquidAI/LFM3-1.7B-MLX-8bit", + "phase": "decode", + "prefilled_history": 63, + "current_row": 63, + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "03d2ff5c67ec8bb02539d7fe526dd0bc4f0e9833a8d6474cd1174874fe79c905", + "reference_full_logits_sha256": "e890b65e52b7071b80f6052d520bca1b857466a6453a3b1fdc43df33ea6089eb" + }, + { + "rows": 1, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.030637681948585384, + "maximum_absolute_error": 0.193359375, + "differences": [], + "worst_rows": [ + { + "row": 63, + "rms_error": 0.06189983119754214, + "maximum_absolute_error": 0.193359375 + } + ], + "worst_columns": [ + { + "column": 6095, + "rms_error": 0.193359375, + "maximum_absolute_error": 0.193359375 + }, + { + "column": 6049, + "rms_error": 0.1796875, + "maximum_absolute_error": 0.1796875 + }, + { + "column": 44706, + "rms_error": 0.17919921875, + "maximum_absolute_error": 0.17919921875 + }, + { + "column": 65418, + "rms_error": 0.17578125, + "maximum_absolute_error": 0.17578125 + }, + { + "column": 105864, + "rms_error": 0.171875, + "maximum_absolute_error": 0.171875 + }, + { + "column": 24030, + "rms_error": 0.171875, + "maximum_absolute_error": 0.171875 + }, + { + "column": 42266, + "rms_error": 0.171875, + "maximum_absolute_error": 0.171875 + }, + { + "column": 3136, + "rms_error": 0.171875, + "maximum_absolute_error": 0.171875 + }, + { + "column": 47240, + "rms_error": 0.171875, + "maximum_absolute_error": 0.171875 + }, + { + "column": 1718, + "rms_error": 0.171875, + "maximum_absolute_error": 0.171875 + }, + { + "column": 40928, + "rms_error": 0.171875, + "maximum_absolute_error": 0.171875 + }, + { + "column": 61448, + "rms_error": 0.1640625, + "maximum_absolute_error": 0.1640625 + }, + { + "column": 108148, + "rms_error": 0.1640625, + "maximum_absolute_error": 0.1640625 + }, + { + "column": 30586, + "rms_error": 0.1640625, + "maximum_absolute_error": 0.1640625 + }, + { + "column": 53574, + "rms_error": 0.1640625, + "maximum_absolute_error": 0.1640625 + }, + { + "column": 12960, + "rms_error": 0.1640625, + "maximum_absolute_error": 0.1640625 + }, + { + "column": 97794, + "rms_error": 0.1640625, + "maximum_absolute_error": 0.1640625 + }, + { + "column": 33311, + "rms_error": 0.1640625, + "maximum_absolute_error": 0.1640625 + }, + { + "column": 27756, + "rms_error": 0.1640625, + "maximum_absolute_error": 0.1640625 + }, + { + "column": 97838, + "rms_error": 0.158203125, + "maximum_absolute_error": 0.158203125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "phase": "decode", + "prefilled_history": 63, + "current_row": 63, + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "2103df493c16eaf3f7e13751dcf6c2f7bdbe9976f12fa84c6e1a7725053c411e", + "reference_full_logits_sha256": "f5c18f0b75b86711e663904ec4011913ee23bf76b565317e8741de74d5b23dd6" + }, + { + "rows": 1, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.013336827988498537, + "maximum_absolute_error": 0.16796875, + "differences": [], + "worst_rows": [ + { + "row": 63, + "rms_error": 0.028049199550743174, + "maximum_absolute_error": 0.16796875 + } + ], + "worst_columns": [ + { + "column": 2819, + "rms_error": 0.16796875, + "maximum_absolute_error": 0.16796875 + }, + { + "column": 28100, + "rms_error": 0.1640625, + "maximum_absolute_error": 0.1640625 + }, + { + "column": 15484, + "rms_error": 0.1640625, + "maximum_absolute_error": 0.1640625 + }, + { + "column": 93, + "rms_error": 0.1640625, + "maximum_absolute_error": 0.1640625 + }, + { + "column": 893, + "rms_error": 0.15625, + "maximum_absolute_error": 0.15625 + }, + { + "column": 74, + "rms_error": 0.15625, + "maximum_absolute_error": 0.15625 + }, + { + "column": 96, + "rms_error": 0.15625, + "maximum_absolute_error": 0.15625 + }, + { + "column": 84, + "rms_error": 0.15625, + "maximum_absolute_error": 0.15625 + }, + { + "column": 8260, + "rms_error": 0.15625, + "maximum_absolute_error": 0.15625 + }, + { + "column": 7511, + "rms_error": 0.15625, + "maximum_absolute_error": 0.15625 + }, + { + "column": 98, + "rms_error": 0.15625, + "maximum_absolute_error": 0.15625 + }, + { + "column": 47335, + "rms_error": 0.15625, + "maximum_absolute_error": 0.15625 + }, + { + "column": 79, + "rms_error": 0.15625, + "maximum_absolute_error": 0.15625 + }, + { + "column": 1384, + "rms_error": 0.142578125, + "maximum_absolute_error": 0.142578125 + }, + { + "column": 7577, + "rms_error": 0.140625, + "maximum_absolute_error": 0.140625 + }, + { + "column": 61735, + "rms_error": 0.140625, + "maximum_absolute_error": 0.140625 + }, + { + "column": 1056, + "rms_error": 0.140625, + "maximum_absolute_error": 0.140625 + }, + { + "column": 51446, + "rms_error": 0.140625, + "maximum_absolute_error": 0.140625 + }, + { + "column": 375, + "rms_error": 0.140625, + "maximum_absolute_error": 0.140625 + }, + { + "column": 59800, + "rms_error": 0.140625, + "maximum_absolute_error": 0.140625 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-6bit", + "phase": "decode", + "prefilled_history": 63, + "current_row": 63, + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "1a033068012e72b9b60be106be6eee6058faaece4b0d439a151b683c1b94c1ef", + "reference_full_logits_sha256": "d16f06c7813c231e4f3b7172456959da94d32adf024381666104fa28c0d3a859" + }, + { + "rows": 1, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.013220059903007179, + "maximum_absolute_error": 0.15625, + "differences": [], + "worst_rows": [ + { + "row": 63, + "rms_error": 0.027668003667474416, + "maximum_absolute_error": 0.15625 + } + ], + "worst_columns": [ + { + "column": 984, + "rms_error": 0.15625, + "maximum_absolute_error": 0.15625 + }, + { + "column": 53, + "rms_error": 0.15625, + "maximum_absolute_error": 0.15625 + }, + { + "column": 42847, + "rms_error": 0.1328125, + "maximum_absolute_error": 0.1328125 + }, + { + "column": 5820, + "rms_error": 0.1328125, + "maximum_absolute_error": 0.1328125 + }, + { + "column": 15460, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 51446, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 48949, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 16019, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 71, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 86, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 15532, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 1443, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 3592, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 3698, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 5872, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 503, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 4353, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 27348, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 1224, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 4507, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-8bit", + "phase": "decode", + "prefilled_history": 63, + "current_row": 63, + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "472b3ad8ec2237267664292acae4be66f78e80271830962d2d4570e088fcec04", + "reference_full_logits_sha256": "c2263cda17da88664fad1fbb499035c33899760ec5cad02a35a4ba1ac1b908c8" + } + ] +} diff --git a/docs/receipts/lfm3/six-width-fp64-prompt.json b/docs/receipts/lfm3/six-width-fp64-prompt.json new file mode 100644 index 0000000000..e9d71c476e --- /dev/null +++ b/docs/receipts/lfm3/six-width-fp64-prompt.json @@ -0,0 +1,1368 @@ +{ + "scope": "All six quantizations, code64 prompt rows, existing post-RoPE native capture against current direct FP64/BF16 oracle; not final tile qualification", + "fidelity_pass": null, + "accepted_threshold": null, + "cells": [ + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.019898213267257773, + "maximum_absolute_error": 0.6875, + "differences": [], + "worst_rows": [ + { + "row": 0, + "rms_error": 0.12301575847128561, + "maximum_absolute_error": 0.4375 + }, + { + "row": 42, + "rms_error": 0.12141064311438426, + "maximum_absolute_error": 0.625 + }, + { + "row": 41, + "rms_error": 0.10940544488449085, + "maximum_absolute_error": 0.578125 + }, + { + "row": 43, + "rms_error": 0.09222703260603499, + "maximum_absolute_error": 0.390625 + }, + { + "row": 28, + "rms_error": 0.0825923186936875, + "maximum_absolute_error": 0.46875 + }, + { + "row": 62, + "rms_error": 0.07925748348396815, + "maximum_absolute_error": 0.6875 + }, + { + "row": 4, + "rms_error": 0.0773653660968508, + "maximum_absolute_error": 0.40625 + }, + { + "row": 40, + "rms_error": 0.0771670764384705, + "maximum_absolute_error": 0.3778076171875 + }, + { + "row": 22, + "rms_error": 0.07286469848686604, + "maximum_absolute_error": 0.4375 + }, + { + "row": 10, + "rms_error": 0.07064260891454813, + "maximum_absolute_error": 0.28125 + }, + { + "row": 44, + "rms_error": 0.06989413816956311, + "maximum_absolute_error": 0.328125 + }, + { + "row": 23, + "rms_error": 0.06668170145228336, + "maximum_absolute_error": 0.34375 + }, + { + "row": 13, + "rms_error": 0.06271886784844971, + "maximum_absolute_error": 0.296875 + }, + { + "row": 30, + "rms_error": 0.06254545915163834, + "maximum_absolute_error": 0.296875 + }, + { + "row": 1, + "rms_error": 0.06233672771987397, + "maximum_absolute_error": 0.28125 + }, + { + "row": 32, + "rms_error": 0.06086335183898396, + "maximum_absolute_error": 0.40625 + }, + { + "row": 37, + "rms_error": 0.05942772461592182, + "maximum_absolute_error": 0.375 + }, + { + "row": 38, + "rms_error": 0.05940583386453879, + "maximum_absolute_error": 0.38134765625 + }, + { + "row": 6, + "rms_error": 0.05880314112372005, + "maximum_absolute_error": 0.25 + }, + { + "row": 47, + "rms_error": 0.0564171984141962, + "maximum_absolute_error": 0.2578125 + } + ], + "worst_columns": [ + { + "column": 5272, + "rms_error": 0.14924881717078994, + "maximum_absolute_error": 0.4375 + }, + { + "column": 20975, + "rms_error": 0.13830767072011435, + "maximum_absolute_error": 0.375 + }, + { + "column": 15960, + "rms_error": 0.13301721391207205, + "maximum_absolute_error": 0.375 + }, + { + "column": 4484, + "rms_error": 0.13052028163702994, + "maximum_absolute_error": 0.40625 + }, + { + "column": 51466, + "rms_error": 0.12749794184669136, + "maximum_absolute_error": 0.375 + }, + { + "column": 1224, + "rms_error": 0.1270584843006156, + "maximum_absolute_error": 0.5 + }, + { + "column": 15136, + "rms_error": 0.12662081140442283, + "maximum_absolute_error": 0.625 + }, + { + "column": 4361, + "rms_error": 0.12336391993992941, + "maximum_absolute_error": 0.375 + }, + { + "column": 893, + "rms_error": 0.12323758317716488, + "maximum_absolute_error": 0.6875 + }, + { + "column": 8328, + "rms_error": 0.12318601764056089, + "maximum_absolute_error": 0.578125 + }, + { + "column": 8260, + "rms_error": 0.12311975651918346, + "maximum_absolute_error": 0.65625 + }, + { + "column": 22155, + "rms_error": 0.12107258273288721, + "maximum_absolute_error": 0.34375 + }, + { + "column": 15582, + "rms_error": 0.1184407464758198, + "maximum_absolute_error": 0.3515625 + }, + { + "column": 4951, + "rms_error": 0.11740358616322723, + "maximum_absolute_error": 0.5 + }, + { + "column": 94813, + "rms_error": 0.11680805446233773, + "maximum_absolute_error": 0.46875 + }, + { + "column": 1396, + "rms_error": 0.11677431639055184, + "maximum_absolute_error": 0.3304443359375 + }, + { + "column": 38765, + "rms_error": 0.11575825461579775, + "maximum_absolute_error": 0.4375 + }, + { + "column": 7511, + "rms_error": 0.11573834151992225, + "maximum_absolute_error": 0.5 + }, + { + "column": 39382, + "rms_error": 0.11526088490486561, + "maximum_absolute_error": 0.46875 + }, + { + "column": 1367, + "rms_error": 0.1150642205596123, + "maximum_absolute_error": 0.328125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "1.7B-4-code-64", + "phase": "prompt", + "checkpoint": "LiquidAI/LFM3-1.7B-MLX-4bit", + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "f15ce195e69a3e054ba60843b37f105739a6e17f7f5facc13a68892ee2bdd32b", + "reference_logits_sha256": "6286092ef4d800dd2fefe310628c1a5270cc489f33c72a373010b02d38164023" + }, + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.026689922049396048, + "maximum_absolute_error": 0.75, + "differences": [], + "worst_rows": [ + { + "row": 10, + "rms_error": 0.17142728645006744, + "maximum_absolute_error": 0.53125 + }, + { + "row": 0, + "rms_error": 0.1422026444976933, + "maximum_absolute_error": 0.75 + }, + { + "row": 42, + "rms_error": 0.14204671379297865, + "maximum_absolute_error": 0.578125 + }, + { + "row": 16, + "rms_error": 0.13838871068371872, + "maximum_absolute_error": 0.4375 + }, + { + "row": 19, + "rms_error": 0.12012588820733641, + "maximum_absolute_error": 0.34375 + }, + { + "row": 15, + "rms_error": 0.11904887812117255, + "maximum_absolute_error": 0.421875 + }, + { + "row": 31, + "rms_error": 0.10889341164425237, + "maximum_absolute_error": 0.390625 + }, + { + "row": 29, + "rms_error": 0.10141750169027551, + "maximum_absolute_error": 0.4375 + }, + { + "row": 43, + "rms_error": 0.10124202752832445, + "maximum_absolute_error": 0.40625 + }, + { + "row": 49, + "rms_error": 0.09557071156115735, + "maximum_absolute_error": 0.4296875 + }, + { + "row": 14, + "rms_error": 0.09401268020082862, + "maximum_absolute_error": 0.5234375 + }, + { + "row": 2, + "rms_error": 0.09175851789989324, + "maximum_absolute_error": 0.3984375 + }, + { + "row": 8, + "rms_error": 0.09048373547123606, + "maximum_absolute_error": 0.34375 + }, + { + "row": 9, + "rms_error": 0.09024813273761305, + "maximum_absolute_error": 0.4921875 + }, + { + "row": 28, + "rms_error": 0.08802512154159117, + "maximum_absolute_error": 0.46875 + }, + { + "row": 33, + "rms_error": 0.08703463336095042, + "maximum_absolute_error": 0.34375 + }, + { + "row": 51, + "rms_error": 0.08688489918955575, + "maximum_absolute_error": 0.375 + }, + { + "row": 22, + "rms_error": 0.08686510053844272, + "maximum_absolute_error": 0.412109375 + }, + { + "row": 3, + "rms_error": 0.08350938284469905, + "maximum_absolute_error": 0.375 + }, + { + "row": 56, + "rms_error": 0.08325712606026651, + "maximum_absolute_error": 0.375 + } + ], + "worst_columns": [ + { + "column": 88140, + "rms_error": 0.21171019029755553, + "maximum_absolute_error": 0.45989990234375 + }, + { + "column": 88304, + "rms_error": 0.20554743382484159, + "maximum_absolute_error": 0.5234375 + }, + { + "column": 115125, + "rms_error": 0.19978058819755923, + "maximum_absolute_error": 0.4375 + }, + { + "column": 49899, + "rms_error": 0.19276769981232406, + "maximum_absolute_error": 0.375 + }, + { + "column": 90359, + "rms_error": 0.19056538429877645, + "maximum_absolute_error": 0.421875 + }, + { + "column": 91957, + "rms_error": 0.18814929803474448, + "maximum_absolute_error": 0.390625 + }, + { + "column": 20975, + "rms_error": 0.18652640335361384, + "maximum_absolute_error": 0.4375 + }, + { + "column": 89889, + "rms_error": 0.18440182377581552, + "maximum_absolute_error": 0.32421875 + }, + { + "column": 22497, + "rms_error": 0.18380758332062028, + "maximum_absolute_error": 0.375 + }, + { + "column": 14310, + "rms_error": 0.18365566751680795, + "maximum_absolute_error": 0.375 + }, + { + "column": 4484, + "rms_error": 0.18060141843418825, + "maximum_absolute_error": 0.34375 + }, + { + "column": 53417, + "rms_error": 0.18054394272788096, + "maximum_absolute_error": 0.4453125 + }, + { + "column": 53842, + "rms_error": 0.1796946342752001, + "maximum_absolute_error": 0.34375 + }, + { + "column": 116115, + "rms_error": 0.17918524830590607, + "maximum_absolute_error": 0.453125 + }, + { + "column": 56370, + "rms_error": 0.1777892499616714, + "maximum_absolute_error": 0.421875 + }, + { + "column": 91849, + "rms_error": 0.17767817334408556, + "maximum_absolute_error": 0.375 + }, + { + "column": 15960, + "rms_error": 0.17582125738297258, + "maximum_absolute_error": 0.375 + }, + { + "column": 87970, + "rms_error": 0.17520242600049366, + "maximum_absolute_error": 0.40625 + }, + { + "column": 117071, + "rms_error": 0.17350036280879913, + "maximum_absolute_error": 0.3515625 + }, + { + "column": 17639, + "rms_error": 0.17306008396153286, + "maximum_absolute_error": 0.375 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "1.7B-6-code-64", + "phase": "prompt", + "checkpoint": "LiquidAI/LFM3-1.7B-MLX-6bit", + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "f94b78c965c874c13fee175b036265f5691b7711b57559f942ecad80e3f1d4a3", + "reference_logits_sha256": "badfeed6f4b81e8f448eb0bd1f523d8dc3f8891e56f0067fe170f566e87eb9e5" + }, + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.0220501312407529, + "maximum_absolute_error": 0.6640625, + "differences": [], + "worst_rows": [ + { + "row": 41, + "rms_error": 0.17345533863740256, + "maximum_absolute_error": 0.59375 + }, + { + "row": 0, + "rms_error": 0.16594398697398952, + "maximum_absolute_error": 0.6640625 + }, + { + "row": 28, + "rms_error": 0.11302524768018539, + "maximum_absolute_error": 0.53125 + }, + { + "row": 22, + "rms_error": 0.0966248427386952, + "maximum_absolute_error": 0.5 + }, + { + "row": 62, + "rms_error": 0.09545741607618395, + "maximum_absolute_error": 0.46875 + }, + { + "row": 63, + "rms_error": 0.09125710667798623, + "maximum_absolute_error": 0.35546875 + }, + { + "row": 15, + "rms_error": 0.09025357831733094, + "maximum_absolute_error": 0.3125 + }, + { + "row": 16, + "rms_error": 0.0888772476755528, + "maximum_absolute_error": 0.3125 + }, + { + "row": 47, + "rms_error": 0.08819708714972854, + "maximum_absolute_error": 0.375 + }, + { + "row": 4, + "rms_error": 0.08449832876546694, + "maximum_absolute_error": 0.53125 + }, + { + "row": 42, + "rms_error": 0.07635969195498178, + "maximum_absolute_error": 0.40625 + }, + { + "row": 32, + "rms_error": 0.0721201318620769, + "maximum_absolute_error": 0.34375 + }, + { + "row": 61, + "rms_error": 0.06953682581797835, + "maximum_absolute_error": 0.34375 + }, + { + "row": 31, + "rms_error": 0.0695276687756236, + "maximum_absolute_error": 0.4375 + }, + { + "row": 33, + "rms_error": 0.06872247151234558, + "maximum_absolute_error": 0.34375 + }, + { + "row": 29, + "rms_error": 0.06841369688082365, + "maximum_absolute_error": 0.38037109375 + }, + { + "row": 43, + "rms_error": 0.06832428159781277, + "maximum_absolute_error": 0.3828125 + }, + { + "row": 21, + "rms_error": 0.06749965844760528, + "maximum_absolute_error": 0.34375 + }, + { + "row": 1, + "rms_error": 0.06727780028766155, + "maximum_absolute_error": 0.28125 + }, + { + "row": 40, + "rms_error": 0.06704998690284182, + "maximum_absolute_error": 0.390625 + } + ], + "worst_columns": [ + { + "column": 42678, + "rms_error": 0.12685415371513498, + "maximum_absolute_error": 0.4296875 + }, + { + "column": 33237, + "rms_error": 0.12621574465901272, + "maximum_absolute_error": 0.5625 + }, + { + "column": 116115, + "rms_error": 0.1260394301754644, + "maximum_absolute_error": 0.3125 + }, + { + "column": 4625, + "rms_error": 0.12559285241247872, + "maximum_absolute_error": 0.375 + }, + { + "column": 20975, + "rms_error": 0.12487812452403631, + "maximum_absolute_error": 0.3125 + }, + { + "column": 61089, + "rms_error": 0.12359072449449683, + "maximum_absolute_error": 0.4375 + }, + { + "column": 46015, + "rms_error": 0.12329067593707133, + "maximum_absolute_error": 0.5 + }, + { + "column": 22155, + "rms_error": 0.12240827791801107, + "maximum_absolute_error": 0.375 + }, + { + "column": 28678, + "rms_error": 0.12169910543090903, + "maximum_absolute_error": 0.484375 + }, + { + "column": 39382, + "rms_error": 0.12169767437400192, + "maximum_absolute_error": 0.59375 + }, + { + "column": 30270, + "rms_error": 0.12051623050711183, + "maximum_absolute_error": 0.5 + }, + { + "column": 116016, + "rms_error": 0.11991592954169991, + "maximum_absolute_error": 0.4375 + }, + { + "column": 91493, + "rms_error": 0.11961819347760586, + "maximum_absolute_error": 0.4375 + }, + { + "column": 92555, + "rms_error": 0.11936493379283061, + "maximum_absolute_error": 0.3125 + }, + { + "column": 9323, + "rms_error": 0.1187778901324646, + "maximum_absolute_error": 0.421875 + }, + { + "column": 10835, + "rms_error": 0.11848167420983581, + "maximum_absolute_error": 0.375 + }, + { + "column": 93981, + "rms_error": 0.11821733014965576, + "maximum_absolute_error": 0.375 + }, + { + "column": 5018, + "rms_error": 0.11756790725059593, + "maximum_absolute_error": 0.5546875 + }, + { + "column": 93924, + "rms_error": 0.11752106955131514, + "maximum_absolute_error": 0.6171875 + }, + { + "column": 119658, + "rms_error": 0.11746209080912955, + "maximum_absolute_error": 0.3046875 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "1.7B-8-code-64", + "phase": "prompt", + "checkpoint": "LiquidAI/LFM3-1.7B-MLX-8bit", + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "c0bc4bfb20a4a24abe52d4b8366c9a4acdabd597b68f627552bc0d9b06c7450b", + "reference_logits_sha256": "e890b65e52b7071b80f6052d520bca1b857466a6453a3b1fdc43df33ea6089eb" + }, + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.013956257000991174, + "maximum_absolute_error": 0.6875, + "differences": [], + "worst_rows": [ + { + "row": 38, + "rms_error": 0.11460334310170639, + "maximum_absolute_error": 0.328125 + }, + { + "row": 29, + "rms_error": 0.07609650779142937, + "maximum_absolute_error": 0.248291015625 + }, + { + "row": 0, + "rms_error": 0.07317533579110798, + "maximum_absolute_error": 0.265625 + }, + { + "row": 57, + "rms_error": 0.06819346713585736, + "maximum_absolute_error": 0.6875 + }, + { + "row": 11, + "rms_error": 0.06719930161309025, + "maximum_absolute_error": 0.296875 + }, + { + "row": 63, + "rms_error": 0.06189983119754214, + "maximum_absolute_error": 0.193359375 + }, + { + "row": 17, + "rms_error": 0.0562844409253447, + "maximum_absolute_error": 0.234375 + }, + { + "row": 55, + "rms_error": 0.055719994631985244, + "maximum_absolute_error": 0.21875 + }, + { + "row": 47, + "rms_error": 0.05486522860095206, + "maximum_absolute_error": 0.21875 + }, + { + "row": 15, + "rms_error": 0.05260483982982732, + "maximum_absolute_error": 0.375 + }, + { + "row": 8, + "rms_error": 0.051856505198323556, + "maximum_absolute_error": 0.25 + }, + { + "row": 21, + "rms_error": 0.05108993972236896, + "maximum_absolute_error": 0.28125 + }, + { + "row": 28, + "rms_error": 0.05107495391341613, + "maximum_absolute_error": 0.46875 + }, + { + "row": 61, + "rms_error": 0.051041773960293906, + "maximum_absolute_error": 0.3125 + }, + { + "row": 12, + "rms_error": 0.050840759407169596, + "maximum_absolute_error": 0.2109375 + }, + { + "row": 35, + "rms_error": 0.050085313313313694, + "maximum_absolute_error": 0.4375 + }, + { + "row": 46, + "rms_error": 0.049754913376188196, + "maximum_absolute_error": 0.375 + }, + { + "row": 39, + "rms_error": 0.04662994047939181, + "maximum_absolute_error": 0.21875 + }, + { + "row": 27, + "rms_error": 0.04651380550464994, + "maximum_absolute_error": 0.19140625 + }, + { + "row": 62, + "rms_error": 0.04637423724016495, + "maximum_absolute_error": 0.21875 + } + ], + "worst_columns": [ + { + "column": 851, + "rms_error": 0.12639365530749455, + "maximum_absolute_error": 0.6875 + }, + { + "column": 33237, + "rms_error": 0.10850725640146129, + "maximum_absolute_error": 0.375 + }, + { + "column": 16033, + "rms_error": 0.10608651587427771, + "maximum_absolute_error": 0.28125 + }, + { + "column": 1011, + "rms_error": 0.10503102966000476, + "maximum_absolute_error": 0.53125 + }, + { + "column": 8373, + "rms_error": 0.10337083627736275, + "maximum_absolute_error": 0.5625 + }, + { + "column": 723, + "rms_error": 0.1002562220633136, + "maximum_absolute_error": 0.4375 + }, + { + "column": 40928, + "rms_error": 0.09699366981465977, + "maximum_absolute_error": 0.5 + }, + { + "column": 1583, + "rms_error": 0.09649414292256268, + "maximum_absolute_error": 0.46875 + }, + { + "column": 295, + "rms_error": 0.09053799629291337, + "maximum_absolute_error": 0.359375 + }, + { + "column": 116016, + "rms_error": 0.09029335147521579, + "maximum_absolute_error": 0.265625 + }, + { + "column": 2690, + "rms_error": 0.08823780691693173, + "maximum_absolute_error": 0.375 + }, + { + "column": 45707, + "rms_error": 0.08812709362843722, + "maximum_absolute_error": 0.265625 + }, + { + "column": 42847, + "rms_error": 0.08731298300569229, + "maximum_absolute_error": 0.3125 + }, + { + "column": 11448, + "rms_error": 0.08659296791732592, + "maximum_absolute_error": 0.25 + }, + { + "column": 21516, + "rms_error": 0.08597371167496128, + "maximum_absolute_error": 0.25 + }, + { + "column": 98225, + "rms_error": 0.08511426945166548, + "maximum_absolute_error": 0.25390625 + }, + { + "column": 11460, + "rms_error": 0.08477921618656097, + "maximum_absolute_error": 0.265625 + }, + { + "column": 17788, + "rms_error": 0.08402464145310369, + "maximum_absolute_error": 0.38671875 + }, + { + "column": 30270, + "rms_error": 0.08306129079244248, + "maximum_absolute_error": 0.28125 + }, + { + "column": 6470, + "rms_error": 0.0826058937474372, + "maximum_absolute_error": 0.4375 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-4-code-64", + "phase": "prompt", + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "14a4249434dcf626b1151fa9d291256ea6dbeb8005c9b2211cff0aefb6e29722", + "reference_logits_sha256": "f5c18f0b75b86711e663904ec4011913ee23bf76b565317e8741de74d5b23dd6" + }, + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 1, + "relative_rms": 0.015377440488712887, + "maximum_absolute_error": 0.96484375, + "differences": [ + { + "row": 13, + "native_top": 22, + "reference_top": 63908, + "native_maximum_set": [ + 22, + 63908 + ], + "reference_maximum_set": [ + 63908 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + }, + { + "row": 22, + "native_top": 16033, + "reference_top": 91, + "native_maximum_set": [ + 16033 + ], + "reference_maximum_set": [ + 91 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.1875, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.1875 + } + ], + "worst_rows": [ + { + "row": 57, + "rms_error": 0.16603965592691475, + "maximum_absolute_error": 0.96484375 + }, + { + "row": 55, + "rms_error": 0.08489186681149496, + "maximum_absolute_error": 0.3046875 + }, + { + "row": 13, + "rms_error": 0.07917376823524733, + "maximum_absolute_error": 0.28125 + }, + { + "row": 2, + "rms_error": 0.06311253390330333, + "maximum_absolute_error": 0.216796875 + }, + { + "row": 32, + "rms_error": 0.060344014781392605, + "maximum_absolute_error": 0.28125 + }, + { + "row": 22, + "rms_error": 0.055970470902545984, + "maximum_absolute_error": 0.375 + }, + { + "row": 43, + "rms_error": 0.053908494189525466, + "maximum_absolute_error": 0.2109375 + }, + { + "row": 53, + "rms_error": 0.05371347456512, + "maximum_absolute_error": 0.2109375 + }, + { + "row": 50, + "rms_error": 0.05305531735124143, + "maximum_absolute_error": 0.1875 + }, + { + "row": 46, + "rms_error": 0.05286167453834549, + "maximum_absolute_error": 0.29296875 + }, + { + "row": 28, + "rms_error": 0.05198421343833805, + "maximum_absolute_error": 0.28125 + }, + { + "row": 56, + "rms_error": 0.050748718923888735, + "maximum_absolute_error": 0.203125 + }, + { + "row": 5, + "rms_error": 0.04965621029864758, + "maximum_absolute_error": 0.1669769287109375 + }, + { + "row": 34, + "rms_error": 0.04927833272243707, + "maximum_absolute_error": 0.21875 + }, + { + "row": 36, + "rms_error": 0.04908556341435462, + "maximum_absolute_error": 0.1875 + }, + { + "row": 4, + "rms_error": 0.0480743780157286, + "maximum_absolute_error": 0.294921875 + }, + { + "row": 62, + "rms_error": 0.04713935283875158, + "maximum_absolute_error": 0.3125 + }, + { + "row": 1, + "rms_error": 0.0456394677394624, + "maximum_absolute_error": 0.1875 + }, + { + "row": 33, + "rms_error": 0.0447680490608537, + "maximum_absolute_error": 0.21875 + }, + { + "row": 38, + "rms_error": 0.044176922297933914, + "maximum_absolute_error": 0.25 + } + ], + "worst_columns": [ + { + "column": 93538, + "rms_error": 0.14026637509168713, + "maximum_absolute_error": 0.96484375 + }, + { + "column": 538, + "rms_error": 0.13238456448014496, + "maximum_absolute_error": 0.953125 + }, + { + "column": 1046, + "rms_error": 0.1223944061953623, + "maximum_absolute_error": 0.5625 + }, + { + "column": 94852, + "rms_error": 0.1214250871538932, + "maximum_absolute_error": 0.828125 + }, + { + "column": 7272, + "rms_error": 0.11949435818918488, + "maximum_absolute_error": 0.84375 + }, + { + "column": 17, + "rms_error": 0.11716969672707923, + "maximum_absolute_error": 0.8125 + }, + { + "column": 6907, + "rms_error": 0.11613526003940478, + "maximum_absolute_error": 0.849609375 + }, + { + "column": 346, + "rms_error": 0.11392491089664815, + "maximum_absolute_error": 0.78125 + }, + { + "column": 16997, + "rms_error": 0.11332787179175711, + "maximum_absolute_error": 0.6875 + }, + { + "column": 3585, + "rms_error": 0.11256296995771509, + "maximum_absolute_error": 0.8125 + }, + { + "column": 647, + "rms_error": 0.11219731048893868, + "maximum_absolute_error": 0.796875 + }, + { + "column": 52421, + "rms_error": 0.10928653461278572, + "maximum_absolute_error": 0.796875 + }, + { + "column": 9673, + "rms_error": 0.10862716394009563, + "maximum_absolute_error": 0.7734375 + }, + { + "column": 13685, + "rms_error": 0.10791157272755565, + "maximum_absolute_error": 0.76953125 + }, + { + "column": 1954, + "rms_error": 0.10729993054205725, + "maximum_absolute_error": 0.767578125 + }, + { + "column": 1062, + "rms_error": 0.10728559453153616, + "maximum_absolute_error": 0.78125 + }, + { + "column": 7415, + "rms_error": 0.10701046101345699, + "maximum_absolute_error": 0.734375 + }, + { + "column": 5728, + "rms_error": 0.1068937588477719, + "maximum_absolute_error": 0.765625 + }, + { + "column": 48564, + "rms_error": 0.10625993545718224, + "maximum_absolute_error": 0.76806640625 + }, + { + "column": 34398, + "rms_error": 0.10505060026867931, + "maximum_absolute_error": 0.7001953125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-code-64", + "phase": "prompt", + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-6bit", + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "02225f1c8c7f5de05ebd44bd49948ad000aac040f3d6025d6655a9772ed08f9b", + "reference_logits_sha256": "d16f06c7813c231e4f3b7172456959da94d32adf024381666104fa28c0d3a859" + }, + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.015466008538336544, + "maximum_absolute_error": 0.6201171875, + "differences": [], + "worst_rows": [ + { + "row": 57, + "rms_error": 0.10084401675170394, + "maximum_absolute_error": 0.6201171875 + }, + { + "row": 54, + "rms_error": 0.08537710665171272, + "maximum_absolute_error": 0.3359375 + }, + { + "row": 4, + "rms_error": 0.0847654133912914, + "maximum_absolute_error": 0.3828125 + }, + { + "row": 28, + "rms_error": 0.07578738582927384, + "maximum_absolute_error": 0.25 + }, + { + "row": 19, + "rms_error": 0.07325385829626402, + "maximum_absolute_error": 0.322265625 + }, + { + "row": 15, + "rms_error": 0.06881209672444469, + "maximum_absolute_error": 0.2587890625 + }, + { + "row": 60, + "rms_error": 0.06712113892820007, + "maximum_absolute_error": 0.21875 + }, + { + "row": 59, + "rms_error": 0.06689059146927472, + "maximum_absolute_error": 0.25 + }, + { + "row": 0, + "rms_error": 0.06643536714740046, + "maximum_absolute_error": 0.28515625 + }, + { + "row": 33, + "rms_error": 0.0644964015744809, + "maximum_absolute_error": 0.203125 + }, + { + "row": 18, + "rms_error": 0.06176497570554598, + "maximum_absolute_error": 0.2734375 + }, + { + "row": 41, + "rms_error": 0.05864792858652117, + "maximum_absolute_error": 0.25 + }, + { + "row": 42, + "rms_error": 0.05249413812563852, + "maximum_absolute_error": 0.28125 + }, + { + "row": 56, + "rms_error": 0.05004946561577254, + "maximum_absolute_error": 0.2578125 + }, + { + "row": 29, + "rms_error": 0.04989588211022091, + "maximum_absolute_error": 0.25 + }, + { + "row": 55, + "rms_error": 0.04815518751565306, + "maximum_absolute_error": 0.296875 + }, + { + "row": 26, + "rms_error": 0.0477324446813435, + "maximum_absolute_error": 0.232421875 + }, + { + "row": 3, + "rms_error": 0.04683582065091273, + "maximum_absolute_error": 0.2734375 + }, + { + "row": 22, + "rms_error": 0.04634210225231695, + "maximum_absolute_error": 0.3125 + }, + { + "row": 46, + "rms_error": 0.04595433784186023, + "maximum_absolute_error": 0.3125 + } + ], + "worst_columns": [ + { + "column": 295, + "rms_error": 0.1137957235809952, + "maximum_absolute_error": 0.6015625 + }, + { + "column": 16033, + "rms_error": 0.1029568109841047, + "maximum_absolute_error": 0.34375 + }, + { + "column": 833, + "rms_error": 0.10202480514875116, + "maximum_absolute_error": 0.40625 + }, + { + "column": 723, + "rms_error": 0.10091368970147231, + "maximum_absolute_error": 0.5 + }, + { + "column": 42847, + "rms_error": 0.1004209925219702, + "maximum_absolute_error": 0.375 + }, + { + "column": 945, + "rms_error": 0.09906024780067336, + "maximum_absolute_error": 0.53515625 + }, + { + "column": 958, + "rms_error": 0.09743958757285678, + "maximum_absolute_error": 0.6201171875 + }, + { + "column": 60912, + "rms_error": 0.09729450010987249, + "maximum_absolute_error": 0.609375 + }, + { + "column": 5081, + "rms_error": 0.09665967271722023, + "maximum_absolute_error": 0.59375 + }, + { + "column": 538, + "rms_error": 0.09560578125073615, + "maximum_absolute_error": 0.56494140625 + }, + { + "column": 1492, + "rms_error": 0.09468376529995495, + "maximum_absolute_error": 0.46875 + }, + { + "column": 90978, + "rms_error": 0.09393744802568715, + "maximum_absolute_error": 0.34375 + }, + { + "column": 2551, + "rms_error": 0.09376275949274417, + "maximum_absolute_error": 0.59375 + }, + { + "column": 2024, + "rms_error": 0.09340610767263444, + "maximum_absolute_error": 0.53125 + }, + { + "column": 3570, + "rms_error": 0.09310916586806178, + "maximum_absolute_error": 0.5 + }, + { + "column": 33237, + "rms_error": 0.09252003646045862, + "maximum_absolute_error": 0.25 + }, + { + "column": 4673, + "rms_error": 0.0914817433096805, + "maximum_absolute_error": 0.4375 + }, + { + "column": 346, + "rms_error": 0.09074473601256952, + "maximum_absolute_error": 0.4375 + }, + { + "column": 8811, + "rms_error": 0.0905887564170753, + "maximum_absolute_error": 0.5546875 + }, + { + "column": 293, + "rms_error": 0.09005452466329847, + "maximum_absolute_error": 0.3984375 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-8-code-64", + "phase": "prompt", + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-8bit", + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "cb2b3a0590dbcb1ee4ea5914786a9a9954fb06e8ef9f16230cc0f097b5c2bb64", + "reference_logits_sha256": "c2263cda17da88664fad1fbb499035c33899760ec5cad02a35a4ba1ac1b908c8" + } + ], + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655" +} diff --git a/docs/receipts/lfm3/structured-token-checks.json b/docs/receipts/lfm3/structured-token-checks.json new file mode 100644 index 0000000000..c9052f82f8 --- /dev/null +++ b/docs/receipts/lfm3/structured-token-checks.json @@ -0,0 +1,401 @@ +{ + "cells": [ + { + "platform": "M4 Pro, macOS 26.5.1", + "checkpoint": "1.7B-4", + "native_binary_sha256": "5c8a04d4af599879797f4f5bed8e00ab7d9d70732e698bd2a1ec2a01ec762e09", + "original_client_exit": 0, + "groups": 32, + "passed_full_token_equality": 32, + "failed_full_token_equality": 0, + "checks": { + "_trimmed_list": { + "count": 32, + "sha256": "dd666bb19c42a2eedab8470043619f903e68261d68ad2d079e5d86c000a1caba", + "first": [ + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "temperature": 0 + }, + "count": 3, + "hashes": [ + "97fc568e1c13add8c4f1c58c38ab18a87039408de0d789a92614693a8f5c73d0", + "97fc568e1c13add8c4f1c58c38ab18a87039408de0d789a92614693a8f5c73d0", + "97fc568e1c13add8c4f1c58c38ab18a87039408de0d789a92614693a8f5c73d0" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "8f3cef2f2dd40372a9f5a5a45465d005711e83e73de634020964ea1164f102fc", + "8f3cef2f2dd40372a9f5a5a45465d005711e83e73de634020964ea1164f102fc", + "8f3cef2f2dd40372a9f5a5a45465d005711e83e73de634020964ea1164f102fc" + ], + "verified": true + } + ], + "last": [ + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Write two sentences about the sea.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "5eaa966d72c48c19e4f5f1d99bbf48d937692e3311587041e7f6d8d604e61494", + "5eaa966d72c48c19e4f5f1d99bbf48d937692e3311587041e7f6d8d604e61494", + "5eaa966d72c48c19e4f5f1d99bbf48d937692e3311587041e7f6d8d604e61494" + ], + "verified": true + } + ] + } + } + }, + { + "platform": "M4 Pro, macOS 26.5.1", + "checkpoint": "VL-2B-4", + "native_binary_sha256": "5c8a04d4af599879797f4f5bed8e00ab7d9d70732e698bd2a1ec2a01ec762e09", + "original_client_exit": 0, + "groups": 32, + "passed_full_token_equality": 32, + "failed_full_token_equality": 0, + "checks": { + "_trimmed_list": { + "count": 32, + "sha256": "a20f779d622b24911707b37eeac4e86f9e7101a81fc01f493892f9c26d42124d", + "first": [ + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "temperature": 0 + }, + "count": 3, + "hashes": [ + "b5c5f176466925a1dc9de8fff4578df4eb4e0efcdac74df82d5df7189f9de5cc", + "b5c5f176466925a1dc9de8fff4578df4eb4e0efcdac74df82d5df7189f9de5cc", + "b5c5f176466925a1dc9de8fff4578df4eb4e0efcdac74df82d5df7189f9de5cc" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "c41981cafaa3da59c92b9e34ce1fd12f59afc0a0fb7229b849c05caf8288a35e", + "c41981cafaa3da59c92b9e34ce1fd12f59afc0a0fb7229b849c05caf8288a35e", + "c41981cafaa3da59c92b9e34ce1fd12f59afc0a0fb7229b849c05caf8288a35e" + ], + "verified": true + } + ], + "last": [ + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Write two sentences about the sea.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "91a1e14a5b9fc3e27a8b4821548c7203af467f0ca1fdea0b4ba3156a2dd27a49", + "91a1e14a5b9fc3e27a8b4821548c7203af467f0ca1fdea0b4ba3156a2dd27a49", + "91a1e14a5b9fc3e27a8b4821548c7203af467f0ca1fdea0b4ba3156a2dd27a49" + ], + "verified": true + } + ] + } + } + }, + { + "platform": "M5 Ultra, macOS 27.0", + "checkpoint": "1.7B-4", + "native_binary_sha256": "251079309919aa9b21ca8cc6a89c58fc3fc9a9baf800fa5ce6a6af313a5b5f65", + "original_client_exit": 0, + "groups": 32, + "passed_full_token_equality": 32, + "failed_full_token_equality": 0, + "checks": { + "_trimmed_list": { + "count": 32, + "sha256": "dd666bb19c42a2eedab8470043619f903e68261d68ad2d079e5d86c000a1caba", + "first": [ + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "temperature": 0 + }, + "count": 3, + "hashes": [ + "97fc568e1c13add8c4f1c58c38ab18a87039408de0d789a92614693a8f5c73d0", + "97fc568e1c13add8c4f1c58c38ab18a87039408de0d789a92614693a8f5c73d0", + "97fc568e1c13add8c4f1c58c38ab18a87039408de0d789a92614693a8f5c73d0" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "8f3cef2f2dd40372a9f5a5a45465d005711e83e73de634020964ea1164f102fc", + "8f3cef2f2dd40372a9f5a5a45465d005711e83e73de634020964ea1164f102fc", + "8f3cef2f2dd40372a9f5a5a45465d005711e83e73de634020964ea1164f102fc" + ], + "verified": true + } + ], + "last": [ + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Write two sentences about the sea.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "5eaa966d72c48c19e4f5f1d99bbf48d937692e3311587041e7f6d8d604e61494", + "5eaa966d72c48c19e4f5f1d99bbf48d937692e3311587041e7f6d8d604e61494", + "5eaa966d72c48c19e4f5f1d99bbf48d937692e3311587041e7f6d8d604e61494" + ], + "verified": true + } + ] + } + } + }, + { + "platform": "M5 Ultra, macOS 27.0", + "checkpoint": "VL-2B-4", + "native_binary_sha256": "251079309919aa9b21ca8cc6a89c58fc3fc9a9baf800fa5ce6a6af313a5b5f65", + "original_client_exit": 0, + "groups": 32, + "passed_full_token_equality": 32, + "failed_full_token_equality": 0, + "checks": { + "_trimmed_list": { + "count": 32, + "sha256": "a20f779d622b24911707b37eeac4e86f9e7101a81fc01f493892f9c26d42124d", + "first": [ + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "temperature": 0 + }, + "count": 3, + "hashes": [ + "b5c5f176466925a1dc9de8fff4578df4eb4e0efcdac74df82d5df7189f9de5cc", + "b5c5f176466925a1dc9de8fff4578df4eb4e0efcdac74df82d5df7189f9de5cc", + "b5c5f176466925a1dc9de8fff4578df4eb4e0efcdac74df82d5df7189f9de5cc" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "c41981cafaa3da59c92b9e34ce1fd12f59afc0a0fb7229b849c05caf8288a35e", + "c41981cafaa3da59c92b9e34ce1fd12f59afc0a0fb7229b849c05caf8288a35e", + "c41981cafaa3da59c92b9e34ce1fd12f59afc0a0fb7229b849c05caf8288a35e" + ], + "verified": true + } + ], + "last": [ + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Write two sentences about the sea.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "91a1e14a5b9fc3e27a8b4821548c7203af467f0ca1fdea0b4ba3156a2dd27a49", + "91a1e14a5b9fc3e27a8b4821548c7203af467f0ca1fdea0b4ba3156a2dd27a49", + "91a1e14a5b9fc3e27a8b4821548c7203af467f0ca1fdea0b4ba3156a2dd27a49" + ], + "verified": true + } + ] + } + } + } + ], + "_trim": { + "note": "Raw capture replaced by counts, first/last items and SHA-256 of the canonical JSON of each trimmed list or string.", + "original_sha256": "3c6ebbe8b8033d9e8cee60b7178005a36df8a2e2f59672cb2059016b60a6c6a7", + "original_bytes": 141286 + } +} diff --git a/docs/receipts/lfm3/text-contract-nf2.json b/docs/receipts/lfm3/text-contract-nf2.json new file mode 100644 index 0000000000..a43401a49d --- /dev/null +++ b/docs/receipts/lfm3/text-contract-nf2.json @@ -0,0 +1,10 @@ +{ + "scope": "Extracted text normal contract build and host verification after compensated-cancellation correction; no GPU run is asserted for this executable.", + "source_commit": "c53423d0edfe11936ce77825a72ba1d45b85c4f2", + "command": "zig build test-kernel-contract tf-kernel-contract -Dcpu=apple_m1 --summary all", + "summary": "Build Summary: 61/61 steps succeeded; 121/121 tests passed", + "binary_sha256": "8adc967eaaa618f957c26f7ef2e8abd6dfe0edeacf53d2f33ce17035b37cf5b7", + "log_sha256": "1a103b6409def0b10b99e6e1b9fb737f9c613cb0e57de8becf5258141a9c011b", + "host": "Apple M4 Pro, macOS 26.5.1", + "compiler": "Zig 0.17.0" +} diff --git a/docs/receipts/lfm3/text-foundation.json b/docs/receipts/lfm3/text-foundation.json new file mode 100644 index 0000000000..2c6b8a82d2 --- /dev/null +++ b/docs/receipts/lfm3/text-foundation.json @@ -0,0 +1,234 @@ +{ + "scope": "Actual text-only extraction; binned projection remains disabled by default", + "binary_sha256": "19da51a24a28b0c1c7c2dac7a77a3badce2789487ba1f6c087baa689bc15ffa2", + "hardware": "Apple M4 Pro, 48 GB, macOS 26.5.1", + "build_summary": "243/243 steps succeeded; 1/1 newly executed tests passed, other tests cached", + "python_tests": 35, + "served": [ + { + "bits": 4, + "evidence": { + "model": "LFM3-1.7B-MLX-4bit", + "binary_sha256": "19da51a24a28b0c1c7c2dac7a77a3badce2789487ba1f6c087baa689bc15ffa2", + "tests": { + "_trimmed_list": { + "count": 40, + "sha256": "ec93990f3e5951b65e9273ee8a6948400e438b6aead1aefa5d4167eacac6e7ea", + "first": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + } + ], + "last": [ + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + } + ] + } + }, + "speed_qualified": false, + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6 + } + }, + { + "bits": 6, + "evidence": { + "model": "LFM3-1.7B-MLX-6bit", + "binary_sha256": "19da51a24a28b0c1c7c2dac7a77a3badce2789487ba1f6c087baa689bc15ffa2", + "tests": { + "_trimmed_list": { + "count": 40, + "sha256": "a896485b9a7998f3c7d1c958fed66fa1333b95158968a66e137a79017ebfe7a7", + "first": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + } + ], + "last": [ + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + } + ] + } + }, + "speed_qualified": false, + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6 + } + }, + { + "bits": 8, + "evidence": { + "model": "LFM3-1.7B-MLX-8bit", + "binary_sha256": "19da51a24a28b0c1c7c2dac7a77a3badce2789487ba1f6c087baa689bc15ffa2", + "tests": { + "_trimmed_list": { + "count": 40, + "sha256": "787b4d0de94565bcfc8d7d2297e3f027f109d5a17dd7476be63f85095b885a81", + "first": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + } + ], + "last": [ + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + } + ] + } + }, + "speed_qualified": false, + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6 + } + } + ], + "vl_refusals": [ + { + "bits": 4, + "vl_refusal": true, + "exit_status": 1 + }, + { + "bits": 6, + "vl_refusal": true, + "exit_status": 1 + }, + { + "bits": 8, + "vl_refusal": true, + "exit_status": 1 + } + ], + "targeted_contract": { + "filter": "LFM3", + "summary": "SUMMARY 49 kernels: 49 pass gates 1-4, 0 fail, 0 known findings; 14 controls: 14 red as planted, 0 missed; 0 limit problems; gates 64304 ms, limits 4564 ms", + "scope": "Registered text-extraction LFM3 probes and limits; not a complete backend run" + }, + "_trim": { + "note": "Raw capture replaced by counts, first/last items and SHA-256 of the canonical JSON of each trimmed list or string.", + "original_sha256": "37ff10575f97eadb77744e937b90e89dd8f28a8eac4c983389c7bc93f8919a88", + "original_bytes": 90093 + } +} diff --git a/docs/receipts/lfm3/text-twin-regression.json b/docs/receipts/lfm3/text-twin-regression.json new file mode 100644 index 0000000000..6a6260f864 --- /dev/null +++ b/docs/receipts/lfm3/text-twin-regression.json @@ -0,0 +1,10 @@ +{ + "scope": "Host/native/developer-tool regression after shared opt-in projection twin. No device/model qualification of this rebuilt binary is asserted.", + "source_commit": "5965e23d8e322275b8c813308c9d7edf72f001aa", + "command": "zig build native dev-tools test test-golden -Dcpu=apple_m1 --summary all", + "summary": "Build Summary: 721/721 steps succeeded; 7233/7278 tests passed (45 skipped)", + "native_binary_sha256": "0027180c77ef97cc3095c7596bd44aa05e9d3fe8ebe87abceb20123f515b627d", + "compiler": "Zig 0.17.0", + "host": "Apple M4 Pro, macOS 26.5.1", + "log_sha256": "89ddf8140264ee52b53be73ee0731252077cccd2f93f9c25e4adbd7c1bc6ee19" +} diff --git a/docs/receipts/lfm3/tool-checks.json b/docs/receipts/lfm3/tool-checks.json new file mode 100644 index 0000000000..4dd647404e --- /dev/null +++ b/docs/receipts/lfm3/tool-checks.json @@ -0,0 +1,958 @@ +{ + "cells": [ + { + "platform": "M5 Ultra, macOS 27.0", + "checkpoint": "1.7B-4", + "native_binary_sha256": "251079309919aa9b21ca8cc6a89c58fc3fc9a9baf800fa5ce6a6af313a5b5f65", + "counts": { + "passed": 34, + "failed": 0 + }, + "checks": [ + { + "name": "required/0/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ] + } + }, + { + "name": "required/0/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "draft_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "required/0/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "named/0/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ] + } + }, + { + "name": "named/0/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "draft_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "named/0/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "none/0/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [] + } + }, + { + "name": "none/0/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "7843d72119d60114d3bfbeeba016494527c318e6b933128fa3a0d190ea191659", + "draft_hash": "7843d72119d60114d3bfbeeba016494527c318e6b933128fa3a0d190ea191659" + } + }, + { + "name": "none/0/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "required/0/concurrent_0_hash", + "passed": true, + "details": { + "http": 200, + "hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "named/0/concurrent_1_hash", + "passed": true, + "details": { + "http": 200, + "hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "none/0/concurrent_2_hash", + "passed": true, + "details": { + "http": 200, + "hash": "7843d72119d60114d3bfbeeba016494527c318e6b933128fa3a0d190ea191659" + } + }, + { + "name": "required/0/concurrent_3_hash", + "passed": true, + "details": { + "http": 200, + "hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "required/0/streamed_calls_and_hash", + "passed": true, + "details": { + "complete": true, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ], + "tensorfold": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lfm3", + "tokens_per_second": 376.55113393239196, + "seconds": 0.058425, + "prefill_seconds": 0.03929575, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.039586625, + "sampling": "greedy", + "drafts": true, + "token_sha": "bd55d22b78c1", + "token_sha256": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "token_ids": [ + 125019, + 207, + 125021, + 2762, + 703, + 125023, + 73, + 703, + 26, + 207, + 125024, + 207, + 125023, + 74, + 703, + 27, + 207, + 125024, + 207, + 125022, + 207, + 125020, + 124900 + ], + "min_rows": 1 + } + } + }, + { + "name": "roundtrip/0/False/valid", + "passed": true, + "details": { + "http": 200, + "answer": { + "result": 5 + } + } + }, + { + "name": "roundtrip/0/True/valid", + "passed": true, + "details": { + "http": 200, + "answer": { + "result": 5 + } + } + }, + { + "name": "roundtrip/0/draft_hash_equals_plain", + "passed": true, + "details": null + }, + { + "name": "required/0.7/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ] + } + }, + { + "name": "required/0.7/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "draft_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "required/0.7/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "named/0.7/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ] + } + }, + { + "name": "named/0.7/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "draft_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "named/0.7/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "none/0.7/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [] + } + }, + { + "name": "none/0.7/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "b42001becb1b1816f0be07fe107d33efd346f492f3e1f0b41fa9314b1fccea51", + "draft_hash": "b42001becb1b1816f0be07fe107d33efd346f492f3e1f0b41fa9314b1fccea51" + } + }, + { + "name": "none/0.7/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "required/0.7/concurrent_0_hash", + "passed": true, + "details": { + "http": 200, + "hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "named/0.7/concurrent_1_hash", + "passed": true, + "details": { + "http": 200, + "hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "none/0.7/concurrent_2_hash", + "passed": true, + "details": { + "http": 200, + "hash": "b42001becb1b1816f0be07fe107d33efd346f492f3e1f0b41fa9314b1fccea51" + } + }, + { + "name": "required/0.7/concurrent_3_hash", + "passed": true, + "details": { + "http": 200, + "hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "required/0.7/streamed_calls_and_hash", + "passed": true, + "details": { + "complete": true, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ], + "tensorfold": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lfm3", + "tokens_per_second": 371.2996571520229, + "seconds": 0.059251334, + "prefill_seconds": 0.039364709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.039642166, + "sampling": "exact", + "drafts": true, + "token_sha": "bd55d22b78c1", + "token_sha256": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "token_ids": [ + 125019, + 207, + 125021, + 2762, + 703, + 125023, + 73, + 703, + 26, + 207, + 125024, + 207, + 125023, + 74, + 703, + 27, + 207, + 125024, + 207, + 125022, + 207, + 125020, + 124900 + ], + "min_rows": 1 + } + } + }, + { + "name": "roundtrip/0.7/False/valid", + "passed": true, + "details": { + "http": 200, + "answer": { + "result": 5 + } + } + }, + { + "name": "roundtrip/0.7/True/valid", + "passed": true, + "details": { + "http": 200, + "answer": { + "result": 5 + } + } + }, + { + "name": "roundtrip/0.7/draft_hash_equals_plain", + "passed": true, + "details": null + } + ] + }, + { + "platform": "M5 Ultra, macOS 27.0", + "checkpoint": "VL-2B-4", + "native_binary_sha256": "251079309919aa9b21ca8cc6a89c58fc3fc9a9baf800fa5ce6a6af313a5b5f65", + "counts": { + "passed": 46, + "failed": 0 + }, + "checks": [ + { + "name": "required/0/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ] + } + }, + { + "name": "required/0/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "draft_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "required/0/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "named/0/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ] + } + }, + { + "name": "named/0/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "draft_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "named/0/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "none/0/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [] + } + }, + { + "name": "none/0/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "7843d72119d60114d3bfbeeba016494527c318e6b933128fa3a0d190ea191659", + "draft_hash": "7843d72119d60114d3bfbeeba016494527c318e6b933128fa3a0d190ea191659" + } + }, + { + "name": "none/0/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "image_red/0/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "report_color", + "arguments": { + "color": "red" + } + } + ] + } + }, + { + "name": "image_red/0/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "488388eba1eefda92bdedab254fa2ce0aa7e3b56d2af834bdaa88e031def833a", + "draft_hash": "488388eba1eefda92bdedab254fa2ce0aa7e3b56d2af834bdaa88e031def833a" + } + }, + { + "name": "image_red/0/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "image_blue/0/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "report_color", + "arguments": { + "color": "blue" + } + } + ] + } + }, + { + "name": "image_blue/0/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "762466499cd3ee7e4587a4911cbd779266f5dd58ca1d744e4c4d3f9d42995b11", + "draft_hash": "762466499cd3ee7e4587a4911cbd779266f5dd58ca1d744e4c4d3f9d42995b11" + } + }, + { + "name": "image_blue/0/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "required/0/concurrent_0_hash", + "passed": true, + "details": { + "http": 200, + "hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "named/0/concurrent_1_hash", + "passed": true, + "details": { + "http": 200, + "hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "none/0/concurrent_2_hash", + "passed": true, + "details": { + "http": 200, + "hash": "7843d72119d60114d3bfbeeba016494527c318e6b933128fa3a0d190ea191659" + } + }, + { + "name": "image_red/0/concurrent_3_hash", + "passed": true, + "details": { + "http": 200, + "hash": "488388eba1eefda92bdedab254fa2ce0aa7e3b56d2af834bdaa88e031def833a" + } + }, + { + "name": "required/0/streamed_calls_and_hash", + "passed": true, + "details": { + "complete": true, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ], + "tensorfold": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lfm3", + "tokens_per_second": 354.15443760661657, + "seconds": 0.062119792, + "prefill_seconds": 0.039358959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.039628792, + "sampling": "greedy", + "drafts": true, + "token_sha": "bd55d22b78c1", + "token_sha256": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "token_ids": [ + 125019, + 207, + 125021, + 2762, + 703, + 125023, + 73, + 703, + 26, + 207, + 125024, + 207, + 125023, + 74, + 703, + 27, + 207, + 125024, + 207, + 125022, + 207, + 125020, + 124900 + ], + "min_rows": 1 + } + } + }, + { + "name": "roundtrip/0/False/valid", + "passed": true, + "details": { + "http": 200, + "answer": { + "result": 5 + } + } + }, + { + "name": "roundtrip/0/True/valid", + "passed": true, + "details": { + "http": 200, + "answer": { + "result": 5 + } + } + }, + { + "name": "roundtrip/0/draft_hash_equals_plain", + "passed": true, + "details": null + }, + { + "name": "required/0.7/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ] + } + }, + { + "name": "required/0.7/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "draft_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "required/0.7/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "named/0.7/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ] + } + }, + { + "name": "named/0.7/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "draft_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "named/0.7/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "none/0.7/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [] + } + }, + { + "name": "none/0.7/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "7843d72119d60114d3bfbeeba016494527c318e6b933128fa3a0d190ea191659", + "draft_hash": "7843d72119d60114d3bfbeeba016494527c318e6b933128fa3a0d190ea191659" + } + }, + { + "name": "none/0.7/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "image_red/0.7/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "report_color", + "arguments": { + "color": "red" + } + } + ] + } + }, + { + "name": "image_red/0.7/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "488388eba1eefda92bdedab254fa2ce0aa7e3b56d2af834bdaa88e031def833a", + "draft_hash": "488388eba1eefda92bdedab254fa2ce0aa7e3b56d2af834bdaa88e031def833a" + } + }, + { + "name": "image_red/0.7/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "image_blue/0.7/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "report_color", + "arguments": { + "color": "blue" + } + } + ] + } + }, + { + "name": "image_blue/0.7/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "762466499cd3ee7e4587a4911cbd779266f5dd58ca1d744e4c4d3f9d42995b11", + "draft_hash": "762466499cd3ee7e4587a4911cbd779266f5dd58ca1d744e4c4d3f9d42995b11" + } + }, + { + "name": "image_blue/0.7/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "required/0.7/concurrent_0_hash", + "passed": true, + "details": { + "http": 200, + "hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "named/0.7/concurrent_1_hash", + "passed": true, + "details": { + "http": 200, + "hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "none/0.7/concurrent_2_hash", + "passed": true, + "details": { + "http": 200, + "hash": "7843d72119d60114d3bfbeeba016494527c318e6b933128fa3a0d190ea191659" + } + }, + { + "name": "image_red/0.7/concurrent_3_hash", + "passed": true, + "details": { + "http": 200, + "hash": "488388eba1eefda92bdedab254fa2ce0aa7e3b56d2af834bdaa88e031def833a" + } + }, + { + "name": "required/0.7/streamed_calls_and_hash", + "passed": true, + "details": { + "complete": true, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ], + "tensorfold": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lfm3", + "tokens_per_second": 348.1274304635238, + "seconds": 0.06319525, + "prefill_seconds": 0.03948375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.039755708, + "sampling": "exact", + "drafts": true, + "token_sha": "bd55d22b78c1", + "token_sha256": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "token_ids": [ + 125019, + 207, + 125021, + 2762, + 703, + 125023, + 73, + 703, + 26, + 207, + 125024, + 207, + 125023, + 74, + 703, + 27, + 207, + 125024, + 207, + 125022, + 207, + 125020, + 124900 + ], + "min_rows": 1 + } + } + }, + { + "name": "roundtrip/0.7/False/valid", + "passed": true, + "details": { + "http": 200, + "answer": { + "result": 5 + } + } + }, + { + "name": "roundtrip/0.7/True/valid", + "passed": true, + "details": { + "http": 200, + "answer": { + "result": 5 + } + } + }, + { + "name": "roundtrip/0.7/draft_hash_equals_plain", + "passed": true, + "details": null + } + ] + } + ] +} diff --git a/docs/receipts/lfm3/vl-twin-regression.json b/docs/receipts/lfm3/vl-twin-regression.json new file mode 100644 index 0000000000..886766e7e6 --- /dev/null +++ b/docs/receipts/lfm3/vl-twin-regression.json @@ -0,0 +1,11 @@ +{ + "scope": "Canonical default-OFF production source rebuild, including trace-only vision snapshots. No device/model pass is asserted for this rebuilt binary.", + "source_commit": "6101b94bf6e8e8e29a6cde588f88b7b5020d5e8c", + "command": "zig build native dev-tools test test-golden -Dcpu=apple_m1 --summary all", + "summary": "Build Summary: 726/726 steps succeeded; 982/1000 tests passed (18 skipped)", + "cached_steps": "Other steps reused cached checks; counts are those newly reported by Zig.", + "native_binary_sha256": "44220f39df174b2364172eac4b8705dbd5f5f47f3175c7ed300a6d47ae3bbcbc", + "log_sha256": "fe4dd4500935a9a64ad0af0e85e090a55dca18e8127889fe9b410617f146f156", + "host": "Apple M4 Pro, macOS 26.5.1", + "compiler": "Zig 0.17.0" +} diff --git a/docs/receipts/lfm3/vl4-code265-rounded-graph.json b/docs/receipts/lfm3/vl4-code265-rounded-graph.json new file mode 100644 index 0000000000..f44895a46f --- /dev/null +++ b/docs/receipts/lfm3/vl4-code265-rounded-graph.json @@ -0,0 +1,383 @@ +{ + "scope": "VL4 four-bit causal code prefix of 265 tokens: independent CPU prescribed-rounding graph with production-options Metal elementary evaluations. Only vocabulary row 264 is compared.", + "runtime_change": false, + "native_activation_injection": false, + "checkpoint": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "revision": "c823fade98e35a3d6942ad4d38b1ea15f675cd12" + }, + "prefix_rows": 265, + "final_row": 264, + "token_ids_sha256": "58ee46fd76e51555ec0fad27675aca85def7ea6406110f7aa9530a11eb94ee4d", + "token_ids": [ + 22497, + 267, + 21516, + 1446, + 395, + 16033, + 273, + 1046, + 38076, + 15493, + 2129, + 8864, + 3520, + 5635, + 22, + 42847, + 851, + 893, + 309, + 2993, + 16318, + 63908, + 3162, + 33237, + 116766, + 92377, + 20, + 1583, + 4673, + 360, + 1224, + 550, + 34289, + 360, + 723, + 550, + 495, + 550, + 229, + 24, + 207, + 22497, + 267, + 21516, + 1446, + 395, + 16033, + 273, + 1046, + 38076, + 15493, + 2129, + 8864, + 3520, + 5635, + 22, + 42847, + 851, + 893, + 309, + 2993, + 16318, + 63908, + 3162, + 33237, + 116766, + 92377, + 20, + 1583, + 4673, + 360, + 1224, + 550, + 34289, + 360, + 723, + 550, + 495, + 550, + 229, + 24, + 207, + 22497, + 267, + 21516, + 1446, + 395, + 16033, + 273, + 1046, + 38076, + 15493, + 2129, + 8864, + 3520, + 5635, + 22, + 42847, + 851, + 893, + 309, + 2993, + 16318, + 63908, + 3162, + 33237, + 116766, + 92377, + 20, + 1583, + 4673, + 360, + 1224, + 550, + 34289, + 360, + 723, + 550, + 495, + 550, + 229, + 24, + 207, + 22497, + 267, + 21516, + 1446, + 395, + 16033, + 273, + 1046, + 38076, + 15493, + 2129, + 8864, + 3520, + 5635, + 22, + 42847, + 851, + 893, + 309, + 2993, + 16318, + 63908, + 3162, + 33237, + 116766, + 92377, + 20, + 1583, + 4673, + 360, + 1224, + 550, + 34289, + 360, + 723, + 550, + 495, + 550, + 229, + 24, + 207, + 22497, + 267, + 21516, + 1446, + 395, + 16033, + 273, + 1046, + 38076, + 15493, + 2129, + 8864, + 3520, + 5635, + 22, + 42847, + 851, + 893, + 309, + 2993, + 16318, + 63908, + 3162, + 33237, + 116766, + 92377, + 20, + 1583, + 4673, + 360, + 1224, + 550, + 34289, + 360, + 723, + 550, + 495, + 550, + 229, + 24, + 207, + 22497, + 267, + 21516, + 1446, + 395, + 16033, + 273, + 1046, + 38076, + 15493, + 2129, + 8864, + 3520, + 5635, + 22, + 42847, + 851, + 893, + 309, + 2993, + 16318, + 63908, + 3162, + 33237, + 116766, + 92377, + 20, + 1583, + 4673, + 360, + 1224, + 550, + 34289, + 360, + 723, + 550, + 495, + 550, + 229, + 24, + 207, + 22497, + 267, + 21516, + 1446, + 395, + 16033, + 273, + 1046, + 38076, + 15493, + 2129, + 8864, + 3520, + 5635, + 22, + 42847, + 851, + 893, + 309 + ], + "measured_identity": { + "final_logits_exact": 128000, + "final_logits_total": 128000, + "native_top": 207, + "rounded_reference_top": 207, + "relative_RMS_vs_native": 0.0, + "compared_stage_count": 0, + "native_stage_trace_available": false + }, + "ideal_FP64_difference": { + "retained": true, + "contenders": { + "207": { + "native": 14.625, + "ideal_FP64": 14.4375, + "projection_only_FP32": 14.625, + "CPU_full_named": 14.625 + }, + "2993": { + "native": 14.125, + "ideal_FP64": 14.8125, + "projection_only_FP32": 14.0625, + "CPU_full_named": 14.125 + }, + "33237": { + "native": 14.375, + "ideal_FP64": 14.25, + "projection_only_FP32": 14.375, + "CPU_full_named": 14.375 + } + }, + "ideal_top": 2993, + "claim": "Reproducing the named rounding sites does not turn disagreement with ideal FP64 into an ideal-FP64 pass." + }, + "elementary_evaluations": { + "batches": 762, + "scalar_values": 74442293, + "operations": { + "rsqrt": 119, + "exp": 577, + "exp2": 22, + "cos": 22, + "sin": 22 + }, + "ordered_chunk_groups": 24, + "max_payload_bytes": 1048448, + "chunking": "Pointwise arrays are split into ordered chunks of at most 262112 FP32 values. Original array input/output hashes are reconstructed and checked.", + "inputs": "Independently computed CPU graph intermediates from checkpoint weights and token IDs; native artifacts are used only for comparison.", + "functions": { + "exp": "metal::precise::exp", + "exp2": "metal::exp2", + "cos": "metal::precise::cos", + "sin": "metal::precise::sin", + "rsqrt": "metal::precise::rsqrt" + }, + "CompileOptions": { + "math_mode": "safe", + "math_functions": "fast", + "language_code": 262144 + }, + "raw_calls_manifest_sha256": "0ed030f0f3e284718e9a3e47c56cd2b7dd63399cca7bf6c03714d4afe10453d7", + "raw_groups_manifest_sha256": "8d1939bd8bfb4a973defcc9824f4f99f32f1ba34e4c9eea9b5204c31ef3c0f15" + }, + "source_sha256": { + "fp64_reference.py": "0fb1128ade095db87b6126792ab14930792ab1f7b41b998570356bc586f16b2e", + "row_projection_reference.py": "d03f224a037a920ba66b9828059d1489e48729cd168077e76789d0ec5c199c0c", + "elementary_reference.py": "3e4db276278867b7683536044ed7bcbe4c7643867d722fdda5b8ad895b3963e7", + "text_rounding_reference.py": "52665eef48570d45a2554861fe8f22b31874f8f464366adf882f40c3d311db27", + "text_attention_rounding.py": "ae67f2b2c26f89f75b1f36822a4b30a9f710df2b42f9f7ec42679373028b89e8" + }, + "reference_C": { + "sha256": "ff336f75fcf0a4f6c0ac92ed627bc1a6abb0b36b8f4cc95d876f2a8bb2f5cc42", + "ABI": "lfm3_row_replay(float*,uint8*,float*,float*,float*,int rows,int N,int K,int group,int bits); recorded binary hash proves artifact identity, not reproducible compiler provenance" + }, + "artifact_sha256": { + "VL-2B-4-code-513.native.bf16": "83013e7646760ead9206c4a53b7cac56a19d11035629c876f9ea6701dddedc08", + "VL-2B-4-code-513.bf16": "61e6cd3d56a41b198ee9877a9a477ec61310ea4826869eef057da42cd8549808", + "code-513.npy": "c932ae46fd83f6e834a4001bd27496af3ebf54ac841a694d03a64558ba0daacb", + "run.py": "7da87e4f18dc43a325a6d3dd20b283d8fb5751e468f95fea843b79751ad9fb0a", + "provider.py": "97aec6e37977bbef32c60e7f86253c29c7b8161159dd6c9aded657e4bd69ddd0", + "row264.bf16": "55ae2380c9a29c927f1ac61d411961770b3039493be450cf0b0e0d2d8cc3484e", + "row.so": "11ad0f66632bfcba548dbede9027ffdb112cd418ff2be80610af56a957424753" + }, + "executed_manifest_sha256": "4d6de7da49b13a1aa07914e8db0ef7555e5894012752c4756a2dc474b426a2d8", + "guarded_execution": { + "lock": "/tmp/tensorfold-model.lock", + "inode": 46835039, + "single_model_admission_passed": true, + "graph_child_exit": 0, + "all_primitive_children_exited": true, + "lock_released_after_completion": true, + "events_sha256": "47f3445ffc6708277117db23cee79a42b3b68111b061ba77dfa974f00d84ad81" + }, + "limits": [ + "Ordinary attention reference lacks selected integer-exponent range repair; observed equality validates only measured inputs.", + "Reference fma uses FP64 multiply/add then FP32 and is not universally correctly rounded fmaf.", + "Masked elementary evaluations may include NaN/-Inf discarded by reference; these are not asserted as native executed sites.", + "No native code-513 stage trace was retained; zero activation stages are compared.", + "Only the four-bit prefix of 265 tokens and final row 264 are covered. No image, 6/8-bit, performance or full-fidelity gate is claimed." + ], + "setup_history": "Two private diagnostic setup attempts stopped before a numerical result: the non-inclusive file-read limit and an incorrect diagnostic language-code assertion. Both were preserved; no runtime arithmetic changed.", + "fidelity_pass": null, + "accepted_threshold": null, + "original_graph_receipt_sha256": "4cd7c5b0e4ac96fc212e81d6b06c7ba2216bbd1ec4033373ea320a636e797dc5" +} diff --git a/docs/receipts/lfm3/vl4-ownvision41-rounded-graph.json b/docs/receipts/lfm3/vl4-ownvision41-rounded-graph.json new file mode 100644 index 0000000000..30cb22922f --- /dev/null +++ b/docs/receipts/lfm3/vl4-ownvision41-rounded-graph.json @@ -0,0 +1,194 @@ +{ + "scope": "Four-bit real-image own-vision replay: independently compute all 70 projected image rows, then decoder prefix41 and vocabulary heads34/40 only.", + "runtime_change": false, + "native_activation_injection": false, + "checkpoint": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "revision": "c823fade98e35a3d6942ad4d38b1ea15f675cd12" + }, + "image": { + "url": "https://placecats.com/300/200", + "jpeg_sha256": "d1b324f529a96d8d8a16d973d88e1e974905d9c2c0e3e8b21a50b1c0e3fe2379", + "canonical_png_sha256": "005500792ee8ea6f35ac369d900fc788cd77667f145e7100c2cff57fae5355d9", + "conversion": "macOS sips JPEG to PNG; canonical PNG is the exact native/reference input" + }, + "input_grid": [ + 14, + 20 + ], + "pixel_patches_shape": [ + 280, + 1536 + ], + "projected_image_rows_shape": [ + 70, + 2048 + ], + "decoder_image_start": 4, + "decoder_independent_image_rows_used": 37, + "decoder_prefix_rows": 41, + "token_ids": [ + 124899, + 5922, + 207, + 125009, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030 + ], + "measured_identity": { + "independent_projected_BF16_words_equal": 143360, + "projected_BF16_words_total": 143360, + "heads": [ + { + "row": 34, + "native_equal_logits": 128000, + "values": 128000, + "native_top": 5594, + "independent_named_top": 5594, + "ideal_FP64_top": 2911, + "contenders": { + "2911": { + "native": 4.40625, + "independent_named": 4.40625, + "ideal_FP64": 4.5625 + }, + "5594": { + "native": 4.65625, + "independent_named": 4.65625, + "ideal_FP64": 4.53125 + } + } + }, + { + "row": 40, + "native_equal_logits": 128000, + "values": 128000, + "native_top": 4899, + "independent_named_top": 4899, + "ideal_FP64_top": 34541, + "contenders": { + "4899": { + "native": 7.375, + "independent_named": 7.375, + "ideal_FP64": 7.125 + }, + "34541": { + "native": 7.25, + "independent_named": 7.25, + "ideal_FP64": 7.1875 + } + } + } + ], + "compared_intermediate_stages": 0, + "native_stage_trace_available": false + }, + "independence": "Only checkpoint weights and frozen preprocessed image pixels enter VisionOracle. All70 projected rows are computed before selecting its first37 rows for the causal decoder prefix. Native image/logit artifacts are read only for comparison.", + "elementary_evaluations": { + "batches": 648, + "scalar_values": 33924773, + "operations": { + "rsqrt": 161, + "exp": 385, + "exp2": 34, + "cos": 34, + "sin": 34 + }, + "ordered_chunk_groups": 39, + "max_payload_bytes": 1048448, + "functions": { + "exp": "metal::precise::exp", + "exp2": "metal::exp2", + "cos": "metal::precise::cos", + "sin": "metal::precise::sin", + "rsqrt": "metal::precise::rsqrt" + }, + "CompileOptions": { + "math_mode": "safe", + "math_functions": "fast", + "language_code": 262144 + }, + "calls_manifest_sha256": "9b23c8b5717c5f4e607b3b2cc6d65ed4514e8c8405c8c69f38e1c15fe45cea59", + "groups_manifest_sha256": "2c02e290c38501ba102db8b629a3e331f62aaaae87d33c2b780a397ec95e323b" + }, + "source_sha256": { + "fp64_reference.py": "0fb1128ade095db87b6126792ab14930792ab1f7b41b998570356bc586f16b2e", + "row_projection_reference.py": "d03f224a037a920ba66b9828059d1489e48729cd168077e76789d0ec5c199c0c", + "elementary_reference.py": "3e4db276278867b7683536044ed7bcbe4c7643867d722fdda5b8ad895b3963e7", + "vision_fp64_reference.py": "742b8b4f44dde824c3a5c2d72a632401a0f9df7b7f23a8d3729d15f4f8c6a4d2", + "text_rounding_reference.py": "52665eef48570d45a2554861fe8f22b31874f8f464366adf882f40c3d311db27", + "vision_rounding_reference.py": "8b2f5679bbd3f05b3e86a5748c3803f744502cffd0159bee2a218da38bd6f1c4", + "text_attention_rounding.py": "ae67f2b2c26f89f75b1f36822a4b30a9f710df2b42f9f7ec42679373028b89e8" + }, + "artifact_sha256": { + "run.py": "866a4c925e7abe43188164cad2fa48c1648db1fca8a6a629f97cfbe4b4dfbbed", + "provider.py": "97aec6e37977bbef32c60e7f86253c29c7b8161159dd6c9aded657e4bd69ddd0", + "lfm3-real-cat-patches.npy": "f1c55a0dd28b91d9929f074995adadea79d7c0862c8cdcd0be6dbfda57383dc1", + "lfm3-real-cat-tokens.npy": "54c970db8c6ba82895cb2f18a48d397ea568ae885241129e9c5ee7dbbc91a24d", + "lfm3-real-cat-native.bf16": "ab2ac7db759e5c4a32fe7d60680ec47f7d1cbc5938fa7d8091bf4756bfeea4fc", + "lfm3-real-cat-native-text.bf16": "d2dc2c1014716cb9ead7543ee00562a1816bf50d4a18b927cd8d8151f0074d54", + "lfm3-real-cat-fp64-text.bf16": "c16af1c1ff0095b2ced285886f4c54e21c48f244cb74b4fa615c8954163e620f", + "row.so": "11ad0f66632bfcba548dbede9027ffdb112cd418ff2be80610af56a957424753", + "row_projection_replay.c": "ff336f75fcf0a4f6c0ac92ed627bc1a6abb0b36b8f4cc95d876f2a8bb2f5cc42" + }, + "independent_image70_sha256": "ab2ac7db759e5c4a32fe7d60680ec47f7d1cbc5938fa7d8091bf4756bfeea4fc", + "two_heads_sha256": "b412201478b2c3335691675e069eb86d92c4db98e901bd67b849e8f86ed4724a", + "executed_manifest_sha256": "bdbf40ef29b656ebe9a91ed282a38f4a76742146dbf326c26ea7542ca315c483", + "original_graph_receipt_sha256": "e23628491a8cbbea05f8746ba97030996d24020428bd328043b7f6acc2018dfc", + "guarded_execution": { + "lock": "/tmp/tensorfold-model.lock", + "inode": 46835039, + "single_model_admission_passed": true, + "graph_child_exit": 0, + "all_primitive_children_exited": true, + "lock_released_after_completion": true, + "events_sha256": "d9c624a97b385a7529f01e8a78c82893a53f6186ebf5758c9ad180e4679ba07c" + }, + "ideal_FP64_difference_retained": true, + "limits": [ + "No intermediate native stage comparison; stage hashes only.", + "Ordinary text attention reference lacks selected integer-exponent repair.", + "FP64 multiply/add then FP32 fma helper is not universally correctly rounded fmaf.", + "Observed graph equality only; ideal FP64 differences are retained without threshold change.", + "The selected decoder rows are inside the image-placeholder span, not generated suffix text.", + "No performance, 6/8-bit, all-position or final fidelity acceptance is claimed.", + "The dense C helper and row helper have pinned source/binary identity; historical binary compiler provenance is limited." + ], + "fidelity_pass": null, + "accepted_threshold": null +} diff --git a/docs/receipts/lfm3/vl4-prefix18-rounded-graph.json b/docs/receipts/lfm3/vl4-prefix18-rounded-graph.json new file mode 100644 index 0000000000..f4a254f121 --- /dev/null +++ b/docs/receipts/lfm3/vl4-prefix18-rounded-graph.json @@ -0,0 +1,2917 @@ +{ + "scope": "VL4 four-bit causal prefix18 independent CPU prescribed-rounding graph with production-options Metal elementary evaluations; finalvocabularyrow17 only", + "runtime_change": false, + "native_activation_injection": false, + "tokens": 18, + "final_row": 17, + "measured_identity": { + "matched_stages": 310, + "stage_BF16_bytes": 37349568, + "all_stages_exact": true, + "final_logits_exact": 128000, + "final_logits_total": 128000, + "native_top": 296, + "rounded_reference_top": 296, + "relative_RMS_vs_native": 0.0 + }, + "ideal_FP64_difference": { + "retained": true, + "contenders": { + "296": { + "native": 12.0625, + "ideal_FP64": 12.125, + "projection_only_FP32": 12.1875, + "CPU_full_named": 12.0625 + }, + "405": { + "native": 11.875, + "ideal_FP64": 12.25, + "projection_only_FP32": 12.125, + "CPU_full_named": 11.875 + } + }, + "claim": "This rounding reproduction does not turn ideal FP64 disagreement into an ideal-FP64 pass." + }, + "elementary_evaluations": { + "batches": 264, + "scalar_values": 3492778, + "operations": { + "rsqrt": 119, + "exp": 79, + "exp2": 22, + "cos": 22, + "sin": 22 + }, + "inputs": "Independently generated CPU graph intermediates from checkpoint and token IDs; native stages are comparison-only.", + "functions": { + "exp": "metal::precise::exp", + "exp2": "metal::exp2", + "cos": "metal::precise::cos", + "sin": "metal::precise::sin", + "rsqrt": "metal::precise::rsqrt" + }, + "CompileOptions": { + "math_mode": "safe", + "math_functions": "fast", + "language_code": 262144 + }, + "all_helper_exits_zero": true, + "raw_calls_manifest_sha256": "7653c5138260c0b5f37d8c6e0b1da6e03b86d810fe7dd4cdbdc92c0361bd8ee8" + }, + "stages": [ + { + "stage": "embedding", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_00_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_00_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_00_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_00_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_00_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_00_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_00_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_00_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_00_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_00_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_00_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_01_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_01_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_01_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_01_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_01_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_01_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_01_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_01_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_01_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_01_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_01_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_attn_q", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_attn_k", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_attn_v", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_attn_queries", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_attn_gate", + "rows": 18, + "width": 16, + "equal": 288, + "values": 288, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_attn_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_03_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_03_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_03_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_03_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_03_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_03_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_03_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_03_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_03_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_03_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_03_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_attn_q", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_attn_k", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_attn_v", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_attn_queries", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_attn_gate", + "rows": 18, + "width": 16, + "equal": 288, + "values": 288, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_attn_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_05_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_05_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_05_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_05_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_05_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_05_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_05_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_05_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_05_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_05_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_05_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_attn_q", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_attn_k", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_attn_v", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_attn_queries", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_attn_gate", + "rows": 18, + "width": 16, + "equal": 288, + "values": 288, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_attn_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_07_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_07_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_07_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_07_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_07_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_07_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_07_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_07_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_07_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_07_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_07_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_attn_q", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_attn_k", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_attn_v", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_attn_queries", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_attn_gate", + "rows": 18, + "width": 16, + "equal": 288, + "values": 288, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_attn_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_09_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_09_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_09_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_09_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_09_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_09_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_09_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_09_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_09_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_09_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_09_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_attn_q", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_attn_k", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_attn_v", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_attn_queries", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_attn_gate", + "rows": 18, + "width": 16, + "equal": 288, + "values": 288, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_attn_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_11_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_11_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_11_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_11_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_11_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_11_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_11_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_11_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_11_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_11_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_11_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_attn_q", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_attn_k", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_attn_v", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_attn_queries", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_attn_gate", + "rows": 18, + "width": 16, + "equal": 288, + "values": 288, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_attn_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_13_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_13_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_13_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_13_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_13_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_13_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_13_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_13_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_13_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_13_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_13_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_attn_q", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_attn_k", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_attn_v", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_attn_queries", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_attn_gate", + "rows": 18, + "width": 16, + "equal": 288, + "values": 288, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_attn_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_15_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_15_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_15_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_15_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_15_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_15_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_15_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_15_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_15_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_15_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_15_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_attn_q", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_attn_k", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_attn_v", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_attn_queries", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_attn_gate", + "rows": 18, + "width": 16, + "equal": 288, + "values": 288, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_attn_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_17_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_17_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_17_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_17_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_17_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_17_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_17_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_17_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_17_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_17_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_17_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_attn_q", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_attn_k", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_attn_v", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_attn_queries", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_attn_gate", + "rows": 18, + "width": 16, + "equal": 288, + "values": 288, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_attn_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_19_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_19_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_19_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_19_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_19_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_19_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_19_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_19_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_19_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_19_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_19_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_attn_q", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_attn_k", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_attn_v", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_attn_queries", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_attn_gate", + "rows": 18, + "width": 16, + "equal": 288, + "values": 288, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_attn_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_21_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_21_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_21_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_21_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_21_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_21_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_21_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_21_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_21_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_21_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_21_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_attn_q", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_attn_k", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_attn_v", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_attn_queries", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_attn_gate", + "rows": 18, + "width": 16, + "equal": 288, + "values": 288, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_attn_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_23_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_23_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_23_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_23_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_23_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_23_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_23_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_23_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_23_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_23_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_23_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "final_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + } + ], + "source_sha256": { + "fp64_reference.py": "0fb1128ade095db87b6126792ab14930792ab1f7b41b998570356bc586f16b2e", + "row_projection_reference.py": "d03f224a037a920ba66b9828059d1489e48729cd168077e76789d0ec5c199c0c", + "elementary_reference.py": "3e4db276278867b7683536044ed7bcbe4c7643867d722fdda5b8ad895b3963e7", + "text_rounding_reference.py": "52665eef48570d45a2554861fe8f22b31874f8f464366adf882f40c3d311db27", + "text_attention_rounding.py": "ae67f2b2c26f89f75b1f36822a4b30a9f710df2b42f9f7ec42679373028b89e8" + }, + "reference_C": { + "sha256": "ff336f75fcf0a4f6c0ac92ed627bc1a6abb0b36b8f4cc95d876f2a8bb2f5cc42", + "ABI": "lfm3_row_replay(float*,uint8*,float*,float*,float*,int rows,int N,int K,int group,int bits); recorded binary hash proves artifact identity, not reproducible compiler provenance" + }, + "artifact_sha256": { + "VL-2B-4-reasoning-257.native.bf16": "118440ccd7fc07a502485a3266a2878ec815f316bcfa9dddd859e5d324fb8654", + "VL-2B-4-reasoning-257.bf16": "eaad83f4ec389f5c731c0d3422f2af47f218f8cdb220a6d0b752641c8f8f9bfe", + "lfm3-rounded-projection-reference.bf16": "b59c43083bfc6008233e7e249ced2f589ace92e1461e95c65300141aea14ab2b", + "reasoning-257.npy": "81449804fdd1a104d6dc9607c27d5cea49237064e50c80a765f68d8eba57f596", + "run.py": "adfd34f5ae830a4e06c1241b2f242da87bdb5d738f55fbfc98507ff8479a2c5e", + "provider.py": "e79e2f0ed4a881ca3cee683764a5eca3d9c4cc0c5aa9d27770bf69f41e922d38", + "row17.bf16": "acae5ff71fd78e34579440ad7609661ae366f423c54d3180b78bb7ff01086e35", + "row.so": "11ad0f66632bfcba548dbede9027ffdb112cd418ff2be80610af56a957424753" + }, + "graph_receipt_sha256": "6f493f08abf49aacbf3b8103d1dfb10ac40656f0c172c002a0f828881688db4c", + "guarded_execution": { + "lock": "/tmp/tensorfold-model.lock", + "inode": 46835039, + "single_model_admission_passed": true, + "graph_child_exit": 0, + "all_primitive_children_exited": true, + "lock_released_after_completion": true, + "events_sha256": "091742c894ffebc8d414eb56d18db1483f1c42bd1bf306523ab8126c6918aedd" + }, + "limits": [ + "Ordinary attention reference lacks selected integer-exponent range repair; observed equality validates only measured inputs.", + "Reference fma uses FP64 multiply/add then FP32 and is not universally correctly rounded fmaf.", + "Masked elementary evaluations may include NaN/-Inf discarded by reference; these are not asserted as native executed sites.", + "Only this four-bit prefix18 and finalrow17 are covered; no realimage, six/eight-bit, long-context, performance, or full-fidelity gate is claimed." + ], + "fidelity_pass": null, + "accepted_threshold": null, + "checkpoint": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "revision": "c823fade98e35a3d6942ad4d38b1ea15f675cd12" + }, + "token_ids": [ + 41, + 9798, + 677, + 1703, + 50474, + 22, + 440, + 1008, + 10494, + 229, + 1325, + 5435, + 20, + 278, + 1840, + 10494, + 10625, + 405 + ], + "executed_manifest_sha256": "670b81847ba268197c5afaea9e5583a0b621bfbb77fa0c9806d9d28718977217" +} diff --git a/docs/recipes/lfm3-native.md b/docs/recipes/lfm3-native.md new file mode 100644 index 0000000000..fdf938297f --- /dev/null +++ b/docs/recipes/lfm3-native.md @@ -0,0 +1,172 @@ +# LFM3 native Metal bring-up + +Tracking issue: [#1491](https://github.com/ashhart/TensorFold/issues/1491). + +The native Metal engine implements LFM3-1.7B text generation and LFM3-VL-2B image-to-text generation. +Both read the official MLX affine 4-, 6-, and 8-bit checkpoints without converting stored weights. +This is an experimental family port. Device and model checks cover an M4 Pro and an M5 Ultra; they do not qualify other Apple GPUs. + +## Weight licence + +The official text and VL MLX exports retain the **LFM Open License v1.0**, rather than TensorFold’s Apache-2.0 code licence. +See the [text weights licence](https://huggingface.co/LiquidAI/LFM3-1.7B-MLX-4bit/blob/main/LICENSE) and +[VL weights licence](https://huggingface.co/LiquidAI/LFM3-VL-2B-MLX-4bit/blob/main/LICENSE). +The checked Macs are an M4 Pro with 48 GB RAM and an M5 Ultra with 256 GiB RAM. + +## Run the models + +Build the native engine with the Zig version in `.zig-version` and Xcode's Metal toolchain: + +```sh +zig build native -Dcpu=apple_m1 +./zig-out/native/bin/tensorfold-native serve LiquidAI/LFM3-1.7B-MLX-4bit --context 8192 +./zig-out/native/bin/tensorfold-native serve LiquidAI/LFM3-VL-2B-MLX-4bit --context 8192 +``` + +Replace `4bit` with `6bit` or `8bit` for those exports. A complete local checkpoint folder also works. +The VL checkpoint loads its own image tower. Send images through OpenAI `image_url` content parts. +The default image budget is 256 tokens per request, with at most four images. +Video input is not implemented. JPEG and WebP decoding require the matching shared decoder libraries. + +The shared lane engine handles target verification, sampling and concurrent streams. +Convolution history and KV state are kept per stream; partial acceptance restores the accepted row's state. +Raw prompts of at least 256 tokens can build a repeat-cache entry: the first request records interest, +the second keeps a prefix state, and a later repeat resumes it. Image cache identity includes image content and grid. + +```mermaid +flowchart LR + I[Image pixels] --> V[Vision encoder and projector] + V --> E[Placeholder embedding rows] + T[Text tokens] --> E + E --> D[Shared LFM3 decoder] + D --> L[Existing lane engine] +``` + +## Inspect a checkpoint + +Use Python 3.11 or newer. The inspector uses the standard library. It resolves a full Hugging Face commit, +reads the config and index, and requests only the safetensors headers. It refuses a server that returns a +complete weight file instead of the requested byte range. + +```sh +python3 tools/lfm3/inspect_checkpoint.py LiquidAI/LFM3-1.7B-MLX-4bit --output text4.json +python3 tools/lfm3/inspect_checkpoint.py LiquidAI/LFM3-VL-2B-MLX-4bit --output vl4.json +``` + +Replace `4bit` with `6bit` or `8bit` to inspect those exports. Pass `--revision COMMIT` to repeat a pinned inspection. +Use `--local-dir MODEL_DIR` instead of a repository name to inspect files already on disk. + +## Check the native text layout + +Use the Zig version in `.zig-version`. + +```sh +zig build test-lfm3-host tf-lfm3-check +./zig-out/bin/tf-lfm3-check text4.json +./zig-out/bin/tf-lfm3-check vl4.json +python3 -m unittest discover -s tools/lfm3 -p 'test_*.py' -v +``` + +The native checker reads the saved metadata, validates shard header geometry, and checks the LFM3 text decoder's +required names, dtypes, shapes, and byte counts. It performs no projection, GPU allocation, or model inference. +For VL, it checks the text decoder only. A passing result does not validate vision weights or image processing. + +The config reader preserves full-attention versus sliding-attention layers, distinct rotary settings, sandwich norms, +and each module's affine width and group size. Unsupported geometry and conflicting quantization declarations fail. + +## Checkpoint details + +All six inspected MLX exports store the decoder under `language_model.model.*`, including the text-only model. +Each decoder has 594 tensors. Convolution kernels are BF16 with shape `[2048, 7, 1]`. + +Text 4/6/8-bit and VL 6/8-bit use affine groups of 64. VL 4-bit uses groups of 32 and an 8-bit embedding override. +The 138 additional VL vision/projector tensors are BF16 in all three exports. Global quantization metadata must not +cause these dense tensors to be treated as packed affine weights. + +The header inspections used these revisions: + +- Text 4-bit: `d0ca73dcfa70ce18733754a492cdc1fb0123262d`. +- Text 6-bit: `26be898c36b545d625d84ac0761c6c3eb664aef3`. +- Text 8-bit: `ee7667f3cfccb2a41725f91820531823fcabc6b7`. +- VL 4-bit: `c823fade98e35a3d6942ad4d38b1ea15f675cd12`. +- VL 6-bit: `6d71bfae121abedc75f6d531fc7f6ceb4f5e3501`. +- VL 8-bit: `83336d5be3874a2bdd3297bde77b2114dbd3a816`. + +## Qualification and remaining gate + +Detailed counts, hashes, numerical evidence and reproducible requests are in the [receipts](../receipts/lfm3/README.md). + +All six checkpoints passed 303 greedy/seeded-sampling, cache, concurrency and cancellation checks on one optimized M4 binary. +The final source build also passed 303/303 on an M5. Text4 and VL4 additionally passed real grammar, thinking, tool, +streaming, history and image-conditioned tool checks with the matching optional grammar library. +The 4-bit text long exactness suite passed all 73 cases through 8,193-token seams. +The final family contract passed 50 kernels and caught all 13 real controls, with zero limit problems. + +Independent projection audits covered 66 class/layout cells and 60,954,624 outputs with no violations of existing FP32/BF16 bounds. +The row-pair optimization preserves the shared arithmetic: 132 checkpoint calls and a full 8,193-token logit comparison were byte-identical. +On the M5, controlled 4-bit cold requests improved text engine prefill from 1.240 to 0.813 seconds at 2k and +73.402 to 61.633 seconds at 64k; VL decoder requests improved from 1.262 to 0.815 and 73.920 to 59.647 seconds. +These are engine prefill measurements; the receipt also retains client TTFT. All compared full output-token hashes matched. + +Native inference retains the checkpoint's BF16 activation boundaries and FP32 reductions. +Text trigonometric values round to BF16 before rotation; vision axial trigonometric values remain FP32. +The CPU FP64 oracle records numerical differences rather than inventing a tie policy. +MLX's vector/matrix quantized paths use different effective reconstruction, and bounded FP32 differences can propagate through history. +See the exact primitive and whole-model evidence before interpreting a next-token mismatch. + +```sh +zig build tf-lfm3-forward tf-lfm3-vision tf-lfm3-project tf-lfm3-profile tf-lfm3-exact -Dcpu=apple_m1 +./zig-out/bin/tf-lfm3-exact MODEL_DIR --long +python3 tools/lfm3/serve_check.py --binary ./zig-out/native/bin/tensorfold-native --model-dir MODEL_DIR --sampled --cancel --output served.json +``` + +Long-context model fidelity, maximum image budgets, 6/8-bit performance and other Apple GPU generations remain separate qualification work. +Structured output requires the native grammar library; logprobs and video are not advertised. +A complete earlier global M5 contract run found one Qwen4 failure that also reproduces on the unchanged pinned base. +It is not a final-source global pass. Agree model-fidelity and performance gates on #1491 before claiming release readiness. + +## Development-default tensor-unit routes + +Development builds enable `lfm3-nax-prompt`, `lfm3-tiled-attention` and 2048 prompt rows by default on tensor-unit +devices. On devices without tensor units serving uses 128 rows and the existing kernels. Release-version builds +default to the existing path and reject either option when explicitly enabled. These remain draft routes; model +fidelity is judged against an independent FP32 forward and the official MLX BF16 run, as agreed on +[#1491](https://github.com/ashhart/TensorFold/issues/1491#issuecomment-6537143418), not against float64. + +```sh +zig build native -Dcpu=apple_m1 -Dlfm3-nax-prompt=true -Dlfm3-tiled-attention=true -Dlfm3-prompt-rows=2048 +``` + +`lfm3-nax-prompt` sends every dense prompt projection (attention q/k/v/out, convolution in/out, FFN gate/up/down) +through `tf_qmmw_t_nax_win`, Flash Next's `tf_qmmw_t_nax` tile with scales and biases loaded eight groups at a time. +It dequantizes each weight to BF16 as MLX does and accumulates in FP32, with the same bytes as `tf_qmmw_t_nax`. 4, 6 +and 8 bits with groups of 32 or 64 are covered, `n` divisible by 64 and `k` by eight groups; other shapes keep the row +kernels. The LM head and decode keep their own kernels. +`lfm3-tiled-attention` adds two attention kernels. Prompt rows use 64-query/32-key tiles with FP32 online softmax and +the probabilities rounded once to BF16 for PV, as MLX does. A prompt segment of up to 2048 rows is one launch, so +where a prompt is cut does not change its state. Decode, verification windows of any width and shared decode rows use a +key-parallel kernel, so a window equals the same rows decoded one at a time: each query's keys split +into at most 64 contiguous parts of at least 128 keys, merged in FP32. Both kernels replay a row through the exact +attention path when its result is nonfinite, so nonfinite classes follow float64. Underflow and overflow of the FP32 +softmax follow MLX, not the earlier extended-range repair. +To select the prior development baseline, build with `-Dlfm3-nax-prompt=false -Dlfm3-tiled-attention=false`. +`tf-lfm3-nax-exact MODEL_DIR` checks these routes on real weights: prompts cut at many offsets, resumed from a saved +prefix, and verification windows of 2 to 32 rows must equal their reference byte for byte. + +`-Dlfm3-prompt-rows=128`, `256`, or `2048` selects the development prompt batch; its tensor-unit default is 2048. +Both options are required for a larger batch, and release-version builds reject them. +Cache marks retain their existing 128-row policy and can split a larger batch. Cancellation and background yield +are checked at chunk boundaries; a 2048-row chunk remains one atomic forward pass. +The server fills prompts one chunk at a time (`prefill_step`), so the lane host runs the shortest remaining prompt next +and decodes live streams between chunks instead of finishing every queued prompt first. Where a prompt is cut does +not change its state (`tf-lfm3-nax-exact`). + +Kernel-contract probes: `metal LFM3 NAX prompt q{4,6,8}g64`, `q4g32` and `metal LFM3 split structural full|sliding`, +each with planted controls (`zig build tf-kernel-contract`, `--only "LFM3 NAX prompt"`). +Developer harnesses (`tf-lfm3-forward ... --nax-prompt`, `tf-lfm3-profile` with `TF_LFM3_NAX_PROMPT=1` and +`TF_LFM3_DECODE_STEPS=N`) enable the same kernels explicitly. + +These are work-in-progress arithmetic changes, not release-qualified replacements. Seam, cancellation, concurrency +and lifecycle validation of the new routes, other Apple GPU generations and the weights' licence statement remain +open. Licence: the LiquidAI LFM3 model cards declare `license: other` with `license_name: lfm1.0` and a LICENSE file in each repo. +Macs checked: an M5 Ultra only. diff --git a/tools/check_lfm3_risk.cpp b/tools/check_lfm3_risk.cpp new file mode 100644 index 0000000000..abfc2bc640 --- /dev/null +++ b/tools/check_lfm3_risk.cpp @@ -0,0 +1,35 @@ +#include +#include +#include +using uint=uint32_t; +#include "predicate.inc" +bool finite(uint a){return(a&0x7f800000u)!=0x7f800000u;} +bool nz(uint a){return(a&0x7fffffffu)!=0;} +bool product(uint a,uint b){uint ae=a>>23&255,be=b>>23&255,s=ae+be;return(finite(a)&finite(b)&nz(a)&nz(b))&((ae==0)|(be==0)|(s<=129)|(s>=380));} +bool sum(uint a,uint b,uint c,uint d,uint r){uint e=r>>23&255;return(finite(a)&finite(b)&finite(c)&finite(d))&((e>=253)|((e==0)&((nz(a)&nz(b))|(nz(c)&nz(d)))));} +bool original(uint x,uint a,uint y,uint b,uint r){return product(x,a)|product(y,b)|sum(x,a,y,b,r);} +void check(uint x,uint a,uint y,uint b,uint r){if(original(x,a,y,b,r)!=l3_pv_risk(x,y,r,l3_risk_decode(a),l3_risk_decode(b))){printf("mismatch %08x %08x %08x %08x %08x\n",x,a,y,b,r);abort();}} +int main(){ + uint64_t pairs=0,classes=0,random=0;uint frac[]={0,1,0x400000,0x7fffff}; + // All 256 exponent classes, both signs, and exact zero/nonzero mantissa categories. + for(uint ae=0;ae<256;++ae)for(uint be=0;be<256;++be)for(uint sa=0;sa<2;++sa)for(uint sb=0;sb<2;++sb)for(uint af:frac)for(uint bf:frac){ + uint a=sa<<31|ae<<23|af,b=sb<<31|be<<23|bf;auto x=l3_risk_decode(a),y=l3_risk_decode(b); + if(x.e!=ae||y.e!=be||x.finite!=finite(a)||y.finite!=finite(b)||x.nonzero!=nz(a)||y.nonzero!=nz(b))abort(); + bool p=(x.finite&y.finite&x.nonzero&y.nonzero)&((x.e==0)|(y.e==0)|((x.e+y.e-130u)>=250u)); + if(p!=product(a,b))abort();check(a,b,0,0,0);++pairs; + } + // The complete Boolean identity factoring the three original predicates, including infeasible states. + for(uint bits=0;bits<256;++bits){bool xf=bits&1,yf=bits&2,xn=bits&4,yn=bits&8,xr=bits&16,yr=bits&32,zero=bits&64,high=bits&128; + bool old=(xf&xn&xr)|(yf&yn&yr)|(xf&yf&(high|(zero&(xn|yn)))); + bool now=((xf&xn)&(xr|(yf&zero)))|((yf&yn)&(yr|(xf&zero)))|(xf&yf&high);if(old!=now)abort(); + } + // Full fused source over Cartesian zero/subnormal/normal/infinity/NaN operand classes and every output exponent. + uint cls[]={0,0x80000000,1,0x80000001,0x3f800000,0xbf800000,0x7f800000,0xff800000,0x7fc00000,0xffc00000}; + for(uint x:cls)for(uint a:cls)for(uint y:cls)for(uint b:cls)for(uint e=0;e<256;++e)for(uint sign=0;sign<2;++sign)for(uint f:frac){check(x,a,y,b,sign<<31|e<<23|f);++classes;} + uint seed=1502;auto next=[&](){seed^=seed<<13;seed^=seed>>17;seed^=seed<<5;return seed;}; + for(uint i=0;i<10000000;++i){uint x=next(),a=next(),y=next(),b=next(),r=next();check(x,a,y,b,r);++random;} + // Predicate controls must distinguish a missing high-result check and a missing subnormal-product check. + if(!original(0x3f800000,0x3f800000,0x3f800000,0x3f800000,253u<<23))abort(); + if(!original(1,0x3f800000,0,0,0x3f800000))abort(); + printf("{\"pair_class_checks\":%llu,\"fused_operand_result_classes\":%llu,\"random_fused_words\":%llu,\"boolean_truth_table_rows\":256,\"control_witnesses\":2,\"mismatches\":0,\"scope\":\"Exhaustive predicate category identity plus deterministic random sample; not exhaustive floating-point arithmetic or GPU qualification\"}\n",pairs,classes,random); +} diff --git a/tools/check_lfm3_risk.py b/tools/check_lfm3_risk.py new file mode 100644 index 0000000000..58bed498f0 --- /dev/null +++ b/tools/check_lfm3_risk.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""Check the compiled Metal predicate source against the original bit predicates.""" +from pathlib import Path +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[1] +source = (root / 'zig/kernels/metal/lfm3/attention.metal').read_text() +start = source.index('struct l3_risk_factor') +end = source.index('inline void l3_x_merge', start) +with tempfile.TemporaryDirectory(prefix='lfm3-risk-proof-') as folder: + work = Path(folder) + (work / 'predicate.inc').write_text(source[start:end]) + (work / 'proof.cpp').write_text((root / 'tools/check_lfm3_risk.cpp').read_text()) + subprocess.run(['clang++', '-O2', '-std=c++17', str(work / 'proof.cpp'), '-o', str(work / 'proof')], check=True) + subprocess.run([str(work / 'proof')], check=True) diff --git a/tools/lfm3/FP32_ENVELOPE.md b/tools/lfm3/FP32_ENVELOPE.md new file mode 100644 index 0000000000..b1ab1edabc --- /dev/null +++ b/tools/lfm3/FP32_ENVELOPE.md @@ -0,0 +1,90 @@ +# Reproduce the bounded FP32/BF16 comparison + +These tools publish the comparison used for the default-off tiled prompt experiment. +They compare independent NumPy FP32 logits (F), actual pinned official MLX-VLM BF16 +logits promoted to FP32 (B), and captured native BF16 logits (N). They do not use +`BF16(F)` as B. The checkpoint reader shares the tensor loader with the older FP64 +reference; the model forward in this tool uses FP32 arithmetic. + +The maintainer's [ratified criterion](https://github.com/ashhart/TensorFold/issues/1491#issuecomment-6537143418) +is no further from FP32 than the reference's own BF16, retaining top-token differences +and margins. Whole-phase logit RMS/max and fixed-F-top-20 probability mean/max are an +explicit executable proposal for that criterion. The exact metric set is not a verbatim +maintainer specification. Row/column exceedances remain visible; no tie waiver applies. + +## Generate the three inputs + +Run from the checkout root. `MODEL_DIR` must be the same admitted TEXT4 checkpoint +for all three runs. The frozen token fixture has 128 prompt tokens and 16 fixed +teacher-forced continuation tokens. The measured rows are absolute positions 127–143. +The MLX reference requires MLX 0.32.4 and the official source files matching the +provided pins. A different installed revision is refused rather than silently reused. + +```sh +python3 tools/lfm3/full_model_reference.py --backend fp32 \ + --model-dir "$MODEL_DIR" --tokens tools/lfm3/fixtures/fp32-envelope-text4-144.npy \ + --prompt-len 128 --chunk 2048 --output fp32.npy + +python3 tools/lfm3/full_model_reference.py --backend mlx \ + --model-dir "$MODEL_DIR" --tokens tools/lfm3/fixtures/fp32-envelope-text4-144.npy \ + --prompt-len 128 --chunk 2048 --expect-mlx 0.32.4 \ + --official-pins tools/lfm3/fixtures/fp32-envelope-mlx-0.32.4-pins.json \ + --output reference-bf16.npy + +zig build tf-lfm3-forward -Dcpu=apple_m1 +./zig-out/bin/tf-lfm3-forward "$MODEL_DIR" \ + tools/lfm3/fixtures/fp32-envelope-text4-144.npy native.bf16 prompt:128 --nax-prompt +``` + +The native flag explicitly enables both development candidates through the public +model APIs. It refuses a device without the optional pipelines, rather than letting +an unsupported device produce a misleading candidate receipt. Omit the flag for the +baseline. Native inference remains native; Python generates development references. +Use the project's GPU lock/admission guard and one model process at a time. The FP32 +reference runs on the CPU. References record model/input/output/source hashes and +NumPy/MLX versions; BLAS implementations can affect the FP32 reduction order. + +## Run the failing check + +```sh +python3 tools/lfm3/score_model_phases.py \ + --fp32 fp32.npy --reference-bf16 reference-bf16.npy \ + --native native.bf16 --output phases.json +``` + +The sidecar reference JSON files must accompany the NPY files. The scorer validates +input, geometry, selected-row and output identities before scoring, and emits prompt +and continuation phases separately. Exit 1 means at least one proposed metric fails; +inspect the JSON to distinguish a probability diagnostic from logit RMS/max failure. +An identity/geometry exception is not a numerical comparison result. Stale output is +removed before validation; a failed identity check cannot reuse an earlier receipt. + +For the retained public `8df18abc4` candidate, phase-level logit RMS/max pass while +prompt probability mean/max and continuation probability max fail. All 17 top choices +match F. This is bounded teacher-forced evidence, not arbitrary-prompt, image, width, +long-context, or production fidelity qualification. The retained raw hashes and actual +metrics are in [the public runtime receipt](../../docs/receipts/lfm3/m5-public-tiled-prompt-runtime.json). + +## Isolate a route and sample more boundary rows + +`tf-lfm3-forward` also accepts `--tiled-attention` or `--nax-prompt` (last argument) to enable one +route at a time (`--nax-prompt` adds the tensor-unit projections and the tiled attention); with neither flag it +runs the baseline route. + +The phase scorer's prompt phase is a single row. `prompt_row_sweep.py` scores many boundary rows of one +causal sequence instead: produce FP32 and official-BF16 rows once with `full_model_reference.py --rows`, capture +native `prompt:N` on `tokens[:N]` for each N, and run +`prompt_row_sweep.py --fp32 ... --reference-bf16 ... --positions ... --route NAME=DIR/PREFIX --lengths ...`. +It reports per-row RMS/max/top-20 probability, top-1 agreement and margins with no pass/fail rule. The retained TEXT6 +result (25 rows, baseline vs tiled attention vs official BF16) is in +[the route-isolation receipt](../../docs/receipts/lfm3/m5-text6-route-isolation-row-sweep.json). + +For vision checkpoints, `tf-lfm3-forward MODEL TOKENS.npy out.bf16 prompt:N IMAGE_ROWS.bf16 START` injects +the projected BF16 image rows at absolute position START; pass the same rows as float32 NPY plus `--image-start` +to `full_model_reference.py`. TEXT8 and VL4/6/8 comparisons (including text after real image rows) are in +[the width/image route receipt](../../docs/receipts/lfm3/m5-width-image-route-comparison.json). + +```sh +python3 -m unittest discover -s tools/lfm3 -p 'test_*envelope.py' +python3 -m unittest discover -s tools/lfm3 -p test_full_model_reference.py +``` diff --git a/tools/lfm3/bench_decode.py b/tools/lfm3/bench_decode.py new file mode 100644 index 0000000000..95b0ba4ef3 --- /dev/null +++ b/tools/lfm3/bench_decode.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Run existing served benchmarks with public code/prose prompts long enough for 256-token comparisons.""" +import json +from pathlib import Path +import sys +import urllib.request + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import bench_concurrent +import bench_openai + +PROMPTS = [ + {'name': 'long-code-chat', 'kind': 'chat', 'prompt': + 'Write a complete Python implementation of a red-black tree with insertion, deletion, searching, ' + 'rotations, and extensive unit tests. Include detailed explanations after each method. ' + 'Continue until the implementation and every test are complete.'}, + {'name': 'long-prose-chat', 'kind': 'chat', 'prompt': + 'Write a detailed 2000-word tutorial explaining how matrix multiplication runs on a GPU, including ' + 'memory layout, tiling, shared memory, synchronization, numerical precision, and several worked ' + 'numerical examples. Explain every step thoroughly.'}, +] + + +def main(): + if len(sys.argv) < 2 or sys.argv[1] not in ('decode', 'concurrent'): + raise SystemExit('Choose decode or concurrent, then pass the existing benchmark client arguments') + mode = sys.argv.pop(1) + module = bench_openai if mode == 'decode' else bench_concurrent + module.PROMPTS = PROMPTS + original = urllib.request.Request + + def request(url, data=None, *args, **kwargs): + if data: + body = json.loads(data) + body.update(top_p=1, top_k=0, repetition_penalty=1, enable_thinking=False) + data = json.dumps(body).encode() + return original(url, data, *args, **kwargs) + + urllib.request.Request = request + try: + module.main() + finally: + urllib.request.Request = original + + +if __name__ == '__main__': + main() diff --git a/tools/lfm3/cold_prompts.py b/tools/lfm3/cold_prompts.py new file mode 100644 index 0000000000..b4af37efe0 --- /dev/null +++ b/tools/lfm3/cold_prompts.py @@ -0,0 +1,60 @@ +"""Regenerate the synthetic cold-prompt request bodies and rendered token arrays, and check them against their hashes.""" +import argparse +import hashlib +import json +import pathlib + +SHA256 = pathlib.Path(__file__).resolve().parents[2] / "docs/receipts/lfm3/cold-prompt-sha256.json" +LENGTHS = (2048, 8192, 32768, 65536) +PREFIX = "A public benchmark fixture. Explain this sequence.\n" +# The chat template's header and the prefix render to 13 tokens, the trailer to 7, each repeated word to token 23018. +HEAD = [124894, 124899, 5922, 207, 41, 1251, 40524, 51639, 22, 42847, 532, 8055, 318] +TAIL = [124900, 207, 124899, 63514, 207, 124901, 124902] +WORD = 23018 +FIXED = { + "max_tokens": 16, "temperature": 0, "top_p": 1, "top_k": 0, "repetition_penalty": 1, "stream": True, + "stream_options": {"include_usage": True}, "enable_thinking": False, + "chat_template_kwargs": {"enable_thinking": False}, +} + + +def body(tokens: int) -> bytes: + """The request body whose rendered prompt is exactly `tokens` long.""" + words = tokens - len(HEAD) - len(TAIL) + return (json.dumps({"messages": [{"role": "user", "content": PREFIX + " alpha" * words}]} | FIXED) + "\n").encode() + + +def ids(tokens: int) -> bytes: + return (json.dumps(HEAD + [WORD] * (tokens - len(HEAD) - len(TAIL)) + TAIL) + "\n").encode() + + +def files() -> dict[str, bytes]: + return {f"{kind}-{n}.{part}.json": make(n) for kind in ("text", "vl") for n in LENGTHS + for part, make in (("body", body), ("ids", ids))} + + +def mismatches(produced: dict[str, bytes], wanted: dict[str, str]) -> list[str]: + """Files whose hash differs from the record, and recorded files nothing produced.""" + bad = [name for name, data in produced.items() if hashlib.sha256(data).hexdigest() != wanted.get(name)] + return bad + [name for name in wanted if name not in produced] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--write", type=pathlib.Path, help="directory to write the 16 files into") + args = parser.parse_args() + wanted = json.loads(SHA256.read_text()) + produced = files() + bad = mismatches(produced, wanted) + for name in bad: + print(f"MISMATCH {name}") + print(f"{len(produced) - len(bad)} of {len(wanted)} files match their recorded SHA-256") + if args.write: + args.write.mkdir(parents=True, exist_ok=True) + for name, data in produced.items(): + (args.write / name).write_bytes(data) + return 1 if bad else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/lfm3/compare_logits.py b/tools/lfm3/compare_logits.py new file mode 100644 index 0000000000..5a981ef0e3 --- /dev/null +++ b/tools/lfm3/compare_logits.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Report every teacher-forced top-token difference without applying an acceptance tolerance.""" +import argparse +import hashlib +import json +from pathlib import Path +import numpy as np + + +def compare(native, reference, row_offset=0): + native, reference = np.asarray(native), np.asarray(reference) + if native.shape != reference.shape or native.ndim != 2 or native.shape[1] < 2 or not len(native): + raise ValueError('Expected equally shaped, nonempty row-by-vocabulary arrays') + if not np.isfinite(native).all() or not np.isfinite(reference).all(): + raise ValueError('Logits must be finite') + differences, worst = [], [] + sum_squared, reference_squared, maximum = 0.0, 0.0, 0.0 + column_squared = np.zeros(native.shape[1], dtype=np.float64) + column_maximum = np.zeros(native.shape[1], dtype=np.float64) + for index, (actual, expected) in enumerate(zip(native, reference)): + actual, expected = actual.astype(np.float64), expected.astype(np.float64) + error = actual - expected + square = float(error @ error) + denominator = float(expected @ expected) + column_squared += error * error + column_maximum = np.maximum(column_maximum, np.abs(error)) + sum_squared += square + reference_squared += denominator + maximum = max(maximum, float(np.abs(error).max())) + worst.append({'row': index + row_offset, 'rms_error': float(np.sqrt(square / len(error))), + 'maximum_absolute_error': float(np.abs(error).max())}) + a, b = int(actual.argmax()), int(expected.argmax()) + if a == b: + continue + aset = np.flatnonzero(actual == actual[a]).tolist() + bset = np.flatnonzero(expected == expected[b]).tolist() + atop = np.partition(actual, -2)[-2:] + btop = np.partition(expected, -2)[-2:] + differences.append({'row': index + row_offset, 'native_top': a, 'reference_top': b, + 'native_maximum_set': aset, 'reference_maximum_set': bset, + 'maximum_sets_overlap': bool(set(aset) & set(bset)), + 'native_margin': float(atop[-1] - atop[-2]), + 'reference_margin': float(btop[-1] - btop[-2]), + 'native_reference_contender_gap': float(actual[a] - actual[b]), + 'reference_native_contender_gap': float(expected[b] - expected[a])}) + columns = np.argsort(column_squared)[-20:][::-1] + worst_columns = [{'column': int(c), 'rms_error': float(np.sqrt(column_squared[c] / len(native))), + 'maximum_absolute_error': float(column_maximum[c])} for c in columns] + return {'rows': len(native), 'vocabulary': native.shape[1], 'top1_differences': len(differences), + 'disjoint_maximum_sets': sum(not item['maximum_sets_overlap'] for item in differences), + 'relative_rms': float(np.sqrt(sum_squared / reference_squared)) if reference_squared else None, + 'maximum_absolute_error': maximum, 'differences': differences, + 'worst_rows': sorted(worst, key=lambda item: item['rms_error'], reverse=True)[:20], + 'worst_columns': worst_columns, 'accepted_threshold': None, 'fidelity_pass': None} + + +def read_bf16(path, vocabulary): + if path.stat().st_size % 2: + raise ValueError('BF16 file contains an incomplete two-byte value') + values = np.fromfile(path, dtype=' len(tokens): + parser.error('Measured rows exceed teacher-forced fixture') + result = compare(native, reference, args.row_offset) + result.update({'phase': args.phase, + 'token_fixture_sha256': hashlib.sha256(args.tokens.read_bytes()).hexdigest(), + 'native_logits_sha256': hashlib.sha256(args.native.read_bytes()).hexdigest(), + 'reference_logits_sha256': hashlib.sha256(args.reference.read_bytes()).hexdigest(), + 'input_identity': 'Caller declares fixture; matching run commands establish execution identity'}) + args.output.write_text(json.dumps(result, indent=2, allow_nan=False) + '\n') + + +if __name__ == '__main__': + main() diff --git a/tools/lfm3/elementary_reference.py b/tools/lfm3/elementary_reference.py new file mode 100644 index 0000000000..6ba6d5848d --- /dev/null +++ b/tools/lfm3/elementary_reference.py @@ -0,0 +1,41 @@ +"""Shared reference-only FP32 elementary functions and activation rounding sites.""" +import numpy as np +from fp64_reference import bf + + +_ELEMENTARY_KERNELS = {} + + +def elementary(values, operation, backend='numpy'): + values = np.asarray(values, dtype=np.float32) + if backend == 'numpy': + if operation == 'rsqrt': + return (1 / np.sqrt(values.astype(np.float64))).astype(np.float32) + return getattr(np, operation)(values) + if backend != 'metal' or operation not in ('exp', 'exp2', 'cos', 'sin', 'rsqrt'): + raise ValueError('Unsupported elementary function backend or operation') + import mlx.core as mx + if operation not in _ELEMENTARY_KERNELS: + function = 'metal::exp2' if operation == 'exp2' else 'metal::precise::' + operation + _ELEMENTARY_KERNELS[operation] = mx.fast.metal_kernel( + name='lfm3_reference_elementary_' + operation, input_names=['X'], output_names=['Y'], + source='uint i=thread_position_in_grid.x;if(i 0)), + 'maximum_pointwise_excess': float(np.max(excess)), + 'pointwise_is_diagnostic_not_invented_gate': True, + 'all_row_errors': [ + { + 'flat_row': i, + 'native_rms': float(rows_n[i]), + 'reference_rms': float(rows_b[i]), + 'native_max': float(np.abs(ne[i]).max()), + 'reference_max': float(np.abs(be[i]).max()), + } + for i in range(len(ne)) + ], + 'worst_columns': [ + { + 'column': int(i), + 'native_rms': float(cols_n[i]), + 'reference_rms': float(cols_b[i]), + 'native_max': float(np.abs(ne[:, i]).max()), + 'reference_max': float(np.abs(be[:, i]).max()), + } + for i in np.argsort(cols_n)[-20:][::-1] + ], + } + result['worst_rows'] = sorted(result['all_row_errors'], key=lambda x: x['native_rms'], reverse=True)[:20] + if space == 'logits': + if n.ndim != 2: + raise ValueError('Logits must be rows,vocabulary') + + def prob(x): + x = x.astype(np.float64) + p = np.exp(x - x.max(axis=1, keepdims=True)) + return p / p.sum(axis=1, keepdims=True) + + pn, pb, pf = map(prob, [n, b, f]) + top = np.argsort(-f, axis=1, kind='stable')[:, : min(20, f.shape[1])] + take = lambda x: np.take_along_axis(x, top, axis=1) + pe_n = np.abs(take(pn) - take(pf)) + pe_b = np.abs(take(pb) - take(pf)) + result['probability_scoring'] = ( + 'Shared stable NumPy binary64 scoring of captured logits only, not FP64 model forward; fixed top20 selected by F, ties use column order' + ) + result['top20_probability'] = { + 'native_mean': float(pe_n.mean()), + 'reference_mean': float(pe_b.mean()), + 'native_max': float(pe_n.max()), + 'reference_max': float(pe_b.max()), + } + result['proposed_metric_results'].update( + top20_probability_mean=pe_n.mean() <= pe_b.mean(), top20_probability_max=pe_n.max() <= pe_b.max() + ) + misses = [] + for row, (x, y, z) in enumerate(zip(n, b, f)): + ni, bi, fi = map(lambda a: int(a.argmax()), [x, y, z]) + if ni == bi == fi: + continue + sets = [np.flatnonzero(a == a.max()).tolist() for a in [x, y, z]] + fs = np.sort(z.astype(np.float64)) + native_sorted = np.sort(x.astype(np.float64)) + reference_sorted = np.sort(y.astype(np.float64)) + misses.append( + { + 'row': row, + 'native_top': ni, + 'reference_bf16_top': bi, + 'fp32_top': fi, + 'native_maximum_set': sets[0], + 'reference_maximum_set': sets[1], + 'fp32_maximum_set': sets[2], + 'native_margin': float(native_sorted[-1] - native_sorted[-2]), + 'reference_bf16_margin': float(reference_sorted[-1] - reference_sorted[-2]), + 'fp32_margin': float(fs[-1] - fs[-2]), + 'fp32_native_contender_gap': float(z[fi]) - float(z[ni]), + 'fp32_reference_contender_gap': float(z[fi]) - float(z[bi]), + 'native_only_miss': ni != fi and bi == fi, + } + ) + result['top_pick_differences'] = misses + result['native_fp32_top1_misses'] = int(np.count_nonzero(n.argmax(-1) != f.argmax(-1))) + result['reference_fp32_top1_misses'] = int(np.count_nonzero(b.argmax(-1) != f.argmax(-1))) + result['top_pick_acceptance'] = ( + 'No invented tie waiver or automatic pick acceptance; root gate must specify rule' + ) + result['proposed_metrics_no_worse'] = all(bool(v) for v in result['proposed_metric_results'].values()) + result['proposed_metric_results'] = {k: bool(v) for k, v in result['proposed_metric_results'].items()} + result['full_model_fidelity_pass'] = None + result['scope'] = ( + 'Finite held-input operator' + if space == 'operator' + else 'Teacher-forced captured logits; scope and top-pick acceptance must be supplied by cell manifest' + ) + return result + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + for n in ['fp32', 'reference-bf16', 'native', 'output']: + p.add_argument('--' + n, type=Path, required=True) + p.add_argument('--native-format', choices=['bf16', 'f32'], default='bf16') + p.add_argument('--space', choices=['operator', 'logits'], default='operator') + a = p.parse_args() + f = np.load(a.fp32, allow_pickle=False) + b = np.load(a.reference_bf16, allow_pickle=False) + if f.dtype != np.float32 or b.dtype != np.float32: + raise ValueError('FP32 files required') + if np.any(b.view(np.uint32) & 0xFFFF): + raise ValueError('Reference file is not actual BF16 values promoted to FP32') + if a.native_format == 'bf16': + n = (np.fromfile(a.native, dtype='> shifts.astype(np.uint64) + cross = shifts + bits > 32 + values[:, cross] |= padded[:, indices[cross] + 1] << (32 - shifts[cross]).astype(np.uint64) + codes = (values & ((1 << bits) - 1)).astype(np.float32) + scale = self.checkpoint.tensor(name + '.scales', rows).astype(np.float32).repeat(group, axis=-1) + bias = self.checkpoint.tensor(name + '.biases', rows).astype(np.float32).repeat(group, axis=-1) + return codes * scale + bias + + def linear(self, x, name): + count = self.checkpoint.files[self.checkpoint.map[name + '.weight']][2][name + '.weight']['shape'][0] + y = np.empty((*x.shape[:-1], count), dtype=np.float32) + for first in range(0, count, 512): + y[..., first:first + 512] = x @ self.weights(name, slice(first, first + 512)).T + return self.cast(y) + + def rms(self, x, gain, epsilon): + mean = np.mean(x * x, axis=-1, keepdims=True, dtype=np.float32) + inverse = np.float32(1) / np.sqrt(mean + np.float32(epsilon)) + return self.cast(self.cast(x * inverse) * gain) + + def norm(self, x, name): + return self.rms(x, self.tensor(name + '.weight'), self.text['norm_eps']) + + def rotary(self, x, kind): + spec = self.text['rope_parameters'][kind] + pairs = int(x.shape[-1] * spec.get('partial_rotary_factor', 1)) // 2 + power = -np.arange(pairs, dtype=np.float32) * np.float32(2 / x.shape[-1]) + frequency = np.concatenate((np.power(np.float32(spec['rope_theta']), power), + np.zeros(x.shape[-1] // 2 - pairs, dtype=np.float32))) + angle = np.arange(len(x), dtype=np.float32)[:, None] * frequency + angle = np.concatenate((angle, angle), axis=-1)[:, None] + first, second = np.split(x, 2, axis=-1) + return self.cast(x * self.cast(np.cos(angle)) + + np.concatenate((-second, first), axis=-1) * self.cast(np.sin(angle))) + + def attention_value(self, q, k, v, window=0, causal=True): + scores = (q @ k.swapaxes(-1, -2)) / np.float32(math.sqrt(q.shape[-1])) + if causal: + distance = np.arange(q.shape[1])[:, None] - np.arange(k.shape[1])[None] + allowed = distance >= 0 + if window: + allowed &= distance < window + scores = np.where(allowed, scores, np.float32(-np.inf)) + probability = np.exp(scores - scores.max(axis=-1, keepdims=True)) + probability = probability / probability.sum(axis=-1, keepdims=True, dtype=np.float32) + return self.cast(probability @ v) + + def attention(self, x, prefix, kind): + heads, kv, width = (self.text[k] for k in ('num_attention_heads', 'num_key_value_heads', 'head_dim')) + q = self.linear(x, prefix + 'self_attn.q_proj').reshape(len(x), heads, width) + k = self.linear(x, prefix + 'self_attn.k_proj').reshape(len(x), kv, width) + v = self.linear(x, prefix + 'self_attn.v_proj').reshape(len(x), kv, width).transpose(1, 0, 2) + q = self.rotary(self.norm(q, prefix + 'self_attn.q_norm'), kind).transpose(1, 0, 2) + k = self.rotary(self.norm(k, prefix + 'self_attn.k_norm'), kind).transpose(1, 0, 2) + k, v = np.repeat(k, heads // kv, axis=0), np.repeat(v, heads // kv, axis=0) + window = self.text['sliding_window'] if kind == 'sliding_attention' else 0 + value = self.attention_value(q, k, v, window).transpose(1, 0, 2) + gate = self.linear(x, prefix + 'self_attn.gate_proj') + gate = self.cast(np.float32(1) / (np.float32(1) + np.exp(-gate))) + return self.linear(self.cast(value * gate[:, :, None]).reshape(len(x), heads * width), + prefix + 'self_attn.o_proj') + + def conv(self, x, prefix): + left, gate, value = np.split(self.linear(x, prefix + 'conv.in_proj'), 3, axis=-1) + mixed = self.cast(left * value) + kernel = self.tensor(prefix + 'conv.conv.weight')[:, :, 0] + padded = np.pad(mixed, ((kernel.shape[1] - 1, 0), (0, 0))) + result = np.zeros_like(mixed) + for tap in range(kernel.shape[1]): + result = result + padded[tap:tap + len(x)] * kernel[:, tap] + return self.linear(self.cast(self.cast(result) * gate), prefix + 'conv.out_proj') + + def swiglu(self, gate, up): + return self.cast(self.cast(gate / (np.float32(1) + np.exp(-gate))) * up) + + def run(self, tokens, image_rows=None, image_start=None): + hidden = self.cast(self.weights(self.prefix + 'embed_tokens', tokens)) + if image_rows is not None: + rows = np.asarray(image_rows, dtype=np.float32) + if image_start is None or rows.ndim != 2 or rows.shape[1] != hidden.shape[1]: + raise ValueError('Expected projected image rows and a matching start') + expected = np.zeros(len(tokens), dtype=bool) + expected[image_start:image_start + len(rows)] = True + if image_start < 0 or image_start + len(rows) > len(tokens) or not np.array_equal(tokens == 125030, expected): + raise ValueError('Image span does not match placeholders') + hidden[image_start:image_start + len(rows)] = self.cast(rows) + elif image_start is not None: + raise ValueError('Image start requires projected rows') + self.take('embedding', hidden) + for index, kind in enumerate(self.text['layer_types']): + prefix = self.prefix + f'layers.{index}.' + x = self.norm(hidden, prefix + 'operator_norm') + result = self.conv(x, prefix) if kind == 'conv' else self.attention(x, prefix, kind) + hidden = self.cast(hidden + self.norm(result, prefix + 'post_operator_norm')) + x = self.norm(hidden, prefix + 'ffn_norm') + gate, up = self.linear(x, prefix + 'feed_forward.w1'), self.linear(x, prefix + 'feed_forward.w3') + result = self.linear(self.swiglu(gate, up), prefix + 'feed_forward.w2') + hidden = self.cast(hidden + self.norm(result, prefix + 'post_ffn_norm')) + self.take(f'text_layer_{index:02d}', hidden) + print('text_layer', index, flush=True) + return self.linear(self.norm(hidden, self.prefix + 'embedding_norm'), self.prefix + 'embed_tokens') + + +class Vision(Graph): + def __init__(self, directory, boundaries='fp32-bf16', trace=None): + super().__init__(directory, boundaries, trace) + self.vision = self.config['vision_config'] + + def dense(self, x, name): + y = x @ self.tensor(name + '.weight').T + if name + '.bias' in self.checkpoint.map: + y = y + self.tensor(name + '.bias') + return self.cast(y) + + def norm(self, x, name): + return self.rms(x, self.tensor(name + '.weight'), self.vision['norm_eps']) + + def conv(self, x, prefix, height, width): + left, gate, value = np.split(self.dense(x, prefix + 'conv.in_proj'), 3, axis=-1) + channels = self.vision['hidden_size'] + grid = self.cast(left * value).reshape(height // 2, width // 2, 2, 2, channels) + grid = grid.transpose(0, 2, 1, 3, 4).reshape(height, width, channels) + padded = np.pad(grid, ((1, 1), (1, 1), (0, 0))) + kernel = self.tensor(prefix + 'conv.spatial_conv.weight')[:, :, :, 0] + result = np.zeros_like(grid) + for row in range(3): + for col in range(3): + result = result + padded[row:row + height, col:col + width] * kernel[:, row, col] + merged = self.cast(result).reshape(height // 2, 2, width // 2, 2, channels) + merged = merged.transpose(0, 2, 1, 3, 4).reshape(height * width, channels) + return self.dense(self.cast(merged * gate), prefix + 'conv.out_proj') + + def rotary(self, x, coordinates): + spatial = x.shape[-1] // 2 + powers = -np.arange(0, spatial, 2, dtype=np.float32) / np.float32(spatial) + frequency = np.power(np.float32(self.vision['rope_parameters']['rope_theta']), powers) + vertical = coordinates[:, 0, None].astype(np.float32) * frequency + horizontal = coordinates[:, 1, None].astype(np.float32) * frequency + angle = np.concatenate((vertical, horizontal, vertical, horizontal), axis=-1)[:, None] + first, second = np.split(x, 2, axis=-1) + return self.cast(x * np.cos(angle) + np.concatenate((-second, first), axis=-1) * np.sin(angle)) + + def attention(self, x, prefix, coordinates): + heads, kv = (self.vision[k] for k in ('num_attention_heads', 'num_key_value_heads')) + dim = self.vision['hidden_size'] // heads + q = self.dense(x, prefix + 'self_attn.q_proj').reshape(len(x), heads, dim) + k = self.dense(x, prefix + 'self_attn.k_proj').reshape(len(x), kv, dim) + v = self.dense(x, prefix + 'self_attn.v_proj').reshape(len(x), kv, dim).transpose(1, 0, 2) + q = self.rotary(self.norm(q, prefix + 'self_attn.q_layernorm'), coordinates).transpose(1, 0, 2) + k = self.rotary(self.norm(k, prefix + 'self_attn.k_layernorm'), coordinates).transpose(1, 0, 2) + k, v = np.repeat(k, heads // kv, axis=0), np.repeat(v, heads // kv, axis=0) + value = self.attention_value(q, k, v, causal=False).transpose(1, 0, 2).reshape(len(x), heads * dim) + return self.dense(value, prefix + 'self_attn.out_proj') + + def run(self, patches, height, width): + if height <= 0 or width <= 0 or height % 2 or width % 2 or patches.shape != (height * width, 1536): + raise ValueError('Expected an even patch grid and1536 elements per patch') + coordinates = np.asarray([(h * 2 + a, w * 2 + b) for h in range(height // 2) + for w in range(width // 2) for a in range(2) for b in range(2)]) + hidden = self.norm(self.dense(self.cast(patches), 'vision_tower.patchifier'), 'vision_tower.embedding_norm') + for index, kind in enumerate(self.vision['layer_types']): + prefix = f'vision_tower.layers.{index}.' + x = self.norm(hidden, prefix + 'operator_norm') + result = self.conv(x, prefix, height, width) if kind == 'conv' else self.attention(x, prefix, coordinates) + hidden = self.cast(hidden + result) + x = self.norm(hidden, prefix + 'ffn_norm') + gate, up = self.dense(x, prefix + 'feed_forward.w1'), self.dense(x, prefix + 'feed_forward.w3') + hidden = self.cast(hidden + self.dense(self.swiglu(gate, up), prefix + 'feed_forward.w2')) + self.take(f'vision_layer_{index:02d}', hidden) + print('vision_layer', index, flush=True) + prefix = 'multi_modal_projector.' + centered = hidden - hidden.mean(axis=-1, keepdims=True, dtype=np.float32) + inverse = np.float32(1) / np.sqrt(np.mean(centered * centered, axis=-1, keepdims=True, dtype=np.float32) + + np.float32(self.config['projector_layer_norm_eps'])) + hidden = self.cast(centered * inverse * self.tensor(prefix + 'layer_norm.weight') + + self.tensor(prefix + 'layer_norm.bias')) + first = self.dense(hidden.reshape(-1, self.vision['hidden_size'] * self.config['downsample_factor'] ** 2), + prefix + 'linear_1') + argument = first / np.sqrt(np.float32(2)) + erf = np.fromiter((math.erf(float(v)) for v in argument.flat), dtype=np.float32).reshape(first.shape) + return self.dense(self.cast(np.float32(0.5) * first * (np.float32(1) + erf)), prefix + 'linear_2') + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--model-dir', type=Path, required=True) + p.add_argument('--tokens', type=Path, required=True) + p.add_argument('--output', type=Path, required=True) + p.add_argument('--boundaries', choices=('fp32', 'fp32-bf16'), required=True) + p.add_argument('--patches', type=Path) + p.add_argument('--grid-h', type=int) + p.add_argument('--grid-w', type=int) + p.add_argument('--image-rows', type=Path) + p.add_argument('--image-start', type=int) + args = p.parse_args() + tokens = np.load(args.tokens, allow_pickle=False) + model = Graph(args.model_dir, args.boundaries) + if tokens.ndim != 1 or not np.issubdtype(tokens.dtype, np.integer) or len(tokens) == 0: + p.error('Expected nonempty integer tokens') + if tokens.min() < 0 or tokens.max() >= model.text['vocab_size']: + p.error('Token IDs exceed vocabulary') + if args.patches and args.image_rows: + p.error('Choose own vision or fixed projected rows') + rows = np.load(args.image_rows, allow_pickle=False) if args.image_rows else None + if args.patches: + rows = Vision(args.model_dir, args.boundaries).run(np.load(args.patches, allow_pickle=False), + args.grid_h, args.grid_w) + np.save(args.output.with_suffix('.image.npy'), rows) + logits = model.run(tokens, rows, args.image_start) + if not np.isfinite(logits).all(): + raise ValueError('Nonfinite FP32 reference logits') + np.save(args.output, logits) + rounded = bf(logits).astype(np.float32) + args.output.with_suffix('.bf16').write_bytes((rounded.view(np.uint32) >> 16).astype('> np.uint64(48)) & np.uint64(0x8000)).astype(np.uint16) + exponent = ((bits >> np.uint64(52)) & np.uint64(0x7ff)).astype(np.int32) + fraction = bits & np.uint64((1 << 52) - 1) + significant = fraction | np.uint64(1 << 52) + high = significant >> np.uint64(45) + tail = significant & np.uint64((1 << 45) - 1) + increment = (tail > np.uint64(1 << 44)) | ((tail == np.uint64(1 << 44)) & ((high & 1) != 0)) + rounded = high + increment.astype(np.uint64) + encoded = ((exponent.astype(np.int64) - 897) * 128 + rounded.astype(np.int64)) + encoded = np.clip(encoded, 0, 0x7f80).astype(np.uint16) + small = np.abs(values) < np.float64(2.0 ** -126) + with np.errstate(over='ignore', invalid='ignore'): + subnormal = np.rint(np.where(small, np.abs(values), 0) * np.float64(2.0 ** 133)).astype(np.uint16) + encoded = np.where(small, subnormal, encoded).astype(np.uint16) + encoded = np.where(exponent == 0x7ff, np.where(fraction != 0, 0x7fc0, 0x7f80), encoded).astype(np.uint16) + return ((encoded | sign).astype(np.uint32) << 16).view(np.float32).astype(np.float64) + + +def inject_image_rows(hidden, tokens, rows, start): + rows = np.asarray(rows, dtype=np.float64) + if start is None or start < 0 or rows.ndim != 2 or not len(rows): + raise ValueError('Expected nonempty projected image rows and nonnegative image start') + if rows.shape[1] != hidden.shape[1] or start + len(rows) > len(hidden): + raise ValueError('Projected image rows exceed decoder geometry') + expected = np.zeros(len(tokens), dtype=bool) + expected[start:start + len(rows)] = True + if not np.array_equal(np.asarray(tokens) == 125030, expected): + raise ValueError('Image placeholder rows do not match projected image span') + if not np.isfinite(rows).all(): + raise ValueError('Projected image rows must be finite') + result = hidden.copy() + result[start:start + len(rows)] = bf(rows) + return result + + +class Oracle: + def __init__(self, directory, weight_mode='exact', dump=None, projection_rounding='fp64', + norm_conv_rounding='fp64', attention_rounding='fp64', elementary_backend='numpy'): + self.directory, self.weight_mode, self.dump = directory, weight_mode, dump + if attention_rounding not in ('fp64', 'metal-fp32') or elementary_backend not in ('numpy', 'metal'): + raise ValueError('Unsupported attention or elementary rounding mode') + self.attention_rounding, self.elementary_backend = attention_rounding, elementary_backend + self.projection_rounding = projection_rounding + if norm_conv_rounding not in ('fp64', 'metal'): + raise ValueError('Unsupported normalization/convolution rounding mode') + self.norm_conv_rounding = norm_conv_rounding + valid_rounding = projection_rounding in ('fp64', 'metal-row') + if not valid_rounding or (projection_rounding == 'metal-row' and weight_mode != 'exact'): + raise ValueError('Metal row replay requires exact affine weights') + directory = directory.resolve() + self.config = json.loads((directory / 'config.json').read_text()) + self.text = self.config.get('text_config', self.config) + if self.text.get('model_type') != 'lfm3' or not self.text.get('tie_word_embeddings'): + raise ValueError('Expected an LFM3 decoder with a tied vocabulary embedding') + index = json.loads((directory / 'model.safetensors.index.json').read_text()) + self.map, self.files = index['weight_map'], {} + for name in set(self.map.values()): + path = (directory / name).resolve() + if not path.is_relative_to(directory): + raise ValueError('Checkpoint shard escapes the model directory') + with path.open('rb') as handle: + length = struct.unpack('> shifts.astype(np.uint64) + cross = shifts + bits > 32 + values[:, cross] |= padded[:, indices[cross] + 1] << (32 - shifts[cross]).astype(np.uint64) + codes = (values & ((1 << bits) - 1)).astype(np.float64) + scale = self.tensor(name + '.scales', rows).repeat(group, axis=-1) + bias = self.tensor(name + '.biases', rows).repeat(group, axis=-1) + weight = scale * codes + bias + return bf(weight) if self.weight_mode == 'bf16' else weight + + def linear(self, x, name): + if self.projection_rounding == 'metal-row': + from row_projection_reference import affine_rows + return bf(affine_rows(self, name, x)) + path, base, header = self.files[self.map[name + '.weight']] + count = header[name + '.weight']['shape'][0] + output = np.empty((*x.shape[:-1], count), dtype=np.float64) + for start in range(0, count, 512): + end = min(start + 512, count) + output[..., start:end] = x @ self.weights(name, slice(start, end)).T + return bf(output) + + def norm(self, values, name): + if self.norm_conv_rounding == 'metal': + from text_rounding_reference import norm_rows + return norm_rows(values, self.tensor(name + '.weight'), self.text['norm_eps'], + self.elementary_backend if self.attention_rounding == 'metal-fp32' else None) + inverse = 1 / np.sqrt(np.mean(values * values, axis=-1, keepdims=True) + self.text['norm_eps']) + return bf(bf(values * inverse) * self.tensor(name + '.weight')) + + def take(self, name, value): + if self.dump: + self.dump(name, value) + + def rotary(self, values, kind): + if getattr(self, 'attention_rounding', 'fp64') == 'metal-fp32': + from text_attention_rounding import rotary + return rotary(values, self.text['rope_parameters'][kind], self.elementary_backend) + width = values.shape[-1] + spec = self.text['rope_parameters'][kind] + pairs = int(width * spec.get('partial_rotary_factor', 1)) // 2 + frequencies = np.concatenate((np.power(float(spec['rope_theta']), -np.arange(pairs) * 2 / width), + np.zeros(width // 2 - pairs))) + angle = np.arange(len(values))[:, None] * frequencies + angle = np.concatenate((angle, angle), axis=-1)[:, None, :] + cosine, sine = bf(np.cos(angle)), bf(np.sin(angle)) + a, b = np.split(values, 2, axis=-1) + return bf(values * cosine + np.concatenate((-b, a), axis=-1) * sine) + + def conv(self, x, prefix, tag): + projected = self.linear(x, prefix + 'conv.in_proj') + self.take(tag + '_conv_in', projected) + b, gate, value = np.split(projected, 3, axis=-1) + gated = bf(b * value) + kernel = self.tensor(prefix + 'conv.conv.weight')[:, :, 0] + taps = kernel.shape[1] + padded = np.pad(gated, ((taps - 1, 0), (0, 0))) + total = np.zeros_like(gated) + for tap in range(taps): + total = total + padded[tap:tap + len(x)] * kernel[:, tap] + if self.norm_conv_rounding == 'metal': + total = total.astype(np.float32).astype(np.float64) + mixed = bf(bf(total) * gate) + self.take(tag + '_conv_gated', mixed) + return self.linear(mixed, prefix + 'conv.out_proj') + + def attention(self, x, prefix, kind): + heads, kv, width = (self.text[key] for key in ('num_attention_heads', 'num_key_value_heads', 'head_dim')) + q = self.linear(x, prefix + 'self_attn.q_proj').reshape(len(x), heads, width) + k = self.linear(x, prefix + 'self_attn.k_proj').reshape(len(x), kv, width) + v = self.linear(x, prefix + 'self_attn.v_proj').reshape(len(x), kv, width) + q = self.rotary(self.norm(q, prefix + 'self_attn.q_norm'), kind).transpose(1, 0, 2) + k = self.rotary(self.norm(k, prefix + 'self_attn.k_norm'), kind).transpose(1, 0, 2) + v = v.transpose(1, 0, 2) + k, v = np.repeat(k, heads // kv, axis=0), np.repeat(v, heads // kv, axis=0) + if self.attention_rounding == 'metal-fp32': + from text_attention_rounding import attention + output = attention(q, k, v, self.text['sliding_window'] if kind == 'sliding_attention' else 0, + self.elementary_backend).transpose(1, 0, 2) + else: + scores = (q @ k.swapaxes(-1, -2)) / math.sqrt(width) + distance = np.arange(len(x))[:, None] - np.arange(len(x))[None, :] + allowed = distance >= 0 + if kind == 'sliding_attention': + allowed &= distance < self.text['sliding_window'] + scores = np.where(allowed, scores, -np.inf) + probabilities = np.exp(scores - scores.max(axis=-1, keepdims=True)) + probabilities /= probabilities.sum(axis=-1, keepdims=True) + output = bf((probabilities @ v).transpose(1, 0, 2)) + raw_gate = self.linear(x, prefix + 'self_attn.gate_proj') + if self.attention_rounding == 'metal-fp32': + from elementary_reference import elementary + gates = bf(np.float32(1) / (np.float32(1) + + elementary(-raw_gate.astype(np.float32), 'exp', self.elementary_backend))) + else: + gates = bf(1 / (1 + np.exp(-raw_gate))) + return self.linear(bf(output * gates[:, :, None]).reshape(len(x), heads * width), prefix + 'self_attn.o_proj') + + def run(self, tokens, image_rows=None, image_start=None): + hidden = bf(self.weights(self.prefix + 'embed_tokens', tokens)) + if image_rows is not None: + hidden = inject_image_rows(hidden, tokens, image_rows, image_start) + elif image_start is not None: + raise ValueError('Image start requires projected image rows') + self.take('embedding', hidden) + for index, kind in enumerate(self.text['layer_types']): + tag = f'layer_{index:02d}' + prefix = self.prefix + f'layers.{index}.' + x = self.norm(hidden, prefix + 'operator_norm') + self.take(tag + '_operator_norm', x) + result = self.conv(x, prefix, tag) if kind == 'conv' else self.attention(x, prefix, kind) + self.take(tag + '_mixer', result) + hidden = bf(hidden + self.norm(result, prefix + 'post_operator_norm')) + self.take(tag + '_post_operator', hidden) + x = self.norm(hidden, prefix + 'ffn_norm') + self.take(tag + '_ffn_norm', x) + gate, up = self.linear(x, prefix + 'feed_forward.w1'), self.linear(x, prefix + 'feed_forward.w3') + self.take(tag + '_gate', gate) + self.take(tag + '_up', up) + if self.attention_rounding == 'metal-fp32': + from elementary_reference import swiglu + act = swiglu(gate, up, self.elementary_backend) + else: + act = bf(bf(gate / (1 + np.exp(-gate))) * up) + self.take(tag + '_act', act) + result = self.linear(act, prefix + 'feed_forward.w2') + self.take(tag + '_down', result) + hidden = bf(hidden + self.norm(result, prefix + 'post_ffn_norm')) + self.take(tag + '_output', hidden) + print(tag, kind, flush=True) + final = self.norm(hidden, self.prefix + 'embedding_norm') + self.take('final_norm', final) + return self.linear(final, self.prefix + 'embed_tokens') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--model-dir', type=Path, required=True) + parser.add_argument('--tokens', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--weight-mode', choices=('exact', 'bf16'), default='exact') + parser.add_argument('--projection-rounding', choices=('fp64', 'metal-row'), default='fp64') + parser.add_argument('--attention-rounding', choices=('fp64', 'metal-fp32'), default='fp64') + parser.add_argument('--elementary-backend', choices=('numpy', 'metal'), default='numpy') + parser.add_argument('--norm-conv-rounding', choices=('fp64', 'metal'), default='fp64') + parser.add_argument('--trace-dir', type=Path) + parser.add_argument('--image-rows', type=Path, help='Projected rows as a two-dimensional numeric NumPy array') + parser.add_argument('--image-start', type=int) + args = parser.parse_args() + def write(path, values): + path.write_bytes((np.asarray(values, dtype=np.float32).view(np.uint32) >> 16).astype('= oracle.text['vocab_size']: + parser.error('Token IDs exceed the decoder vocabulary') + if tokens.size > oracle.text['max_position_embeddings']: + parser.error('Token array exceeds the decoder context') + image_rows = np.load(args.image_rows, allow_pickle=False) if args.image_rows else None + if (args.image_rows is None) != (args.image_start is None): + parser.error('--image-rows and --image-start must be supplied together') + logits = oracle.run(tokens, image_rows, args.image_start) + if not np.isfinite(logits).all(): + raise ValueError('Reference produced nonfinite logits') + write(args.output, logits) + arithmetic = 'FP64 reductions with direct binary64-to-BF16 RNE activation boundaries' + if args.projection_rounding == 'metal-row': + arithmetic += '; affine projections instead replay the Metal FP32 lane rounding, other operations remain FP64' + if args.norm_conv_rounding == 'metal': + arithmetic += '; normalization and convolution replay the production FP32 reduction sites' + if args.attention_rounding == 'metal-fp32': + arithmetic += ('; bounded decoder attention/rotary/activations replay FP32 sites with explicit ' + + args.elementary_backend + ' elementary functions') + metadata = {'dtype': 'BF16', 'arithmetic': arithmetic, 'weight_mode': args.weight_mode, + 'attention_rounding': args.attention_rounding, 'elementary_backend': args.elementary_backend, + 'projection_rounding': args.projection_rounding, 'norm_conv_rounding': args.norm_conv_rounding, + 'shape': list(logits.shape), + 'accepted_threshold': None} + args.output.with_suffix('.json').write_text(json.dumps(metadata, indent=2) + '\n') + +if __name__ == '__main__': + main() diff --git a/tools/lfm3/full_model_envelope.py b/tools/lfm3/full_model_envelope.py new file mode 100644 index 0000000000..8af4b6902c --- /dev/null +++ b/tools/lfm3/full_model_envelope.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Validate propagated full-logit reference identities, then apply the unchanged canonical scorer.""" +import argparse, json, subprocess, sys +from pathlib import Path + +p = argparse.ArgumentParser() +p.add_argument('--fp32', type=Path, required=True) +p.add_argument('--reference-bf16', type=Path, required=True) +p.add_argument('--native', type=Path, required=True) +p.add_argument('--output', type=Path, required=True) +a = p.parse_args() +f = json.loads(a.fp32.with_suffix('.json').read_text()) +b = json.loads(a.reference_bf16.with_suffix('.json').read_text()) +for k in [ + 'operator', + 'selected_token_positions', + 'prompt_len', + 'teacher_forced_continuation', + 'reference_chunk', + 'image_start', + 'shape', +]: + if f[k] != b[k]: + raise ValueError(('Full-model reference mismatch', k)) +if f['operator'] != 'full_model_logits' or not b['official_sources_sha256']: + raise ValueError('Need actual official propagated full-model reference') +subprocess.run( + [ + sys.executable, + str(Path(__file__).with_name('fp32_bf16_envelope.py')), + '--fp32', + str(a.fp32), + '--reference-bf16', + str(a.reference_bf16), + '--native', + str(a.native), + '--output', + str(a.output), + '--space', + 'logits', + ], + check=True, +) +d = json.loads(a.output.read_text()) +d['selected_token_positions'] = f['selected_token_positions'] +d['prompt_len'] = f['prompt_len'] +d['teacher_forced_continuation'] = f['teacher_forced_continuation'] +d['scope'] = ( + 'Propagated decoder selected full-vocabulary logits; no standalone operator substitution. Aggregate whole-cell RMS/max and fixed-F-top20 mean/max gate; row/column and pick differences explicit.' +) +d['row_max_exceedances'] = [r for r in d['all_row_errors'] if r['native_max'] > r['reference_max']] +d['row_rms_exceedances'] = [r for r in d['all_row_errors'] if r['native_rms'] > r['reference_rms']] +a.output.write_text(json.dumps(d, indent=2) + '\n') +if not d['proposed_metrics_no_worse']: + raise SystemExit('FullModelMetricEnvelopeFailure') diff --git a/tools/lfm3/full_model_reference.py b/tools/lfm3/full_model_reference.py new file mode 100644 index 0000000000..5c8f4c35bc --- /dev/null +++ b/tools/lfm3/full_model_reference.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +"""Independent FP32 decoder or actual installed official MLX BF16 decoder, propagated inputs.""" +import argparse, hashlib, importlib, importlib.metadata, inspect, json, math, os, struct +from pathlib import Path +import numpy as np +from fp32_graph_reference import Graph +from fp64_reference import Oracle + + +def sha(path): + h = hashlib.sha256() + with Path(path).open('rb') as f: + for block in iter(lambda: f.read(8 << 20), b''): + h.update(block) + return h.hexdigest() + + +class Checkpoint(Oracle): + def __init__(self, directory): + directory = Path(directory).resolve() + self.directory = directory + self.config = json.loads((directory / 'config.json').read_text()) + self.text = self.config.get('text_config', self.config) + if self.text.get('model_type') != 'lfm3' or not self.text.get('tie_word_embeddings'): + raise ValueError('Expected tied LFM3') + index = directory / 'model.safetensors.index.json' + self.map = {} + self.files = {} + declared = json.loads(index.read_text())['weight_map'] if index.exists() else None + names = ( + sorted(set(declared.values())) + if declared is not None + else sorted(p.name for p in directory.glob('*.safetensors')) + ) + if not names: + raise ValueError('No checkpoint shards') + for name in names: + p = (directory / name).resolve() + if not p.is_relative_to(directory): + raise ValueError('Escaping checkpoint path') + with p.open('rb') as f: + length = struct.unpack(' 64 << 20: + raise ValueError('Unbounded header') + header = json.loads(f.read(length)) + self.files[name] = (p, 8 + length, header) + for key in header: + if key == '__metadata__': + continue + if key in self.map: + raise ValueError('Duplicate tensor') + self.map[key] = name + if declared is not None and self.map != declared: + raise ValueError('Index/header mismatch') + prefixes = [p for p in ['model.', 'language_model.model.'] if p + 'embed_tokens.weight' in self.map] + if len(prefixes) != 1: + raise ValueError('Ambiguous decoder namespace') + self.prefix = prefixes[0] + + def pins(self): + paths = [self.directory / 'config.json'] + [p for p, _, _ in self.files.values()] + index = self.directory / 'model.safetensors.index.json' + if index.exists(): + paths.append(index) + return {str(p): sha(p) for p in paths} + + +class FP32(Graph): + def __init__(self, directory, query_tile=128): + self.checkpoint = Checkpoint(directory) + self.config = self.checkpoint.config + self.text = self.checkpoint.text + self.prefix = self.checkpoint.prefix + self.boundaries = 'fp32' + self.trace = None + self.query_tile = query_tile + + def attention_value(self, q, k, v, window=0, causal=True): + # Query tiling bounds the score allocation. Each query still uses the full original K reduction. + out = np.empty_like(q) + key_positions = np.arange(k.shape[1])[None, :] + for start in range(0, q.shape[1], self.query_tile): + stop = min(start + self.query_tile, q.shape[1]) + scores = (q[:, start:stop] @ k.swapaxes(-1, -2)) / np.float32(math.sqrt(q.shape[-1])) + if causal: + distance = np.arange(start, stop)[:, None] - key_positions + allowed = distance >= 0 + if window: + allowed &= distance < window + scores = np.where(allowed, scores, np.float32(-np.inf)) + p = np.exp(scores - scores.max(axis=-1, keepdims=True)) + p = p / p.sum(axis=-1, keepdims=True, dtype=np.float32) + out[:, start:stop] = p @ v + return out + + def selected(self, tokens, indices, image_rows=None, image_start=None): + hidden = self.weights(self.prefix + 'embed_tokens', tokens) + if image_rows is not None: + inject_check(tokens, image_rows, image_start, hidden.shape[-1]) + hidden[image_start : image_start + len(image_rows)] = image_rows + elif image_start is not None: + raise ValueError('Image start without rows') + for index, kind in enumerate(self.text['layer_types']): + prefix = self.prefix + f'layers.{index}.' + x = self.norm(hidden, prefix + 'operator_norm') + result = self.conv(x, prefix) if kind == 'conv' else self.attention(x, prefix, kind) + hidden = hidden + ( + self.norm(result, prefix + 'post_operator_norm') + if self.text.get('use_sandwich_norm', True) + else result + ) + x = self.norm(hidden, prefix + 'ffn_norm') + gate = self.linear(x, prefix + 'feed_forward.w1') + up = self.linear(x, prefix + 'feed_forward.w3') + result = self.linear(self.swiglu(gate, up), prefix + 'feed_forward.w2') + hidden = hidden + ( + self.norm(result, prefix + 'post_ffn_norm') + if self.text.get('use_sandwich_norm', True) + else result + ) + if not np.isfinite(hidden).all(): + raise ValueError('Nonfinite FP32 hidden') + print('FP32 layer', index, flush=True) + return self.linear( + self.norm(hidden[indices], self.prefix + 'embedding_norm'), self.prefix + 'embed_tokens' + ) + + +def inject_check(tokens, rows, start, width): + if ( + start is None + or start < 0 + or rows.ndim != 2 + or rows.shape[1] != width + or len(rows) == 0 + or start + len(rows) > len(tokens) + ): + raise ValueError('Invalid image rows') + expected = np.zeros(len(tokens), bool) + expected[start : start + len(rows)] = True + if not np.array_equal(tokens == 125030, expected) or not np.isfinite(rows).all(): + raise ValueError('Invalid image placeholder span') + + +def official(a, tokens, indices, image_rows): + import mlx.core as mx + import mlx_vlm.utils as utils + + if importlib.metadata.version('mlx') != a.expect_mlx: + raise ValueError('Wrong MLX version') + if a.official_pins: + expected = json.loads(a.official_pins.read_text()) + site = Path(utils.__file__).resolve().parent.parent + for rel, h in expected.items(): + path = (site / rel).resolve() + if not path.is_relative_to(site) or sha(path) != h: + raise ValueError(('Official source pin mismatch', rel)) + model = utils.load_model(a.model_dir, lazy=False) + lm = model.language_model + source_paths = { + str(Path(inspect.getfile(type(x))).resolve()) + for x in [model, lm, lm.model, lm.model.layers[0], lm.model.embed_tokens] + } + source_paths.add(str(Path(utils.__file__).resolve())) + for layer in lm.model.layers: + source_paths.add( + str( + Path( + inspect.getfile(type(layer.self_attn if layer.is_attention_layer else layer.conv)) + ).resolve() + ) + ) + modules = ['mlx_vlm.models.base', 'mlx_vlm.models.cache'] + for name in modules: + source_paths.add(str(Path(importlib.import_module(name).__file__).resolve())) + sources = {p: sha(p) for p in sorted(source_paths)} + cache = lm.make_cache() + answers = {} + ends = ( + list(range(a.chunk, a.prompt_len, a.chunk)) + + [a.prompt_len] + + list(range(a.prompt_len + 1, len(tokens) + 1)) + ) + start = 0 + for end in ends: + ids = mx.array(tokens[None, start:end], dtype=mx.int32) + emb = None + if image_rows is not None: + emb = lm.model.embed_tokens(ids) + lo = max(start, a.image_start) + hi = min(end, a.image_start + len(image_rows)) + if lo < hi: + emb[:, lo - start : hi - start] = mx.array( + image_rows[lo - a.image_start : hi - a.image_start] + ).astype(mx.bfloat16)[None] + output = lm(ids, cache=cache, inputs_embeds=emb) + pick = [int(i) for i in indices if start <= i < end] + if pick: + y = output.logits[:, [i - start for i in pick]] + mx.eval(y) + if y.dtype != mx.bfloat16: + raise ValueError(('Official logits not BF16', y.dtype)) + values = np.asarray(y.astype(mx.float32))[0] + for i, row in zip(pick, values): + answers[i] = row + mx.eval([c.state for c in cache]) + start = end + return np.stack([answers[int(i)] for i in indices]), sources + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--backend', choices=['fp32', 'mlx'], required=True) + p.add_argument('--model-dir', type=Path, required=True) + p.add_argument('--tokens', type=Path, required=True) + p.add_argument('--output', type=Path, required=True) + p.add_argument('--prompt-len', type=int, required=True) + p.add_argument( + '--rows', + help='Comma separated absolute token positions; default last prompt plus every teacher-forced continuation position', + ) + p.add_argument('--chunk', type=int, default=2048) + p.add_argument('--query-tile', type=int, default=128) + p.add_argument('--expect-mlx', default='0.32.4') + p.add_argument('--official-pins', type=Path) + p.add_argument('--image-rows', type=Path) + p.add_argument('--image-start', type=int) + a = p.parse_args() + tokens = np.load(a.tokens, allow_pickle=False) + if ( + tokens.ndim != 1 + or not np.issubdtype(tokens.dtype, np.integer) + or not 0 < a.prompt_len <= len(tokens) + or a.chunk <= 0 + or a.query_tile <= 0 + ): + raise ValueError('Invalid sequence geometry') + indices = np.array( + [int(i) for i in a.rows.split(',')] if a.rows else list(range(a.prompt_len - 1, len(tokens))), + dtype=np.int64, + ) + if ( + len(indices) == 0 + or len(np.unique(indices)) != len(indices) + or indices.min() < 0 + or indices.max() >= len(tokens) + ): + raise ValueError('Invalid selected rows') + ck = Checkpoint(a.model_dir) + if tokens.min() < 0 or tokens.max() >= ck.text['vocab_size']: + raise ValueError('Invalid token ID') + rows = np.load(a.image_rows, allow_pickle=False).astype(np.float32) if a.image_rows else None + if rows is not None: + inject_check(tokens, rows, a.image_start, ck.text['hidden_size']) + if np.any(rows.view(np.uint32) & 65535): + raise ValueError('Fixed image rows must be identical BF16 values in both references') + elif a.image_start is not None: + raise ValueError('Image start requires rows') + pins = ck.pins() + pins[str(a.tokens)] = sha(a.tokens) + if a.image_rows: + pins[str(a.image_rows)] = sha(a.image_rows) + y, sources = ( + (FP32(a.model_dir, a.query_tile).selected(tokens, indices, rows, a.image_start), {}) + if a.backend == 'fp32' + else official(a, tokens, indices, rows) + ) + if y.dtype != np.float32 or not np.isfinite(y).all(): + raise ValueError('Invalid captured logits') + a.output.parent.mkdir(parents=True, exist_ok=True) + np.save(a.output, y) + meta = { + 'operator': 'full_model_logits', + 'backend': a.backend, + 'rows': len(indices), + 'shape': list(y.shape), + 'selected_token_positions': indices.tolist(), + 'prompt_len': a.prompt_len, + 'teacher_forced_continuation': len(tokens) - a.prompt_len, + 'reference_chunk': a.chunk, + 'image_start': a.image_start, + 'inputs_sha256': pins, + 'output_sha256': sha(a.output), + 'source_sha256': sha(__file__), + 'support_sources_sha256': { + str(Path(__file__).parent / n): sha(Path(__file__).parent / n) + for n in ['fp32_graph_reference.py', 'fp64_reference.py'] + }, + 'official_sources_sha256': sources, + 'numpy': np.__version__, + 'thread_environment': { + k: os.environ.get(k) + for k in ['VECLIB_MAXIMUM_THREADS', 'OPENBLAS_NUM_THREADS', 'OMP_NUM_THREADS'] + }, + 'query_tile': a.query_tile, + 'official_pins_sha256': sha(a.official_pins) if a.official_pins else None, + 'mlx_version': importlib.metadata.version('mlx') if a.backend == 'mlx' else None, + 'arithmetic': ( + 'Independent CPU FP32 propagated decoder, no BF16 intermediate rounding' + if a.backend == 'fp32' + else 'Installed official MLX-VLM propagated BF16 decoder; official cache; prompt chunks then one token teacher-forced continuation' + ), + 'acceptance': None, + 'scope': 'Selected full-vocabulary logits after propagated decoder; fixed projected image input only if supplied; no independent vision claim', + } + a.output.with_suffix('.json').write_text(json.dumps(meta, indent=2) + '\n') + + +if __name__ == '__main__': + main() diff --git a/tools/lfm3/inspect_checkpoint.py b/tools/lfm3/inspect_checkpoint.py new file mode 100644 index 0000000000..2fec63457d --- /dev/null +++ b/tools/lfm3/inspect_checkpoint.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +"""Inspect pinned Hugging Face configs and safetensors headers without downloading tensor data.""" + +import argparse +import collections +import json +import math +import os +import re +import struct +import sys +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path, PurePosixPath + +MAX_JSON = 16 * 1024 * 1024 +MAX_HEADER = 16 * 1024 * 1024 + + +class InspectionError(Exception): + pass + + +def safe_name(name): + path = PurePosixPath(name) + if not name or path.is_absolute() or '..' in path.parts or '\\' in name: + raise InspectionError(f'Unsafe checkpoint file name: {name!r}') + return name + + +def read_json_bytes(data, description): + try: + return json.loads(data) + except (ValueError, UnicodeError) as exc: + raise InspectionError(f'Invalid JSON in {description}: {exc}') from exc + + +class Remote: + def __init__(self, repository, revision): + if not re.fullmatch(r'[\w.-]+/[\w.-]+', repository): + raise InspectionError('Repository must have owner/name form') + api = 'https://huggingface.co/api/models/' + repository + '/revision/' + metadata = self.json_url(api + urllib.parse.quote(revision, safe='')) + self.revision = metadata.get('sha', '') + if not re.fullmatch(r'[0-9a-f]{40}', self.revision): + raise InspectionError('Hugging Face did not resolve a full commit revision') + self.repository = repository + self.files = {entry['rfilename'] for entry in metadata.get('siblings', [])} + self.base = f'https://huggingface.co/{repository}/resolve/{self.revision}/' + + @staticmethod + def fetch(url, limit, byte_range=None, with_total=False): + headers = {'User-Agent': 'TensorFold-checkpoint-inspector', 'Accept-Encoding': 'identity'} + if byte_range is not None: + start, end = byte_range + headers['Range'] = f'bytes={start}-{end}' + try: + with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=60) as response: + if byte_range is not None: + match = re.fullmatch(r'bytes (\d+)-(\d+)/(\d+)', response.headers.get('Content-Range', '')) + if response.status != 206 or not match: + raise InspectionError('Server refused an exact HTTP byte range; refusing tensor download') + first, last, total = map(int, match.groups()) + if (first, last) != byte_range or total <= last: + raise InspectionError('Server returned an unexpected byte range') + if response.headers.get('Content-Encoding', 'identity') != 'identity': + raise InspectionError('Compressed range response is not supported') + length = response.headers.get('Content-Length') + if length is not None and int(length) > limit: + raise InspectionError(f'Response exceeds {limit}-byte body limit') + data = response.read(limit + 1) + if len(data) > limit: + raise InspectionError(f'Response exceeds {limit}-byte body limit') + if byte_range is not None and len(data) != byte_range[1] - byte_range[0] + 1: + raise InspectionError('Truncated byte-range response') + return (data, total) if with_total else data + except (urllib.error.URLError, TimeoutError, OSError, ValueError) as exc: + raise InspectionError(f'Cannot fetch {url}: {exc}') from exc + + @classmethod + def json_url(cls, url): + return read_json_bytes(cls.fetch(url, MAX_JSON), url) + + def json(self, name): + return self.json_url(self.base + urllib.parse.quote(safe_name(name), safe='/')) + + def header(self, name): + url = self.base + urllib.parse.quote(safe_name(name), safe='/') + prefix, total = self.fetch(url, 8, (0, 7), with_total=True) + size = header_size(prefix) + if total < 8 + size: + raise InspectionError('Safetensors header extends beyond shard size') + data, second_total = self.fetch(url, size, (8, 7 + size), with_total=True) + if total != second_total: + raise InspectionError('Shard size changed between header requests') + return read_json_bytes(data, name), total - 8 - size + + +class Local: + def __init__(self, directory): + self.directory = Path(directory).resolve() + self.repository = str(self.directory) + self.revision = None + self.files = {str(path.relative_to(self.directory)) for path in self.directory.rglob('*.safetensors')} + if (self.directory / 'model.safetensors.index.json').is_file(): + self.files.add('model.safetensors.index.json') + + def path(self, name): + path = (self.directory / safe_name(name)).resolve() + if not path.is_relative_to(self.directory): + raise InspectionError('Checkpoint file escapes local directory') + return path + + def json(self, name): + with self.path(name).open('rb') as stream: + data = stream.read(MAX_JSON + 1) + if len(data) > MAX_JSON: + raise InspectionError('Local JSON exceeds body limit') + return read_json_bytes(data, name) + + def header(self, name): + with self.path(name).open('rb') as stream: + size = header_size(stream.read(8)) + data = stream.read(size) + total = os.fstat(stream.fileno()).st_size + if len(data) != size: + raise InspectionError('Truncated local safetensors header') + return read_json_bytes(data, name), total - 8 - size + + +def header_size(prefix): + if len(prefix) != 8: + raise InspectionError('Truncated safetensors size prefix') + size = struct.unpack(' mc)), + 'columns_native_better': int(np.count_nonzero(nc < mc)), + 'worst_column_increases': [{'column': int(i), 'native_rms': float(nc[i]), + 'mlx_rms': float(mc[i]), 'increase': float(increase[i])} for i in worst], + }) + np.savez_compressed(output / (case['name'] + '-' + label + '-columns.npz'), native=nc, mlx=mc) + return { + 'name': case['name'], 'model': case['model_directory'], 'module': case['module'], + 'shape': [len(x), case['k'], case['n']], 'quantization': list(model.quant(case['module'])), + 'input_sha256': hashlib.sha256((inputs / case['input_file']).read_bytes()).hexdigest(), + 'native_prompt_sha256': hashlib.sha256((candidate / (case['name'] + '.bf16')).read_bytes()).hexdigest(), + 'native_decode_sha256': hashlib.sha256((candidate / (case['name'] + '-one.bf16')).read_bytes()).hexdigest(), + 'native_prompt_decode_equal': bool(np.array_equal(native[:1], one)), + 'mlx_prompt_decode_equal': bool(np.array_equal(mlx[:1], mlx_one)), 'comparisons': records, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--models', type=Path, required=True) + parser.add_argument('--inputs', type=Path, required=True) + parser.add_argument('--candidate', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + import mlx.core as mx + args.output.mkdir(parents=True, exist_ok=True) + cases = json.loads((args.inputs / 'manifest.json').read_text()) + records, current = [], None + for case in cases: + if current != case['model_directory']: + if current is not None: + del reference, model + mx.clear_cache() + current = case['model_directory'] + model = Oracle(args.models / current) + reference = Reference(args.models / current, mx) + record = compare(case, model, reference, args.inputs, args.candidate, args.output, mx) + records.append(record) + (args.output / 'results.json').write_text(json.dumps(records, indent=2) + '\n') + print(json.dumps(record), flush=True) + summary = { + 'cells': len(records), 'projection_outputs': sum(c['shape'][0] * c['shape'][2] for c in records), + 'native_prompt_decode_equal_cells': sum(c['native_prompt_decode_equal'] for c in records), + 'mlx_prompt_decode_equal_cells': sum(c['mlx_prompt_decode_equal'] for c in records), + 'acceptance_threshold': None, + } + (args.output / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n') + print(json.dumps(summary), flush=True) + + +if __name__ == '__main__': + main() diff --git a/tools/lfm3/prompt_row_sweep.py b/tools/lfm3/prompt_row_sweep.py new file mode 100755 index 0000000000..269bdb4238 --- /dev/null +++ b/tools/lfm3/prompt_row_sweep.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Per-prompt-length boundary-row error versus independent FP32 for several native routes and official BF16. + +A single prompt-boundary row is a very noisy statistic: the official BF16 row can land well below its own +long-run error by chance. This tool scores many boundary rows of one causal sequence instead. For prompt length +N the native capture for `tokens[:N]` with `prompt:N` has one row (position N-1); FP32 and official BF16 are +produced once for the whole sequence with `full_model_reference.py --rows ...` and indexed by absolute position. +It reports facts only and applies no pass/fail rule. +""" +import argparse +import hashlib +import json +from pathlib import Path + +import numpy as np + +VOCAB = 128000 + + +def sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def bf16(path): + raw = np.fromfile(path, dtype=' r['official_bf16']['rms'] for r in rows) for name in routes + } + out = { + 'scope': 'Per-row boundary error versus independent FP32; descriptive statistics, no pass/fail rule', + 'rows': len(rows), + 'summary': summary, + 'per_row': rows, + 'files_sha256': hashes, + } + a.output.write_text(json.dumps(out, indent=1) + '\n') + print(json.dumps(summary, indent=1)) + + +if __name__ == '__main__': + main() diff --git a/tools/lfm3/reference.py b/tools/lfm3/reference.py new file mode 100644 index 0000000000..f536b6a98a --- /dev/null +++ b/tools/lfm3/reference.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +"""Developer-only MLX teacher-forced LFM3 text reference, independent of native kernels.""" + +import argparse +import json +import math +from pathlib import Path + + +def canonical(name): + if name.endswith('.weight'): + name = name[:-7] + for prefix in ('model.language_model.', 'language_model.', 'text_model.', 'model.'): + if name.startswith(prefix): + name = name[len(prefix):] + return name + + +def quantization(config, name): + spec = config.get('quantization', config.get('quantization_config')) + if not isinstance(spec, dict): + raise ValueError('Checkpoint needs affine quantization metadata') + width = spec + for key, value in spec.items(): + if isinstance(value, dict) and canonical(key) == canonical(name): + width = value + break + bits, group = width['bits'], width['group_size'] + if bits not in (4, 6, 8) or group not in (32, 64) or width.get('mode', 'affine') != 'affine': + raise ValueError(f'Unsupported quantization for {name}') + return bits, group + + +class Reference: + def __init__(self, directory, mx, attention='sdpa'): + self.mx = mx + self.attention_mode = attention + self.config = json.loads((directory / 'config.json').read_text()) + self.text = self.config.get('text_config', self.config) + if self.text.get('model_type') != 'lfm3' or not self.text.get('tie_word_embeddings'): + raise ValueError('Expected LFM3 with a tied output embedding') + self.weights = {} + index = directory / 'model.safetensors.index.json' + files = sorted(set(json.loads(index.read_text())['weight_map'].values())) if index.exists() else [ + path.name for path in directory.glob('*.safetensors') + ] + if not files: + raise ValueError('No safetensors checkpoint found') + for name in files: + path = (directory / name).resolve() + if not path.is_relative_to(directory.resolve()): + raise ValueError('Shard path escapes checkpoint directory') + for key, value in mx.load(str(path)).items(): + if key in self.weights: + raise ValueError(f'Duplicate tensor {key}') + self.weights[key] = value + wrapped = 'language_model.model.embed_tokens.weight' in self.weights + bare = 'model.embed_tokens.weight' in self.weights + if wrapped == bare: + raise ValueError('Checkpoint must use exactly one text namespace') + self.prefix = 'language_model.model.' if wrapped else 'model.' + + def round(self, x): + return x.astype(self.mx.bfloat16) + + def linear(self, x, module): + bits, group = quantization(self.config, module) + return self.round(self.mx.quantized_matmul( + x, self.weights[module + '.weight'], self.weights[module + '.scales'], + self.weights[module + '.biases'], transpose=True, group_size=group, bits=bits, + )) + + def embedding(self, tokens): + mx = self.mx + name = self.prefix + 'embed_tokens' + bits, group = quantization(self.config, name) + ids = mx.array(tokens, dtype=mx.int32) + return self.round(mx.dequantize( + mx.take(self.weights[name + '.weight'], ids, axis=0), + mx.take(self.weights[name + '.scales'], ids, axis=0), + mx.take(self.weights[name + '.biases'], ids, axis=0), group_size=group, bits=bits, + )) + + def norm(self, x, name): + mx = self.mx + values = x.astype(mx.float32) + scale = mx.rsqrt(mx.mean(values * values, axis=-1, keepdims=True) + self.text['norm_eps']) + normalized = self.round(values * scale) + return self.round(normalized * self.weights[name + '.weight']) + + def convolution(self, x, prefix, dump=None): + mx = self.mx + width = self.text['hidden_size'] + projected = self.linear(x, prefix + 'conv.in_proj') + if dump: + dump('conv_in', projected) + b, gate, values = mx.split(projected, 3, axis=-1) + gated = self.round(b * values) + kernel = self.weights[prefix + 'conv.conv.weight'].astype(mx.float32)[:, :, 0] + taps = kernel.shape[1] + padded = mx.pad(gated.astype(mx.float32), ((taps - 1, 0), (0, 0))) + total = mx.zeros((x.shape[0], width), dtype=mx.float32) + for tap in range(taps): + total = total + padded[tap:tap + x.shape[0]] * kernel[:, tap] + result = self.round(self.round(total) * gate) + if dump: + dump('conv_gated', result) + return self.linear(result, prefix + 'conv.out_proj') + + def rotary(self, x, kind): + mx = self.mx + dim = self.text['head_dim'] + spec = self.text['rope_parameters'][kind] + pairs = int(dim * spec.get('partial_rotary_factor', 1.0)) // 2 + frequencies = mx.power(mx.array(spec['rope_theta'], dtype=mx.float32), + -mx.arange(0, pairs * 2, 2, dtype=mx.float32) / dim) + frequencies = mx.concatenate((frequencies, mx.zeros((dim // 2 - pairs,), dtype=mx.float32))) + angles = mx.arange(x.shape[0], dtype=mx.float32)[:, None] * frequencies[None, :] + angles = mx.concatenate((angles, angles), axis=-1) + cosine = self.round(mx.cos(angles)).astype(mx.float32)[:, None, :] + sine = self.round(mx.sin(angles)).astype(mx.float32)[:, None, :] + values = x.astype(mx.float32) + first, second = mx.split(values, 2, axis=-1) + rotated = mx.concatenate((-second, first), axis=-1) + return self.round(values * cosine + rotated * sine) + + def attention(self, x, prefix, kind): + mx = self.mx + length = x.shape[0] + heads, kv, dim = (self.text[key] for key in ('num_attention_heads', 'num_key_value_heads', 'head_dim')) + q = self.linear(x, prefix + 'self_attn.q_proj').reshape(length, heads, dim) + k = self.linear(x, prefix + 'self_attn.k_proj').reshape(length, kv, dim) + v = self.linear(x, prefix + 'self_attn.v_proj').reshape(length, kv, dim) + q = self.rotary(self.norm(q, prefix + 'self_attn.q_norm'), kind).transpose(1, 0, 2)[None] + k = self.rotary(self.norm(k, prefix + 'self_attn.k_norm'), kind).transpose(1, 0, 2)[None] + v = v.transpose(1, 0, 2)[None] + positions = mx.arange(length) + distance = positions[:, None] - positions[None, :] + allowed = distance >= 0 + if kind == 'sliding_attention': + allowed = allowed & (distance < self.text['sliding_window']) + if self.attention_mode == 'sdpa': + output = mx.fast.scaled_dot_product_attention(q, k, v, scale=1 / math.sqrt(dim), mask=allowed) + else: + k = mx.repeat(k, heads // kv, axis=1) + v = mx.repeat(v, heads // kv, axis=1) + scores = self.round(mx.matmul(q, k.swapaxes(-1, -2))) + scores = self.round(scores * (1 / math.sqrt(dim))) + scores = mx.where(allowed, scores.astype(mx.float32), -float('inf')) + probabilities = self.round(mx.softmax(scores, axis=-1)) + output = mx.matmul(probabilities, v) + output = self.round(output[0].transpose(1, 0, 2)) + logits = self.linear(x, prefix + 'self_attn.gate_proj') + gate = self.round(mx.sigmoid(logits.astype(mx.float32))) + output = self.round(output * gate[:, :, None]).reshape(length, heads * dim) + return self.linear(output, prefix + 'self_attn.o_proj') + + def run(self, tokens, dump=None, last_only=False, image_rows=None, image_start=0): + mx = self.mx + hidden = self.embedding(tokens) + if image_rows is not None: + hidden = mx.concatenate((hidden[:image_start], image_rows, hidden[image_start + image_rows.shape[0]:]), axis=0) + if dump: + dump('embedding', hidden) + for index, kind in enumerate(self.text['layer_types']): + prefix = self.prefix + f'layers.{index}.' + x = self.norm(hidden, prefix + 'operator_norm') + if dump: + dump(f'layer_{index:02d}_operator_norm', x) + conv_dump = (lambda suffix, value: dump(f'layer_{index:02d}_{suffix}', value)) if dump else None + result = self.convolution(x, prefix, conv_dump) if kind == 'conv' else self.attention(x, prefix, kind) + if dump: + dump(f'layer_{index:02d}_mixer', result) + hidden = self.round(hidden + self.norm(result, prefix + 'post_operator_norm')) + if dump: + dump(f'layer_{index:02d}_post_operator', hidden) + x = self.norm(hidden, prefix + 'ffn_norm') + if dump: + dump(f'layer_{index:02d}_ffn_norm', x) + gate = self.linear(x, prefix + 'feed_forward.w1') + up = self.linear(x, prefix + 'feed_forward.w3') + if dump: + dump(f'layer_{index:02d}_gate', gate) + dump(f'layer_{index:02d}_up', up) + silu = self.round(gate.astype(mx.float32) * mx.sigmoid(gate.astype(mx.float32))) + act = self.round(silu * up) + if dump: + dump(f'layer_{index:02d}_act', act) + result = self.linear(act, prefix + 'feed_forward.w2') + if dump: + dump(f'layer_{index:02d}_down', result) + hidden = self.round(hidden + self.norm(result, prefix + 'post_ffn_norm')) + mx.eval(hidden) + if dump: + dump(f'layer_{index:02d}', hidden) + dump(f'layer_{index:02d}_output', hidden) + final = self.norm(hidden[-1:] if last_only else hidden, self.prefix + 'embedding_norm') + if dump: + dump('final_norm', final) + return self.linear(final, self.prefix + 'embed_tokens') + + +def write_bf16(path, values, mx, np): + floats = np.asarray(values.astype(mx.float32)) + raw = (floats.view(np.uint32) >> 16).astype('= model.text['vocab_size']: + parser.error('Token IDs exceed the checkpoint vocabulary') + if tokens.size > model.text['max_position_embeddings']: + parser.error('Token sequence exceeds the checkpoint context') + image_rows = None + image_count = 0 + if args.image_rows is not None: + if model.config.get('model_type') != 'lfm3_vl': + parser.error('Projected image rows require an LFM3-VL checkpoint') + width = model.text['hidden_size'] + if args.image_rows.stat().st_size == 0 or args.image_rows.stat().st_size % (width * 2): + parser.error('Expected complete BF16 projected image rows') + raw = np.fromfile(args.image_rows, dtype=' tokens.size or not np.isfinite(values).all(): + parser.error('Image rows are nonfinite or exceed the prompt') + positions = np.flatnonzero(tokens == model.config['image_token_id']) + if not np.array_equal(positions, np.arange(args.image_start, end)): + parser.error('Image placeholders must match the supplied row span exactly') + image_rows = mx.array(values).astype(mx.bfloat16) + elif args.image_start != 0: + parser.error('--image-start requires --image-rows') + intermediates = {} + + def dump(name, values): + write_bf16(args.intermediates_dir / (name + '.bf16'), values, mx, np) + intermediates[name] = {'shape': list(values.shape), 'dtype': 'BF16'} + + if args.intermediates_dir: + args.intermediates_dir.mkdir(parents=True, exist_ok=True) + logits = model.run(tokens, dump if args.intermediates_dir else None, args.last_only, image_rows, args.image_start) + mx.eval(logits) + write_bf16(args.output, logits, mx, np) + metadata = { + 'shape': list(logits.shape), 'dtype': 'BF16', 'byte_order': 'little', + 'attention': args.attention, 'token_count': int(tokens.size), 'intermediates': intermediates, + 'image_start': args.image_start, 'image_rows': image_count, + } + args.output.with_suffix(args.output.suffix + '.json').write_text(json.dumps(metadata, indent=2) + '\n') + print(json.dumps(metadata)) + + +if __name__ == '__main__': + main() diff --git a/tools/lfm3/row_projection_reference.py b/tools/lfm3/row_projection_reference.py new file mode 100644 index 0000000000..b8737a56c4 --- /dev/null +++ b/tools/lfm3/row_projection_reference.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Independent affine row-projection replay with the shared Metal helper's FP32 rounding sites.""" +import argparse +import ctypes +import os +import json +from pathlib import Path +import numpy as np +from fp64_reference import Oracle, bf + + +def affine_rows(oracle, name, inputs): + original = np.asarray(inputs) + if not np.isfinite(original).all() or not np.array_equal(original, bf(original)): + raise ValueError('Expected finite BF16-representable inputs') + inputs = np.asarray(original, dtype=np.float32) + bits, group = oracle.quant(name) + words = oracle.tensor(name + '.weight') + n = words.shape[0] + k = words.shape[1] * 32 // bits + if inputs.ndim != 2 or inputs.shape[1] != k or k % 512: + raise ValueError('Expected row-by-K inputs with a complete 512-value lane pass') + positions = np.arange(k) * bits + indices, shifts = positions // 32, positions % 32 + padded = np.pad(words.astype(np.uint64), ((0, 0), (0, 1))) + codes = padded[:, indices] >> shifts.astype(np.uint64) + cross = shifts + bits > 32 + codes[:, cross] |= padded[:, indices[cross] + 1] << (32 - shifts[cross]).astype(np.uint64) + codes = (codes & ((1 << bits) - 1)).astype(np.uint8) + scales = oracle.tensor(name + '.scales').astype(np.float32) + biases = oracle.tensor(name + '.biases').astype(np.float32) + output = np.empty((len(inputs), n), dtype=np.float32) + library = os.environ.get('LFM3_ROW_REPLAY_LIBRARY') + if library: + function = ctypes.CDLL(library).lfm3_row_replay + float_pointer, byte_pointer = ctypes.POINTER(ctypes.c_float), ctypes.POINTER(ctypes.c_uint8) + function.argtypes = [float_pointer, byte_pointer, float_pointer, float_pointer, float_pointer] + [ctypes.c_int] * 5 + function.restype = ctypes.c_int + inputs = np.ascontiguousarray(inputs) + codes = np.ascontiguousarray(codes) + scales, biases = np.ascontiguousarray(scales), np.ascontiguousarray(biases) + status = function(inputs.ctypes.data_as(float_pointer), codes.ctypes.data_as(byte_pointer), + scales.ctypes.data_as(float_pointer), biases.ctypes.data_as(float_pointer), + output.ctypes.data_as(float_pointer), len(inputs), n, k, group, bits) + if status != 0: + raise MemoryError('CPU projection replay scratch allocation failed') + return output + for first in range(0, len(inputs), 8): + x = inputs[first:first + 8] + for column in range(0, n, 256): + count = min(256, n - column) + lanes = np.zeros((len(x), count, 32), dtype=np.float32) + for at in range(0, k, 512): + xx = x[:, at:at + 512].reshape(len(x), 1, 32, 16) + qq = codes[column:column + count, at:at + 512].reshape(1, count, 32, 16) + sums = np.zeros((len(x), 1, 32), dtype=np.float32) + for i in range(16): + sums = sums + xx[..., i] + if bits == 4: + place = np.tile(np.array([1, 16, 256, 4096], dtype=np.float32), 4) + xx, qq = xx / place, qq * place + dot = np.zeros_like(lanes) + for i in range(0, 16, 4): + term = xx[..., i] * qq[..., i] + for j in range(1, 4): + term = term + xx[..., i + j] * qq[..., i + j] + dot = dot + term + groups = (np.arange(32) * 16 + at) // group + scale = scales[column:column + count, groups][None] + bias = biases[column:column + count, groups][None] + bias_product = sums * bias + fused = (scale.astype(np.float64) * dot.astype(np.float64) + bias_product).astype(np.float32) + lanes = lanes + fused + for distance in (1, 2, 4, 8, 16): + lanes = lanes + lanes[..., np.arange(32) ^ distance] + output[first:first + len(x), column:column + count] = lanes[..., 0] + return output + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--model-dir', type=Path, required=True) + parser.add_argument('--projection', required=True) + parser.add_argument('--inputs', type=Path, required=True, help='BF16-representable numeric NumPy rows') + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--native-raw', type=Path, help='Optional captured pre-BF16 FP32 NumPy output') + args = parser.parse_args() + result = affine_rows(Oracle(args.model_dir), args.projection, np.load(args.inputs, allow_pickle=False)) + np.save(args.output, result) + receipt = {'shape': list(result.shape), 'projection': args.projection, + 'scope': 'Projection rounding replay; not a whole-model fidelity pass', + 'affine_fusion': ('Explicit CPU fmaf' if os.environ.get('LFM3_ROW_REPLAY_LIBRARY') else + 'FP64 multiply-add then FP32 round; not arbitrary IEEE fmaf proof'), + 'accepted_threshold': None} + if args.native_raw: + native = np.load(args.native_raw, allow_pickle=False) + if native.shape != result.shape or native.dtype != np.float32: + raise ValueError('Captured native output must have the same FP32 geometry') + receipt.update(raw_equal=int(np.count_nonzero(native == result)), values=result.size, + bf16_equal=int(np.count_nonzero(bf(native) == bf(result))), + maximum_absolute_error=float(np.max(np.abs(native.astype(np.float64) - result)))) + args.output.with_suffix('.json').write_text(json.dumps(receipt, indent=2) + '\n') + + +def dense_rows(inputs, weight): + inputs, weight = np.asarray(inputs), np.asarray(weight) + if inputs.ndim != 2 or weight.ndim != 2 or inputs.shape[1] != weight.shape[1] or inputs.shape[1] % 32: + raise ValueError('Expected compatible dense row matrices with complete 32-lane passes') + if any(not np.isfinite(x).all() or not np.array_equal(x, bf(x)) for x in (inputs, weight)): + raise ValueError('Dense replay requires finite BF16 operands') + inputs, weight = np.ascontiguousarray(inputs, dtype=np.float32), np.ascontiguousarray(weight, dtype=np.float32) + output = np.empty((len(inputs), len(weight)), dtype=np.float32) + library = os.environ.get('LFM3_ROW_REPLAY_LIBRARY') + if library: + function = ctypes.CDLL(library).lfm3_dense_replay + pointer = ctypes.POINTER(ctypes.c_float) + function.argtypes = [pointer, pointer, pointer] + [ctypes.c_int] * 3 + function.restype = None + function(inputs.ctypes.data_as(pointer), weight.ctypes.data_as(pointer), output.ctypes.data_as(pointer), + len(inputs), len(weight), inputs.shape[1]) + return output + for row in range(0, len(inputs), 8): + x = inputs[row:row + 8] + for column in range(0, len(weight), 256): + w = weight[column:column + 256] + lanes = np.zeros((len(x), len(w), 32), dtype=np.float32) + for at in range(0, inputs.shape[1], 32): + lanes = (x[:, None, at:at + 32].astype(np.float64) + * w[None, :, at:at + 32].astype(np.float64) + lanes).astype(np.float32) + for distance in (1, 2, 4, 8, 16): + lanes = lanes + lanes[..., np.arange(32) ^ distance] + output[row:row + len(x), column:column + len(w)] = lanes[..., 0] + return output + + +if __name__ == '__main__': + main() diff --git a/tools/lfm3/row_projection_replay.c b/tools/lfm3/row_projection_replay.c new file mode 100644 index 0000000000..9eb61c54d0 --- /dev/null +++ b/tools/lfm3/row_projection_replay.c @@ -0,0 +1,77 @@ +// Optional CPU diagnostic accelerator for the independent Metal row-reduction replay, never native inference. +#include +#include +#include + +// Compile with -ffp-contract=off: only the explicitly named affine fmaf is contracted. +int lfm3_row_replay(const float *x, const uint8_t *codes, const float *scales, const float *biases, + float *output, int rows, int n, int k, int group, int bits) { + float *placed = malloc((int64_t)rows * k * sizeof(float)); + float *sums = malloc((int64_t)rows * (k / 16) * sizeof(float)); + if (!placed || !sums) { free(placed); free(sums); return 1; } + for (int row = 0; row < rows; ++row) { + for (int start = 0; start < k; start += 16) { + float sum = 0.0f; + for (int i = 0; i < 16; ++i) { + const int64_t at = (int64_t)row * k + start + i; + sum = sum + x[at]; + const float place = bits == 4 ? (float)(1u << (4 * (i % 4))) : 1.0f; + placed[at] = x[at] / place; + } + sums[(int64_t)row * (k / 16) + start / 16] = sum; + } + } + for (int row = 0; row < rows; ++row) { + for (int column = 0; column < n; ++column) { + float lanes[32] = {0}; + for (int base = 0; base < k; base += 512) { + for (int lane = 0; lane < 32; ++lane) { + const int start = base + lane * 16; + const float *input = placed + (int64_t)row * k + start; + const uint8_t *weight = codes + (int64_t)column * k + start; + const float sum = sums[(int64_t)row * (k / 16) + start / 16]; + float dot = 0.0f; + for (int i = 0; i < 16; i += 4) { + float products[4]; + for (int j = 0; j < 4; ++j) { + const float place = bits == 4 ? (float)(1u << (4 * j)) : 1.0f; + const float placed_input = input[i + j]; + const float placed_code = (float)weight[i + j] * place; + products[j] = placed_input * placed_code; + } + float four = products[0] + products[1]; + four = four + products[2]; + four = four + products[3]; + dot = dot + four; + } + const int64_t at = (int64_t)column * (k / group) + start / group; + const float bias_product = sum * biases[at]; + lanes[lane] = lanes[lane] + fmaf(scales[at], dot, bias_product); + } + } + for (int distance = 1; distance < 32; distance *= 2) + for (int lane = 0; lane < 32; lane += 2 * distance) + lanes[lane] = lanes[lane] + lanes[lane + distance]; + output[(int64_t)row * n + column] = lanes[0]; + } + } + free(placed); + free(sums); + return 0; +} + +// BF16 operands' dense projection uses one FP32 accumulator per lane and the same adjacent butterfly. +void lfm3_dense_replay(const float *x, const float *weight, float *output, int rows, int n, int k) { + for (int row = 0; row < rows; ++row) { + for (int column = 0; column < n; ++column) { + float lanes[32] = {0}; + for (int lane = 0; lane < 32; ++lane) + for (int i = lane; i < k; i += 32) + lanes[lane] = fmaf(x[(int64_t)row * k + i], weight[(int64_t)column * k + i], lanes[lane]); + for (int distance = 1; distance < 32; distance *= 2) + for (int lane = 0; lane < 32; lane += 2 * distance) + lanes[lane] = lanes[lane] + lanes[lane + distance]; + output[(int64_t)row * n + column] = lanes[0]; + } + } +} diff --git a/tools/lfm3/score_model_phases.py b/tools/lfm3/score_model_phases.py new file mode 100644 index 0000000000..ac4c19b3b6 --- /dev/null +++ b/tools/lfm3/score_model_phases.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Score prompt boundary and propagated teacher-forced continuation separately; never pool away a failing phase.""" +import argparse, hashlib, json, subprocess, sys +from pathlib import Path +import numpy as np +from fp32_bf16_envelope import compare + +p = argparse.ArgumentParser() +for key in ['fp32', 'reference-bf16', 'native', 'output']: + p.add_argument('--' + key, type=Path, required=True) +a = p.parse_args() +aggregate = a.output.with_suffix('.aggregate.json') +inputs = {path.resolve() for path in [a.fp32, a.reference_bf16, a.native]} +inputs |= {path.with_suffix('.json').resolve() for path in [a.fp32, a.reference_bf16]} +if a.output.resolve() in inputs or aggregate.resolve() in inputs: + raise ValueError('Output must not overwrite an input or reference metadata') +aggregate.unlink(missing_ok=True) +a.output.unlink(missing_ok=True) +r = subprocess.run( + [ + sys.executable, + str(Path(__file__).with_name('full_model_envelope.py')), + '--fp32', + str(a.fp32), + '--reference-bf16', + str(a.reference_bf16), + '--native', + str(a.native), + '--output', + str(aggregate), + ] +) +if not aggregate.exists(): + raise SystemExit('Reference identity validation failed') +d = json.loads(aggregate.read_text()) +f = np.load(a.fp32, allow_pickle=False) +b = np.load(a.reference_bf16, allow_pickle=False) +n = (np.fromfile(a.native, dtype='= prompt), +]: + if not mask.any(): + continue + result = compare(n[mask], b[mask], f[mask], space='logits') + result['absolute_token_positions'] = positions[mask].tolist() + result['row_max_exceedances'] = [ + x for x in result['all_row_errors'] if x['native_max'] > x['reference_max'] + ] + result['row_rms_exceedances'] = [ + x for x in result['all_row_errors'] if x['native_rms'] > x['reference_rms'] + ] + phases[label] = result +out = { + 'scope': 'Independent propagated full-vocabulary prompt-boundary and fixed teacher-forced continuation comparison. Not greedy decode acceptance; no top-pick tie waiver.', + 'files_sha256': d['files_sha256'], + 'aggregate_metrics_pass': d['proposed_metrics_no_worse'], + 'phases': phases, + 'all_phase_metrics_pass': all(x['proposed_metrics_no_worse'] for x in phases.values()), + 'top_pick_acceptance': None, + 'scorer_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), +} +a.output.write_text( + json.dumps(out, indent=2, default=lambda x: x.item() if isinstance(x, np.generic) else x) + '\n' +) +print( + json.dumps( + { + 'aggregate': out['aggregate_metrics_pass'], + 'all_phases': out['all_phase_metrics_pass'], + 'phases': { + k: { + 'pass': v['proposed_metrics_no_worse'], + 'native': v['native'], + 'reference': v['reference_bf16'], + 'native_top1_misses': v['native_fp32_top1_misses'], + 'reference_top1_misses': v['reference_fp32_top1_misses'], + } + for k, v in phases.items() + }, + }, + default=lambda x: x.item() if isinstance(x, np.generic) else x, + ) +) +if r.returncode or not out['all_phase_metrics_pass']: + raise SystemExit(1) diff --git a/tools/lfm3/serve_check.py b/tools/lfm3/serve_check.py new file mode 100644 index 0000000000..feaf54fa02 --- /dev/null +++ b/tools/lfm3/serve_check.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +"""Run owned loopback servers to check LFM3 served token equality, prompt reuse and image behavior.""" + +import argparse +import base64 +import concurrent.futures +import hashlib +import json +import re +import socket +import struct +import subprocess +import sys +import tempfile +import threading +import time +import urllib.error +import urllib.request +import zlib +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from bench_evidence import Evidence, token_equal + + +def solid_png(red, green, blue): + def chunk(kind, data): + return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data)) + rows = (b'\0' + bytes((red, green, blue)) * 256) * 256 + return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', 256, 256, 8, 2, 0, 0, 0)) + chunk( + b'IDAT', zlib.compress(rows), + ) + chunk(b'IEND', b'') + + +def image_url(data): + return {'type': 'image_url', 'image_url': {'url': 'data:image/png;base64,' + base64.b64encode(data).decode()}} + + +class Server: + def __init__(self, binary, directory, cache, log_directory): + self.binary, self.directory, self.cache = binary, directory, cache + self.log_directory = log_directory + self.process = None + + def __enter__(self): + with socket.socket() as sock: + sock.bind(('127.0.0.1', 0)) + port = sock.getsockname()[1] + self.base = f'http://127.0.0.1:{port}' + self.log = tempfile.NamedTemporaryFile(mode='w+b', prefix='server-', suffix='.log', + dir=self.log_directory, delete=False) + command = [str(self.binary), 'serve', str(self.directory), '--host', '127.0.0.1', '--port', str(port), + '--name', 'lfm3-check', '--context', '2048', '--parallel', '4', '--temperature', '0', + '--prompt-cache-gib', str(self.cache), '--no-thinking', '--no-update-check'] + self.command = command + self.process = subprocess.Popen(command, stdout=self.log, stderr=subprocess.STDOUT) + try: + deadline = time.monotonic() + 180 + while time.monotonic() < deadline: + if self.process.poll() is not None: + raise RuntimeError(f'Server exited {self.process.returncode}; see {self.log.name}') + try: + with urllib.request.urlopen(self.base + '/v1/models', timeout=1) as response: + if response.status == 200: + return self + except (urllib.error.URLError, TimeoutError): + pass + time.sleep(0.2) + raise TimeoutError(f'Server startup timed out; see {self.log.name}') + except BaseException: + self.__exit__(None, None, None) + raise + + def __exit__(self, *_): + if self.process is not None and self.process.poll() is None: + self.process.terminate() + try: + self.process.wait(timeout=20) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(timeout=10) + self.log.close() + + def request(self, item, draft=False, gate=None): + body = {'model': 'lfm3-check', 'max_tokens': 24, 'temperature': item.get('temperature', 0), + 'seed': 1234, 'stream': True, 'stream_options': {'include_usage': True}, + 'ignore_eos': True, 'draft': draft} + if body['temperature']: + body.update(top_k=20, top_p=0.95) + chat = 'messages' in item + if chat: + body['messages'] = item['messages'] + body['chat_template_kwargs'] = {'enable_thinking': False} + else: + body['prompt'] = item['prompt'] + url = self.base + ('/v1/chat/completions' if chat else '/v1/completions') + request = urllib.request.Request(url, data=json.dumps(body).encode(), + headers={'Content-Type': 'application/json'}) + if gate: + gate.wait(timeout=30) + evidence, usage, runtime, text = Evidence(), {}, {}, [] + complete = False + try: + with urllib.request.urlopen(request, timeout=180) as response: + for raw in response: + line = raw.decode().strip() + if line == 'data: [DONE]': + complete = True + break + if not line.startswith('data:'): + continue + event = json.loads(line[5:]) + if 'error' in event: + raise RuntimeError(str(event['error'])) + usage = event.get('usage') or usage + runtime = event.get('tensorfold') or runtime + for choice in event.get('choices', []): + evidence.add(choice) + text.append(choice.get('text') or (choice.get('delta') or {}).get('content') or '') + result = evidence.finish(usage, runtime, complete) + result['text'] = ''.join(text) + if not complete: + result['error'] = 'incomplete_stream' + return result + except urllib.error.HTTPError as exc: + return {'error': exc.read(8192).decode(errors='replace'), 'status': exc.code, 'complete': False} + except Exception as exc: + return {'error': f'{type(exc).__name__}: {exc}', 'complete': False} + + + def disconnections(self): + with urllib.request.urlopen(self.base + '/metrics', timeout=5) as response: + text = response.read(1 << 20).decode() + match = re.search(r'^tensorfold:client_disconnections_total (\d+)$', text, re.MULTILINE) + if match is None: + raise RuntimeError('Missing server client-disconnection counter') + return int(match.group(1)) + + def cancel_stream(self, item): + before = self.disconnections() + body = {'model': 'lfm3-check', 'max_tokens': 512, 'temperature': 0, 'seed': 1234, + 'stream': True, 'ignore_eos': True, 'draft': True} + is_chat = 'messages' in item + if is_chat: + body['messages'] = item['messages'] + body['chat_template_kwargs'] = {'enable_thinking': False} + else: + body['prompt'] = item['prompt'] + endpoint = '/v1/chat/completions' if is_chat else '/v1/completions' + request = urllib.request.Request(self.base + endpoint, data=json.dumps(body).encode(), + headers={'Content-Type': 'application/json'}) + first_content, request_id, already_finished = None, None, False + with urllib.request.urlopen(request, timeout=30) as response: + for raw in response: + line = raw.decode().strip() + if line == 'data: [DONE]': + already_finished = True + break + if not line.startswith('data:'): + continue + event = json.loads(line[5:]) + if 'error' in event: + raise RuntimeError(str(event['error'])) + request_id = event.get('id') or request_id + for choice in event.get('choices', []): + delta = choice.get('delta') or {} + content = choice.get('text') or delta.get('content') or delta.get('reasoning_content') + already_finished = already_finished or choice.get('finish_reason') is not None + if content: + first_content = content + break + if first_content: + break + deadline = time.monotonic() + 20 + after = before + while time.monotonic() < deadline and after <= before and first_content and not already_finished: + after = self.disconnections() + if after <= before: + time.sleep(0.05) + return {'request_id': request_id, 'first_content': first_content, 'requested_tokens': 512, + 'closed_before_done': bool(first_content) and not already_finished, + 'server_disconnections_before': before, 'server_disconnections_after': after, + 'server_confirmed_cancel': after > before} + + +def concurrent_requests(server, items): + gate = threading.Barrier(len(items)) + with concurrent.futures.ThreadPoolExecutor(max_workers=len(items)) as executor: + futures = [executor.submit(server.request, item, True, gate) for item in items] + return [future.result() for future in futures] + + +def chat(text, images=()): + content = [image_url(image) for image in images] + [{'type': 'text', 'text': text}] + return {'messages': [{'role': 'user', 'content': content if images else text}]} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--binary', type=Path, required=True) + parser.add_argument('--model-dir', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--skip-images', action='store_true', help='Diagnostic omission, recorded as skipped') + parser.add_argument('--cancel', action='store_true', help='Also check real HTTP cancellation and recovery') + parser.add_argument('--cancel-only', action='store_true', help='Run only fresh baselines and cancellation checks') + parser.add_argument('--sampled', action='store_true', help='Also test seeded sampling') + args = parser.parse_args() + args.binary, args.model_dir = args.binary.resolve(), args.model_dir.resolve() + config = json.loads((args.model_dir / 'config.json').read_text()) + vision = config.get('model_type') == 'lfm3_vl' + receipt = {'model': args.model_dir.name, 'binary_sha256': hashlib.sha256(args.binary.read_bytes()).hexdigest(), + 'tests': [], 'logs': [], 'commands': [], 'speed_qualified': False} + + def check(name, passed, evidence=None, skip=False): + status = 'skipped' if skip else 'passed' if passed is True else 'failed' + receipt['tests'].append({'name': name, 'status': status, 'evidence': evidence}) + print(f'{status}: {name}', flush=True) + + code = {'prompt': 'Write a Python function that returns the larger of two integers.'} + prose = chat('Explain why the sky is blue in two sentences.') + story = ('The river flows past a quiet village. Children watch birds near the bridge. ' * 60) + long = {'prompt': story + '\nSummarize the story in one sentence.'} + edited = {'prompt': 'A mountain rises beside a busy town. ' + story[70:] + '\nSummarize in one sentence.'} + items = {'code': code, 'chat': prose, 'long_prefix': long, 'edited_prefix': edited} + if args.sampled: + items['sampled_code'] = dict(code, temperature=0.7) + items['sampled_chat'] = dict(prose, temperature=0.7) + if vision and not args.skip_images: + red, blue = solid_png(255, 0, 0), solid_png(0, 0, 255) + question = 'What single solid color fills this image? Reply with only the color name.' + items.update(red=chat(question, [red]), blue=chat(question, [blue])) + order = 'Name the solid color of the first image, then the second image. Reply with two color names in order.' + items.update(red_blue=chat(order, [red, blue]), blue_red=chat(order, [blue, red])) + elif vision: + check('images', None, 'Explicit --skip-images diagnostic run', skip=True) + if args.cancel_only: + items = {name: items[name] for name in ('code', 'chat', 'red') if name in items} + baselines = {} + log_directory = tempfile.mkdtemp(prefix='lfm3-serve-check-') + try: + with Server(args.binary, args.model_dir, 0, log_directory) as server: + receipt['logs'].append(server.log.name) + receipt['commands'].append(server.command) + for name, item in items.items(): + result = server.request(item) + baselines[name] = result + check(name + '/fresh_plain', result.get('complete') and bool(result.get('token_sha256')), result) + if 'long_prefix' in baselines: + check('long_prefix/length_over_256', (baselines['long_prefix'].get('prompt_tokens') or 0) > 256, + baselines['long_prefix'].get('prompt_tokens')) + with Server(args.binary, args.model_dir, 0.1, log_directory) as server: + receipt['logs'].append(server.log.name) + receipt['commands'].append(server.command) + if not args.cancel_only: + for name, item in items.items(): + for draft in (False, True): + result = server.request(item, draft=draft) + if name == 'edited_prefix' and not draft: + check('edited_prefix/no_stale_prefix_reuse', result.get('cached_tokens') == 0, result) + check(name + ('/drafted_equals_fresh' if draft else '/plain_equals_fresh'), + token_equal(result, baselines[name]), result) + if name == 'long_prefix': + warm = server.request(item, draft=False) + check('long_prefix/warm_reuses_and_matches_fresh', + (warm.get('cached_tokens') or 0) > 0 and token_equal(warm, baselines[name]) is True, warm) + if name == 'edited_prefix': + result = server.request(item) + check('edited_prefix/matches_cache_off', token_equal(result, baselines[name]), result) + for level in (1, 2, 4): + names = ['code', 'chat'] if not args.sampled else ['code', 'chat', 'sampled_code', 'sampled_chat'] + for name in names: + results = concurrent_requests(server, [items[name]] * level) + check(f'{name}/concurrent_{level}_equals_solo', + all(token_equal(result, baselines[name]) is True for result in results), results) + if vision and not args.skip_images: + for name, expected in [('red', ['red']), ('blue', ['blue']), + ('red_blue', ['red', 'blue']), ('blue_red', ['blue', 'red'])]: + found = re.findall(r'\b(?:red|blue)\b', baselines[name].get('text', '').lower()) + check(name + '/color_semantics', found[:len(expected)] == expected, + {'expected': expected, 'observed': found, 'text': baselines[name].get('text')}) + image_items = [items['red'], items['blue'], items['red_blue'], items['blue_red']] + results = concurrent_requests(server, image_items) + check('images/concurrent_equals_solos', all( + token_equal(result, baselines[name]) is True + for result, name in zip(results, ('red', 'blue', 'red_blue', 'blue_red')) + ), results) + corrupt = server.request(chat('Describe this image.', [b'\x89PNG\r\n\x1a\ninvalid'])) + check('images/corrupt_png_rejected', 400 <= corrupt.get('status', 0) < 500, corrupt) + if args.cancel or args.cancel_only: + for name in ('code', 'chat', 'red'): + if name not in items: + continue + cancelled = server.cancel_stream(items[name]) + check(name + '/cancel_server_confirmed', + cancelled['closed_before_done'] and cancelled['server_confirmed_cancel'], cancelled) + recovered = server.request(items[name], draft=False) + check(name + '/cancel_next_plain_equals_fresh', token_equal(recovered, baselines[name]), recovered) + recovered = server.request(items[name], draft=True) + check(name + '/cancel_next_drafted_equals_fresh', + token_equal(recovered, baselines[name]), recovered) + + except Exception as exc: + check('harness_execution', False, f'{type(exc).__name__}: {exc}') + receipt['counts'] = {status: sum(test['status'] == status for test in receipt['tests']) + for status in ('passed', 'failed', 'skipped')} + receipt['cancellation_checks'] = sum('/cancel_' in test['name'] for test in receipt['tests']) + args.output.write_text(json.dumps(receipt, indent=2) + '\n') + print(json.dumps(receipt['counts'])) + return int(receipt['counts']['failed'] > 0) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/lfm3/test_cold_prompts.py b/tools/lfm3/test_cold_prompts.py new file mode 100644 index 0000000000..e32c6b2195 --- /dev/null +++ b/tools/lfm3/test_cold_prompts.py @@ -0,0 +1,21 @@ +"""The generated cold-prompt files must reproduce the committed hashes.""" +import json +import unittest + +import cold_prompts + + +class ColdPromptTests(unittest.TestCase): + def test_regenerated_files_match_recorded_hashes(self): + wanted = json.loads(cold_prompts.SHA256.read_text()) + self.assertEqual(cold_prompts.mismatches(cold_prompts.files(), wanted), []) + + def test_a_changed_file_is_reported(self): + wanted = json.loads(cold_prompts.SHA256.read_text()) + produced = cold_prompts.files() + produced["text-2048.ids.json"] += b" " + self.assertEqual(cold_prompts.mismatches(produced, wanted), ["text-2048.ids.json"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/lfm3/test_compare_logits.py b/tools/lfm3/test_compare_logits.py new file mode 100644 index 0000000000..19ddae8c54 --- /dev/null +++ b/tools/lfm3/test_compare_logits.py @@ -0,0 +1,44 @@ +"""Ensure fidelity receipts preserve real differences, ties and phase-independent row identities.""" +import unittest +try: + import numpy as np +except ImportError: + np = None +if np is not None: + from compare_logits import compare + + +@unittest.skipIf(np is None, 'NumPy is required for fidelity reports') +class CompareTests(unittest.TestCase): + def test_disjoint_contenders_and_both_margins_are_retained(self): + result = compare([[2, 1, 0], [0, 3, 1]], [[1, 2, 0], [0, 3, 1]], 7) + self.assertEqual(result['rows'], 2) + self.assertEqual(result['disjoint_maximum_sets'], 1) + row = result['differences'][0] + self.assertEqual((row['row'], row['native_top'], row['reference_top']), (7, 0, 1)) + self.assertEqual(row['native_margin'], 1) + self.assertEqual(row['reference_margin'], 1) + self.assertEqual(row['native_reference_contender_gap'], 1) + self.assertEqual(row['reference_native_contender_gap'], 1) + self.assertIsNone(result['fidelity_pass']) + + def test_overlapping_tie_is_reported_without_waiving_it(self): + result = compare([[1, 2, 0]], [[2, 2, 0]]) + self.assertEqual(result['top1_differences'], 1) + self.assertEqual(result['disjoint_maximum_sets'], 0) + self.assertEqual(result['differences'][0]['reference_margin'], 0) + self.assertIsNone(result['accepted_threshold']) + + def test_worst_rows_are_not_hidden_by_average(self): + result = compare([[0, 0], [9, 1]], [[0, 0], [1, 1]]) + self.assertEqual(result['worst_rows'][0]['row'], 1) + self.assertEqual(result['maximum_absolute_error'], 8) + + def test_invalid_geometry_and_nonfinite_inputs_are_refused(self): + for a, b in (([[1]], [[1]]), ([[1, 2]], [[1, 2], [1, 2]]), ([[np.nan, 1]], [[1, 2]])): + with self.assertRaises(ValueError): + compare(a, b) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/lfm3/test_fp32_bf16_envelope.py b/tools/lfm3/test_fp32_bf16_envelope.py new file mode 100644 index 0000000000..8b950240dc --- /dev/null +++ b/tools/lfm3/test_fp32_bf16_envelope.py @@ -0,0 +1,102 @@ +"""Failure controls for the finite FP32/reference-BF16 comparison proposal.""" + +import json +import hashlib +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +import numpy as np +from fp32_bf16_envelope import compare, input_pins + + +class EnvelopeTests(unittest.TestCase): + def test_actual_reference_is_its_own_envelope(self): + f = np.array([[1.001, -0.501, 0.1], [0.25, 0.125, -0.75]], dtype=np.float32) + b = np.array([[1, -0.5, 0.1], [0.25, 0.125, -0.75]], dtype=np.float32) + result = compare(b, b, f, 'logits') + self.assertTrue(result['proposed_metrics_no_worse']) + self.assertEqual(result['top_pick_differences'], []) + + def test_better_mean_does_not_hide_worse_maximum(self): + f = np.zeros((2, 8), dtype=np.float32) + b = np.full_like(f, 0.125) + n = f.copy() + n[0, 0] = 0.25 + result = compare(n, b, f) + self.assertTrue(result['proposed_metric_results']['rms_error']) + self.assertFalse(result['proposed_metric_results']['maximum_absolute_error']) + self.assertFalse(result['proposed_metrics_no_worse']) + + def test_duplicate_input_basenames_are_refused(self): + with self.assertRaises(ValueError): + input_pins({'inputs_sha256': {'a/tokens.npy': 'first', 'b/tokens.npy': 'second'}}) + + def test_nonfinite_is_not_a_pass(self): + f = np.ones((1, 4), dtype=np.float32) + n = f.copy() + n[0, 0] = np.nan + with self.assertRaises(ValueError): + compare(n, f, f) + + def test_top_difference_is_retained_with_margin(self): + f = np.array([[1, 0.5, 0]], dtype=np.float32) + n = np.array([[0.5, 1, 0]], dtype=np.float32) + result = compare(n, f, f, 'logits') + self.assertEqual(result['native_fp32_top1_misses'], 1) + self.assertEqual(result['top_pick_differences'][0]['fp32_margin'], 0.5) + self.assertFalse(result['proposed_metrics_no_worse']) + + def test_reference_identity_failure_cannot_reuse_stale_aggregate(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + f = np.ones((2, 4), dtype=np.float32) + for name, backend in [('f', 'fp32'), ('b', 'mlx')]: + path = root / (name + '.npy') + np.save(path, f) + metadata = dict( + operator='full_model_logits', + selected_token_positions=[0, 1], + prompt_len=1, + teacher_forced_continuation=1, + reference_chunk=128, + image_start=None, + shape=[2, 4], + backend=backend, + inputs_sha256={'tokens.npy': 'matching-input'}, + output_sha256=hashlib.sha256(path.read_bytes()).hexdigest(), + official_sources_sha256={'model.py': 'pinned-source'}, + ) + if name == 'b': + metadata['prompt_len'] = 2 + path.with_suffix('.json').write_text(json.dumps(metadata)) + native = root / 'native.bf16' + (f.view(np.uint32) >> 16).astype('> 16).astype(np.uint16) + + +@unittest.skipIf(np is None, 'NumPy is required for the independent FP64 reference') +class ReferenceTests(unittest.TestCase): + def test_all_finite_values_and_adjacent_rounding_boundaries(self): + raw = np.arange(0x7f80, dtype=np.uint16) + values = (raw.astype(np.uint32) << 16).view(np.float32).astype(np.float64) + np.testing.assert_array_equal(bits(values), raw) + np.testing.assert_array_equal(bits(-values), raw | 0x8000) + middle = (values[:-1] + values[1:]) / 2 + even = np.where((raw[:-1] & 1) == 0, raw[:-1], raw[1:]) + np.testing.assert_array_equal(bits(middle), even) + np.testing.assert_array_equal(bits(-middle), even | 0x8000) + np.testing.assert_array_equal(bits(np.nextafter(middle, np.inf)), raw[1:]) + np.testing.assert_array_equal(bits(np.nextafter(middle, -np.inf)), raw[:-1]) + + def test_fp32_intermediate_would_double_round_the_fp64_result(self): + value = 1 + 2 ** -8 + 2 ** -30 + self.assertEqual(int(bits(value)), 0x3f81) + self.assertEqual(int(bits(np.float32(value))), 0x3f80) + self.assertEqual(int(bits(1 + 2 ** -8 - 2 ** -30)), 0x3f80) + + def test_subnormal_overflow_and_nonfinite_values(self): + self.assertEqual(int(bits(2 ** -134)), 0) + self.assertEqual(int(bits(np.nextafter(np.float64(2 ** -134), np.inf))), 1) + maximum = (np.uint32(0x7f7f) << np.uint32(16)).view(np.float32).astype(np.float64) + threshold = maximum + 2 ** 119 + self.assertEqual(int(bits(np.nextafter(threshold, -np.inf))), 0x7f7f) + self.assertEqual(int(bits(threshold)), 0x7f80) + self.assertEqual(int(bits(float('inf'))), 0x7f80) + self.assertEqual(int(bits(-float('inf'))), 0xff80) + self.assertEqual(int(bits(float('nan'))), 0x7fc0) + self.assertEqual(int(bits(-0.0)), 0x8000) + + def test_six_bit_codes_cross_word_boundaries_before_weight_rounding(self): + oracle = Oracle.__new__(Oracle) + oracle.config = {'quantization': {'bits': 6, 'group_size': 32}} + oracle.weight_mode = 'exact' + tensors = { + 'projection.weight': np.array([[0x440c2040, 0xa2481c61, 0x3ce34c2c, + 0x544d2450, 0xa6585d65, 0x7de75c6d]], dtype=np.uint32), + 'projection.scales': np.array([[1 / 256]], dtype=np.float64), + 'projection.biases': np.array([[8]], dtype=np.float64), + } + oracle.tensor = lambda name, rows=None: tensors[name] if rows is None else tensors[name][rows] + expected = 8 + np.arange(32, dtype=np.float64)[None] / 256 + np.testing.assert_array_equal(oracle.weights('projection', slice(0, 1)), expected) + oracle.weight_mode = 'bf16' + rounded = oracle.weights('projection', slice(0, 1)) + self.assertEqual(rounded[0, 1], 8) + self.assertNotEqual(rounded[0, 1], expected[0, 1]) + + def test_projected_image_rows_replace_only_matching_placeholders(self): + hidden = np.arange(12, dtype=np.float64).reshape(3, 4) + rows = np.array([[1 + 2 ** -8 + 2 ** -30, 2, 3, 4]]) + result = inject_image_rows(hidden, [10, 125030, 11], rows, 1) + np.testing.assert_array_equal(result[[0, 2]], hidden[[0, 2]]) + np.testing.assert_array_equal(hidden, np.arange(12).reshape(3, 4)) + self.assertEqual(result[1, 0], float(bf(rows[0, 0]))) + for tokens, value, start in (([10, 125030, 11], rows, 0), + ([10, 125030, 125030], rows, 1), + ([10, 125030, 11], rows[:, :3], 1), + ([10, 125030, 11], rows * np.nan, 1)): + with self.assertRaises(ValueError): + inject_image_rows(hidden, tokens, value, start) + + def test_embedding_override_does_not_change_other_projections(self): + oracle = Oracle.__new__(Oracle) + oracle.config = {'quantization': {'bits': 4, 'group_size': 32, + 'language_model.model.embed_tokens': {'bits': 8, 'group_size': 32}}} + self.assertEqual(oracle.quant('language_model.model.embed_tokens'), (8, 32)) + self.assertEqual(oracle.quant('language_model.model.layers.0.conv.in_proj'), (4, 32)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/lfm3/test_full_model_reference.py b/tools/lfm3/test_full_model_reference.py new file mode 100644 index 0000000000..6511de39e9 --- /dev/null +++ b/tools/lfm3/test_full_model_reference.py @@ -0,0 +1,56 @@ +import unittest, tempfile, json, struct +from pathlib import Path +import numpy as np +from full_model_reference import Checkpoint, FP32, inject_check +from fp32_graph_reference import Graph + + +class Tests(unittest.TestCase): + def checkpoint(self, prefix, index): + with tempfile.TemporaryDirectory() as t: + d = Path(t) + (d / 'config.json').write_text(json.dumps({'model_type': 'lfm3', 'tie_word_embeddings': True})) + name = prefix + 'embed_tokens.weight' + h = {name: {'dtype': 'U32', 'shape': [1, 2], 'data_offsets': [0, 8]}} + raw = json.dumps(h).encode() + (d / 'model.safetensors').write_bytes( + struct.pack('> shift) & 0xffffffff for shift in range(0, 512 * bits, 32)] + for value in packed], dtype=np.uint32) + return Fixture() + + def test_all_three_packed_widths_match_independent_integer_sum(self): + for bits in (4, 6, 8): + result = affine_rows(self.oracle(bits), 'projection', np.ones((2, 512), dtype=np.float32)) + expected = 2 * sum(i % (1 << bits) for i in range(512)) - 512 + np.testing.assert_array_equal(result, np.full((2, 1), expected)) + + def test_noncontiguous_unpacked_columns_keep_their_identity(self): + values = np.arange(512, dtype=np.float32) % 64 - 32 + for bits in (4, 6, 8): + actual = affine_rows(self.oracle(bits, 3), 'projection', values[None]) + expected = [sum(float(values[i]) * ((i + 3 * c) % (1 << bits)) * (2 + c) + for i in range(512)) - float(values.sum()) for c in range(3)] + np.testing.assert_array_equal(actual, [expected]) + + def test_dense_fp32_rounding_occurs_before_final_storage(self): + values = np.zeros((1, 96), dtype=np.float32) + values[0, [0, 32, 64]] = [2 ** 24, 1, -(2 ** 24)] + weight = np.ones((1, 96), dtype=np.float32) + self.assertEqual((values.astype(np.float64) @ weight.astype(np.float64).T)[0, 0], 1) + self.assertEqual(float(dense_rows(values, weight)[0, 0]), 0) + + def test_dense_columns_and_input_guards(self): + values = (np.arange(64) % 8).astype(np.float32)[None] + weights = np.stack([np.ones(64), np.arange(64) % 4]).astype(np.float32)[::-1] + np.testing.assert_array_equal(dense_rows(values, weights), values @ weights.T) + with self.assertRaises(ValueError): + dense_rows(values[:, :-1], weights) + with self.assertRaises(ValueError): + dense_rows(np.full((1, 64), 1 + 2 ** -30), weights) + + def test_inputs_must_be_finite_bf16_rows_with_served_k_geometry(self): + for value in (np.ones((1, 511)), np.full((1, 512), 1.001), + np.full((1, 512), 1 + 2 ** -30), np.full((1, 512), np.inf)): + with self.assertRaises(ValueError): + affine_rows(self.oracle(4), 'projection', value) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/lfm3/test_text_attention_rounding.py b/tools/lfm3/test_text_attention_rounding.py new file mode 100644 index 0000000000..352e19cba6 --- /dev/null +++ b/tools/lfm3/test_text_attention_rounding.py @@ -0,0 +1,34 @@ +"""Check causal geometry and rotary identity for the optional FP32 replay.""" +import unittest +try: + import numpy as np +except ImportError: + np = None +if np is not None: + from text_attention_rounding import attention, rotary + + + +@unittest.skipIf(np is None, 'NumPy is required for rounding reference tests') +class TextAttentionRoundingTests(unittest.TestCase): + def test_causal_uniform_attention_uses_only_prefix(self): + q = np.zeros((2, 4, 128), dtype=np.float32) + v = np.broadcast_to(np.arange(1, 5, dtype=np.float32)[None, :, None], q.shape) + expected = np.broadcast_to(np.array([1, 1.5, 2, 2.5])[None, :, None], q.shape) + np.testing.assert_array_equal(attention(q, q, v, 0, 'numpy'), expected) + sliding = np.broadcast_to(np.array([1, 1.5, 2.5, 3.5])[None, :, None], q.shape) + np.testing.assert_array_equal(attention(q, q, v, 2, 'numpy'), sliding) + + def test_zero_position_is_identity_with_partial_rotary(self): + values = np.arange(128, dtype=np.float32).reshape(1, 1, 128) + spec = {'rope_theta': 10000, 'partial_rotary_factor': .5} + np.testing.assert_array_equal(rotary(values, spec, 'numpy'), values) + + def test_unimplemented_multi_partition_replay_is_rejected(self): + q = np.zeros((1, 1025, 128), dtype=np.float32) + with self.assertRaises(ValueError): + attention(q, q, q, 0, 'numpy') + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/lfm3/test_text_rounding_reference.py b/tools/lfm3/test_text_rounding_reference.py new file mode 100644 index 0000000000..af9e9b11d4 --- /dev/null +++ b/tools/lfm3/test_text_rounding_reference.py @@ -0,0 +1,53 @@ +"""The production reduction tree must not be replaced by an FP64 sum followed by one FP32 cast.""" +import unittest +try: + import numpy as np +except ImportError: + np = None +if np is not None: + from text_rounding_reference import norm_rows, sum32 + + +@unittest.skipIf(np is None, 'NumPy is required for the normalization rounding replay') +class TextRoundingTests(unittest.TestCase): + def test_positive_lane_sums_round_at_each_butterfly(self): + values = np.ones(32, dtype=np.float32) + values[0] = 2 ** 24 + self.assertEqual(sum32(values), 2 ** 24 + 30) + self.assertEqual(np.float32(values.astype(np.float64).sum()), 2 ** 24 + 32) + self.assertEqual(values[0], 2 ** 24) + + def test_hidden_and_head_norms_keep_rows_independent(self): + for width in (128, 2048): + values = np.ones((2, width), dtype=np.float32) + values[1] *= 2 + result = norm_rows(values, np.ones(width), 1e-5) + np.testing.assert_array_equal(result, np.ones_like(values)) + np.testing.assert_array_equal(values[1], np.full(width, 2)) + + def test_bad_geometry_or_non_bf16_inputs_are_refused(self): + for value, gain in ((np.ones((1, 64)), np.ones(64)), + (np.full((1, 128), 1 + 2 ** -30), np.ones(128)), + (np.ones((1, 128)), np.ones(127))): + with self.assertRaises(ValueError): + norm_rows(value, gain, 1e-5) + +@unittest.skipIf(np is None, 'NumPy is required for rounding reference tests') +class MetalFlushBoundaryTests(unittest.TestCase): + def test_named_metal_mode_flushes_fp32_subnormal_gain_product(self): + from unittest.mock import patch + values = np.ones((1, 128)) + values[0, 0] = .5 + gain = np.ones(128) + gain[0] = 2 ** -126 + ideal = norm_rows(values, gain, 1e-5) + self.assertGreater(ideal[0, 0], 0) + with patch('elementary_reference.elementary', + side_effect=lambda x, operation, backend: (1 / np.sqrt(x.astype(np.float64))).astype(np.float32)): + actual = norm_rows(values, gain, 1e-5, 'metal') + self.assertEqual(actual[0, 0], 0) + np.testing.assert_array_equal(actual[:, 1:], ideal[:, 1:]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/lfm3/test_vision_fp64_reference.py b/tools/lfm3/test_vision_fp64_reference.py new file mode 100644 index 0000000000..998099571b --- /dev/null +++ b/tools/lfm3/test_vision_fp64_reference.py @@ -0,0 +1,33 @@ +"""Verify vision oracle rounding and reject malformed patch grids before evaluating weights.""" +import unittest +from types import SimpleNamespace +try: + import numpy as np +except ImportError: + np = None +if np is not None: + from vision_fp64_reference import VisionOracle + + +@unittest.skipIf(np is None, 'NumPy is required for the FP64 vision reference') +class VisionOracleTests(unittest.TestCase): + def test_dense_rounds_the_fp64_sum_directly_to_bf16(self): + oracle = VisionOracle.__new__(VisionOracle) + oracle.loader = SimpleNamespace(map={}) + oracle.weight = lambda name: np.ones((1, 3), dtype=np.float64) + actual = oracle.dense(np.array([[1, 2 ** -8, 2 ** -30]], dtype=np.float64), 'projection') + self.assertEqual(actual[0, 0], 1 + 2 ** -7) + self.assertEqual(float(np.float32(1 + 2 ** -8 + 2 ** -30)), 1 + 2 ** -8) + + def test_grid_and_patch_layout_are_validated_before_weight_access(self): + oracle = VisionOracle.__new__(VisionOracle) + for height, width, patches in ((0, 2, np.zeros((0, 1536))), + (3, 2, np.zeros((6, 1536))), + (2, 2, np.zeros((4, 768))), + (2, 2, np.full((4, 1536), np.nan))): + with self.assertRaises(ValueError): + oracle.run(patches, height, width) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/lfm3/test_vision_rounding_reference.py b/tools/lfm3/test_vision_rounding_reference.py new file mode 100644 index 0000000000..e569e3f95e --- /dev/null +++ b/tools/lfm3/test_vision_rounding_reference.py @@ -0,0 +1,65 @@ +"""Check vision replay indexing and intermediate FP32 rounding independently.""" +import unittest +try: + import numpy as np +except ImportError: + np = None +if np is not None: + from elementary_reference import fma, swiglu + from vision_rounding_reference import rms_rows, convolution, rotary, attention, layer_norm, gelu + + + +@unittest.skipIf(np is None, 'NumPy is required for rounding reference tests') +class VisionRoundingTests(unittest.TestCase): + def test_fma_retains_product_residual_before_round(self): + a, b = np.float32(1 + 2 ** -23), np.float32(1 - 2 ** -23) + self.assertEqual(float(fma(a, b, -1)), -2 ** -46) + self.assertEqual(float(np.float32(a * b) - np.float32(1)), 0) + + def test_vision_rms_keeps_strided_row_channels_and_gain(self): + values = np.ones((2, 1024)) + gain = np.ones(1024) + gain[513] = -2 + actual = rms_rows(values, gain, 1e-5) + np.testing.assert_array_equal(actual, np.tile(gain, (2, 1))) + with self.assertRaises(ValueError): + rms_rows(np.ones((1, 128)), np.ones(128), 1e-5) + + def test_spatial_convolution_zero_padding_and_channel_axis(self): + grid = np.arange(24).reshape(2, 3, 4) + kernel = np.zeros((4, 3, 3)) + kernel[:, 1, 1] = np.arange(1, 5) + np.testing.assert_array_equal(convolution(grid, kernel), grid * np.arange(1, 5)) + kernel[:] = 0 + kernel[:, 0, 1] = 1 + expected = np.zeros_like(grid) + expected[1] = grid[0] + np.testing.assert_array_equal(convolution(grid, kernel), expected) + + def test_zero_position_rotary_is_identity(self): + values = np.arange(128, dtype=np.float32).reshape(2, 1, 64) + np.testing.assert_array_equal(rotary(values, np.zeros((2, 2))), values) + + def test_attention_constant_visible_values_are_preserved(self): + q = np.zeros((2, 5, 64), dtype=np.float32) + v = np.broadcast_to(np.arange(64, dtype=np.float32), q.shape) + np.testing.assert_array_equal(attention(q, q, v), v) + with self.assertRaises(ValueError): + attention(q[..., :32], q, v) + + def test_projector_constant_row_and_gelu_origin(self): + np.testing.assert_array_equal(layer_norm(np.ones((2, 1024)), np.ones(1024), + np.zeros(1024), 1e-6), np.zeros((2, 1024))) + np.testing.assert_array_equal(gelu(np.array([0], dtype=np.float32)), [0]) + + def test_swiglu_rounds_silu_before_up_product(self): + from fp64_reference import bf + gate = np.array([0, 1, -1], dtype=np.float32) + up = np.array([3, 2, 2], dtype=np.float32) + silu = bf(gate / (np.float32(1) + np.exp(-gate))) + np.testing.assert_array_equal(swiglu(gate, up), bf(silu * up)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/lfm3/text_attention_rounding.py b/tools/lfm3/text_attention_rounding.py new file mode 100644 index 0000000000..667b423a2c --- /dev/null +++ b/tools/lfm3/text_attention_rounding.py @@ -0,0 +1,57 @@ +"""Independent bounded FP32 replay of decoder rotary and one-partition attention.""" +import math +import numpy as np +from fp64_reference import bf +from elementary_reference import elementary, fma +from text_rounding_reference import sum32 + + +def rotary(values, spec, backend): + values = np.asarray(values, dtype=np.float32) + pairs = int(values.shape[-1] * spec.get('partial_rotary_factor', 1)) // 2 + frequency = np.arange(64, dtype=np.float32) + exponent = -frequency / np.float32(64) * np.float32(math.log2(spec['rope_theta'])) + frequency = elementary(exponent, 'exp2', backend) + frequency[pairs:] = 0 + angle = np.arange(len(values), dtype=np.float32)[:, None] * frequency + cosine = bf(elementary(angle, 'cos', backend)).astype(np.float32)[:, None] + sine = bf(elementary(angle, 'sin', backend)).astype(np.float32)[:, None] + a, b = values[..., :64], values[..., 64:] + return bf(np.concatenate((fma(a, cosine, -(b * sine)), fma(b, cosine, a * sine)), axis=-1)) + + +def attention(q, k, v, window, backend): + q, k, v = (np.asarray(value, dtype=np.float32) for value in (q, k, v)) + heads, rows, width = q.shape + if width != 128 or rows > 1024 or k.shape != q.shape or v.shape != q.shape: + raise ValueError('Named text attention replay requires at most 1024 equally expanded 128-channel rows') + output = np.empty_like(q) + for first in range(0, rows, 16): + query = q[:, first:first + 16] + products = np.zeros((heads, len(query[0]), rows, 32), dtype=np.float32) + for part in range(4): + products = fma(query[:, :, None, part * 32:(part + 1) * 32], + k[:, None, :, part * 32:(part + 1) * 32], products) + scores = sum32(products) * np.float32(1 / math.sqrt(128)) + distance = np.arange(first, first + len(query[0]))[:, None] - np.arange(rows)[None] + allowed = distance >= 0 + if window: + allowed &= distance < window + scores = np.where(allowed[None], scores, np.float32(-np.inf)) + maxima = np.maximum.accumulate(scores, axis=-1) + previous = np.concatenate((np.full((*scores.shape[:2], 1), -np.inf, np.float32), maxima[..., :-1]), axis=-1) + with np.errstate(invalid='ignore'): + alpha = elementary(previous - maxima, 'exp', backend) + probability = elementary(scores - maxima, 'exp', backend) + alpha = np.where(np.isneginf(previous), np.float32(0), alpha) + probability = np.where(np.isneginf(scores), np.float32(0), probability) + total = np.zeros(scores.shape[:2], dtype=np.float32) + result = np.zeros((*total.shape, 128), dtype=np.float32) + for key in range(rows): + active = allowed[None, :, key] + next_total = fma(total, alpha[..., key], probability[..., key]) + next_result = fma(result, alpha[..., key, None], probability[..., key, None] * v[:, key, None]) + total = np.where(active, next_total, total) + result = np.where(active[..., None], next_result, result) + output[:, first:first + len(query[0])] = result / total[..., None] + return bf(output) diff --git a/tools/lfm3/text_rounding_reference.py b/tools/lfm3/text_rounding_reference.py new file mode 100644 index 0000000000..5322e0c764 --- /dev/null +++ b/tools/lfm3/text_rounding_reference.py @@ -0,0 +1,52 @@ +"""Independent replay of the text RMS normalization's prescribed FP32 partial and lane reductions.""" +import numpy as np +from fp64_reference import bf + + +def sum32(values): + result = np.asarray(values, dtype=np.float32) + if result.shape[-1] != 32: + raise ValueError('Expected exactly 32 reduction lanes') + for distance in (1, 2, 4, 8, 16): + result = result + result[..., np.arange(32) ^ distance] + return result[..., 0] + + +def norm_rows(values, gain, epsilon, elementary_backend=None): + original, gain = np.asarray(values), np.asarray(gain) + if original.ndim == 0 or not np.isfinite(original).all() or not np.array_equal(original, bf(original)): + raise ValueError('Expected finite BF16 normalization inputs') + values = np.asarray(original, dtype=np.float32) + width = values.shape[-1] + if gain.shape != (width,) or not np.isfinite(gain).all() or not np.array_equal(gain, bf(gain)): + raise ValueError('Expected one finite BF16 normalization gain per column') + if not np.isfinite(epsilon) or epsilon <= 0: + raise ValueError('Expected positive finite normalization epsilon') + if width == 2048: + partial = values.reshape(*values.shape[:-1], 256, 8) + square = np.zeros(partial.shape[:-1], dtype=np.float32) + for at in range(8): + square = (partial[..., at].astype(np.float64) ** 2 + square).astype(np.float32) + groups = sum32(square.reshape(*values.shape[:-1], 8, 32)) + padded = np.zeros((*values.shape[:-1], 32), dtype=np.float32) + padded[..., :8] = groups + total = sum32(padded) + elif width == 128: + partial = values.reshape(*values.shape[:-1], 4, 32) + square = np.zeros((*values.shape[:-1], 32), dtype=np.float32) + for at in range(4): + square = (partial[..., at, :].astype(np.float64) ** 2 + square).astype(np.float32) + total = sum32(square) + else: + raise ValueError('Expected a text hidden row or attention head') + variance = total / np.float32(width) + np.float32(epsilon) + inverse = (1 / np.sqrt(variance.astype(np.float64))).astype(np.float32)[..., None] + if elementary_backend is not None: + from elementary_reference import elementary + inverse = elementary(variance, 'rsqrt', elementary_backend)[..., None] + first = bf(values * inverse).astype(np.float32) + result = first * np.asarray(gain, dtype=np.float32) + if elementary_backend == 'metal': + # Metal's FP32 arithmetic flushes subnormal products, before the BF16 store. + result = np.where(np.abs(result) < np.finfo(np.float32).tiny, np.copysign(np.float32(0), result), result) + return bf(result) diff --git a/tools/lfm3/vision_attention_rounding.py b/tools/lfm3/vision_attention_rounding.py new file mode 100644 index 0000000000..5db27c723c --- /dev/null +++ b/tools/lfm3/vision_attention_rounding.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Diagnose production elementary rounding; never declare an independent FP64 fidelity pass.""" +import argparse +import hashlib +import json +from pathlib import Path +import numpy as np +from elementary_reference import fma +from text_rounding_reference import sum32 + + +def read(directory, name, rows, width): + words = np.fromfile(directory / (name + '.bf16'), dtype=' 16384: + raise ValueError('Expected a valid even patch grid') + if patches.shape != (height * width, 1536) or not np.isfinite(patches).all(): + raise ValueError('Expected finite merge-ordered patches with 1536 elements each') + coordinates = np.array([(h * 2 + a, w * 2 + b) for h in range(height // 2) + for w in range(width // 2) for a in range(2) for b in range(2)]) + hidden = self.dense(bf(patches), 'vision_tower.patchifier') + self.take('patchifier', hidden) + hidden = self.rms(hidden, 'vision_tower.embedding_norm') + self.take('embedding_norm', hidden) + for index, kind in enumerate(self.vision['layer_types']): + prefix = f'vision_tower.layers.{index}.' + x = self.rms(hidden, prefix + 'operator_norm') + if index == 0: + self.take('first_operator_norm', x) + result = (self.convolution(x, prefix, height, width) if kind == 'conv' + else self.attention(x, prefix, coordinates)) + if index == 0: + self.take('first_operator', result) + hidden = bf(hidden + result) + if index == 0: + self.take('first_residual', hidden) + x = self.rms(hidden, prefix + 'ffn_norm') + if index == 0: + self.take('first_ffn_norm', x) + gate, up = self.dense(x, prefix + 'feed_forward.w1'), self.dense(x, prefix + 'feed_forward.w3') + if index == 0: + self.take('first_ffn_gate', gate) + self.take('first_ffn_up', up) + if getattr(self, 'operator_rounding', 'fp64') == 'metal-fp32': + from elementary_reference import swiglu + act = swiglu(gate, up, self.elementary_backend) + else: + act = bf(bf(gate / (1 + np.exp(-gate))) * up) + if index == 0: + self.take('first_ffn_act', act) + down = self.dense(act, prefix + 'feed_forward.w2') + if index == 0: + self.take('first_ffn_down', down) + hidden = bf(hidden + down) + self.take(f'layer_{index:02d}', hidden) + print(f'vision_layer_{index:02d}', flush=True) + prefix = 'multi_modal_projector.' + centered = hidden - hidden.mean(axis=-1, keepdims=True) + inverse = 1 / np.sqrt(np.mean(centered * centered, axis=-1, keepdims=True) + + self.config['projector_layer_norm_eps']) + normalized = bf(centered * inverse * self.weight(prefix + 'layer_norm.weight') + + self.weight(prefix + 'layer_norm.bias')) + if getattr(self, 'operator_rounding', 'fp64') == 'metal-fp32': + from vision_rounding_reference import layer_norm + normalized = layer_norm(hidden, self.weight(prefix + 'layer_norm.weight'), + self.weight(prefix + 'layer_norm.bias'), self.config['projector_layer_norm_eps'], + self.elementary_backend) + self.take('projector_norm', normalized) + merged = normalized.reshape(-1, self.vision['hidden_size'] * self.config['downsample_factor'] ** 2) + first = self.dense(merged, prefix + 'linear_1') + erf = np.fromiter((math.erf(v / math.sqrt(2)) for v in first.flat), dtype=np.float64).reshape(first.shape) + activated = bf(0.5 * first * (1 + erf)) + if getattr(self, 'operator_rounding', 'fp64') == 'metal-fp32': + from vision_rounding_reference import gelu + activated = gelu(first, self.elementary_backend) + self.take('projector_gelu', activated) + return self.dense(activated, prefix + 'linear_2') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--model-dir', type=Path, required=True) + parser.add_argument('--patches', type=Path, required=True) + parser.add_argument('--grid-h', type=int, required=True) + parser.add_argument('--grid-w', type=int, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--trace-dir', type=Path) + parser.add_argument('--elementary-backend', choices=('numpy', 'metal'), default='numpy') + parser.add_argument('--operator-rounding', choices=('fp64', 'metal-fp32'), default='fp64') + parser.add_argument('--dense-rounding', choices=('fp64', 'metal-row'), default='fp64') + args = parser.parse_args() + def write(path, values): + path.write_bytes((np.asarray(values, dtype=np.float32).view(np.uint32) >> 16).astype('