From b160bcdd6d466e8d75736de3ba51cc57c043504f Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Mon, 12 Oct 2026 21:09:43 -0700 Subject: [PATCH 01/33] DeepSeek V4.1 Flash: configuration, Engram tables and resident weight staging Config parsing, the Engram hash layout, table plans over the original shards' embed weight and scale, and the EXL3 resident weight staging behind paced reads. Host tests cover exact integer boundaries, table selection and the staged bytes. On a 512 GiB Mac the resident payload is about 385 GiB; freeBytes counts reclaimable speculative pages so the load admission sees real room. No kernels; the served engine comes with the lane backend. --- zig/build/deepseek_v41.zig | 22 ++ zig/deepseek_v41_weights_checks.zig | 8 + zig/src/families/deepseek_v41/view.zig | 25 ++ zig/src/families/deepseek_v41/weights.zig | 273 ++++++++++++++++++++++ zig/src/tensorfold.zig | 5 + 5 files changed, 333 insertions(+) create mode 100644 zig/build/deepseek_v41.zig create mode 100644 zig/deepseek_v41_weights_checks.zig create mode 100644 zig/src/families/deepseek_v41/view.zig create mode 100644 zig/src/families/deepseek_v41/weights.zig diff --git a/zig/build/deepseek_v41.zig b/zig/build/deepseek_v41.zig new file mode 100644 index 0000000000..6b196fedff --- /dev/null +++ b/zig/build/deepseek_v41.zig @@ -0,0 +1,22 @@ +//! Experimental DS4.1 weight-loading qualification targets. +const std = @import("std"); + +pub fn weights( + b: *std.Build, + target: std.Build.ResolvedTarget, + optimize: std.builtin.OptimizeMode, + metal: *std.Build.Module, +) *std.Build.Step { + const host = b.addTest(.{ .root_module = b.createModule(.{ + .root_source_file = b.path("zig/deepseek_v41_weights_checks.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + .imports = &.{ + .{ .name = "metal", .module = metal }, + }, + }) }); + const step = b.step("test-deepseek-v41-weights", "V4.1 configuration, Engram table plans and resident weight staging"); + step.dependOn(&b.addRunArtifact(host).step); + return step; +} diff --git a/zig/deepseek_v41_weights_checks.zig b/zig/deepseek_v41_weights_checks.zig new file mode 100644 index 0000000000..b3d4fedcac --- /dev/null +++ b/zig/deepseek_v41_weights_checks.zig @@ -0,0 +1,8 @@ +//! Host checks for V4.1 configuration, Engram table plans and resident weight staging; no kernels compiled. +test { + _ = @import("src/families/deepseek_v41/config.zig"); + _ = @import("src/families/deepseek_v41/view.zig"); + _ = @import("src/families/deepseek_v41/engram_hash.zig"); + _ = @import("src/families/deepseek_v41/load.zig"); + _ = @import("src/families/deepseek_v41/weights.zig"); +} diff --git a/zig/src/families/deepseek_v41/view.zig b/zig/src/families/deepseek_v41/view.zig new file mode 100644 index 0000000000..fbaa08ce39 --- /dev/null +++ b/zig/src/families/deepseek_v41/view.zig @@ -0,0 +1,25 @@ +//! A borrowed Metal buffer region; element precision is carried by each operator's explicit descriptor. +const std = @import("std"); +const mtl = @import("metal"); + +pub const View = struct { + buffer: mtl.Buffer, + offset: usize = 0, + + pub fn shifted(self: View, bytes: usize) View { + return .{ .buffer = self.buffer, .offset = self.offset + bytes }; + } + + pub fn require(self: View, bytes: usize, alignment: usize) !void { + if (self.offset % alignment != 0 or self.offset > self.buffer.length() or + bytes > self.buffer.length() - self.offset) return error.InvalidTensorView; + } + + /// Rows are strided elements; only the final row's used width is required. + pub fn requireRows(self: View, rows: usize, stride: usize, width: usize, element_bytes: usize) !void { + if (rows == 0 or width == 0 or stride < width) return error.InvalidTensorView; + const preceding = try std.math.mul(usize, rows - 1, stride); + const elements = try std.math.add(usize, preceding, width); + try self.require(try std.math.mul(usize, elements, element_bytes), element_bytes); + } +}; diff --git a/zig/src/families/deepseek_v41/weights.zig b/zig/src/families/deepseek_v41/weights.zig new file mode 100644 index 0000000000..10afd22356 --- /dev/null +++ b/zig/src/families/deepseek_v41/weights.zig @@ -0,0 +1,273 @@ +//! Strict pinned backbone schema and borrowed resident bindings; only expert descriptor tables are owned. +const std = @import("std"); +const mtl = @import("metal"); +const cp = @import("../../core/checkpoint_metal.zig"); +const st = @import("../../core/safetensors.zig"); +const exl3 = @import("../../core/exl3_projection.zig"); +const load = @import("load.zig"); + +pub const Tensor = cp.Tensor; +pub const Matrix = exl3.Matrix; +pub const Hc = struct { projection: Tensor, scale: Tensor, base: Tensor }; +pub const Compressor = struct { wkv: Matrix, wgate: ?Matrix, norm: Tensor }; +pub const Indexer = struct { wq_b: Matrix, wk: ?Matrix, weights_proj: Tensor, k_norm: ?Tensor }; +pub const Attention = struct { + wq_a: Matrix, + wq_b: Matrix, + wkv: Matrix, + wo_b: Matrix, + wo_a: [8]Matrix, + sink: Tensor, + q_norm: Tensor, + kv_norm: Tensor, + compressor: ?Compressor, + indexer: ?Indexer, +}; +/// Projection order is w1 (gate), w3 (up), w2 (down), for shared and routed experts alike. +pub const Moe = struct { + gate: Tensor, + bias: Tensor, + shared: [3]Matrix, + descriptors: [3]mtl.Buffer, +}; +pub const Engram = struct { table: Tensor, scales: Tensor, q_weight: Tensor, k_weight: Tensor, wkv: Matrix }; +pub const Layer = struct { + attn_norm: Tensor, + ffn_norm: Tensor, + hc_attn: Hc, + hc_ffn: Hc, + attn: Attention, + moe: Moe, + engram: ?Engram, +}; +pub const Model = struct { + embed: Tensor, + head: Matrix, + norm: Tensor, + layers: [40]Layer, + + /// Checkpoint buffers must outlive this borrowed model and every submitted command using it. + pub fn deinit(self: *Model) void { + for (&self.layers) |*layer| for (layer.moe.descriptors) |buffer| buffer.deinit(); + self.* = undefined; + } + pub fn registerResidency(self: *const Model, set: mtl.ResidencySet) void { + for (&self.layers) |*layer| for (layer.moe.descriptors) |buffer| set.add(buffer); + } + pub fn residentBytes(_: *const Model) usize { + return sizeofDescriptors(); + } +}; + +/// Admission adds these bytes to load.Plan.bytes; no dequantized weights or duplicate projections are allocated. +pub fn sizeofDescriptors() usize { + return 40 * 3 * 384 * @sizeOf(exl3.ExpertDescriptor); +} + +/// Every executed tensor is checked using headers only, before checkpoint materialization or model allocation. +pub fn validate(plan: *const load.Plan) !void { + try populate(false, plan, null); +} + +/// Bind resident tensors and marker payloads; route descriptors replace redundant host matrix metadata. +pub fn bind(device: mtl.Device, checkpoint: *const cp.Checkpoint) !Model { + var model: Model = undefined; + try populate(true, checkpoint, &model); + var owned: usize = 0; + errdefer for (0..owned) |i| model.layers[i / 3].moe.descriptors[i % 3].deinit(); + var prefix: [128]u8 = undefined; + for (&model.layers, 0..) |*layer, layer_id| { + inline for (.{ "w1", "w3", "w2" }, 0..) |part, projection| { + const bytes = 384 * @sizeOf(exl3.ExpertDescriptor); + const buffer = try device.buffer(bytes, mtl.ResourceOptions.shared | mtl.ResourceOptions.untracked); + layer.moe.descriptors[projection] = buffer; + owned += 1; + const out: [*]exl3.ExpertDescriptor = @ptrCast(@alignCast(buffer.contents())); + for (0..384) |expert| { + var m: Matrix = undefined; + const k: usize = if (projection == 2) 2304 else 5120; + const n: usize = if (projection == 2) 5120 else 2304; + try matrix(true, checkpoint, try named(&prefix, "layers.{d}.ffn.experts.{d}.{s}", .{ layer_id, expert, part }), k, n, &m); + out[expert] = try exl3.descriptor(m); + } + } + } + return model; +} + +/// A borrowed CPU/GPU matrix view for checkpoint qualification, using the same strict storage contract. +pub fn lookup(checkpoint: *const cp.Checkpoint, prefix: []const u8, k: usize, n: usize) !Matrix { + var out: Matrix = undefined; + try matrix(true, checkpoint, prefix, k, n, &out); + return out; +} + +fn named(buffer: []u8, comptime fmt: []const u8, args: anytype) ![]const u8 { + return std.fmt.bufPrint(buffer, fmt, args); +} + +fn plain(comptime binding: bool, source: anytype, name: []const u8, dtype: st.DType, shape: []const usize, destination: ?*Tensor) !void { + const tensor = try source.get(name); + if (tensor.dtype != dtype or tensor.rank != shape.len or + !std.mem.eql(usize, tensor.shape[0..tensor.rank], shape)) + { + std.log.err("unsupported backbone tensor {s}: {s} {any}", .{ name, @tagName(tensor.dtype), tensor.shape[0..tensor.rank] }); + return error.UnsupportedTensor; + } + const offset = if (binding) tensor.offset else tensor.begin; + if (offset % dtype.size() != 0) return error.InvalidTensorView; + if (binding) { + var count: usize = 1; + for (shape) |dim| count = try std.math.mul(usize, count, dim); + if (tensor.bytes != try std.math.mul(usize, count, dtype.size())) return error.UnsupportedTensor; + if (tensor.offset % dtype.size() != 0 or tensor.offset > tensor.buffer.length() or + tensor.bytes > tensor.buffer.length() - tensor.offset) return error.InvalidTensorView; + destination.?.* = tensor; + } +} + +fn has(source: anytype, name: []const u8) bool { + return source.has(name); +} + +const Scale = struct { name: []const u8, format: exl3.ScaleFormat }; +fn scale(comptime binding: bool, source: anytype, prefix: []const u8, size: usize, comptime fp: []const u8, comptime signs: []const u8, storage: []u8) !Scale { + var alternative: [160]u8 = undefined; + const f = try named(storage, "{s}.{s}", .{ prefix, fp }); + const s = try named(&alternative, "{s}.{s}", .{ prefix, signs }); + const have_fp = has(source, f); + if (have_fp == has(source, s)) return error.MissingOrAmbiguousExl3Scale; + const name = if (have_fp) f else try named(storage, "{s}.{s}", .{ prefix, signs }); + const t = try source.get(name); + if (t.rank != 1 or t.shape[0] != (if (have_fp) size else size / 16) or + t.dtype != (if (have_fp) st.DType.f16 else st.DType.i16)) return error.UnsupportedExl3Scale; + const offset = if (binding) t.offset else t.begin; + if (offset % 2 != 0) return error.InvalidTensorView; + if (binding) { + const bytes = (if (have_fp) size else size / 16) * 2; + if (t.bytes != bytes or t.offset % 2 != 0 or t.offset > t.buffer.length() or + bytes > t.buffer.length() - t.offset) return error.InvalidTensorView; + } + return .{ .name = name, .format = if (have_fp) .f16 else .packed_signs }; +} + +fn matrix(comptime binding: bool, source: anytype, prefix: []const u8, k: usize, n: usize, destination: ?*Matrix) !void { + var name: [160]u8 = undefined; + const trellis = try source.get(try named(&name, "{s}.trellis", .{prefix})); + if (trellis.dtype != .i16 or trellis.rank != 3 or trellis.shape[0] != k / 16 or + trellis.shape[1] != n / 16 or trellis.shape[2] % 8 != 0) return error.UnsupportedExl3Shape; + if ((if (binding) trellis.offset else trellis.begin) % 2 != 0) return error.InvalidTensorView; + const halves = trellis.shape[2] / 8; + if (halves < 2 or halves > 16 or (halves % 2 != 0 and halves > 7)) return error.UnsupportedExl3Width; + // Pinned checkpoint uses mul1 throughout; missing markers cannot silently select the legacy codebook. + const marker = try source.get(try named(&name, "{s}.mul1", .{prefix})); + if (marker.dtype != .i32 or marker.rank != 0) return error.UnsupportedExl3Marker; + if ((if (binding) marker.offset else marker.begin) % 4 != 0) return error.InvalidTensorView; + if (has(source, try named(&name, "{s}.mcg", .{prefix})) or + has(source, try named(&name, "{s}.3inst", .{prefix})) or + has(source, try named(&name, "{s}.bias", .{prefix})) or + has(source, try named(&name, "{s}.weight", .{prefix}))) return error.UnsupportedExl3Parts; + var su_name: [160]u8 = undefined; + var sv_name: [160]u8 = undefined; + const su = try scale(binding, source, prefix, k, "suh", "su", &su_name); + const sv = try scale(binding, source, prefix, n, "svh", "sv", &sv_name); + if (binding) { + const expected = k / 16 * (n / 16) * halves * 16; + if (trellis.bytes != expected or trellis.offset % 2 != 0 or + trellis.offset > trellis.buffer.length() or expected > trellis.buffer.length() - trellis.offset) + return error.InvalidTensorView; + if (marker.bytes != 4 or marker.offset % 4 != 0 or marker.offset > marker.buffer.length() or + 4 > marker.buffer.length() - marker.offset) return error.InvalidTensorView; + const marker_bytes = marker.buffer.contents()[marker.offset..][0..4]; + if (std.mem.readInt(u32, marker_bytes, .little) != exl3.format.Mul1Mul) return error.UnsupportedExl3Marker; + const suh = try source.get(su.name); + const svh = try source.get(sv.name); + destination.?.* = .{ + .trellis = .{ .buffer = trellis.buffer, .offset = trellis.offset }, + .suh = .{ .buffer = suh.buffer, .offset = suh.offset }, + .svh = .{ .buffer = svh.buffer, .offset = svh.offset }, + .k = k, + .n = n, + .width = .{ .halves = @intCast(halves) }, + .codebook = .mul1, + .suh_format = su.format, + .svh_format = sv.format, + }; + try destination.?.validate(); + } +} + +fn populate(comptime binding: bool, source: anytype, model: ?*Model) !void { + try plain(binding, source, "embed.weight", .bf16, &.{ 129280, 5120 }, if (binding) &model.?.embed else null); + try plain(binding, source, "norm.weight", .bf16, &.{5120}, if (binding) &model.?.norm else null); + try matrix(binding, source, "head", 5120, 129280, if (binding) &model.?.head else null); + var name: [160]u8 = undefined; + var prefix: [128]u8 = undefined; + for (0..40) |layer| { + const out: ?*Layer = if (binding) &model.?.layers[layer] else null; + try plain(binding, source, try named(&name, "layers.{d}.attn_norm.weight", .{layer}), .bf16, &.{5120}, if (binding) &out.?.attn_norm else null); + try plain(binding, source, try named(&name, "layers.{d}.ffn_norm.weight", .{layer}), .bf16, &.{5120}, if (binding) &out.?.ffn_norm else null); + inline for (.{ "attn", "ffn" }) |kind| { + const hc: ?*Hc = if (binding) (if (std.mem.eql(u8, kind, "attn")) &out.?.hc_attn else &out.?.hc_ffn) else null; + try plain(binding, source, try named(&name, "layers.{d}.hc_{s}_fn", .{ layer, kind }), .f32, &.{ 24, 20480 }, if (binding) &hc.?.projection else null); + try plain(binding, source, try named(&name, "layers.{d}.hc_{s}_scale", .{ layer, kind }), .f32, &.{3}, if (binding) &hc.?.scale else null); + try plain(binding, source, try named(&name, "layers.{d}.hc_{s}_base", .{ layer, kind }), .f32, &.{24}, if (binding) &hc.?.base else null); + } + const attn: ?*Attention = if (binding) &out.?.attn else null; + inline for (.{ "wq_a", "wq_b", "wkv", "wo_b" }, .{ 5120, 1280, 5120, 8192 }, .{ 1280, 32768, 512, 5120 }) |part, k, n| { + try matrix(binding, source, try named(&prefix, "layers.{d}.attn.{s}", .{ layer, part }), k, n, if (binding) &@field(attn.?, part) else null); + } + for (0..8) |slice| try matrix(binding, source, try named(&prefix, "layers.{d}.attn.wo_a.slice.{d}", .{ layer, slice }), 4096, 1024, if (binding) &attn.?.wo_a[slice] else null); + try plain(binding, source, try named(&name, "layers.{d}.attn.attn_sink", .{layer}), .f32, &.{64}, if (binding) &attn.?.sink else null); + try plain(binding, source, try named(&name, "layers.{d}.attn.q_norm.weight", .{layer}), .bf16, &.{1280}, if (binding) &attn.?.q_norm else null); + try plain(binding, source, try named(&name, "layers.{d}.attn.kv_norm.weight", .{layer}), .bf16, &.{512}, if (binding) &attn.?.kv_norm else null); + const owner = layer == 2 or layer == 8 or layer == 14 or layer == 20; + if (binding) { + attn.?.compressor = if (owner) @as(Compressor, undefined) else null; + attn.?.indexer = if (owner or layer == 24 or layer == 28 or layer == 32 or layer == 36) + @as(Indexer, undefined) + else + null; + } + if (owner) { + const compressor: ?*Compressor = if (binding) &attn.?.compressor.? else null; + try matrix(binding, source, try named(&prefix, "layers.{d}.attn.compressor.wkv", .{layer}), 5120, 512, if (binding) &compressor.?.wkv else null); + try plain(binding, source, try named(&name, "layers.{d}.attn.compressor.norm.weight", .{layer}), .bf16, &.{512}, if (binding) &compressor.?.norm else null); + if (binding) compressor.?.wgate = if (layer < 20) @as(Matrix, undefined) else null; + if (layer < 20) try matrix(binding, source, try named(&prefix, "layers.{d}.attn.compressor.wgate", .{layer}), 5120, 512, if (binding) &compressor.?.wgate.? else null); + } + if (owner or layer == 24 or layer == 28 or layer == 32 or layer == 36) { + const indexer: ?*Indexer = if (binding) &attn.?.indexer.? else null; + try matrix(binding, source, try named(&prefix, "layers.{d}.attn.indexer.wq_b", .{layer}), 1280, 4096, if (binding) &indexer.?.wq_b else null); + try plain(binding, source, try named(&name, "layers.{d}.attn.indexer.weights_proj.weight", .{layer}), .f16, &.{ 32, 5120 }, if (binding) &indexer.?.weights_proj else null); + if (binding) { + indexer.?.wk = if (owner) @as(Matrix, undefined) else null; + indexer.?.k_norm = if (owner) @as(Tensor, undefined) else null; + } + if (owner) { + try matrix(binding, source, try named(&prefix, "layers.{d}.attn.indexer.wk", .{layer}), 512, 128, if (binding) &indexer.?.wk.? else null); + try plain(binding, source, try named(&name, "layers.{d}.attn.indexer.k_norm.weight", .{layer}), .bf16, &.{128}, if (binding) &indexer.?.k_norm.? else null); + } + } + const moe: ?*Moe = if (binding) &out.?.moe else null; + try plain(binding, source, try named(&name, "layers.{d}.ffn.gate.weight", .{layer}), .f16, &.{ 384, 5120 }, if (binding) &moe.?.gate else null); + try plain(binding, source, try named(&name, "layers.{d}.ffn.gate.bias", .{layer}), .f16, &.{384}, if (binding) &moe.?.bias else null); + inline for (.{ "w1", "w3", "w2" }, 0..) |part, projection| { + const k: usize = if (projection == 2) 2304 else 5120; + const n: usize = if (projection == 2) 5120 else 2304; + try matrix(binding, source, try named(&prefix, "layers.{d}.ffn.shared_experts.{s}", .{ layer, part }), k, n, if (binding) &moe.?.shared[projection] else null); + if (!binding) for (0..384) |expert| try matrix(false, source, try named(&prefix, "layers.{d}.ffn.experts.{d}.{s}", .{ layer, expert, part }), k, n, null); + } + const engram = layer == 1 or layer == 14; + if (binding) out.?.engram = if (engram) @as(Engram, undefined) else null; + if (engram) { + const e: ?*Engram = if (binding) &out.?.engram.? else null; + const rows: usize = if (layer == 1) 384006168 else 384016682; + try plain(binding, source, try named(&name, "layers.{d}.engram.embed.weight", .{layer}), .f8_e4m3, &.{ rows, 256 }, if (binding) &e.?.table else null); + try plain(binding, source, try named(&name, "layers.{d}.engram.embed.scale", .{layer}), .f8_e8m0, &.{ rows, 8 }, if (binding) &e.?.scales else null); + try plain(binding, source, try named(&name, "layers.{d}.engram.q_weight", .{layer}), .f32, &.{ 4, 5120 }, if (binding) &e.?.q_weight else null); + try plain(binding, source, try named(&name, "layers.{d}.engram.k_weight", .{layer}), .f32, &.{ 4, 5120 }, if (binding) &e.?.k_weight else null); + try matrix(binding, source, try named(&prefix, "layers.{d}.engram.wkv", .{layer}), 6144, 25600, if (binding) &e.?.wkv else null); + } + } +} diff --git a/zig/src/tensorfold.zig b/zig/src/tensorfold.zig index 2ed6d0a1af..9c660ed577 100644 --- a/zig/src/tensorfold.zig +++ b/zig/src/tensorfold.zig @@ -22,6 +22,11 @@ pub const row_projection = @import("core/row_projection.zig"); pub const affine_mm = @import("core/affine_mm.zig"); /// The checkpoint formats (MLX affine and its fp64 reader for the device checks, exact widen). pub const quant = @import("core/quant/quant.zig"); +pub const deepseek_v41_config = @import("families/deepseek_v41/config.zig"); +pub const deepseek_v41_load = @import("families/deepseek_v41/load.zig"); +pub const deepseek_v41_weights = @import("families/deepseek_v41/weights.zig"); +pub const deepseek_v41_view = @import("families/deepseek_v41/view.zig"); +pub const deepseek_v41_engram_hash = @import("families/deepseek_v41/engram_hash.zig"); pub const exl3_format = @import("core/exl3_format.zig"); pub const exl3_projection = @import("core/exl3_projection.zig"); pub const exlr8 = @import("core/exlr8.zig"); From c53616ff891c1bd8d99c59a40a1857c8c56df035 Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Mon, 12 Oct 2026 21:12:42 -0700 Subject: [PATCH 02/33] DeepSeek V4.1 Flash: Metal operators with fp64-dot references and the qualification runner The Engram hash/commit, sparse and compressed attention (CSA2), the hyperconnection mix, routed and shared experts, the cache and rotary glue as source Metal kernels, with zig/tests/deepseek_v41_reference as the independent CPU oracle: plain reductions in fp64, no native statistics, no injected coefficients. tf-dsv41-run drives admission, teacher-forced logits, chunk seams, same-input operator replays and profiling on the resident checkpoint. The retained whole-model gate stays failed at 4.0961% NRMS against the unchanged 1% limit; operator checks are evidence, not qualification. --- zig/build/deepseek_v41.zig | 38 ++ zig/deepseek_v41_checks.zig | 13 + zig/deepseek_v41_reference_checks.zig | 7 + zig/deepseek_v41_run.zig | 306 +++++++++++++ zig/kernels/metal/core/expert_groups.metal | 36 ++ zig/kernels/metal/deepseek_v41/csa2.metal | 102 +++++ zig/kernels/metal/deepseek_v41/engram.metal | 150 +++++++ zig/kernels/metal/deepseek_v41/index.metal | 103 +++++ zig/kernels/metal/deepseek_v41/kv.metal | 141 ++++++ zig/kernels/metal/deepseek_v41/mhc.metal | 120 +++++ zig/kernels/metal/deepseek_v41/ops.metal | 124 ++++++ zig/kernels/metal/deepseek_v41/rope.metal | 17 + zig/kernels/metal/deepseek_v41/route.metal | 53 +++ zig/kernels/metal/sources.zig | 11 + zig/src/core/expert_groups.zig | 101 +++++ zig/src/core/gpu_counters.zig | 78 +++- zig/src/families/deepseek_v41/attention.zig | 258 +++++++++++ zig/src/families/deepseek_v41/csa2.zig | 77 ++++ zig/src/families/deepseek_v41/csa2_index.zig | 139 ++++++ zig/src/families/deepseek_v41/csa2_test.zig | 309 +++++++++++++ zig/src/families/deepseek_v41/engram.zig | 203 +++++++++ zig/src/families/deepseek_v41/engram_test.zig | 247 +++++++++++ .../fixtures/hc_attention_rms.json | 25 ++ .../deepseek_v41/index_precision_test.zig | 57 +++ zig/src/families/deepseek_v41/kv.zig | 162 +++++++ zig/src/families/deepseek_v41/kv_test.zig | 234 ++++++++++ zig/src/families/deepseek_v41/mhc.zig | 95 ++++ zig/src/families/deepseek_v41/mhc_test.zig | 209 +++++++++ zig/src/families/deepseek_v41/ops.zig | 110 +++++ zig/src/families/deepseek_v41/ops_check.zig | 191 ++++++++ zig/src/families/deepseek_v41/profiler.zig | 93 ++++ zig/src/families/deepseek_v41/rope.zig | 134 ++++++ zig/src/families/deepseek_v41/route.zig | 39 ++ zig/src/families/deepseek_v41/route_test.zig | 152 +++++++ .../families/deepseek_v41/test_support.zig | 40 ++ zig/src/families/deepseek_v41/workspace.zig | 107 +++++ zig/src/metal/counters.zig | 4 +- zig/src/metal/metal.zig | 1 + zig/src/tensorfold.zig | 4 + .../deepseek_v41_reference/attention_math.zig | 298 +++++++++++++ .../deepseek_v41_reference/engram_math.zig | 106 +++++ zig/tests/deepseek_v41_reference/math.zig | 289 +++++++++++++ .../operator_attention.zig | 155 +++++++ .../operator_diagnostic.zig | 124 ++++++ .../deepseek_v41_reference/operator_hc.zig | 98 +++++ .../operator_history.zig | 298 +++++++++++++ .../deepseek_v41_reference/operator_moe.zig | 409 ++++++++++++++++++ .../deepseek_v41_reference/operators.zig | 92 ++++ zig/tests/deepseek_v41_reference/profile.zig | 52 +++ .../deepseek_v41_reference/projection.zig | 261 +++++++++++ zig/tests/deepseek_v41_reference/qualify.zig | 97 +++++ .../deepseek_v41_reference/reference.zig | 176 ++++++++ .../rope_frequencies.json | 14 + zig/tests/deepseek_v41_reference/trace.zig | 123 ++++++ 54 files changed, 6859 insertions(+), 23 deletions(-) create mode 100644 zig/deepseek_v41_checks.zig create mode 100644 zig/deepseek_v41_reference_checks.zig create mode 100644 zig/deepseek_v41_run.zig create mode 100644 zig/kernels/metal/core/expert_groups.metal create mode 100644 zig/kernels/metal/deepseek_v41/csa2.metal create mode 100644 zig/kernels/metal/deepseek_v41/engram.metal create mode 100644 zig/kernels/metal/deepseek_v41/index.metal create mode 100644 zig/kernels/metal/deepseek_v41/kv.metal create mode 100644 zig/kernels/metal/deepseek_v41/mhc.metal create mode 100644 zig/kernels/metal/deepseek_v41/ops.metal create mode 100644 zig/kernels/metal/deepseek_v41/rope.metal create mode 100644 zig/kernels/metal/deepseek_v41/route.metal create mode 100644 zig/src/core/expert_groups.zig create mode 100644 zig/src/families/deepseek_v41/attention.zig create mode 100644 zig/src/families/deepseek_v41/csa2.zig create mode 100644 zig/src/families/deepseek_v41/csa2_index.zig create mode 100644 zig/src/families/deepseek_v41/csa2_test.zig create mode 100644 zig/src/families/deepseek_v41/engram.zig create mode 100644 zig/src/families/deepseek_v41/engram_test.zig create mode 100644 zig/src/families/deepseek_v41/fixtures/hc_attention_rms.json create mode 100644 zig/src/families/deepseek_v41/index_precision_test.zig create mode 100644 zig/src/families/deepseek_v41/kv.zig create mode 100644 zig/src/families/deepseek_v41/kv_test.zig create mode 100644 zig/src/families/deepseek_v41/mhc.zig create mode 100644 zig/src/families/deepseek_v41/mhc_test.zig create mode 100644 zig/src/families/deepseek_v41/ops.zig create mode 100644 zig/src/families/deepseek_v41/ops_check.zig create mode 100644 zig/src/families/deepseek_v41/profiler.zig create mode 100644 zig/src/families/deepseek_v41/rope.zig create mode 100644 zig/src/families/deepseek_v41/route.zig create mode 100644 zig/src/families/deepseek_v41/route_test.zig create mode 100644 zig/src/families/deepseek_v41/test_support.zig create mode 100644 zig/src/families/deepseek_v41/workspace.zig create mode 100644 zig/tests/deepseek_v41_reference/attention_math.zig create mode 100644 zig/tests/deepseek_v41_reference/engram_math.zig create mode 100644 zig/tests/deepseek_v41_reference/math.zig create mode 100644 zig/tests/deepseek_v41_reference/operator_attention.zig create mode 100644 zig/tests/deepseek_v41_reference/operator_diagnostic.zig create mode 100644 zig/tests/deepseek_v41_reference/operator_hc.zig create mode 100644 zig/tests/deepseek_v41_reference/operator_history.zig create mode 100644 zig/tests/deepseek_v41_reference/operator_moe.zig create mode 100644 zig/tests/deepseek_v41_reference/operators.zig create mode 100644 zig/tests/deepseek_v41_reference/profile.zig create mode 100644 zig/tests/deepseek_v41_reference/projection.zig create mode 100644 zig/tests/deepseek_v41_reference/qualify.zig create mode 100644 zig/tests/deepseek_v41_reference/reference.zig create mode 100644 zig/tests/deepseek_v41_reference/rope_frequencies.json create mode 100644 zig/tests/deepseek_v41_reference/trace.zig diff --git a/zig/build/deepseek_v41.zig b/zig/build/deepseek_v41.zig index 6b196fedff..2e419e9755 100644 --- a/zig/build/deepseek_v41.zig +++ b/zig/build/deepseek_v41.zig @@ -20,3 +20,41 @@ pub fn weights( step.dependOn(&b.addRunArtifact(host).step); return step; } + +pub fn checks( + b: *std.Build, + target: std.Build.ResolvedTarget, + optimize: std.builtin.OptimizeMode, + metal: *std.Build.Module, + sources: *std.Build.Module, + engine: *std.Build.Module, +) struct { device: *std.Build.Step, reference: *std.Build.Step } { + const device = b.addTest(.{ .root_module = b.createModule(.{ + .root_source_file = b.path("zig/deepseek_v41_checks.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + .imports = &.{ + .{ .name = "metal", .module = metal }, + .{ .name = "kernel_sources", .module = sources }, + }, + }) }); + const device_step = b.step("test-deepseek-v41", "Experimental DS4.1 EXL3, Engram, mHC and sparse attention on Metal"); + device_step.dependOn(&b.addRunArtifact(device).step); + const reference = b.addTest(.{ + .root_module = b.createModule(.{ + .root_source_file = b.path("zig/deepseek_v41_reference_checks.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + .imports = &.{ + .{ .name = "tensorfold", .module = engine }, + .{ .name = "metal", .module = metal }, + }, + }), + .filters = &.{"tests.deepseek_v41_reference."}, + }); + const reference_step = b.step("test-deepseek-v41-reference", "Independent CPU teacher-oracle arithmetic boundaries"); + reference_step.dependOn(&b.addRunArtifact(reference).step); + return .{ .device = device_step, .reference = reference_step }; +} diff --git a/zig/deepseek_v41_checks.zig b/zig/deepseek_v41_checks.zig new file mode 100644 index 0000000000..755a35c618 --- /dev/null +++ b/zig/deepseek_v41_checks.zig @@ -0,0 +1,13 @@ +//! Source-built V4.1 operator qualification; real Metal execution, no checkpoint needed. +test { + _ = @import("src/core/exl3_projection_gpu_test.zig"); + _ = @import("src/families/deepseek_v41/engram.zig"); + _ = @import("src/families/deepseek_v41/csa2.zig"); + _ = @import("src/families/deepseek_v41/ops_check.zig"); + _ = @import("src/families/deepseek_v41/load.zig"); + _ = @import("src/families/deepseek_v41/index_precision_test.zig"); + _ = @import("src/families/deepseek_v41/config.zig"); + _ = @import("src/families/deepseek_v41/profiler.zig"); + _ = @import("src/core/gpu_counters.zig"); + _ = @import("src/core/expert_groups.zig"); +} diff --git a/zig/deepseek_v41_reference_checks.zig b/zig/deepseek_v41_reference_checks.zig new file mode 100644 index 0000000000..61aa30f788 --- /dev/null +++ b/zig/deepseek_v41_reference_checks.zig @@ -0,0 +1,7 @@ +//! Independent CPU arithmetic boundaries used by the resident-checkpoint teacher oracle. +test { + _ = @import("tests/deepseek_v41_reference/projection.zig"); + _ = @import("tests/deepseek_v41_reference/math.zig"); + _ = @import("tests/deepseek_v41_reference/attention_math.zig"); + _ = @import("tests/deepseek_v41_reference/qualify.zig"); +} diff --git a/zig/deepseek_v41_run.zig b/zig/deepseek_v41_run.zig new file mode 100644 index 0000000000..e6ca81e783 --- /dev/null +++ b/zig/deepseek_v41_run.zig @@ -0,0 +1,306 @@ +//! Full-checkpoint admission and execution receipts; no synthetic weights or replay substitute. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const ds = tf.deepseek_v41; +const config = tf.deepseek_v41_config; +const load = tf.deepseek_v41_load; +const weights = tf.deepseek_v41_weights; +const process_memory = @import("src/server/process_memory.zig"); +const oracle = @import("tests/deepseek_v41_reference/qualify.zig"); +const profile = @import("tests/deepseek_v41_reference/profile.zig"); +const trace = @import("tests/deepseek_v41_reference/trace.zig"); +const operators = @import("tests/deepseek_v41_reference/operators.zig"); +const attention_operators = @import("tests/deepseek_v41_reference/operator_attention.zig"); +const history_operators = @import("tests/deepseek_v41_reference/operator_history.zig"); +const hc_operators = @import("tests/deepseek_v41_reference/operator_hc.zig"); +const moe_operators = @import("tests/deepseek_v41_reference/operator_moe.zig"); + +fn usage(code: u8) noreturn { + std.debug.print("usage: tf-dsv41-run plan MODEL TABLES\n" ++ + " tf-dsv41-run teacher MODEL TABLES IDS_JSON OUT_DIR [CHUNK]\n" ++ + " tf-dsv41-run seams MODEL TABLES IDS_JSON OUT_DIR\n" ++ + " tf-dsv41-run oracle MODEL TABLES IDS_JSON OUT_DIR\n" ++ + " tf-dsv41-run profile MODEL TABLES IDS_JSON OUT_DIR\n" ++ + " tf-dsv41-run trace MODEL TABLES IDS_JSON OUT_DIR\n" ++ + " tf-dsv41-run operators MODEL TABLES OUT_DIR\n" ++ + " tf-dsv41-run attention-sameinput MODEL TABLES TRACE_CPU_BIN OUT_DIR [LAYER]\n" ++ + " tf-dsv41-run attention-history-sameinput MODEL TABLES TRACE_DIR OUT_DIR TOKEN LAYER\n" ++ + " tf-dsv41-run hc-sameinput MODEL TABLES TRACE_CPU_BIN OUT_DIR [LAYER]\n" ++ + " tf-dsv41-run moe-sameinput MODEL TABLES TRACE_GPU_BIN OUT_DIR [LAYER]\n" ++ + " tf-dsv41-run generate MODEL TABLES IDS_JSON OUT_DIR [TOKENS]\n" ++ + "IDS_JSON: nonempty JSON array of token IDs below 129280.\n" ++ + "Limits: oracle/trace 16 IDs; profile 256 IDs; teacher/seams 4096 IDs.\n" ++ + "History: TOKEN below 4096; retained normalized prefix at most 256 MiB.\n" ++ + "Local diagnostics do not qualify the model. Oracle retains the independent 1% NRMS gate.\n", .{}); + std.process.exit(code); +} +fn number(s: []const u8) !u32 { + return std.fmt.parseInt(u32, s, 10); +} +fn validArguments(args: []const [:0]const u8) bool { + if (args.len < 2) return false; + const mode = args[1]; + for ([_][]const u8{ "plan", "operators" }, [_]usize{ 4, 5 }) |name, count| + if (std.mem.eql(u8, mode, name)) return args.len == count; + if (std.mem.eql(u8, mode, "attention-history-sameinput")) return args.len == 8; + for ([_][]const u8{ "teacher", "generate", "attention-sameinput", "hc-sameinput", "moe-sameinput" }) |name| + if (std.mem.eql(u8, mode, name)) return args.len == 6 or args.len == 7; + for ([_][]const u8{ "seams", "oracle", "profile", "trace" }) |name| + if (std.mem.eql(u8, mode, name)) return args.len == 6; + return false; +} +fn argmax(row: []const f32) !u32 { + var best: u32 = 0; + if (row.len == 0) return error.EmptyLogits; + var score = -std.math.inf(f32); + for (row, 0..) |v, i| { + if (!std.math.isFinite(v)) return error.NonfiniteLogits; + if (v > score) { + best = @intCast(i); + score = v; + } + } + return best; +} +fn write(io: std.Io, a: std.mem.Allocator, dir: []const u8, name: []const u8, data: []const u8) !void { + const path = try std.fs.path.join(a, &.{ dir, name }); + defer a.free(path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = data }); +} +const Counters = struct { disk_read: u64, pageins: u64, footprint: u64 }; +/// Darwin's rusage_info_v4 fields the runner reads around execution (page-ins, disk reads, footprint). +const RUsage = extern struct { + uuid: [16]u8, + user_time: u64, + system_time: u64, + pkg_idle_wkups: u64, + interrupt_wkups: u64, + pageins: u64, + wired_size: u64, + resident_size: u64, + phys_footprint: u64, + proc_start_abstime: u64, + proc_exit_abstime: u64, + child_user_time: u64, + child_system_time: u64, + child_pkg_idle_wkups: u64, + child_interrupt_wkups: u64, + child_pageins: u64, + child_elapsed_abstime: u64, + diskio_bytesread: u64, +}; +fn counters() !Counters { + const proc_pid_rusage = struct { + extern "c" fn proc_pid_rusage(pid: c_int, flavor: c_int, buffer: *anyopaque) c_int; + }.proc_pid_rusage; + const getpid = struct { + extern "c" fn getpid() c_int; + }.getpid; + var r: RUsage = undefined; + if (proc_pid_rusage(getpid(), 4, &r) != 0) return error.MemoryCountersUnavailable; + return .{ .disk_read = r.diskio_bytesread, .pageins = r.pageins, .footprint = r.phys_footprint }; +} +fn teacher(m: *ds.Model, ids: []const u32, chunk: u32, out: []f32) !f64 { + if (chunk == 0 or chunk > m.workspace.plan.rows or ids.len > m.cache.plan.capacity) + return error.InvalidTeacherWindow; + if (out.len != ids.len * 129280) return error.InvalidTeacherOutputLength; + try m.reset(); + var elapsed: f64 = 0; + var start: usize = 0; + while (start < ids.len) { + const end = @min(start + chunk, ids.len); + const row = try m.forward(ids[start..end], .all); + @memcpy(out[start * 129280 .. end * 129280], row); + for (row) |value| if (!std.math.isFinite(value)) return error.NonfiniteLogits; + elapsed += m.last_gpu_seconds; + start = end; + } + return elapsed; +} + +pub fn main(init: std.process.Init) !void { + const a = init.gpa; + const io = init.io; + const args = try init.minimal.args.toSlice(init.arena.allocator()); + if (args.len == 2 and (std.mem.eql(u8, args[1], "--help") or std.mem.eql(u8, args[1], "-h"))) usage(0); + if (!validArguments(args)) usage(2); + const arena = init.arena.allocator(); + const mode = args[1]; + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + if (std.mem.eql(u8, mode, "plan")) { + const cfg = try config.Config.load(arena, io, args[2]); + var plan = try load.Plan.open(a, io, args[2], args[3]); + defer plan.deinit(); + try weights.validate(&plan); + const device = try mtl.Device.init(); + defer device.deinit(); + std.debug.print("{{\"mode\":\"plan\",\"weight_bytes\":{d},\"shards\":{d},\"tensors\":{d},\"context\":{d},\"metal_working_set_bytes\":{d}}}\n", .{ plan.bytes, plan.shards.items.len, plan.tensors.count(), cfg.text_config.max_position_embeddings, device.maxWorkingSet() }); + return; + } + const is_operators = std.mem.eql(u8, mode, "operators"); + const is_attention = std.mem.eql(u8, mode, "attention-sameinput"); + const is_attention_history = std.mem.eql(u8, mode, "attention-history-sameinput"); + const is_hc = std.mem.eql(u8, mode, "hc-sameinput"); + const is_moe = std.mem.eql(u8, mode, "moe-sameinput"); + const max_ids: usize = if (std.mem.eql(u8, mode, "oracle") or std.mem.eql(u8, mode, "trace")) 16 else if (std.mem.eql(u8, mode, "profile")) ds.prefill_step else if (std.mem.eql(u8, mode, "teacher") or std.mem.eql(u8, mode, "seams")) 4096 else 1048576; + const input = if (is_operators or is_attention or is_attention_history or is_hc or is_moe) &.{} else try std.Io.Dir.cwd().readFileAlloc(io, args[4], arena, .limited(max_ids * 12 + 2)); + const ids: []const u32 = if (is_operators or is_attention or is_attention_history or is_hc or is_moe) &.{} else try std.json.parseFromSliceLeaky([]const u32, arena, input, .{}); + if (!is_operators and !is_attention and !is_attention_history and !is_hc and !is_moe and (ids.len == 0 or ids.len > 1048576)) return error.InvalidPrompt; + if (ids.len > max_ids) return error.InvalidPrompt; + for (ids) |id| if (id >= 129280) return error.InvalidToken; + const is_oracle = std.mem.eql(u8, mode, "oracle"); + const is_trace = std.mem.eql(u8, mode, "trace"); + const dir = args[if (is_operators) @as(usize, 4) else 5]; + try std.Io.Dir.cwd().createDirPath(io, dir); + const map = try std.fs.path.join(arena, &.{ args[2], "engram-token-map.bin" }); + const map_blob = if (is_oracle or is_trace) + try std.Io.Dir.cwd().readFileAlloc(io, map, arena, .limited(2 << 20)) + else + &.{}; + const model = try a.create(ds.Model); + defer a.destroy(model); + const begin = mtl.clock.seconds(); + model.* = try ds.Model.open(a, io, .{ .dir = args[2], .table_dir = args[3], .token_map = map }); + defer model.deinit(); + const ready = mtl.clock.seconds(); + const before = try counters(); + var execution_after: ?Counters = null; + var execution_error: ?anyerror = null; + execute(a, io, model, args, ids, map_blob, dir, before, &execution_after) catch |err| { + execution_error = err; + }; + const after = execution_after orelse try counters(); + if (after.disk_read < before.disk_read or after.pageins < before.pageins) return error.NonmonotonicCounters; + const resident_gate_passed = after.disk_read == before.disk_read; + const receipt = try std.json.Stringify.valueAlloc(a, .{ + .mode = mode, + .evidence_origin = "current_invocation_not_historical_qualification", + .context_capacity = model.cache.plan.capacity, + .consumed_tokens = model.cache.position, + .weight_bytes = model.checkpoint.residentBytes(), + .resident_allocation_bytes = model.residency.allocatedSize(), + .ready_footprint_bytes = before.footprint, + .final_footprint_bytes = after.footprint, + .load_seconds = ready - begin, + .measurement_scope = if (execution_after != null) "model_execution" else "execution_and_diagnostic_io", + .execution_disk_read_bytes = after.disk_read - before.disk_read, + .execution_pageins = after.pageins - before.pageins, + .resident_gate_passed = resident_gate_passed, + .execution_error = if (execution_error) |err| @as(?[]const u8, @errorName(err)) else null, + }, .{}); + defer a.free(receipt); + std.debug.print("{s}\n", .{receipt}); + try write(io, a, dir, "residency.json", receipt); + if (!resident_gate_passed) std.debug.print("residency gate failed: ResidentExecutionReadFromDisk\n", .{}); + if (execution_error) |err| return err; + if (!resident_gate_passed) return error.ResidentExecutionReadFromDisk; +} + +fn execute(a: std.mem.Allocator, io: std.Io, model: *ds.Model, args: []const [:0]const u8, ids: []const u32, map_blob: []const u8, dir: []const u8, before: Counters, execution_after: *?Counters) !void { + const mode = args[1]; + const is_operators = std.mem.eql(u8, mode, "operators"); + const is_attention = std.mem.eql(u8, mode, "attention-sameinput"); + const is_attention_history = std.mem.eql(u8, mode, "attention-history-sameinput"); + const is_hc = std.mem.eql(u8, mode, "hc-sameinput"); + const is_moe = std.mem.eql(u8, mode, "moe-sameinput"); + const is_oracle = std.mem.eql(u8, mode, "oracle"); + const is_profile = std.mem.eql(u8, mode, "profile"); + const is_trace = std.mem.eql(u8, mode, "trace"); + const is_teacher = std.mem.eql(u8, mode, "teacher"); + const is_seams = std.mem.eql(u8, mode, "seams"); + if (is_operators) { + try operators.run(a, io, model, dir); + } else if (is_attention) { + try attention_operators.runAttention(a, io, model, args[4], dir, if (args.len > 6) try number(args[6]) else 0); + } else if (is_attention_history) { + try history_operators.runAttentionHistory(a, io, model, args[4], dir, try number(args[6]), try number(args[7])); + } else if (is_hc) { + try hc_operators.runHyperconnections(a, io, model, args[4], dir, if (args.len > 6) try number(args[6]) else 0); + } else if (is_moe) { + try moe_operators.runMoe(a, io, model, args[4], dir, if (args.len > 6) try number(args[6]) else 21); + } else if (is_oracle) { + try operators.run(a, io, model, dir); + _ = try oracle.run(a, io, model, ids, map_blob, dir); + } else if (is_profile) { + try profile.run(a, io, model, ids, dir); + } else if (is_trace) { + try trace.run(a, io, model, ids, map_blob, dir); + } else if (is_teacher or is_seams) { + const chunk = if (args.len > 6) try number(args[6]) else model.workspace.plan.rows; + if (chunk == 0 or chunk > model.workspace.plan.rows) return error.InvalidPrefillChunk; + const outputs = try a.alloc(f32, ids.len * 129280); + defer a.free(outputs); + const elapsed = try teacher(model, ids, chunk, outputs); + // Capture execution before writing the receipt, not filesystem metadata reads from the receipt itself. + if (is_seams) { + const alternative = try a.alloc(f32, outputs.len); + defer a.free(alternative); + for ([_]u32{ 1, 2, 3, 7, 31, 127, 128, 129, 255, 256 }) |split| { + if (split > model.workspace.plan.rows) continue; + _ = try teacher(model, ids, split, alternative); + var error_max: f64 = 0; + var disagreements: usize = 0; + var bit_disagreements: usize = 0; + for (outputs, alternative) |x, y| { + error_max = @max(error_max, @abs(@as(f64, x) - y)); + bit_disagreements += @intFromBool(@as(u32, @bitCast(x)) != @as(u32, @bitCast(y))); + } + for (0..ids.len) |i| { + if (try argmax(outputs[i * 129280 ..][0..129280]) != try argmax(alternative[i * 129280 ..][0..129280])) + disagreements += 1; + } + std.debug.print("{{\"mode\":\"seam\",\"chunk\":{d},\"max_logit_error\":{d},\"bit_disagreements\":{d},\"greedy_disagreements\":{d}}}\n", .{ split, error_max, bit_disagreements, disagreements }); + if (disagreements != 0 or bit_disagreements != 0) { + var name: [48]u8 = undefined; + const file = try std.fmt.bufPrint(&name, "seam-{d}-logits.f32", .{split}); + execution_after.* = try counters(); + try write(io, a, dir, file, std.mem.sliceAsBytes(alternative)); + return error.ChunkSeamMismatch; + } + } + } + std.debug.print("{{\"mode\":\"teacher\",\"tokens\":{d},\"gpu_seconds\":{d}}}\n", .{ ids.len, elapsed }); + execution_after.* = try counters(); + try write(io, a, dir, "logits.f32", std.mem.sliceAsBytes(outputs)); + } else if (std.mem.eql(u8, mode, "generate")) { + const n = if (args.len > 6) try number(args[6]) else 64; + if (ids.len > model.cache.plan.capacity or n == 0 or n > model.cache.plan.capacity - ids.len) + return error.ContextFull; + try model.reset(); + const prefill_start = mtl.clock.seconds(); + var row: []const f32 = &.{}; + var start: usize = 0; + var progress_at = prefill_start; + while (start < ids.len) { + const end = @min(start + model.workspace.plan.rows, ids.len); + row = try model.forward(ids[start..end], if (end == ids.len) .last else .none); + start = end; + const now = mtl.clock.seconds(); + if (now - progress_at >= 60 or start == ids.len) { + const memory = try counters(); + std.debug.print("{{\"mode\":\"prefill_progress\",\"completed_tokens\":{d},\"prompt_tokens\":{d},\"elapsed_seconds\":{d},\"footprint_bytes\":{d},\"disk_read_bytes\":{d},\"pageins\":{d}}}\n", .{ start, ids.len, now - prefill_start, memory.footprint, memory.disk_read - before.disk_read, memory.pageins - before.pageins }); + progress_at = now; + } + } + const prefill_end = mtl.clock.seconds(); + const generated = try a.alloc(u32, n); + defer a.free(generated); + var gpu_seconds: f64 = 0; + const decode_start = mtl.clock.seconds(); + for (generated, 0..) |*token, i| { + token.* = try argmax(row); + if (i + 1 < generated.len) { + row = try model.forward(&.{token.*}, .last); + gpu_seconds += model.last_gpu_seconds; + } + } + const decode_end = mtl.clock.seconds(); + execution_after.* = try counters(); + const json = try std.json.Stringify.valueAlloc(a, generated, .{}); + defer a.free(json); + try write(io, a, dir, "tokens.json", json); + std.debug.print("{{\"mode\":\"generate\",\"prompt_tokens\":{d},\"completion_tokens\":{d},\"prefill_seconds\":{d},\"decode_seconds\":{d},\"decode_gpu_seconds\":{d}}}\n", .{ ids.len, n, prefill_end - prefill_start, decode_end - decode_start, gpu_seconds }); + } else usage(2); +} diff --git a/zig/kernels/metal/core/expert_groups.metal b/zig/kernels/metal/core/expert_groups.metal new file mode 100644 index 0000000000..be543d5780 --- /dev/null +++ b/zig/kernels/metal/core/expert_groups.metal @@ -0,0 +1,36 @@ +// Group routed slots into padded eight-row expert tiles; scatter preserves original token and output IDs. +#include +using namespace metal; +struct TfExpertGroups { uint rows, experts, repeat, padded; }; +#define TF_GROUP_ARGS \ + const device uint* ids [[buffer(0)]], device atomic_uint* counts [[buffer(1)]], \ + device uint* offsets [[buffer(2)]], device uint* ranks [[buffer(3)]], \ + device uint* expert_ids [[buffer(4)]], device uint* token_ids [[buffer(5)]], \ + device uint* slot_ids [[buffer(6)]], constant TfExpertGroups& d [[buffer(7)]], \ + uint i [[thread_position_in_grid]] + +kernel void tf_expert_group_clear(TF_GROUP_ARGS) { + if (i < d.experts) atomic_store_explicit(counts + i, 0u, memory_order_relaxed); + if (i < d.padded) expert_ids[i] = 0xffffffffu; +} +kernel void tf_expert_group_count(TF_GROUP_ARGS) { + if (i >= d.rows || ids[i] >= d.experts) return; + ranks[i] = atomic_fetch_add_explicit(counts + ids[i], 1u, memory_order_relaxed); +} +kernel void tf_expert_group_prefix(TF_GROUP_ARGS) { + if (i != 0) return; + uint offset = 0; + for (uint expert = 0; expert < d.experts; ++expert) { + offsets[expert] = offset; + const uint count = atomic_load_explicit(counts + expert, memory_order_relaxed); + offset += (count + 7u) & ~7u; + } +} +kernel void tf_expert_group_scatter(TF_GROUP_ARGS) { + if (i >= d.rows || ids[i] >= d.experts) return; + const uint destination = offsets[ids[i]] + ranks[i]; + if (destination >= d.padded) return; + expert_ids[destination] = ids[i]; + token_ids[destination] = i / d.repeat; + slot_ids[destination] = i; +} diff --git a/zig/kernels/metal/deepseek_v41/csa2.metal b/zig/kernels/metal/deepseek_v41/csa2.metal new file mode 100644 index 0000000000..9d16cfcce5 --- /dev/null +++ b/zig/kernels/metal/deepseek_v41/csa2.metal @@ -0,0 +1,102 @@ +// Source is compiled after kv.metal, which provides exact packed-cache decoders. +struct Dsv41Attention { + uint rows, heads, width, q_stride, position, chunk_rows, window, ring_stride; + uint chunk_stride, main_stride, main_block, ratio, compressed_rows, topk, index_stride; + float scale; +}; +// BF16 query/key products are exact in FP32; compensate accumulation and lane reduction. +inline void dsv41_dot_add(thread float &sum, thread float &error, float value) { + float next = sum + value; + float recovered = next - sum; + error += (sum - (next - recovered)) + (value - recovered); + sum = next; +} +inline float dsv41_dot_reduce(float sum, float error) { + for (uint offset = 16; offset != 0; offset >>= 1) { + float other = simd_shuffle_xor(sum, offset); + float other_error = simd_shuffle_xor(error, offset); + dsv41_dot_add(sum, error, other); + dsv41_dot_add(sum, error, other_error); + } + return sum + error; +} +// BF16 probability rounding uses fixed query-based tiles, never a chunk-dependent physical key layout. +inline uint dsv41_window_columns(constant Dsv41Attention &p, uint row) { + return p.position + row == 0 ? 1 : p.window; +} +inline device const uchar *dsv41_attn_key(uint slot, uint row, device const uchar *ring, + device const uchar *chunk, device const uchar *main, device const int *indices, + constant Dsv41Attention &p, thread uint &kind) { + uint query = p.position + row; + uint columns = dsv41_window_columns(p, row); + if (slot < columns) { + uint pos; + if (query == 0) { + pos = 0; + } else if (query + 1 < p.window) { + pos = (query + 1 + slot) % p.window; + } else { + uint begin = query + 1 > p.window ? query + 1 - p.window : 0; + pos = begin + slot; + } + if (pos > query) return nullptr; + kind = 0; + return pos >= p.position ? chunk + (pos - p.position) * p.chunk_stride + : ring + (pos % p.window) * p.ring_stride; + } + if (!p.ratio) return nullptr; + int pos = indices[row * p.index_stride + slot - columns]; + if (pos < 0 || uint(pos) >= min(p.compressed_rows, (query + 1) / p.ratio)) return nullptr; + kind = 1; + return main + uint(pos) * p.main_stride; +} +kernel void dsv41_sparse_attention(device const ushort *q [[buffer(0)]], device const uchar *ring [[buffer(1)]], + device const uchar *chunk [[buffer(2)]], device const uchar *main [[buffer(3)]], + device const int *indices [[buffer(4)]], device const float *sink [[buffer(5)]], + device ushort *out [[buffer(6)]], constant Dsv41Attention &p [[buffer(7)]], + uint2 g [[threadgroup_position_in_grid]], uint lane [[thread_index_in_threadgroup]]) { + uint head = g.x, row = g.y; + if (head >= p.heads || row >= p.rows) return; + threadgroup float scores[64]; + float query[16], acc[16]; + for (uint j = 0; j < p.width / 32; ++j) { + query[j] = dsv41_bf(q[row * p.q_stride + head * p.width + lane + j * 32]); acc[j] = 0; + } + float mx = -1e30f, sum = 0; + uint query_position = p.position + row; + uint compressed_columns = p.ratio ? min(p.topk, min(p.compressed_rows, (query_position + 1) / p.ratio)) : 0; + uint count = dsv41_window_columns(p, row) + compressed_columns; + for (uint base = 0; base < count; base += 64) { + float next_max = mx; + for (uint j = 0; j < 64; ++j) { + uint kind = 0; + device const uchar *key = base + j < count ? dsv41_attn_key(base + j, row, ring, chunk, main, indices, p, kind) : nullptr; + float dot = 0, dot_error = 0; + if (key) for (uint d = 0; d < p.width / 32; ++d) + dsv41_dot_add(dot, dot_error, query[d] * dsv41_unpack(key, lane + d * 32, p.width, kind ? p.main_block : 32, kind)); + dot = key ? dsv41_dot_reduce(dot, dot_error) * p.scale : -INFINITY; + next_max = max(next_max, dot); + if (lane == 0) scores[j] = dot; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + float correction = exp(mx - next_max); + float tile_sum = 0; + for (uint d = 0; d < p.width / 32; ++d) acc[d] *= correction; + for (uint j = 0; j < 64; ++j) { + float probability = exp(scores[j] - next_max); + tile_sum += probability; + float rounded = dsv41_bf(dsv41_round(probability)); + uint kind = 0; + device const uchar *key = base + j < count ? dsv41_attn_key(base + j, row, ring, chunk, main, indices, p, kind) : nullptr; + if (key) for (uint d = 0; d < p.width / 32; ++d) + acc[d] += rounded * dsv41_unpack(key, lane + d * 32, p.width, kind ? p.main_block : 32, kind); + } + sum = sum * correction + tile_sum; + mx = next_max; + threadgroup_barrier(mem_flags::mem_threadgroup); + } + // The source adds sink mass to the completed online denominator before the FP32 division. + float denom = sum + exp(sink[head] - mx); + for (uint d = 0; d < p.width / 32; ++d) + out[row * p.q_stride + head * p.width + lane + d * 32] = dsv41_round(acc[d] / denom); +} diff --git a/zig/kernels/metal/deepseek_v41/engram.metal b/zig/kernels/metal/deepseek_v41/engram.metal new file mode 100644 index 0000000000..532bbd01ee --- /dev/null +++ b/zig/kernels/metal/deepseek_v41/engram.metal @@ -0,0 +1,150 @@ +// Math/layout attribution: DeepSeek AI DeepSeek-V4.1-Flash inference/{engram,model}.py (MIT). +#include +using namespace metal; + +struct EngramLayer { + uint layer_id, rows; + ulong multipliers[4]; + uint primes[24], offsets[24]; +}; +struct EngramHistory { int tail[3]; uint available; }; +struct EngramHashArgs { uint tokens, pad, masked; }; + +inline int engram_token(const device uint* ids, const device uint* map, + const device uchar* mask, uint pos, uint masked) { + return masked && mask[pos] == 0 ? -1 : int(map[ids[pos]]); +} + +// One lane per token; history is immutable until all layers have hashed this chunk. +kernel void tf_v41_engram_hash(const device uint* ids [[buffer(0)]], + const device uint* map [[buffer(1)]], + const device uchar* mask [[buffer(2)]], + const device EngramHistory& history [[buffer(3)]], + device uint* out [[buffer(4)]], + constant EngramLayer& layer [[buffer(5)]], + constant EngramHashArgs& args [[buffer(6)]], + uint pos [[thread_position_in_grid]]) { + if (pos >= args.tokens) return; + bool blocked = false; + ulong rolling = 0; + for (uint shift = 0; shift < 4; shift++) { + int token; + if (shift <= pos) token = engram_token(ids, map, mask, pos - shift, args.masked); + else { + const uint back = shift - pos - 1; + token = back < history.available ? history.tail[back] : -1; + } + blocked = blocked || token == -1; + const ulong compressed = blocked ? args.pad : uint(token); + rolling ^= compressed * layer.multipliers[shift]; + if (shift != 0) for (uint head = 0; head < 8; head++) { + const uint col = (shift - 1) * 8 + head; + out[size_t(pos) * 24 + col] = uint(rolling % layer.primes[col]) + layer.offsets[col]; + } + } +} + +// Called exactly once after all Engram layers hash a chunk, never once per layer. +kernel void tf_v41_engram_commit(const device uint* ids [[buffer(0)]], + const device uint* map [[buffer(1)]], + const device uchar* mask [[buffer(2)]], + device EngramHistory& history [[buffer(3)]], + constant EngramHashArgs& args [[buffer(4)]]) { + int tail[3]; + for (uint i = 0; i < 3; i++) { + tail[i] = i < args.tokens ? engram_token(ids, map, mask, args.tokens - i - 1, args.masked) + : i - args.tokens < history.available ? history.tail[i - args.tokens] : -1; + } + for (uint i = 0; i < 3; i++) history.tail[i] = tail[i]; + history.available = min(3u, history.available + args.tokens); +} + +kernel void tf_v41_engram_reset(device EngramHistory& history [[buffer(0)]]) { + for (uint i = 0; i < 3; i++) history.tail[i] = -1; + history.available = 0; +} + +inline float engram_e4m3(uchar bits) { + const uint magnitude = bits & 127, exponent = magnitude >> 3, mantissa = magnitude & 7; + float value = magnitude == 127 ? NAN : exponent == 0 ? float(mantissa) * 0x1p-9f + : ldexp(1.0f + float(mantissa) * 0.125f, int(exponent) - 7); + return bits & 128 ? -value : value; +} +inline float engram_ue8m0(uchar bits) { + return bits == 255 ? NAN : as_type(bits == 0 ? 0x00400000u : uint(bits) << 23); +} + +struct EngramGatherArgs { + ulong weight_stride, scale_stride, output_stride; + uint tokens, columns, head_dim, scale_block; +}; + +// Read only the selected E4M3FN rows; one UE8M0 scale per 32 values of each row. +kernel void tf_v41_engram_gather(const device uint* ids [[buffer(0)]], + const device uchar* table [[buffer(1)]], + const device uchar* scales [[buffer(2)]], + device bfloat* out [[buffer(3)]], + constant EngramGatherArgs& args [[buffer(4)]], + uint2 pos [[thread_position_in_grid]]) { + if (pos.y >= args.tokens || pos.x >= args.columns * args.head_dim) return; + const uint col = pos.x / args.head_dim, d = pos.x % args.head_dim; + const size_t row = ids[size_t(pos.y) * args.columns + col]; + const float value = engram_e4m3(table[row * args.weight_stride + d]); + const float scale = engram_ue8m0(scales[row * args.scale_stride + d / args.scale_block]); + out[size_t(pos.y) * args.output_stride + pos.x] = bfloat(value * scale); +} + +struct EngramGateArgs { + ulong hidden_stride, kv_stride, output_stride; + uint dim, copies, masked; + float eps; +}; + +// Each copy gets its own RMS dot gate; all copies share the value following the flattened keys. +kernel void tf_v41_engram_gate(const device bfloat* hidden [[buffer(0)]], + const device bfloat* kv [[buffer(1)]], + const device float* q_weight [[buffer(2)]], + const device float* k_weight [[buffer(3)]], + const device uchar* mask [[buffer(4)]], + device bfloat* out [[buffer(5)]], + constant EngramGateArgs& args [[buffer(6)]], + uint2 group [[threadgroup_position_in_grid]], + uint t [[thread_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]], + uint sg [[simdgroup_index_in_threadgroup]]) { + threadgroup float3 sums[8]; + threadgroup float gate; + const size_t hbase = size_t(group.x) * args.hidden_stride + size_t(group.y) * args.dim; + const size_t kbase = size_t(group.x) * args.kv_stride + size_t(group.y) * args.dim; + const size_t wbase = size_t(group.y) * args.dim; + float3 acc = 0; + for (uint d = t; d < args.dim; d += 256) { + const float h = float(hidden[hbase + d]), k = float(kv[kbase + d]); + const float weight = q_weight[wbase + d] * k_weight[wbase + d]; + acc += float3(h * h, k * k, (h * weight) * k); + } + acc = float3(simd_sum(acc.x), simd_sum(acc.y), simd_sum(acc.z)); + if (lane == 0) sums[sg] = acc; + threadgroup_barrier(mem_flags::mem_threadgroup); + if (sg == 0) { + acc = lane < 8 ? sums[lane] : float3(0); + acc = float3(simd_sum(acc.x), simd_sum(acc.y), simd_sum(acc.z)); + if (lane == 0) { + float rstd; + { +#pragma clang fp contract(off) + const float factor = 1.0f / float(args.dim); + rstd = metal::precise::rsqrt(acc.x * factor + args.eps) + * metal::precise::rsqrt(acc.y * factor + args.eps); + } + const float dot = (acc.z * rstd) * metal::precise::rsqrt(float(args.dim)); + const float signed_root = copysign(metal::precise::sqrt(max(abs(dot), 1e-6f)), dot); + gate = args.masked && mask[group.x] == 0 ? 0.0f : 1.0f / (1.0f + metal::precise::exp(-signed_root)); + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + const size_t vbase = size_t(group.x) * args.kv_stride + size_t(args.copies) * args.dim; + const size_t obase = size_t(group.x) * args.output_stride + size_t(group.y) * args.dim; + for (uint d = t; d < args.dim; d += 256) + out[obase + d] = bfloat(float(hidden[hbase + d]) + gate * float(kv[vbase + d])); +} diff --git a/zig/kernels/metal/deepseek_v41/index.metal b/zig/kernels/metal/deepseek_v41/index.metal new file mode 100644 index 0000000000..8d62337624 --- /dev/null +++ b/zig/kernels/metal/deepseek_v41/index.metal @@ -0,0 +1,103 @@ +// Source is compiled after kv.metal; no query-by-key-by-head score tensor is materialized. +struct Dsv41Index { + uint rows, heads, width, q_stride, weight_stride, position, ratio, keys, key_stride; + uint tile_start, tile_width, topk, candidate_topk, block_size, use_candidates, make_candidates; + float scale; +}; +struct Dsv41Pick { float score; int position; }; +inline bool dsv41_less(Dsv41Pick a, Dsv41Pick b) { + return a.score < b.score || (a.score == b.score && a.position > b.position); +} +inline void dsv41_heap(device Dsv41Pick *h, uint size, Dsv41Pick v) { + if (!dsv41_less(h[0], v)) return; + uint at = 0; + while (2 * at + 1 < size) { + uint child = 2 * at + 1; + if (child + 1 < size && dsv41_less(h[child + 1], h[child])) ++child; + if (!dsv41_less(h[child], v)) break; + h[at] = h[child]; at = child; + } + h[at] = v; +} +inline int dsv41_index_position(uint local, uint row, device const Dsv41Pick *candidates, + constant Dsv41Index &p) { + uint slot = p.tile_start + local; + if (!p.use_candidates) return int(slot); + uint block = slot / p.block_size; + if (block >= p.candidate_topk) return -1; + int id = candidates[row * p.candidate_topk + block].position; + return id < 0 ? -1 : id * int(p.block_size) + int(slot % p.block_size); +} +kernel void dsv41_index_score(device const ushort *q [[buffer(0)]], device const ushort *weight [[buffer(1)]], + device const uchar *key [[buffer(2)]], device const Dsv41Pick *candidates [[buffer(3)]], + device float *scores [[buffer(4)]], constant Dsv41Index &p [[buffer(5)]], + uint2 g [[threadgroup_position_in_grid]], uint lane [[thread_index_in_threadgroup]]) { + uint local = g.x, row = g.y; + if (local >= p.tile_width || row >= p.rows) return; + int pos = dsv41_index_position(local, row, candidates, p); + uint visible = min(p.keys, (p.position + row + 1) / p.ratio); + if (pos < 0 || uint(pos) >= visible) { + if (lane == 0) scores[row * p.tile_width + local] = -INFINITY; + return; + } + device const uchar *k = key + uint(pos) * p.key_stride; + float key_values[4]; + for (uint d = 0; d < p.width / 32; ++d) key_values[d] = dsv41_unpack(k, lane + 32 * d, p.width, 32, 2); + float total = 0; + for (uint head = 0; head < p.heads; ++head) { + float dot = 0; + for (uint d = 0; d < p.width / 32; ++d) + dot += dsv41_bf(q[row * p.q_stride + head * p.width + lane + 32 * d]) * key_values[d]; + dot = dsv41_bf(dsv41_round(simd_sum(dot))); + const float scaled_weight = dsv41_bf(dsv41_round(dsv41_bf(weight[row * p.weight_stride + head]) * p.scale)); + total += dsv41_bf(dsv41_round(max(dot, 0.0f) * scaled_weight)); + } + if (lane == 0) scores[row * p.tile_width + local] = dsv41_bf(dsv41_round(total)); +} +kernel void dsv41_index_clear(device Dsv41Pick *top [[buffer(0)]], device Dsv41Pick *candidate [[buffer(1)]], + constant Dsv41Index &p [[buffer(2)]], uint2 i [[thread_position_in_grid]]) { + if (i.y >= p.rows) return; + if (i.x < p.topk) top[i.y * p.topk + i.x] = {-INFINITY, -1}; + if (p.make_candidates && i.x < p.candidate_topk) candidate[i.y * p.candidate_topk + i.x] = {-INFINITY, -1}; +} +kernel void dsv41_index_merge(device const float *scores [[buffer(0)]], device Dsv41Pick *top [[buffer(1)]], + device Dsv41Pick *candidate [[buffer(2)]], constant Dsv41Index &p [[buffer(3)]], + uint row [[thread_position_in_grid]]) { + if (row >= p.rows) return; + device Dsv41Pick *heap = top + row * p.topk; + for (uint i = 0; i < p.tile_width; ++i) { + float score = scores[row * p.tile_width + i]; + int pos = dsv41_index_position(i, row, candidate, p); + if (isfinite(score)) dsv41_heap(heap, p.topk, {score, pos}); + } + if (!p.make_candidates) return; + uint visible = min(p.keys, (p.position + row + 1) / p.ratio); + device Dsv41Pick *blocks = candidate + row * p.candidate_topk; + for (uint base = 0; base < p.tile_width; base += p.block_size) { + float score = -INFINITY; + for (uint j = 0; j < p.block_size; ++j) score = max(score, scores[row * p.tile_width + base + j]); + uint block = (p.tile_start + base) / p.block_size; + if (visible && block == (visible - 1) / p.block_size) score = INFINITY; + if (score > -INFINITY) dsv41_heap(blocks, p.candidate_topk, {score, int(block)}); + } +} +inline void dsv41_sort_positions(device Dsv41Pick *p, uint size) { + // Selection is score-ordered; attention sees the reference's ascending position order. + for (uint i = 1; i < size; ++i) { + Dsv41Pick value = p[i]; uint j = i; + while (j > 0 && (p[j - 1].position < 0 || + (value.position >= 0 && p[j - 1].position > value.position))) { + if (value.position < 0) break; + p[j] = p[j - 1]; --j; + } + p[j] = value; + } +} +kernel void dsv41_index_finish(device Dsv41Pick *top [[buffer(0)]], device Dsv41Pick *candidate [[buffer(1)]], + device int *positions [[buffer(2)]], constant Dsv41Index &p [[buffer(3)]], uint row [[thread_position_in_grid]]) { + if (row >= p.rows) return; + device Dsv41Pick *heap = top + row * p.topk; + dsv41_sort_positions(heap, p.topk); + for (uint i = 0; i < p.topk; ++i) positions[row * p.topk + i] = heap[i].position; + if (p.make_candidates) dsv41_sort_positions(candidate + row * p.candidate_topk, p.candidate_topk); +} diff --git a/zig/kernels/metal/deepseek_v41/kv.metal b/zig/kernels/metal/deepseek_v41/kv.metal new file mode 100644 index 0000000000..6ac2017efb --- /dev/null +++ b/zig/kernels/metal/deepseek_v41/kv.metal @@ -0,0 +1,141 @@ +// DeepSeek V4.1 cache quantization and pooling, following the official inference kernels. +#include +using namespace metal; +inline float dsv41_bf(ushort x) { return as_type(uint(x) << 16); } +inline ushort dsv41_round(float x) { + uint u = as_type(x); + return ushort((u + 0x7fff + ((u >> 16) & 1)) >> 16); +} +inline float dsv41_e4(uchar c) { + uint e = (c >> 3) & 15, m = c & 7; + float v = e == 0 ? float(m) * 0x1p-9f : ldexp(1.0f + float(m) / 8.0f, int(e) - 7); + return (c & 128) ? -v : v; +} +inline uchar dsv41_e4code(float x) { + float a = min(abs(x), 448.0f); + if (a == 0) return signbit(x) ? 128 : 0; + int e = max(-6, int((as_type(a) >> 23) & 255) - 127); + int q = int(rint(ldexp(a, 3 - e))); + uint c = e == -6 && q < 8 ? uint(q) : uint((e + 7) * 8 + q - 8); + return uchar(min(c, 126u) | (signbit(x) ? 128u : 0u)); +} +inline float dsv41_e2(uchar c) { + const float v[8] = {0, .5f, 1, 1.5f, 2, 3, 4, 6}; + return (c & 8) ? -v[c & 7] : v[c & 7]; +} +inline uchar dsv41_e2code(float x) { + float a = min(abs(x), 6.0f); + uint pick = a <= .25f ? 0 : a < .75f ? 1 : a <= 1.25f ? 2 : a < 1.75f ? 3 + : a <= 2.5f ? 4 : a < 3.5f ? 5 : a <= 5.0f ? 6 : 7; + return uchar(pick | (signbit(x) ? 8 : 0)); +} +inline float dsv41_unpack(device const uchar *p, uint d, uint width, uint block, uint kind) { + if (kind == 0) { + float s = as_type(uint(p[width + d / 32]) << 23); + return dsv41_bf(dsv41_round(dsv41_e4(p[d]) * s)); + } + uchar c = p[d / 2]; + float s = kind == 1 ? dsv41_e4(p[width / 2 + d / block]) + : as_type(uint(p[width / 2 + d / block]) << 23); + return dsv41_bf(dsv41_round(dsv41_e2((d & 1) ? c >> 4 : c & 15) * s)); +} +struct Dsv41Pack { uint rows, width, stride, packed_stride, block, kind, first, ring, inplace; }; +kernel void dsv41_pack(device ushort *x [[buffer(0)]], device uchar *out [[buffer(1)]], + constant Dsv41Pack &p [[buffer(2)]], uint2 g [[threadgroup_position_in_grid]], + uint lane [[thread_index_in_threadgroup]]) { + uint row = g.y, group = g.x, d = group * p.block + lane; + if (row >= p.rows) return; + float v = lane < p.block ? dsv41_bf(x[row * p.stride + d]) : 0; + float a = simd_max(abs(v)); + float scale; uchar sc; + if (p.kind == 1) { + sc = dsv41_e4code(max(a, 6.0f * 0x1p-9f) / 6.0f); scale = dsv41_e4(sc); + } else { + float minimum = p.kind == 0 ? 1e-4f : 6.0f * 0x1p-126f; + float z = max(a, minimum) / (p.kind == 0 ? 448.0f : 6.0f); + uint bits = as_type(z); + int exp = int((bits >> 23) & 255) - 127 + ((bits & 0x7fffff) != 0); + sc = uchar(exp + 127); scale = ldexp(1.0f, exp); + } + uint dst = p.ring ? (p.first + row) % p.ring : p.first + row; + device uchar *packed = out + dst * p.packed_stride; + if (lane == 0) packed[(p.kind == 0 ? p.width : p.width / 2) + group] = sc; + if (lane >= p.block) return; + if (p.kind == 0) { + uchar c = dsv41_e4code(v / scale); packed[d] = c; + if (p.inplace) x[row * p.stride + d] = dsv41_round(dsv41_e4(c) * scale); + } else { + uchar c = dsv41_e2code(v / scale); + uchar other = uchar(simd_shuffle_xor(uint(c), 1)); + if (!(lane & 1)) packed[d / 2] = c | (other << 4); + if (p.inplace) x[row * p.stride + d] = dsv41_round(dsv41_e2(c) * scale); + } +} +struct Dsv41Pool { uint rows, width, ratio, position, stride, input_f32; float eps; }; +kernel void dsv41_pool(device const uchar *values [[buffer(0)]], device const float *gate [[buffer(1)]], + device float *partial [[buffer(2)]], device const ushort *weight [[buffer(3)]], + device ushort *latent [[buffer(4)]], constant Dsv41Pool &p [[buffer(5)]], + uint group [[threadgroup_position_in_grid]], uint lane [[thread_index_in_threadgroup]]) { + // A group reads the prior chunk's tail before the independent tail-update dispatch. + uint start = (p.position / p.ratio + group) * p.ratio; + threadgroup float pooled[512]; + float square = 0; + for (uint d = lane; d < p.width; d += 128) { + float mx = -INFINITY, sum = 0, total = 0; + for (uint j = 0; j < p.ratio; ++j) { + uint pos = start + j; + float score = p.ratio == 1 ? 0 : (pos < p.position ? partial[p.ratio * p.width + (pos % p.ratio) * p.width + d] + : gate[(pos - p.position) * p.stride + d]); + mx = max(mx, score); + } + float slots[2], exponent[2]; + for (uint j = 0; j < p.ratio; ++j) { + uint pos = start + j; + slots[j] = pos < p.position ? partial[(pos % p.ratio) * p.width + d] + : (p.input_f32 ? reinterpret_cast(values)[(pos - p.position) * p.stride + d] + : dsv41_bf(reinterpret_cast(values)[(pos - p.position) * p.stride + d])); + float score = p.ratio == 1 ? 0 : (pos < p.position ? partial[p.ratio * p.width + (pos % p.ratio) * p.width + d] + : gate[(pos - p.position) * p.stride + d]); + exponent[j] = exp(score - mx); sum += exponent[j]; + } + for (uint j = 0; j < p.ratio; ++j) total += slots[j] * (exponent[j] / sum); + float v = dsv41_bf(dsv41_round(total)); pooled[d] = v; square += v * v; + } + threadgroup float sums[4]; + float ss = simd_sum(square); + if (!(lane % 32)) sums[lane / 32] = ss; + threadgroup_barrier(mem_flags::mem_threadgroup); + float inv = rsqrt((sums[0] + sums[1] + sums[2] + sums[3]) / p.width + p.eps); + for (uint d = lane; d < p.width; d += 128) + latent[group * p.width + d] = dsv41_round(pooled[d] * inv * dsv41_bf(weight[d])); +} +kernel void dsv41_pool_tail(device const float *values [[buffer(0)]], device const float *gate [[buffer(1)]], + device float *partial [[buffer(2)]], constant Dsv41Pool &p [[buffer(3)]], + uint d [[thread_position_in_grid]]) { + if (d >= p.width || p.ratio == 1) return; + uint end = p.position + p.rows, tail = end % p.ratio; + for (uint j = 0; j < tail; ++j) { + uint pos = end - tail + j; + if (pos < p.position) continue; + partial[j * p.width + d] = values[(pos - p.position) * p.stride + d]; + partial[p.ratio * p.width + j * p.width + d] = gate[(pos - p.position) * p.stride + d]; + } +} +kernel void dsv41_copy(device const uchar *src [[buffer(0)]], device uchar *dst [[buffer(1)]], + constant uint &bytes [[buffer(2)]], uint i [[thread_position_in_grid]]) { + if (i < bytes) dst[i] = src[i]; +} +kernel void dsv41_publish_window(device const uchar *chunk [[buffer(0)]], device uchar *ring [[buffer(1)]], + constant uint4 &p [[buffer(2)]], uint2 i [[thread_position_in_grid]]) { + uint count = min(p.x, 128u), first = p.x - count; + if (i.x >= p.z || i.y >= count) return; + uint row = first + i.y, slot = (p.y + row) % 128; + ring[slot * p.w + i.x] = chunk[row * p.z + i.x]; +} +kernel void dsv41_unpack_rows(device const uchar *packed [[buffer(0)]], device ushort *out [[buffer(1)]], + constant Dsv41Pack &p [[buffer(2)]], uint2 i [[thread_position_in_grid]]) { + if (i.x >= p.width || i.y >= p.rows) return; + uint row = p.ring ? (p.first + i.y) % p.ring : p.first + i.y; + out[i.y * p.stride + i.x] = dsv41_round(dsv41_unpack(packed + row * p.packed_stride, + i.x, p.width, p.block, p.kind)); +} diff --git a/zig/kernels/metal/deepseek_v41/mhc.metal b/zig/kernels/metal/deepseek_v41/mhc.metal new file mode 100644 index 0000000000..8b6203a4cf --- /dev/null +++ b/zig/kernels/metal/deepseek_v41/mhc.metal @@ -0,0 +1,120 @@ +// Reference: DeepSeek-V4.1-Flash inference/model.py and kernel.py, revision 2cba9e42aa026125f3ed06c6d98c1db82f7ca027. +#include +using namespace metal; +inline float bf(ushort x) { return as_type(uint(x)<<16); } +inline ushort rb(float x) { + uint u=as_type(x); + if ((u&0x7fffffff)>0x7f800000) return ushort((u>>16)|0x40); + return ushort((u+0x7fff+((u>>16)&1))>>16); +} +// Torch CUDA materializes FP32 squares before MeanOps accumulation. +#pragma clang fp contract(off) +struct Rms { uint rows,width,stream_stride; float eps; }; +kernel void dsv41_hc_rms(const device ushort* x [[buffer(0)]], device float* inv [[buffer(1)]], + constant Rms& d [[buffer(2)]], uint row [[threadgroup_position_in_grid]], + uint t [[thread_index_in_threadgroup]], uint lane [[thread_index_in_simdgroup]], + uint sg [[simdgroup_index_in_threadgroup]]) { + // Fixed single-row CUDA tree: four vector accumulators, stride 512, independent of prefill chunks. + threadgroup float sums[512]; + float4 accum=0.0f; + for (uint i=4*t;i<4*d.width;i+=4*512) { + float4 a=float4(bf(x[ulong(row)*d.stream_stride+i]), + bf(x[ulong(row)*d.stream_stride+i+1]), + bf(x[ulong(row)*d.stream_stride+i+2]), + bf(x[ulong(row)*d.stream_stride+i+3])); + float4 square=a*a; + accum+=square; + } + float s=((accum.x+accum.y)+accum.z)+accum.w; + sums[t]=s; + for (uint offset=256;offset>=32;offset>>=1) { + threadgroup_barrier(mem_flags::mem_threadgroup); + if (t0;offset>>=1) s+=simd_shuffle_down(s,offset); + // MeanOps projects with a rounded FP32 factor, not a division of the sum. + float factor=1.0f/float(4*d.width); + if (lane==0) inv[row]=rsqrt(s*factor+d.eps); + } +} +#pragma clang fp contract(on) +struct Split { uint rows,mix_stride,coefficient_stride,comb_stride; float eps; }; +kernel void dsv41_hc_split(const device float* mixes [[buffer(0)]], const device float* inv [[buffer(1)]], + const device float* scale [[buffer(2)]], const device float* base [[buffer(3)]], + device float* pre [[buffer(4)]], device float* post [[buffer(5)]], + device float* comb [[buffer(6)]], constant Split& d [[buffer(7)]], + uint row [[thread_position_in_grid]]) { + if (row>=d.rows) return; + float m[16]; + for (uint j=0;j<4;j++) { + float a=fma(mixes[ulong(row)*d.mix_stride+j]*inv[row],scale[0],base[j]); + float b=fma(mixes[ulong(row)*d.mix_stride+4+j]*inv[row],scale[1],base[4+j]); + pre[ulong(row)*d.coefficient_stride+j]=1.0f/(1.0f+exp(-a))+d.eps; + post[ulong(row)*d.coefficient_stride+j]=2.0f/(1.0f+exp(-b)); + for (uint k=0;k<4;k++) { + uint q=4*j+k; + m[q]=fma(mixes[ulong(row)*d.mix_stride+8+q]*inv[row],scale[2],base[8+q]); + } + } + // Initial row softmax + eps, then a column normalization. This counts as iteration one. + for (uint j=0;j<4;j++) { + float mx=max(max(m[4*j],m[4*j+1]),max(m[4*j+2],m[4*j+3])); + for (uint k=0;k<4;k++) m[4*j+k]=exp(m[4*j+k]-mx); + // Pinned TileLang HC row/column sums use FP32 butterfly pairs. + float s=(m[4*j]+m[4*j+2])+(m[4*j+1]+m[4*j+3]); + for (uint k=0;k<4;k++) m[4*j+k]=m[4*j+k]/s+d.eps; + } + for (uint it=0;it<20;it++) { + if (it>0) { + for (uint j=0;j<4;j++) { + float s=((m[4*j]+m[4*j+2])+(m[4*j+1]+m[4*j+3]))+d.eps; + for (uint k=0;k<4;k++) m[4*j+k]/=s; + } + } + for (uint k=0;k<4;k++) { + float s=((m[k]+m[8+k])+(m[4+k]+m[12+k]))+d.eps; + for (uint j=0;j<4;j++) m[4*j+k]/=s; + } + } + for (uint q=0;q<16;q++) comb[ulong(row)*d.comb_stride+q]=m[q]; +} +// PyTorch materializes these FP32 products before reduction and branch addition. +#pragma clang fp contract(off) +struct Collapse { uint rows,width,stream_stride,y_stride,pre_stride; }; +kernel void dsv41_hc_collapse(const device ushort* x [[buffer(0)]], const device float* previous_pre [[buffer(1)]], + device ushort* y [[buffer(2)]], constant Collapse& d [[buffer(3)]], + uint2 i [[thread_position_in_grid]]) { + if (i.x>=d.width || i.y>=d.rows) return; + float s=0; + for (uint c=0;c<4;c++) s+=previous_pre[ulong(i.y)*d.pre_stride+c]*bf(x[ulong(i.y)*d.stream_stride+ulong(c)*d.width+i.x]); + y[ulong(i.y)*d.y_stride+i.x]=rb(s); +} +struct Post { uint rows,width,branch_stride,residual_stride,y_stride,coefficient_stride,comb_stride; }; +kernel void dsv41_hc_post(const device ushort* branch [[buffer(0)]], const device ushort* residual [[buffer(1)]], + const device float* post [[buffer(2)]], const device float* comb [[buffer(3)]], + device ushort* y [[buffer(4)]], constant Post& d [[buffer(5)]], + uint2 i [[thread_position_in_grid]]) { + if (i.x>=d.width || i.y>=d.rows) return; + float r[4]; + for (uint j=0;j<4;j++) r[j]=bf(residual[ulong(i.y)*d.residual_stride+ulong(j)*d.width+i.x]); + float b=bf(branch[ulong(i.y)*d.branch_stride+i.x]); + for (uint k=0;k<4;k++) { + float s=0; + for (uint j=0;j<4;j++) s+=comb[ulong(i.y)*d.comb_stride+4*j+k]*r[j]; + y[ulong(i.y)*d.y_stride+ulong(k)*d.width+i.x]=rb(post[ulong(i.y)*d.coefficient_stride+k]*b+s); + } +} +#pragma clang fp contract(on) +struct Expand { uint rows,width,x_stride,stream_stride,pre_stride; }; +kernel void dsv41_hc_expand(const device ushort* x [[buffer(0)]], device ushort* stream [[buffer(1)]], + device float* pre [[buffer(2)]], constant Expand& d [[buffer(3)]], + uint2 i [[thread_position_in_grid]]) { + if (i.x>=d.width || i.y>=d.rows) return; + for (uint c=0;c<4;c++) stream[ulong(i.y)*d.stream_stride+ulong(c)*d.width+i.x]=x[ulong(i.y)*d.x_stride+i.x]; + if (i.x==0) for (uint c=0;c<4;c++) pre[ulong(i.y)*d.pre_stride+c]=c==0?1.0f:0.0f; +} diff --git a/zig/kernels/metal/deepseek_v41/ops.metal b/zig/kernels/metal/deepseek_v41/ops.metal new file mode 100644 index 0000000000..d831901dd5 --- /dev/null +++ b/zig/kernels/metal/deepseek_v41/ops.metal @@ -0,0 +1,124 @@ +// Math: deepseek-ai/DeepSeek-V4.1-Flash inference/model.py, revision 2cba9e42aa026125f3ed06c6d98c1db82f7ca027. +#include +using namespace metal; +inline float bf(ushort x) { return as_type(uint(x) << 16); } +inline ushort rb(float x) { + uint u = as_type(x); + if ((u & 0x7fffffff) > 0x7f800000) return ushort((u >> 16) | 0x40); + return ushort((u + 0x7fff + ((u >> 16) & 1)) >> 16); +} +inline float rd(const device uchar* p, ulong i, uint dtype) { + if (dtype==1) return ((const device float*)p)[i]; + if (dtype==2) return float(((const device half*)p)[i]); + return bf(((const device ushort*)p)[i]); +} +struct Rms { uint rows, width, x_stride, y_stride; float eps; uint weight_f32; }; +#pragma clang fp contract(off) +kernel void dsv41_rms(const device ushort* x [[buffer(0)]], const device uchar* w [[buffer(1)]], + device ushort* y [[buffer(2)]], constant Rms& d [[buffer(3)]], + uint row [[threadgroup_position_in_grid]], uint t [[thread_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]], uint sg [[simdgroup_index_in_threadgroup]]) { + threadgroup float sums[8]; + float s = 0; + for (uint i = t; i < d.width; i += 256) { float a = bf(x[ulong(row)*d.x_stride+i]); s += a*a; } + s = simd_sum(s); + if (lane == 0) sums[sg] = s; + threadgroup_barrier(mem_flags::mem_threadgroup); + if (sg == 0) { + float total = simd_sum(lane < 8 ? sums[lane] : 0.0f); + const float factor = 1.0f / float(d.width); + if (lane == 0) sums[0] = rsqrt(total * factor + d.eps); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint i = t; i < d.width; i += 256) + y[ulong(row)*d.y_stride+i] = rb((bf(x[ulong(row)*d.x_stride+i])*sums[0])*rd(w,i,d.weight_f32)); +} +#pragma clang fp contract(on) +// Each SIMD group emits four columns; compensated dots do not depend on token count. + +// Precision: 0=BF16, 1=FP32, 2=FP16; FP16 weights never round through BF16. +inline void dense_add(thread float& high, thread float& low, float value) { + float next = high + value; + float borrowed = next - high; + low += (high - (next - borrowed)) + (value - borrowed); + high = next; +} +struct Dense { uint rows, k, n, groups, x_stride, w_stride, y_stride, x_dtype, w_dtype, y_dtype; }; +kernel void dsv41_dense(const device uchar* x [[buffer(0)]], const device uchar* w [[buffer(1)]], + device uchar* y [[buffer(2)]], constant Dense& d [[buffer(3)]], + uint3 g [[threadgroup_position_in_grid]], uint lane [[thread_index_in_simdgroup]], + uint sg [[simdgroup_index_in_threadgroup]]) { + uint col = g.x*16 + sg*4; + float a[4] = {0,0,0,0}, residual[4] = {0,0,0,0}; + for (uint k = lane; k < d.k; k += 32) { + float v = rd(x,ulong(g.y)*d.x_stride+ulong(g.z)*d.k+k,d.x_dtype); + for (uint j=0;j<4;j++) if (col+j>= 1) { + float other = simd_shuffle_xor(a[j], offset); + float other_residual = simd_shuffle_xor(residual[j], offset); + dense_add(a[j], residual[j], other); + residual[j] += other_residual; + } + float v = a[j] + residual[j]; + if (lane==0 && col+j=d.width || i.y>=d.rows) return; + ushort v=table[ulong(ids[i.y])*d.table_stride+i.x]; + for (uint c=0;c=d.width || i.y>=d.rows) return; + ulong k=ulong(i.y)*d.input_stride+i.x; + float a=min(rd(gate,k,d.input_f32),10.0f), b=clamp(rd(up,k,d.input_f32),-10.0f,10.0f); + float v=(a/(1.0f+exp(-a)))*b; + if (d.weighted) v=weights[i.y]*v; + y[ulong(i.y)*d.output_stride+i.x]=rb(v); +} +struct Combine { uint rows,width,routed_row_stride,slot_stride,shared_stride,y_stride,input_f32,id_stride; }; +kernel void dsv41_combine(const device uchar* routed [[buffer(0)]], const device uchar* shared [[buffer(1)]], + const device uint* ids [[buffer(2)]], device ushort* y [[buffer(3)]], + constant Combine& d [[buffer(4)]], + uint2 i [[thread_position_in_grid]], uint t [[thread_index_in_threadgroup]]) { + threadgroup uint slots[6]; + if (t==0) { + for (uint a=0;a<6;a++) slots[a]=a; + for (uint a=1;a<6;a++) { + uint slot=slots[a], j=a; + while (j>0 && ids[ulong(i.y)*d.id_stride+slots[j-1]]>ids[ulong(i.y)*d.id_stride+slot]) { + slots[j]=slots[j-1]; j--; + } + slots[j]=slot; + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (i.x>=d.width || i.y>=d.rows) return; + for (uint slot=0;slot<6;slot++) if (ids[ulong(i.y)*d.id_stride+slot]>=384) { + y[ulong(i.y)*d.y_stride+i.x]=rb(NAN); + return; + } + float s=0; + for (uint rank=0;rank<6;rank++) + s+=rd(routed,ulong(i.y)*d.routed_row_stride+ulong(slots[rank])*d.slot_stride+i.x,d.input_f32); + s+=rd(shared,ulong(i.y)*d.shared_stride+i.x,d.input_f32); + y[ulong(i.y)*d.y_stride+i.x]=rb(s); +} diff --git a/zig/kernels/metal/deepseek_v41/rope.metal b/zig/kernels/metal/deepseek_v41/rope.metal new file mode 100644 index 0000000000..09c4d1cb2e --- /dev/null +++ b/zig/kernels/metal/deepseek_v41/rope.metal @@ -0,0 +1,17 @@ +// Adjacent-pair RoPE and YaRN match DeepSeek V4.1's in-place BF16 rotation. +#include +using namespace metal; +struct Dsv41Rope { uint rows, heads, width, rotary, stride, position, step, inverse, spectrum; }; +kernel void dsv41_rope(device ushort *x [[buffer(0)]], constant Dsv41Rope &p [[buffer(1)]], + device const float2 *factors [[buffer(2)]], + uint3 i [[thread_position_in_grid]]) { + if (i.x >= p.rotary / 2 || i.y >= p.heads || i.z >= p.rows) return; + // The encoder binds the selected spectrum; rows are absolute source positions. + float2 rotation = factors[(ulong(p.position) + ulong(i.z) * p.step) * (p.rotary / 2) + i.x]; + float c = rotation.x, s = p.inverse ? -rotation.y : rotation.y; + uint at = i.z * p.stride + i.y * p.width + p.width - p.rotary + 2 * i.x; + float a = as_type(uint(x[at]) << 16), b = as_type(uint(x[at + 1]) << 16); + uint u = as_type(a * c - b * s), v = as_type(a * s + b * c); + x[at] = ushort((u + 0x7fff + ((u >> 16) & 1)) >> 16); + x[at + 1] = ushort((v + 0x7fff + ((v >> 16) & 1)) >> 16); +} diff --git a/zig/kernels/metal/deepseek_v41/route.metal b/zig/kernels/metal/deepseek_v41/route.metal new file mode 100644 index 0000000000..51f5454dd3 --- /dev/null +++ b/zig/kernels/metal/deepseek_v41/route.metal @@ -0,0 +1,53 @@ +// Reference: DeepSeek-V4.1-Flash inference/model.py Gate, revision 2cba9e42aa026125f3ed06c6d98c1db82f7ca027. +#include +using namespace metal; +inline float lp(float x) { float u=1.0f+x; return u==1.0f?x:x*(log(u)/(u-1.0f)); } +inline float score(float x) { return sqrt(x>20.0f?x:lp(exp(x))); } +struct Shape { uint rows,logit_stride,output_stride; float temperature; uint bias_dtype; }; +kernel void dsv41_route(const device float* logits [[buffer(0)]], const device uchar* bias [[buffer(1)]], + device uint* ids [[buffer(2)]], device float* weights [[buffer(3)]], + constant Shape& d [[buffer(4)]], uint row [[threadgroup_position_in_grid]], + uint t [[thread_index_in_threadgroup]], uint lane [[thread_index_in_simdgroup]], + uint sg [[simdgroup_index_in_threadgroup]]) { + threadgroup float scores[384], select[384], best[4], picked[6]; + threadgroup uint best_id[4], winner; + for (uint e=t;e<384;e+=128) { + float b; + if (d.bias_dtype==1) b=((const device float*)bias)[e]; + else if (d.bias_dtype==2) b=float(((const device half*)bias)[e]); + else b=as_type(uint(((const device ushort*)bias)[e])<<16); + scores[e]=score(logits[ulong(row)*d.logit_stride+e]/d.temperature); + select[e]=scores[e]+b; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + // Every expert participates in every rank selection; ties use the lower expert id deterministically. + for (uint slot=0;slot<6;slot++) { + float v=-INFINITY; + uint id=0xffffffffu; + for (uint e=t;e<384;e+=128) if (select[e]>v || (select[e]==v && e std.math.maxInt(u32) or experts > std.math.maxInt(u32)) + return error.InvalidExpertGrouping; + const active = @min(rows, experts); + const tiles = try std.math.add(usize, active, (rows - active) / 8); + const padded = try std.math.mul(usize, tiles, 8); + if (padded > std.math.maxInt(u32)) return error.ExpertGroupingTooLarge; + const sizes = [_]usize{ experts * 4, experts * 4, rows * 4, padded * 4, padded * 4, padded * 4 }; + var plan = Plan{ .rows = @intCast(rows), .experts = @intCast(experts), .padded = @intCast(padded), .offsets = undefined, .bytes = 0 }; + for (sizes, 0..) |size, i| { + plan.offsets[i] = std.mem.alignForward(usize, plan.bytes, 16); + plan.bytes = try std.math.add(usize, plan.offsets[i], size); + } + plan.bytes = std.mem.alignForward(usize, plan.bytes, 16); + return plan; + } +}; + +pub const Scratch = struct { + buffer: mtl.Buffer, + offset: usize = 0, + plan: Plan, + + pub fn view(self: Scratch, field: Field) exl3.View { + return .{ .buffer = self.buffer, .offset = self.offset + self.plan.offsets[@backingInt(field)] }; + } +}; + +pub const Pipelines = struct { + stages: [4]mtl.Pipeline, + + pub fn load(device: mtl.Device) !Pipelines { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const lib = try mtl.Library.fromSource(device, sources.core_expert_groups, mtl.CompileOptions.mlx()); + defer lib.deinit(); + var stages: [4]mtl.Pipeline = undefined; + var loaded: usize = 0; + errdefer for (stages[0..loaded]) |pipeline| pipeline.deinit(); + inline for (.{ "tf_expert_group_clear", "tf_expert_group_count", "tf_expert_group_prefix", "tf_expert_group_scatter" }, 0..) |name, i| { + stages[i] = try mtl.Pipeline.init(device, lib, name, false); + loaded += 1; + } + return .{ .stages = stages }; + } + pub fn deinit(self: Pipelines) void { + for (self.stages) |pipeline| pipeline.deinit(); + } +}; +const Args = extern struct { rows: u32, experts: u32, repeat: u32, padded: u32 }; + +pub fn encode(enc: mtl.ComputeEncoder, pipelines: *const Pipelines, ids: exl3.View, scratch: Scratch, repeat: u32) !void { + // Plan must come from Plan.init; route IDs remain GPU-resident and are bounds-checked by the kernels. + if (repeat == 0 or scratch.plan.rows == 0 or scratch.plan.experts == 0 or + scratch.plan.rows % repeat != 0) return error.InvalidExpertGrouping; + if (ids.offset % 4 != 0 or scratch.offset % 4 != 0 or + ids.offset > ids.buffer.length() or @as(usize, scratch.plan.rows) * 4 > ids.buffer.length() - ids.offset or + scratch.offset > scratch.buffer.length() or scratch.plan.bytes > scratch.buffer.length() - scratch.offset) + return error.InvalidExpertGroupingStorage; + const args = Args{ .rows = scratch.plan.rows, .experts = scratch.plan.experts, .repeat = repeat, .padded = scratch.plan.padded }; + for (pipelines.stages, 0..) |pipeline, phase| { + enc.setPipeline(pipeline); + enc.setBuffer(ids.buffer, ids.offset, 0); + inline for (std.enums.values(Field), 1..) |field, i| { + const view = scratch.view(field); + enc.setBuffer(view.buffer, view.offset, i); + } + enc.setValue(args, 7); + enc.dispatchThreads(.{ .width = switch (phase) { + 0 => @max(args.experts, args.padded), + 2 => 1, + else => args.rows, + } }, .{ .width = 256 }); + enc.barrier(); + } +} + +test "expert grouping bounds padding without context-sized allocation" { + const small = try Plan.init(6, 384); + try std.testing.expectEqual(@as(u32, 48), small.padded); + const full = try Plan.init(1536, 384); + try std.testing.expectEqual(@as(u32, 4224), full.padded); + try std.testing.expect(full.bytes < 64 * 1024); + try std.testing.expectError(error.InvalidExpertGrouping, Plan.init(0, 384)); + try std.testing.expectError(error.ExpertGroupingTooLarge, Plan.init(std.math.maxInt(u32), 384)); +} diff --git a/zig/src/core/gpu_counters.zig b/zig/src/core/gpu_counters.zig index 2e0b216165..72edee1343 100644 --- a/zig/src/core/gpu_counters.zig +++ b/zig/src/core/gpu_counters.zig @@ -1,4 +1,4 @@ -//! Same-command-buffer GPU timestamps, bounded resources and no per-span CPU waits. +//! Bounded same-command GPU timestamps; disabled init allocates nothing, no per-span CPU waits or CPU timing fallback. const std = @import("std"); const mtl = @import("metal"); const counters = mtl.counters; @@ -9,6 +9,8 @@ pub const max_span_capacity: usize = 65536; pub const Options = struct { enabled: bool = false, max_spans: usize = 512 }; pub const Span = struct { kind: Class = .dense, + /// Caller-owned diagnostic label; does not affect sampling or generic class aggregation. + metadata: u32 = 0, begin_timestamp: u64 = 0, end_timestamp: u64 = 0, gpu_seconds: f64 = 0, @@ -16,7 +18,7 @@ pub const Span = struct { pub const Aggregate = struct { calls: u64 = 0, gpu_seconds: f64 = 0 }; const State = enum { idle, recording, resolved, read, broken }; -/// Two reference samples convert GPU ticks to nanoseconds; never use CPU span time. +/// Apple's paired reference samples convert GPU ticks to CPU nanoseconds, not elapsed operator time. pub const Calibration = struct { before: counters.ClockSample, after: counters.ClockSample, @@ -29,7 +31,8 @@ pub const Calibration = struct { } pub fn duration(self: Calibration, begin: u64, end: u64) !f64 { - if (begin == 0 or end == 0 or begin == counters.error_value or end == counters.error_value) return error.InvalidCounterSample; + if (begin == 0 or end == 0 or begin == counters.error_value or end == counters.error_value) + return error.InvalidCounterSample; if (end < begin) return error.ReversedCounterSample; if (begin < self.before.gpu or end > self.after.gpu) return error.CounterOutsideCalibration; const ticks: f64 = @floatFromInt(end - begin); @@ -79,7 +82,7 @@ pub const Recorder = struct { calibration: ?Calibration = null, totals: [class_count]Aggregate = @splat(.{}), - /// Init-only bounded storage; disabled init allocates nothing. + /// Init-only storage, descriptors and fence; 1..65536 spans, at most 16 CPU-resolve bytes per span. pub fn init(allocator: std.mem.Allocator, device: mtl.Device, options: Options) !Recorder { var self = Recorder{ .allocator = allocator }; if (!options.enabled) return self; @@ -120,8 +123,8 @@ pub const Recorder = struct { self.state = .recording; } - /// Non-nested spans; stage mode rotates fenced encoders, requiring resource and pipeline rebinding. - pub fn beginSpan(self: *Recorder, encoder: *mtl.ComputeEncoder, kind: Class) !void { + /// No nesting; stage sampling rotates fenced serial encoders: callers must rebind and redeclare resources. + pub fn beginSpan(self: *Recorder, encoder: *mtl.ComputeEncoder, kind: Class, metadata: u32) !void { if (!self.enabled()) return; if (self.state != .recording or !self.encoder_open) return error.CounterNotRecording; if (self.active_span != null) return error.NestedCounterSpan; @@ -141,7 +144,7 @@ pub const Recorder = struct { self.encoder_open = true; resources.fence.waitCompute(encoder.*); } - self.spans[index] = .{ .kind = kind }; + self.spans[index] = .{ .kind = kind, .metadata = metadata }; self.span_count += 1; self.active_span = index; } @@ -172,7 +175,7 @@ pub const Recorder = struct { self.active_span = null; } - /// Ends the final encoder before the caller's existing commit/wait; resolution requires completion. + /// End the final encoder before existing commit/wait; resolve counters only after command completion. pub fn finishEncoding(self: *Recorder, encoder: *mtl.ComputeEncoder) !void { if (!self.enabled()) { encoder.end(); @@ -186,7 +189,7 @@ pub const Recorder = struct { self.state = .resolved; } - /// Cancels unsubmitted work, closing any surviving encoder, including after encoder creation failure. + /// Close and discard failed, unsubmitted encoding without committing; safe after encoder creation failure. pub fn cancelEncoding(self: *Recorder, encoder: *mtl.ComputeEncoder) !void { if (self.command_buffer) |cb| { if (cb.status() != .not_enqueued) return error.CounterCommandAlreadySubmitted; @@ -198,7 +201,7 @@ pub const Recorder = struct { try self.discardAfterCompletion(); } - /// No wait or fallback; results are borrowed until beginCommand, discardAfterCompletion or deinit. + /// No wait or fallback; results are borrowed until reuse/discard/deinit, errors and invalid calibration propagate. pub fn readAfterCompletion(self: *Recorder) ![]const Span { if (!self.enabled()) return &.{}; if (self.state == .read) return self.spans[0..self.span_count]; @@ -244,7 +247,7 @@ pub const Recorder = struct { return self.totals; } - /// Discards closed, unsubmitted or completed recordings; in-flight resources are never reused. + /// Discard completed/unsubmitted recordings after closing encoders; in-flight reuse is rejected, never waited. pub fn discardAfterCompletion(self: *Recorder) !void { if (self.encoder_open) return error.CounterEncoderStillOpen; if (self.command_buffer) |cb| { @@ -288,14 +291,14 @@ test "disabled recorder allocates nothing and never touches a device or encoder var recorder = try Recorder.init(std.testing.failing_allocator, undefined, .{ .enabled = false }); defer recorder.deinit() catch unreachable; var encoder: mtl.ComputeEncoder = undefined; - try recorder.beginSpan(&encoder, .attention); + try recorder.beginSpan(&encoder, .attention, 0); try recorder.endSpan(&encoder); try std.testing.expect(!recorder.enabled()); try std.testing.expectEqual(@as(usize, 0), (try recorder.readAfterCompletion()).len); for (recorder.aggregates()) |aggregate| try std.testing.expectEqual(@as(u64, 0), aggregate.calls); } -test "calibration rejects missing samples, both reversed clocks and both range boundaries" { +test "calibration rejects missing samples and both clock and range boundaries" { const c = Calibration{ .before = .{ .cpu = 1000, .gpu = 100 }, .after = .{ .cpu = 5000, .gpu = 300 } }; try std.testing.expectError(error.InvalidCounterSample, c.duration(0, 200)); try std.testing.expectError(error.InvalidCounterSample, c.duration(100, counters.error_value)); @@ -308,12 +311,45 @@ test "calibration rejects missing samples, both reversed clocks and both range b } test "enabled capacity validation occurs before any device access" { - try std.testing.expectError(error.InvalidSpanCapacity, Recorder.init(std.testing.failing_allocator, undefined, .{ - .enabled = true, - .max_spans = 0, - })); - try std.testing.expectError(error.InvalidSpanCapacity, Recorder.init(std.testing.failing_allocator, undefined, .{ - .enabled = true, - .max_spans = max_span_capacity + 1, - })); + try std.testing.expectError(error.InvalidSpanCapacity, Recorder.init(std.testing.failing_allocator, undefined, .{ .enabled = true, .max_spans = 0 })); + try std.testing.expectError(error.InvalidSpanCapacity, Recorder.init(std.testing.failing_allocator, undefined, .{ .enabled = true, .max_spans = max_span_capacity + 1 })); +} + +test "counter ownership rejects open and in-flight reuse without a CPU span wait" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + if (counters.query(device).mode() == .unsupported) return error.SkipZigTest; + const queue = try device.queue(); + defer queue.deinit(); + const gate = try device.sharedEvent(); + defer gate.deinit(); + var recorder = try Recorder.init(std.testing.allocator, device, .{ .enabled = true, .max_spans = 1 }); + defer recorder.deinit() catch unreachable; + const cb = queue.commandBuffer(); + cb.waitFor(gate, 1); + var encoder = cb.compute(.serial); + try recorder.beginCommand(cb); + var submitted = false; + defer { + if (submitted) { + gate.set(1); + cb.wait(); + recorder.discardAfterCompletion() catch unreachable; + } else recorder.cancelEncoding(&encoder) catch unreachable; + } + try std.testing.expectError(error.CounterEncoderStillOpen, recorder.deinit()); + try recorder.beginSpan(&encoder, .dense, 42); + try std.testing.expectError(error.NestedCounterSpan, recorder.beginSpan(&encoder, .copy, 43)); + try std.testing.expectError(error.UnclosedCounterSpan, recorder.finishEncoding(&encoder)); + try recorder.endSpan(&encoder); + try std.testing.expectError(error.CounterSpanCapacityExceeded, recorder.beginSpan(&encoder, .copy, 43)); + try recorder.finishEncoding(&encoder); + cb.commit(); + submitted = true; + try std.testing.expectError(error.CounterCommandNotCompleted, recorder.readAfterCompletion()); + try std.testing.expectError(error.CounterRecordingInFlight, recorder.beginCommand(cb)); + try std.testing.expectError(error.CounterCommandNotCompleted, recorder.discardAfterCompletion()); + try std.testing.expectError(error.CounterCommandNotCompleted, recorder.deinit()); } diff --git a/zig/src/families/deepseek_v41/attention.zig b/zig/src/families/deepseek_v41/attention.zig new file mode 100644 index 0000000000..6d7cfe5da7 --- /dev/null +++ b/zig/src/families/deepseek_v41/attention.zig @@ -0,0 +1,258 @@ +//! Pinned DeepSeek V4.1 attention, resident EXL3 projections and causal packed-cache chunk execution. +const mtl = @import("metal"); +const exl3 = @import("../../core/exl3_projection.zig"); +const checkpoint = @import("../../core/checkpoint_metal.zig"); +const weights = @import("weights.zig"); +const config = @import("config.zig"); +const ops = @import("ops.zig"); +const csa2 = @import("csa2.zig"); +const kv = @import("kv.zig"); +const index = @import("csa2_index.zig"); +const rope = @import("rope.zig"); +pub const View = @import("view.zig").View; +pub const State = kv.cache.State; + +/// Opt-in bounded intermediate capture; ordinary execution does not install an observer. +pub const DiagnosticStage = enum { + qr_projection, + qr_norm, + q_projection, + q, + kv_projection, + kv_norm, + kv_rope, + kv_quantized, + sparse_o, + o, + grouped, + wo_b, +}; +pub const DiagnosticObserver = struct { + ctx: *anyopaque, + capture: *const fn (*anyopaque, mtl.ComputeEncoder, DiagnosticStage, View) anyerror!void, + + fn observe(comptime enabled: bool, self: ?DiagnosticObserver, e: mtl.ComputeEncoder, stage: DiagnosticStage, source: View) !void { + if (enabled) if (self) |observer| try observer.capture(observer.ctx, e, stage, source); + } +}; + +pub const Pipelines = struct { + plain: ops.Pipelines, + cache: kv.Pipelines, + select: index.Pipelines, + rotary: rope.Pipelines, + sparse: csa2.Pipelines, + + pub fn load(device: mtl.Device, capacity: u32, c: config.Text) !Pipelines { + var plain = try ops.Pipelines.load(device); + errdefer plain.deinit(); + var cache = try kv.Pipelines.load(device); + errdefer cache.deinit(); + var select = try index.Pipelines.load(device); + errdefer select.deinit(); + var rotary = try rope.Pipelines.load(device, capacity, c.qk_rope_head_dim, .{ .theta = c.rope_theta, .original = 0 }, .{ .theta = c.compress_rope_theta, .factor = c.rope_scaling.factor, .original = c.rope_scaling.original_max_position_embeddings, .beta_fast = c.rope_scaling.beta_fast, .beta_slow = c.rope_scaling.beta_slow }); + errdefer rotary.deinit(); + const sparse = try csa2.Pipelines.load(device); + return .{ .plain = plain, .cache = cache, .select = select, .rotary = rotary, .sparse = sparse }; + } + pub fn residentBytes(capacity: u32, c: config.Text) usize { + return rope.Pipelines.residentBytes(capacity, c.qk_rope_head_dim); + } + pub fn registerResidency(p: *const Pipelines, set: mtl.ResidencySet) void { + p.rotary.registerResidency(set); + } + pub fn deinit(p: *Pipelines) void { + p.plain.deinit(); + p.cache.deinit(); + p.select.deinit(); + p.rotary.deinit(); + p.sparse.deinit(); + } +}; + +// Row-sized regions never contain dequantized weights or a query-by-context tensor. +const regions = [_]usize{ 1280 * 2, 32768 * 2, 512 * 2, 512 * 4, 512 * 4, 512 * 2, 128 * 2, 4096 * 2, 32 * 2, 8192 * 2 }; +const Region = enum(usize) { qr, query, raw_kv, values, gates, latent, index_k, index_q, index_weights, grouped }; + +pub const Workspace = struct { + buffer: mtl.Buffer, + selection: index.Scratch, + max_rows: u32, + selection_layer: ?u32 = null, + selection_position: u32 = 0, + selection_rows: u32 = 0, + candidate_position: u32 = 0, + candidate_rows: u32 = 0, + + pub fn bytes(max_rows: u32) usize { + var per_row: usize = 0; + for (regions) |size| per_row += size; + return @as(usize, max_rows) * per_row + index.Scratch.bytes(max_rows); + } + pub fn init(device: mtl.Device, max_rows: u32) !Workspace { + if (max_rows == 0 or max_rows > 4096) return error.InvalidAttentionWorkspace; + const buffer = try device.buffer(bytes(max_rows) - index.Scratch.bytes(max_rows), mtl.ResourceOptions.shared | mtl.ResourceOptions.untracked); + errdefer buffer.deinit(); + const selection = try index.Scratch.init(device, max_rows); + return .{ .buffer = buffer, .selection = selection, .max_rows = max_rows }; + } + pub fn deinit(w: *Workspace) void { + w.buffer.deinit(); + w.selection.deinit(); + } + pub fn residentBytes(w: *const Workspace) usize { + return bytes(w.max_rows); + } + pub fn registerResidency(w: *const Workspace, set: mtl.ResidencySet) void { + set.add(w.buffer); + set.add(w.selection.buffer); + } + fn view(w: *const Workspace, region: Region) View { + var offset: usize = 0; + for (regions[0..@backingInt(region)]) |size| offset += size * w.max_rows; + return .{ .buffer = w.buffer, .offset = offset }; + } +}; + +fn tensor(t: checkpoint.Tensor) View { + return .{ .buffer = t.buffer, .offset = t.offset }; +} +fn core(v: View) exl3.View { + return .{ .buffer = v.buffer, .offset = v.offset }; +} +fn projection(p: *const exl3.Pipelines, e: mtl.ComputeEncoder, m: exl3.Matrix, x: View, y: View, scratch: exl3.View, rows: u32, xs: u32, ys: u32, output: exl3.Output) !void { + try exl3.encode(p, e, m, core(x), core(y), scratch, .{ .count = rows, .x_stride = xs, .y_stride = ys, .output = output }); +} +fn norm(p: *const Pipelines, e: mtl.ComputeEncoder, x: View, weight: checkpoint.Tensor, rows: u32, width: u32, eps: f32) void { + ops.encodeRms(e, &p.plain, x, tensor(weight), x, .{ .rows = rows, .width = width, .x_stride = width, .y_stride = width, .eps = eps, .weight_f32 = weight.dtype == .f32 }); + e.barrier(); +} +fn rotate(p: *const Pipelines, e: mtl.ComputeEncoder, x: View, rows: u32, heads: u32, width: u32, position: u32, ratio: u32, step: u32, inverse: bool, c: config.Text) !void { + try p.rotary.encode(e, x, .{ .rows = rows, .heads = heads, .width = width, .rotary = c.qk_rope_head_dim, .stride = heads * width, .position = position, .step = step, .inverse = @intFromBool(inverse), .spectrum = if (ratio == 0) .normal else .compressed }); +} +fn precedingIndex(layer: u32) u32 { + var source: u32 = 2; + for (kv.cache.index_sources) |id| { + if (id > layer) break; + source = id; + } + return source; +} + +/// BF16 x/y [rows,5120]; layers 0..39 share the chunk-start cursor, appended by the caller after all layers. + +// No extra RMS after wq_b or index-query Hadamard outside EXL3. +pub fn encode(p: *const Pipelines, e: mtl.ComputeEncoder, w: *const weights.Attention, state: *State, work: *Workspace, x: View, y: View, ep: *const exl3.Pipelines, scratch: exl3.View, layer: u32, rows: u32, c: config.Text) !void { + return encodeDiagnostic(p, e, w, state, work, x, y, ep, scratch, layer, rows, c, false, null); +} + +/// Same encoder and arithmetic as encode, with optional copies at operator boundaries. +pub fn encodeDiagnostic(p: *const Pipelines, e: mtl.ComputeEncoder, w: *const weights.Attention, state: *State, work: *Workspace, x: View, y: View, ep: *const exl3.Pipelines, scratch: exl3.View, layer: u32, rows: u32, c: config.Text, comptime diagnostic: bool, observer: ?DiagnosticObserver) !void { + if (layer >= 40 or rows == 0 or rows > work.max_rows or + state.position > state.plan.capacity or rows > state.plan.capacity - state.position) + return error.InvalidAttentionChunk; + const position = state.position; + const ratio = c.compress_ratios[layer]; + const end = position + rows; + const q_width = c.num_attention_heads * c.head_dim; + const qr = work.view(.qr); + const q = work.view(.query); + const raw = work.view(.raw_kv); + const ring = try state.ring(layer); + const chunk = work.selection.windowChunk(); + e.barrier(); + try projection(ep, e, w.wq_a, x, qr, scratch, rows, c.hidden_size, c.q_lora_rank, .bf16); + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .qr_projection, qr); + norm(p, e, qr, w.q_norm, rows, c.q_lora_rank, c.rms_norm_eps); + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .qr_norm, qr); + try projection(ep, e, w.wq_b, qr, q, scratch, rows, c.q_lora_rank, q_width, .bf16); + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .q_projection, q); + try rotate(p, e, q, rows, c.num_attention_heads, c.head_dim, position, ratio, 1, false, c); + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .q, q); + try projection(ep, e, w.wkv, x, raw, scratch, rows, c.hidden_size, c.head_dim, .bf16); + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .kv_projection, raw); + norm(p, e, raw, w.kv_norm, rows, c.head_dim, c.rms_norm_eps); + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .kv_norm, raw); + try rotate(p, e, raw, rows, 1, c.head_dim, position, ratio, 1, false, c); + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .kv_rope, raw); + try p.cache.encodePack(e, raw, chunk, .{ .rows = rows, .width = c.head_dim, .stride = c.head_dim, .packed_stride = kv.cache.ring_stride, .block = 32, .kind = .window_fp8 }); + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .kv_quantized, chunk); + + var main = chunk; + var compressed_rows: u32 = 0; + if (ratio != 0) { + const source = try state.source(layer, ratio); + main = source.main; + compressed_rows = end / ratio; + const complete = compressed_rows - position / ratio; + const latent = work.view(.latent); + if (source.owner == layer) { + const compressor = if (w.compressor) |*value| value else return error.MissingAttentionCompressor; + const output: exl3.Output = if (ratio == 1) .bf16 else .f32; + const values = work.view(.values); + const gates = work.view(.gates); + try projection(ep, e, compressor.wkv, x, values, scratch, rows, c.hidden_size, c.head_dim, output); + if (ratio > 1) { + const gate = compressor.wgate orelse return error.MissingAttentionCompressorGate; + try projection(ep, e, gate, x, gates, scratch, rows, c.hidden_size, c.head_dim, .f32); + } + try p.cache.encodePool(e, values, gates, source.partial, tensor(compressor.norm), latent, .{ .rows = rows, .ratio = ratio, .position = position, .stride = c.head_dim, .input_f32 = @intFromBool(ratio > 1), .eps = c.rms_norm_eps }); + if (complete != 0) { + const selector = if (w.indexer) |*value| value else return error.MissingAttentionIndexer; + const wk = selector.wk orelse return error.MissingAttentionIndexKeyProjection; + const k_norm = selector.k_norm orelse return error.MissingAttentionIndexKeyNorm; + const key = work.view(.index_k); + try projection(ep, e, wk, latent, key, scratch, complete, c.head_dim, c.index_head_dim, .bf16); + norm(p, e, key, k_norm, complete, c.index_head_dim, c.rms_norm_eps); + try rotate(p, e, key, complete, 1, c.index_head_dim, position / ratio * ratio, ratio, ratio, false, c); + try p.cache.encodePack(e, key, source.index, .{ .rows = complete, .width = c.index_head_dim, .stride = c.index_head_dim, .packed_stride = kv.cache.index_stride, .block = 32, .kind = .index_fp4, .first = position / ratio }); + } + } + if (kv.cache.isIndexSource(layer)) { + const selector = if (w.indexer) |*value| value else return error.MissingAttentionIndexer; + const iq = work.view(.index_q); + const iw = work.view(.index_weights); + const iq_width = c.index_n_heads * c.index_head_dim; + if (compressed_rows != 0) { + try projection(ep, e, selector.wq_b, qr, iq, scratch, rows, c.q_lora_rank, iq_width, .bf16); + try rotate(p, e, iq, rows, c.index_n_heads, c.index_head_dim, position, ratio, 1, false, c); + try p.cache.encodePack(e, iq, work.selection.queryPacked(), .{ .rows = rows * c.index_n_heads, .width = c.index_head_dim, .stride = c.index_head_dim, .packed_stride = kv.cache.index_stride, .block = 32, .kind = .index_fp4, .inplace = 1 }); + ops.encodeDense(e, &p.plain, x, tensor(selector.weights_proj), iw, .{ .rows = rows, .k = c.hidden_size, .n = c.index_n_heads, .x_stride = c.hidden_size, .w_stride = c.hidden_size, .y_stride = c.index_n_heads, .w_dtype = .f16 }); + e.barrier(); + } + const make_candidates = layer == c.candidate_source_layer_id; + const use_candidates = layer > c.candidate_source_layer_id; + if (use_candidates and (work.candidate_position != position or work.candidate_rows != rows)) + return error.MissingAttentionCandidates; + try p.select.encode(e, iq, iw, source.index, work.selection, .{ .rows = rows, .q_stride = iq_width, .weight_stride = c.index_n_heads, .position = position, .ratio = ratio, .keys = compressed_rows, .make_candidates = @intFromBool(make_candidates), .use_candidates = @intFromBool(use_candidates) }); + work.selection_layer = layer; + work.selection_position = position; + work.selection_rows = rows; + if (make_candidates) { + work.candidate_position = position; + work.candidate_rows = rows; + } + } else if (work.selection_layer != precedingIndex(layer) or + work.selection_position != position or work.selection_rows != rows) return error.MissingAttentionSelection; + // Index K consumes the normalized latent before this in-place RoPE mutates it. + if (source.owner == layer and complete != 0) { + try rotate(p, e, latent, complete, 1, c.head_dim, position / ratio * ratio, ratio, ratio, false, c); + try p.cache.encodePack(e, latent, source.main, .{ .rows = complete, .width = c.head_dim, .stride = c.head_dim, .packed_stride = kv.cache.main_stride, .block = 16, .kind = .main_fp4, .first = position / ratio }); + } + } + try p.sparse.encode(e, q, ring, chunk, main, work.selection.positions(), tensor(w.sink), q, .{ .rows = rows, .q_stride = q_width, .position = position, .chunk_rows = rows, .ratio = ratio, .compressed_rows = compressed_rows, .topk = @min(c.index_topk, compressed_rows) }); + e.barrier(); + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .sparse_o, q); + try p.cache.publishWindow(e, chunk, ring, rows, position); + try rotate(p, e, q, rows, c.num_attention_heads, c.head_dim, position, ratio, 1, true, c); + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .o, q); + const grouped = work.view(.grouped); + const group_input = q_width / c.o_groups; + const group_output = c.o_groups * c.o_lora_rank; + for (w.wo_a, 0..) |matrix, group| { + try projection(ep, e, matrix, q.shifted(group * group_input * 2), grouped.shifted(group * c.o_lora_rank * 2), scratch, rows, q_width, group_output, .bf16); + } + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .grouped, grouped); + try projection(ep, e, w.wo_b, grouped, y, scratch, rows, group_output, c.hidden_size, .bf16); + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .wo_b, y); +} diff --git a/zig/src/families/deepseek_v41/csa2.zig b/zig/src/families/deepseek_v41/csa2.zig new file mode 100644 index 0000000000..395de9107a --- /dev/null +++ b/zig/src/families/deepseek_v41/csa2.zig @@ -0,0 +1,77 @@ +//! Exact causal SWA plus shared compressed KV; fixed-buffer chunked prefill and decode use one path. +const std = @import("std"); +const mtl = @import("metal"); +const sources = @import("kernel_sources"); +const View = @import("view.zig").View; +pub const index = @import("csa2_index.zig"); +pub const kv = @import("kv.zig"); +pub const rope = @import("rope.zig"); +pub const Params = extern struct { + rows: u32, + heads: u32 = 64, + width: u32 = 512, + q_stride: u32, + position: u32, + chunk_rows: u32, + window: u32 = 128, + ring_stride: u32 = 528, + chunk_stride: u32 = 528, + main_stride: u32 = 288, + main_block: u32 = 16, + ratio: u32, + compressed_rows: u32, + topk: u32 = 512, + index_stride: u32 = 512, + scale: f32 = 0.04419417382415922, + pub fn validate(p: Params) !void { + if (p.rows == 0 or p.heads == 0 or p.width == 0 or p.width > 512 or p.width % 32 != 0 or + @as(u64, p.q_stride) < @as(u64, p.heads) * p.width or + p.chunk_rows < p.rows or p.window != 128 or + p.ring_stride < p.width + p.width / 32 or p.chunk_stride < p.width + p.width / 32 or + p.main_stride < p.width / 2 + p.width / 16 or p.main_block != 16 or + (p.ratio != 0 and p.ratio != 1 and p.ratio != 2) or p.topk > 512 or p.index_stride < p.topk or + @as(u64, p.position) + p.rows > kv.cache.max_context or + p.compressed_rows > kv.cache.max_context or !std.math.isFinite(p.scale) or p.scale <= 0 or + @as(u64, p.rows -| 1) * p.q_stride + @as(u64, p.heads) * p.width > std.math.maxInt(u32) or + @as(u64, p.rows) * p.index_stride > std.math.maxInt(u32) or + @as(u64, p.rows) * p.chunk_stride > std.math.maxInt(u32) or + @as(u64, p.window) * p.ring_stride > std.math.maxInt(u32) or + @as(u64, p.compressed_rows) * p.main_stride > std.math.maxInt(u32)) + return error.InvalidSparseAttentionShape; + } +}; +pub const Pipelines = struct { + attention: mtl.Pipeline, + pub fn load(device: mtl.Device) !Pipelines { + var options = mtl.CompileOptions.mlx(); + options.math_functions = .precise; + const lib = try mtl.Library.fromSource(device, sources.deepseek_v41_kv ++ sources.deepseek_v41_csa2, options); + defer lib.deinit(); + return .{ .attention = try mtl.Pipeline.init(device, lib, "dsv41_sparse_attention", false) }; + } + pub fn deinit(p: *Pipelines) void { + p.attention.deinit(); + } + /// BF16 Q/output [rows,heads,width], FP32 sinks, I32 indices; publish this chunk to the ring only after dispatch. + pub fn encode(p: *const Pipelines, enc: mtl.ComputeEncoder, q: View, ring: View, chunk: View, compressed: View, positions: View, sink: View, out: View, args: Params) !void { + try args.validate(); + try q.requireRows(args.rows, args.q_stride, @as(usize, args.heads) * args.width, 2); + try out.requireRows(args.rows, args.q_stride, @as(usize, args.heads) * args.width, 2); + try sink.require(@as(usize, args.heads) * 4, 4); + if (args.position != 0) try ring.requireRows(@min(args.position, args.window), args.ring_stride, args.width + args.width / 32, 1); + try chunk.requireRows(args.rows, args.chunk_stride, args.width + args.width / 32, 1); + if (args.ratio != 0 and args.compressed_rows != 0 and args.topk != 0) { + try compressed.requireRows(args.compressed_rows, args.main_stride, args.width / 2 + args.width / 16, 1); + try positions.requireRows(args.rows, args.index_stride, args.topk, 4); + } + enc.setPipeline(p.attention); + for ([_]View{ q, ring, chunk, compressed, positions, sink, out }, 0..) |v, i| + enc.setBuffer(v.buffer, v.offset, i); + enc.setValue(args, 7); + enc.dispatchGroups(mtl.Size.of(args.heads, args.rows, 1), mtl.Size.of(32, 1, 1)); + } +}; + +test { + _ = @import("kv_test.zig"); +} diff --git a/zig/src/families/deepseek_v41/csa2_index.zig b/zig/src/families/deepseek_v41/csa2_index.zig new file mode 100644 index 0000000000..2ac845ef07 --- /dev/null +++ b/zig/src/families/deepseek_v41/csa2_index.zig @@ -0,0 +1,139 @@ +//! Tiled GPU scores and bounded online heaps; the memory footprint does not grow with context length. +const std = @import("std"); +const max_context = @import("kv_state.zig").max_context; +const mtl = @import("metal"); +const sources = @import("kernel_sources"); +const View = @import("view.zig").View; +pub const score_tile_width = 4096; +pub const Params = extern struct { + rows: u32, + heads: u32 = 32, + width: u32 = 128, + q_stride: u32, + weight_stride: u32, + position: u32, + ratio: u32, + keys: u32, + key_stride: u32 = 68, + tile_start: u32 = 0, + tile_width: u32 = score_tile_width, + topk: u32 = 512, + candidate_topk: u32 = 2048, + block_size: u32 = 8, + use_candidates: u32 = 0, + make_candidates: u32 = 0, + scale: f32 = 0.015625, + pub fn validate(p: Params) !void { + if (p.rows == 0 or p.heads == 0 or p.heads > 32 or p.width == 0 or p.width > 128 or p.width % 32 != 0 or + p.q_stride < p.heads * p.width or p.weight_stride < p.heads or + (p.ratio != 1 and p.ratio != 2) or p.key_stride < p.width / 2 + p.width / 32 or + p.topk != 512 or p.candidate_topk != 2048 or p.block_size != 8 or + p.tile_width != score_tile_width or p.use_candidates > 1 or p.make_candidates > 1 or + (p.use_candidates != 0 and p.make_candidates != 0) or + @as(u64, p.position) + p.rows > max_context or p.keys > max_context or + !std.math.isFinite(p.scale) or p.scale <= 0 or + @as(u64, p.rows) * p.q_stride > std.math.maxInt(u32) or + @as(u64, p.rows) * p.weight_stride > std.math.maxInt(u32) or + @as(u64, p.keys) * p.key_stride > std.math.maxInt(u32)) + return error.InvalidIndexerShape; + } +}; +pub const Scratch = struct { + buffer: mtl.Buffer, + rows: u32, + pub fn bytes(rows: u32) usize { + return @as(usize, rows) * (score_tile_width * 4 + 512 * 8 + 2048 * 8 + 512 * 4 + 528 + 32 * 68); + } + pub fn init(device: mtl.Device, rows: u32) !Scratch { + if (rows == 0 or rows > 4096) return error.InvalidIndexerScratch; + return .{ .buffer = try device.buffer(bytes(rows), mtl.ResourceOptions.shared), .rows = rows }; + } + pub fn deinit(s: *Scratch) void { + s.buffer.deinit(); + } + pub fn scores(s: Scratch) View { + return .{ .buffer = s.buffer }; + } + pub fn heap(s: Scratch) View { + return s.scores().shifted(@as(usize, s.rows) * score_tile_width * 4); + } + pub fn candidates(s: Scratch) View { + return s.heap().shifted(@as(usize, s.rows) * 512 * 8); + } + pub fn positions(s: Scratch) View { + return s.candidates().shifted(@as(usize, s.rows) * 2048 * 8); + } + pub fn windowChunk(s: Scratch) View { + return s.positions().shifted(@as(usize, s.rows) * 512 * 4); + } + pub fn queryPacked(s: Scratch) View { + return s.windowChunk().shifted(@as(usize, s.rows) * 528); + } +}; +pub const Pipelines = struct { + score: mtl.Pipeline, + clear: mtl.Pipeline, + merge: mtl.Pipeline, + finish: mtl.Pipeline, + pub fn load(device: mtl.Device) !Pipelines { + var options = mtl.CompileOptions.mlx(); + options.math_functions = .precise; + const lib = try mtl.Library.fromSource(device, sources.deepseek_v41_kv ++ sources.deepseek_v41_index, options); + defer lib.deinit(); + var p: Pipelines = undefined; + p.score = try mtl.Pipeline.init(device, lib, "dsv41_index_score", false); + errdefer p.score.deinit(); + p.clear = try mtl.Pipeline.init(device, lib, "dsv41_index_clear", false); + errdefer p.clear.deinit(); + p.merge = try mtl.Pipeline.init(device, lib, "dsv41_index_merge", false); + errdefer p.merge.deinit(); + p.finish = try mtl.Pipeline.init(device, lib, "dsv41_index_finish", false); + return p; + } + pub fn deinit(p: *Pipelines) void { + p.score.deinit(); + p.clear.deinit(); + p.merge.deinit(); + p.finish.deinit(); + } + /// Q has already undergone BF16 RoPE and in-place FP4/E8M0 quantization; weights are BF16 projection output. + pub fn encode(p: *const Pipelines, enc: mtl.ComputeEncoder, q: View, weights: View, keys: View, scratch: Scratch, args: Params) !void { + try args.validate(); + if (args.rows > scratch.rows) return error.IndexerScratchExceeded; + try (View{ .buffer = scratch.buffer }).require(Scratch.bytes(scratch.rows), 4); + if (args.keys != 0) { + try q.requireRows(args.rows, args.q_stride, @as(usize, args.heads) * args.width, 2); + try weights.requireRows(args.rows, args.weight_stride, args.heads, 2); + try keys.requireRows(args.keys, args.key_stride, args.width / 2 + args.width / 32, 1); + } + enc.setPipeline(p.clear); + for ([_]View{ scratch.heap(), scratch.candidates() }, 0..) |v, i| enc.setBuffer(v.buffer, v.offset, i); + enc.setValue(args, 2); + enc.dispatchThreads(mtl.Size.of(@max(args.topk, args.candidate_topk), args.rows, 1), mtl.Size.of(128, 1, 1)); + enc.barrier(); + const count = if (args.use_candidates != 0) args.candidate_topk * args.block_size else args.keys; + var at: u32 = 0; + while (at < count) : (at += score_tile_width) { + var tile = args; + tile.tile_start = at; + enc.setPipeline(p.score); + for ([_]View{ q, weights, keys, scratch.candidates(), scratch.scores() }, 0..) |v, i| + enc.setBuffer(v.buffer, v.offset, i); + enc.setValue(tile, 5); + enc.dispatchGroups(mtl.Size.of(score_tile_width, args.rows, 1), mtl.Size.of(32, 1, 1)); + enc.barrier(); + enc.setPipeline(p.merge); + for ([_]View{ scratch.scores(), scratch.heap(), scratch.candidates() }, 0..) |v, i| + enc.setBuffer(v.buffer, v.offset, i); + enc.setValue(tile, 3); + enc.dispatchThreads(mtl.Size.of(args.rows, 1, 1), mtl.Size.of(1, 1, 1)); + enc.barrier(); + } + enc.setPipeline(p.finish); + for ([_]View{ scratch.heap(), scratch.candidates(), scratch.positions() }, 0..) |v, i| + enc.setBuffer(v.buffer, v.offset, i); + enc.setValue(args, 3); + enc.dispatchThreads(mtl.Size.of(args.rows, 1, 1), mtl.Size.of(1, 1, 1)); + enc.barrier(); + } +}; diff --git a/zig/src/families/deepseek_v41/csa2_test.zig b/zig/src/families/deepseek_v41/csa2_test.zig new file mode 100644 index 0000000000..0ac81cacca --- /dev/null +++ b/zig/src/families/deepseek_v41/csa2_test.zig @@ -0,0 +1,309 @@ +const std = @import("std"); +const mtl = @import("metal"); +const csa2 = @import("csa2.zig"); +const index = @import("csa2_index.zig"); +const rope = @import("rope.zig"); +const View = @import("view.zig").View; +fn bf(x: f32) u16 { + const bits: u32 = @bitCast(x); + return @intCast((bits +% 0x7fff +% ((bits >> 16) & 1)) >> 16); +} +fn view(b: mtl.Buffer) View { + return .{ .buffer = b }; +} +fn submit(cb: mtl.CommandBuffer, enc: mtl.ComputeEncoder) !void { + enc.end(); + cb.commit(); + cb.wait(); + try std.testing.expect(cb.failure() == null); +} + +test "sparse attention preserves a causal window across ring wrap and masks future compressed keys" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try csa2.Pipelines.load(device); + defer p.deinit(); + const q = try device.buffer(3 * 512 * 2, mtl.ResourceOptions.shared); + defer q.deinit(); + const ring = try device.buffer(128 * 528, mtl.ResourceOptions.shared); + defer ring.deinit(); + const chunk = try device.buffer(3 * 528, mtl.ResourceOptions.shared); + defer chunk.deinit(); + const main = try device.buffer(201 * 288, mtl.ResourceOptions.shared); + defer main.deinit(); + const positions = try device.buffer(6 * 4, mtl.ResourceOptions.shared); + defer positions.deinit(); + const sink = try device.buffer(4, mtl.ResourceOptions.shared); + defer sink.deinit(); + const out = try device.buffer(3 * 512 * 2, mtl.ResourceOptions.shared); + defer out.deinit(); + @memset(q.contents()[0 .. 3 * 512 * 2], 0); + sink.slice(f32, 1)[0] = 0; + for (0..128) |r| { + @memset(ring.contents()[r * 528 .. r * 528 + 512], 0x38); + @memset(ring.contents()[r * 528 + 512 .. (r + 1) * 528], 127); + } + for (0..3) |r| { + @memset(chunk.contents()[r * 528 .. r * 528 + 512], 0x40); + @memset(chunk.contents()[r * 528 + 512 .. (r + 1) * 528], 127); + positions.slice(i32, 6)[2 * r] = 0; + positions.slice(i32, 6)[2 * r + 1] = 200; + } + @memset(main.contents()[0..256], 0x66); + @memset(main.contents()[256..288], 0x38); + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encode(enc, view(q), view(ring), view(chunk), view(main), view(positions), view(sink), view(out), .{ .rows = 3, .heads = 1, .q_stride = 512, .position = 129, .chunk_rows = 3, .ratio = 2, .compressed_rows = 201, .topk = 2, .index_stride = 2 }); + try submit(cb, enc); + for (0..3) |r| { + const expected = bf((133 + @as(f32, @floatFromInt(r))) / 130); + for (out.slice(u16, 3 * 512)[r * 512 .. (r + 1) * 512]) |v| try std.testing.expectEqual(expected, v); + } +} + +test "sparse QK cancellation retains a small score before sink normalization" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try csa2.Pipelines.load(device); + defer p.deinit(); + const q = try device.buffer(512 * 2, mtl.ResourceOptions.shared); + defer q.deinit(); + const chunk = try device.buffer(528, mtl.ResourceOptions.shared); + defer chunk.deinit(); + const sink = try device.buffer(4, mtl.ResourceOptions.shared); + defer sink.deinit(); + const out = try device.buffer(512 * 2, mtl.ResourceOptions.shared); + defer out.deinit(); + @memset(q.slice(u16, 512), 0); + q.slice(u16, 512)[0] = bf(0x1p24); + q.slice(u16, 512)[1] = bf(-0x1p24); + q.slice(u16, 512)[32] = bf(0x1p-6); + @memset(chunk.contents()[0..512], 0x38); + @memset(chunk.contents()[512..528], 127); + sink.slice(f32, 1)[0] = 0; + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + // No prior window or compressed keys: their unused bindings share the immutable chunk. + try p.encode(enc, view(q), view(chunk), view(chunk), view(chunk), view(chunk), view(sink), view(out), .{ .rows = 1, .heads = 1, .q_stride = 512, .position = 0, .chunk_rows = 1, .ratio = 0, .compressed_rows = 0, .scale = 1 }); + try submit(cb, enc); + // Exact dot = 2^-6; 1 / (1 + exp(-2^-6)) rounds to this BF16 value. + for (out.slice(u16, 512)) |v| try std.testing.expectEqual(bf(0.50390625), v); +} + +test "the first query retains BF16 probability arithmetic regardless of future prefill rows" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try csa2.Pipelines.load(device); + defer p.deinit(); + const q = try device.buffer(32 * 2, mtl.ResourceOptions.shared); + defer q.deinit(); + const ring = try device.buffer(128 * 33, mtl.ResourceOptions.shared); + defer ring.deinit(); + const chunk = try device.buffer(128 * 33, mtl.ResourceOptions.shared); + defer chunk.deinit(); + const main = try device.buffer(18, mtl.ResourceOptions.shared); + defer main.deinit(); + const positions = try device.buffer(4, mtl.ResourceOptions.shared); + defer positions.deinit(); + const sink = try device.buffer(4, mtl.ResourceOptions.shared); + defer sink.deinit(); + const out = try device.buffer(32 * 2, mtl.ResourceOptions.shared); + defer out.deinit(); + @memset(q.slice(u16, 32), 0); + q.slice(u16, 32)[0] = bf(1); + @memset(chunk.contents()[0..32], 0x38); + chunk.contents()[32] = 127; + @memset(main.contents()[0..16], 0x44); + @memset(main.contents()[16..18], 0x38); + positions.slice(i32, 1)[0] = 0; + sink.slice(f32, 1)[0] = 0.005; + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encode(enc, view(q), view(ring), view(chunk), view(main), view(positions), view(sink), view(out), .{ .rows = 1, .heads = 1, .width = 32, .q_stride = 32, .position = 0, .chunk_rows = 1, .ring_stride = 33, .chunk_stride = 33, .main_stride = 18, .ratio = 1, .compressed_rows = 1, .topk = 1, .index_stride = 1, .scale = 1 }); + try submit(cb, enc); + // BF16(exp(-1))*1 + 1*2; padded windows would move key 2 into a later online-softmax tile. + for (out.slice(u16, 32)) |v| try std.testing.expectEqual(bf(1.5703125), v); + const cb2 = queue.commandBuffer(); + const enc2 = cb2.compute(.serial); + try p.encode(enc2, view(q), view(ring), view(chunk), view(main), view(positions), view(sink), view(out), .{ .rows = 1, .heads = 1, .width = 32, .q_stride = 32, .position = 0, .chunk_rows = 128, .ring_stride = 33, .chunk_stride = 33, .main_stride = 18, .ratio = 1, .compressed_rows = 1, .topk = 1, .index_stride = 1, .scale = 1 }); + try submit(cb2, enc2); + for (out.slice(u16, 32)) |v| try std.testing.expectEqual(bf(1.5703125), v); +} + +test "hierarchical candidate selection pins the newest partial block and subsequent reindex uses those blocks" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try index.Pipelines.load(device); + defer p.deinit(); + var scratch = try index.Scratch.init(device, 1); + defer scratch.deinit(); + const q = try device.buffer(32 * 2, mtl.ResourceOptions.shared); + defer q.deinit(); + const weights = try device.buffer(2, mtl.ResourceOptions.shared); + defer weights.deinit(); + const keys_count = 2048 * 8 + 3; + const keys = try device.buffer(keys_count * 17, mtl.ResourceOptions.shared); + defer keys.deinit(); + @memset(q.slice(u16, 32), bf(1)); + weights.slice(u16, 1)[0] = bf(1); + for (0..keys_count) |r| { + @memset(keys.contents()[r * 17 .. r * 17 + 16], 0x22); + keys.contents()[r * 17 + 16] = if (r >= 2048 * 8) 126 else 127; + } + var args = index.Params{ .rows = 1, .heads = 1, .width = 32, .q_stride = 32, .weight_stride = 1, .position = keys_count - 1, .ratio = 1, .keys = keys_count, .key_stride = 17, .make_candidates = 1, .scale = 1 }; + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encode(enc, view(q), view(weights), view(keys), scratch, args); + try submit(cb, enc); + const Pick = extern struct { score: f32, position: i32 }; + const candidates = @as([*]Pick, @ptrCast(@alignCast(scratch.candidates().buffer.contents() + scratch.candidates().offset)))[0..2048]; + try std.testing.expectEqual(@as(i32, 2048), candidates[2047].position); + for (candidates[0..2047], 0..) |pick, i| try std.testing.expectEqual(@as(i32, @intCast(i)), pick.position); + const cb2 = queue.commandBuffer(); + const enc2 = cb2.compute(.serial); + args.make_candidates = 0; + args.use_candidates = 1; + try p.encode(enc2, view(q), view(weights), view(keys), scratch, args); + try submit(cb2, enc2); + const selected = @as([*]i32, @ptrCast(@alignCast(scratch.positions().buffer.contents() + scratch.positions().offset)))[0..512]; + for (selected, 0..) |pos, i| try std.testing.expectEqual(@as(i32, @intCast(i)), pos); +} + +test "RoPE rotates adjacent tail pairs with BF16 assignment and compressed positions use group starts" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try rope.Pipelines.load(device, 3, 32, .{ .theta = 10000 }, .{ .theta = 160000 }); + defer p.deinit(); + const x = try device.buffer(2 * 64 * 2, mtl.ResourceOptions.shared); + defer x.deinit(); + @memset(x.slice(u16, 128), bf(1)); + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try std.testing.expectError(error.InvalidRotaryShape, p.encode(enc, view(x), .{ .rows = 2, .heads = 1, .width = 64, .rotary = 32, .stride = 64, .position = 1, .step = 2 })); + try std.testing.expectError(error.InvalidRotaryShape, p.encode(enc, view(x), .{ .rows = 1, .heads = 1, .width = 64, .rotary = 16, .stride = 64, .position = 0 })); + try std.testing.expectError(error.InvalidRotaryShape, p.encode(enc, view(x), .{ .rows = 2, .heads = 1, .width = 64, .rotary = 32, .stride = 64, .position = std.math.maxInt(u32), .step = std.math.maxInt(u32) })); + try p.encode(enc, view(x), .{ .rows = 2, .heads = 1, .width = 64, .rotary = 32, .stride = 64, .position = 0, .step = 2 }); + try submit(cb, enc); + for (x.slice(u16, 128)[0..96]) |v| try std.testing.expectEqual(bf(1), v); + for (0..16) |pair| { + const power: f32 = @floatCast(std.math.pow(f64, 10000, @as(f32, @floatFromInt(pair * 2)) / 32)); + const frequency: f32 = 1 / power; + const angle: f32 = 2 * frequency; + const c: f32 = @floatCast(@cos(@as(f64, angle))); + const s: f32 = @floatCast(@sin(@as(f64, angle))); + const a = bf(c - s); + const b = bf(s + c); + try std.testing.expectEqual(a, x.slice(u16, 128)[96 + pair * 2]); + try std.testing.expectEqual(b, x.slice(u16, 128)[97 + pair * 2]); + } +} + +test "indexer visibility follows completed compressor groups for every query in a chunk" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try index.Pipelines.load(device); + defer p.deinit(); + var scratch = try index.Scratch.init(device, 3); + defer scratch.deinit(); + const q = try device.buffer(3 * 32 * 2, mtl.ResourceOptions.shared); + defer q.deinit(); + const weights = try device.buffer(3 * 2, mtl.ResourceOptions.shared); + defer weights.deinit(); + const keys = try device.buffer(3 * 17, mtl.ResourceOptions.shared); + defer keys.deinit(); + @memset(q.slice(u16, 3 * 32), bf(1)); + @memset(weights.slice(u16, 3), bf(1)); + for (0..3) |r| { + @memset(keys.contents()[r * 17 .. r * 17 + 16], 0x22); + keys.contents()[r * 17 + 16] = 127; + } + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encode(enc, view(q), view(weights), view(keys), scratch, .{ .rows = 3, .heads = 1, .width = 32, .q_stride = 32, .weight_stride = 1, .position = 0, .ratio = 2, .keys = 3, .key_stride = 17, .scale = 1 }); + try submit(cb, enc); + const selected = @as([*]i32, @ptrCast(@alignCast(scratch.positions().buffer.contents() + + scratch.positions().offset)))[0 .. 3 * 512]; + for (selected[0..512]) |pos| try std.testing.expectEqual(@as(i32, -1), pos); + for (1..3) |r| { + try std.testing.expectEqual(@as(i32, 0), selected[r * 512]); + for (selected[r * 512 + 1 .. (r + 1) * 512]) |pos| try std.testing.expectEqual(@as(i32, -1), pos); + } +} + +test "YaRN interpolates its correction band and inverse rotation conjugates the frequency" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try rope.Pipelines.load(device, 65537, 32, .{ .theta = 10000 }, .{ .theta = 160000, .original = 65536 }); + defer p.deinit(); + const x = try device.buffer(64 * 2, mtl.ResourceOptions.shared); + defer x.deinit(); + @memset(x.slice(u16, 64), bf(1)); + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encode(enc, view(x), .{ .rows = 1, .heads = 1, .width = 64, .rotary = 32, .stride = 64, .position = 65536, .inverse = 1, .spectrum = .compressed }); + try submit(cb, enc); + const low: f32 = @floatCast(@max(@floor(32 * @log(@as(f64, 65536) / (32 * 2 * std.math.pi)) / (2 * @log(@as(f64, 160000)))), 0)); + const high: f32 = @floatCast(@min(@ceil(32 * @log(@as(f64, 65536) / (2 * std.math.pi)) / (2 * @log(@as(f64, 160000)))), 31)); + for ([_]usize{ 0, 10, 15 }) |pair| { + const power: f32 = @floatCast(std.math.pow(f64, 160000, @as(f32, @floatFromInt(pair * 2)) / 32)); + const frequency: f32 = 1 / power; + const smooth: f32 = 1 - std.math.clamp((@as(f32, @floatFromInt(pair)) - low) / @max(high - low, 1e-3), 0, 1); + const slow: f32 = frequency / 16 * (1 - smooth); + const fast: f32 = frequency * smooth; + const angle: f32 = 65536 * (slow + fast); + const c: f32 = @floatCast(@cos(@as(f64, angle))); + const s: f32 = -@as(f32, @floatCast(@sin(@as(f64, angle)))); + try std.testing.expectEqual(bf(c - s), x.slice(u16, 64)[32 + pair * 2]); + try std.testing.expectEqual(bf(s + c), x.slice(u16, 64)[33 + pair * 2]); + } +} + +test "RoPE retained source pair rounds the position-five imaginary consumer to pinned CUDA BF16" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try rope.Pipelines.load(device, 6, 64, .{ .theta = 10000 }, .{ .theta = 160000, .original = 65536 }); + defer p.deinit(); + const x = try device.buffer(512 * 2, mtl.ResourceOptions.shared); + defer x.deinit(); + @memset(x.slice(u16, 512), bf(0)); + // Token-5/layer-0 rotary pair 21 crosses a BF16 tie under per-head cos; pinned source factors give .546875. + x.slice(u16, 512)[490] = bf(1.15625); + x.slice(u16, 512)[491] = bf(0.53125); + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encode(enc, view(x), .{ .rows = 1, .heads = 1, .width = 512, .rotary = 64, .stride = 512, .position = 5 }); + try submit(cb, enc); + try std.testing.expectEqual(bf(0.546875), x.slice(u16, 512)[491]); +} diff --git a/zig/src/families/deepseek_v41/engram.zig b/zig/src/families/deepseek_v41/engram.zig new file mode 100644 index 0000000000..0f5a0287c2 --- /dev/null +++ b/zig/src/families/deepseek_v41/engram.zig @@ -0,0 +1,203 @@ +//! Resident Engram FP8 gathers and signed-sqrt gates; WKV is projected separately by the caller. +const std = @import("std"); +const mtl = @import("metal"); +const sources = @import("kernel_sources"); +pub const View = @import("view.zig").View; +pub const hash = @import("engram_hash.zig"); + +pub const head_dim = 256; +pub const gather_dim = hash.columns * head_dim; +pub const history_bytes = 16; + +/// Borrowed full-resident E4M3FN rows and UE8M0 rowwise scale blocks, never a 32x32 scale matrix. +pub const Table = struct { + weight: View, + scale: View, + rows: u32, + weight_stride_bytes: usize = head_dim, + scale_stride_bytes: usize = head_dim / 32, + scale_block: u32 = 32, + + pub fn validate(self: Table) !void { + if (self.rows == 0 or self.weight_stride_bytes < head_dim or self.scale_stride_bytes < head_dim / 32 or self.scale_block != 32) return error.InvalidEngramTable; + try self.weight.requireRows(self.rows, self.weight_stride_bytes, head_dim, 1); + try self.scale.requireRows(self.rows, self.scale_stride_bytes, head_dim / 32, 1); + } +}; + +/// BF16 hidden [tokens,copies,dim], WKV [tokens,(copies+1)*dim]; norm weights contiguous FP32 [copies,dim]. +pub const Gate = struct { + q_weight: View, + k_weight: View, + dim: u32, + copies: u32, + hidden_stride: usize, + kv_stride: usize, + output_stride: usize, + eps: f32, + + pub fn validate(self: Gate) !void { + const width = try std.math.mul(usize, self.dim, self.copies); + const projected_width = try std.math.add(usize, width, self.dim); + if (self.dim == 0 or self.copies == 0 or self.hidden_stride < width or self.output_stride < width or self.kv_stride < projected_width or !std.math.isFinite(self.eps) or self.eps <= 0 or + self.dim > std.math.maxInt(u32) - 255) return error.InvalidEngramGate; + const weight_bytes = try std.math.mul(usize, width, 4); + try self.q_weight.require(weight_bytes, 4); + try self.k_weight.require(weight_bytes, 4); + } +}; + +pub const Pipelines = struct { + hash_ids: mtl.Pipeline, + commit: mtl.Pipeline, + reset: mtl.Pipeline, + gather: mtl.Pipeline, + gate: mtl.Pipeline, + + pub fn load(device: mtl.Device) !Pipelines { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const lib = try mtl.Library.fromSource(device, sources.deepseek_v41_engram, mtl.CompileOptions.mlx()); + defer lib.deinit(); + const h = try mtl.Pipeline.init(device, lib, "tf_v41_engram_hash", false); + errdefer h.deinit(); + const c = try mtl.Pipeline.init(device, lib, "tf_v41_engram_commit", false); + errdefer c.deinit(); + const r = try mtl.Pipeline.init(device, lib, "tf_v41_engram_reset", false); + errdefer r.deinit(); + const g = try mtl.Pipeline.init(device, lib, "tf_v41_engram_gather", false); + errdefer g.deinit(); + const gate = try mtl.Pipeline.init(device, lib, "tf_v41_engram_gate", false); + return .{ .hash_ids = h, .commit = c, .reset = r, .gather = g, .gate = gate }; + } + + pub fn deinit(self: *Pipelines) void { + self.hash_ids.deinit(); + self.commit.deinit(); + self.reset.deinit(); + self.gather.deinit(); + self.gate.deinit(); + } +}; + +const HashArgs = extern struct { tokens: u32, pad: u32, masked: u32 }; +const GatherArgs = extern struct { + weight_stride: u64, + scale_stride: u64, + output_stride: u64, + tokens: u32, + columns: u32, + head_dim: u32, + scale_block: u32, +}; +const GateArgs = extern struct { + hidden_stride: u64, + kv_stride: u64, + output_stride: u64, + dim: u32, + copies: u32, + masked: u32, + eps: f32, +}; + +fn bind(e: mtl.ComputeEncoder, view: View, at: usize) void { + e.setBuffer(view.buffer, view.offset, at); +} + +/// Each history View is one sequence's 16-byte GPU record; initialize before its first hash. +pub fn encodeReset(e: mtl.ComputeEncoder, p: *const Pipelines, history: View) void { + e.setPipeline(p.reset); + bind(e, history, 0); + e.dispatchGroups(.{ .width = 1 }, .{ .width = 1 }); + e.barrier(); +} + +/// Snapshot/rollback are GPU copies; caller ends compute encoding before beginning a blit encoder. +pub fn encodeSnapshot(e: mtl.BlitEncoder, history: View, saved: View) void { + e.copy(history.buffer, history.offset, saved.buffer, saved.offset, history_bytes); +} + +pub fn encodeRollback(e: mtl.BlitEncoder, saved: View, history: View) void { + encodeSnapshot(e, saved, history); +} + +/// Raw u32 IDs -> [tokens,24] hashes; map covers valid IDs, optional 0=DEAD mask, history is unchanged. +pub fn encodeHash(e: mtl.ComputeEncoder, p: *const Pipelines, meta: hash.Layer, ids: View, token_map: View, mask: ?View, history: View, out: View, tokens: u32) !void { + if (tokens == 0) return error.EmptyEngramChunk; + if (!std.mem.eql(u8, std.mem.asBytes(&meta), std.mem.asBytes(&hash.layers[0])) and + !std.mem.eql(u8, std.mem.asBytes(&meta), std.mem.asBytes(&hash.layers[1]))) return error.InvalidEngramMetadata; + try ids.requireRows(tokens, 1, 1, 4); + try token_map.require(4, 4); + try history.require(history_bytes, 4); + try out.requireRows(tokens, hash.columns, hash.columns, 4); + if (mask) |m| try m.require(tokens, 1); + e.setPipeline(p.hash_ids); + bind(e, ids, 0); + bind(e, token_map, 1); + bind(e, mask orelse ids, 2); + bind(e, history, 3); + bind(e, out, 4); + e.setValue(meta, 5); + e.setValue(HashArgs{ .tokens = tokens, .pad = hash.pad_id, .masked = @intFromBool(mask != null) }, 6); + e.dispatchThreads(.{ .width = tokens }, .{ .width = 64 }); + e.barrier(); +} + +/// Commit exactly once per chunk after both layers have hashed; remains bounded at three past tokens. +pub fn encodeCommit(e: mtl.ComputeEncoder, p: *const Pipelines, ids: View, token_map: View, mask: ?View, history: View, tokens: u32) !void { + if (tokens == 0) return error.EmptyEngramChunk; + try ids.requireRows(tokens, 1, 1, 4); + try token_map.require(4, 4); + try history.require(history_bytes, 4); + if (mask) |m| try m.require(tokens, 1); + e.barrier(); + e.setPipeline(p.commit); + bind(e, ids, 0); + bind(e, token_map, 1); + bind(e, mask orelse ids, 2); + bind(e, history, 3); + e.setValue(HashArgs{ .tokens = tokens, .pad = hash.pad_id, .masked = @intFromBool(mask != null) }, 4); + e.dispatchGroups(.{ .width = 1 }, .{ .width = 1 }); + e.barrier(); +} + +/// Gather uint32 [tokens,24] row IDs into BF16 [tokens,6144]; IDs must be within the borrowed table. +pub fn encodeGather(e: mtl.ComputeEncoder, p: *const Pipelines, table: Table, ids: View, out: View, tokens: u32, output_stride: usize) !void { + try table.validate(); + if (tokens == 0 or output_stride < gather_dim) return error.InvalidEngramGather; + try ids.requireRows(tokens, hash.columns, hash.columns, 4); + try out.requireRows(tokens, output_stride, gather_dim, 2); + e.setPipeline(p.gather); + bind(e, ids, 0); + bind(e, table.weight, 1); + bind(e, table.scale, 2); + bind(e, out, 3); + e.setValue(GatherArgs{ .weight_stride = table.weight_stride_bytes, .scale_stride = table.scale_stride_bytes, .output_stride = output_stride, .tokens = tokens, .columns = hash.columns, .head_dim = head_dim, .scale_block = table.scale_block }, 4); + e.dispatchThreads(.{ .width = gather_dim, .height = tokens }, .{ .width = 256 }); + e.barrier(); +} + +/// Add per-copy gate times shared value to BF16 hidden; out may alias hidden, never the projected KV. +pub fn encodeGate(e: mtl.ComputeEncoder, p: *const Pipelines, gate: Gate, hidden: View, projected_kv: View, mask: ?View, out: View, tokens: u32) !void { + try gate.validate(); + if (tokens == 0) return error.EmptyEngramChunk; + const width = @as(usize, gate.dim) * gate.copies; + try hidden.requireRows(tokens, gate.hidden_stride, width, 2); + try projected_kv.requireRows(tokens, gate.kv_stride, width + gate.dim, 2); + try out.requireRows(tokens, gate.output_stride, width, 2); + if (mask) |m| try m.require(tokens, 1); + e.setPipeline(p.gate); + bind(e, hidden, 0); + bind(e, projected_kv, 1); + bind(e, gate.q_weight, 2); + bind(e, gate.k_weight, 3); + bind(e, mask orelse hidden, 4); + bind(e, out, 5); + e.setValue(GateArgs{ .hidden_stride = gate.hidden_stride, .kv_stride = gate.kv_stride, .output_stride = gate.output_stride, .dim = gate.dim, .copies = gate.copies, .masked = @intFromBool(mask != null), .eps = gate.eps }, 6); + e.dispatchGroups(.{ .width = tokens, .height = gate.copies }, .{ .width = 256 }); + e.barrier(); +} + +test { + _ = @import("engram_test.zig"); +} diff --git a/zig/src/families/deepseek_v41/engram_test.zig b/zig/src/families/deepseek_v41/engram_test.zig new file mode 100644 index 0000000000..dd085a8744 --- /dev/null +++ b/zig/src/families/deepseek_v41/engram_test.zig @@ -0,0 +1,247 @@ +//! Behavioral hash, rollback, selected-row dequantization and per-copy signed-sqrt gate regressions. +const std = @import("std"); +const hash = @import("engram_hash.zig"); +const engram = @import("engram.zig"); +const mtl = @import("metal"); +const View = engram.View; + +// Generated offline with the official NumPy multipliers and prime-bucket algorithm, not the native code. +const first = [_]u32{ + 8939594, 28321402, 40265357, 55817216, 71147020, 86924254, 110813108, 121427189, + 128454377, 156247504, 165979524, 188168291, 198564788, 211763979, 239563936, 241631034, + 263018334, 286373017, 297618161, 319241270, 333656732, 339279600, 364226324, 373889199, +}; +const third = [_]u32{ + 13176822, 30752650, 42532473, 56771564, 70133176, 85254602, 108815600, 115334181, + 138863141, 149049926, 167314083, 191090346, 204680718, 211476972, 237672992, 245604111, + 269210034, 282954889, 298312790, 313992267, 329778842, 345458689, 359433926, 374584261, +}; +const after_dead = [_]u32{ + 762556, 18016262, 38494153, 58317688, 64056565, 81970637, 106927994, 122902996, + 129060773, 153088767, 160206490, 178949865, 205667218, 211027556, 235070602, 251994813, + 263310652, 278975004, 292595948, 315407209, 325281398, 348093105, 353903080, 377405725, +}; + +test "official hash IDs preserve order, padding and globally unique layer buckets" { + var history: hash.History = .{}; + const a = try history.push(7, hash.layers[0]); + try std.testing.expectEqualSlices(u32, &first, &a); + _ = try history.push(9, hash.layers[0]); + const c = try history.push(11, hash.layers[0]); + try std.testing.expectEqualSlices(u32, &third, &c); + for (hash.layers) |meta| for (meta.primes, meta.offsets) |p, offset| { + try std.testing.expect(p >= 16000000); + try std.testing.expect(offset + p <= meta.rows); + }; + var high: hash.History = .{ .tail = .{ 1234, 5678, 42 }, .position = 3 }; + const result = try high.push(99091, hash.layers[1]); + const expected = [_]u32{ + 3315102, 30010869, 33829090, 51274514, 77267621, 84401643, 106522093, 127296527, + 137402994, 151593930, 162661867, 180852958, 195297507, 213361283, 228864003, 255457679, + 262256667, 282063670, 300635716, 304966933, 321424067, 350721784, 358839309, 370654406, + }; + try std.testing.expectEqualSlices(u32, &expected, &result); +} + +test "dead image positions block all older lookbacks and snapshot rollback restores the exact boundary" { + var history: hash.History = .{}; + _ = try history.push(7, hash.layers[0]); + _ = try history.push(9, hash.layers[0]); + const saved = history.snapshot(); + _ = try history.push(hash.dead, hash.layers[0]); + const got = try history.push(13, hash.layers[0]); + try std.testing.expectEqualSlices(u32, &after_dead, &got); + history.rollback(saved); + const replay = try history.push(11, hash.layers[0]); + try std.testing.expectEqualSlices(u32, &third, &replay); + history.reset(); + const reset = try history.push(7, hash.layers[0]); + try std.testing.expectEqualSlices(u32, &first, &reset); + try std.testing.expectError(error.InvalidEngramCompressedId, history.push(-2, hash.layers[0])); + try std.testing.expectEqual(@as(u64, 1), history.position); +} + +test "all prefill/decode partitions preserve hashes including a dead token at the chunk boundary" { + var bytes: [20 * 4]u8 = undefined; + for (0..20) |i| std.mem.writeInt(u32, bytes[i * 4 ..][0..4], @intCast(i), .little); + const map = hash.TokenMap{ .bytes = &bytes }; + const ids = [_]u32{ 7, 9, 11, 3, 13, 15, 17 }; + const mask = [_]u8{ 1, 1, 1, 0, 1, 1, 1 }; + for (hash.layers) |meta| { + var whole: hash.History = .{}; + var expected: [ids.len][hash.columns]u32 = undefined; + try hash.hashChunk(&whole, map, meta, &ids, &mask, &expected); + for (0..ids.len + 1) |split| { + var chunked: hash.History = .{}; + var actual: [ids.len][hash.columns]u32 = undefined; + try hash.hashChunk(&chunked, map, meta, ids[0..split], mask[0..split], actual[0..split]); + try hash.hashChunk(&chunked, map, meta, ids[split..], mask[split..], actual[split..]); + try std.testing.expectEqualDeep(expected, actual); + try std.testing.expectEqualDeep(whole, chunked); + } + const saved = whole.snapshot(); + var invalid: [2][hash.columns]u32 = undefined; + try std.testing.expectError(error.InvalidEngramToken, hash.hashChunk(&whole, map, meta, &.{ 1, 20 }, null, &invalid)); + try std.testing.expectEqualDeep(saved, whole); + } +} + +test "raw tokenizer IDs are compressed before hashing and normalization-equivalent tokens share hashes" { + var bytes: [4 * 4]u8 = undefined; + for ([_]u32{ 7, 9, 11, 9 }, 0..) |id, i| std.mem.writeInt(u32, bytes[i * 4 ..][0..4], id, .little); + const map = hash.TokenMap{ .bytes = &bytes }; + var a: hash.History = .{}; + var b: hash.History = .{}; + var out_a: [3][hash.columns]u32 = undefined; + var out_b: [3][hash.columns]u32 = undefined; + try hash.hashChunk(&a, map, hash.layers[0], &.{ 0, 1, 2 }, null, &out_a); + try hash.hashChunk(&b, map, hash.layers[0], &.{ 0, 3, 2 }, null, &out_b); + try std.testing.expectEqualDeep(out_a, out_b); + try std.testing.expectEqualSlices(u32, &first, &out_a[0]); + try std.testing.expectEqualSlices(u32, &third, &out_a[2]); +} + +fn view(buffer: mtl.Buffer) View { + return .{ .buffer = buffer }; +} +fn bf(value: f32) u16 { + const bits: u32 = @bitCast(value); + return @truncate((bits + 0x7fff + ((bits >> 16) & 1)) >> 16); +} +fn float(bits: u16) f32 { + return @bitCast(@as(u32, bits) << 16); +} +fn completed(cb: mtl.CommandBuffer) !void { + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.EngramGPUFailure; +} + +test "Metal hashes chunks with image breaks and rolls back bounded GPU history" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + var p = try engram.Pipelines.load(device); + defer p.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + const ids = try device.buffer(7 * 4, 0); + defer ids.deinit(); + @memcpy(ids.slice(u32, 7), &[_]u32{ 7, 9, 11, 3, 13, 15, 17 }); + const map = try device.buffer(20 * 4, 0); + defer map.deinit(); + for (map.slice(u32, 20), 0..) |*id, i| id.* = @intCast(i); + const mask = try device.buffer(7, 0); + defer mask.deinit(); + @memcpy(mask.slice(u8, 7), &[_]u8{ 1, 1, 1, 0, 1, 1, 1 }); + const history = try device.buffer(engram.history_bytes, 0); + defer history.deinit(); + const saved = try device.buffer(engram.history_bytes, 0); + defer saved.deinit(); + const out = try device.buffer(7 * 24 * 4, 0); + defer out.deinit(); + const replay = try device.buffer(24 * 4, 0); + defer replay.deinit(); + const cb = queue.commandBuffer(); + var e = cb.compute(.serial); + engram.encodeReset(e, &p, view(history)); + try engram.encodeHash(e, &p, hash.layers[0], view(ids), view(map), view(mask), view(history), view(out), 3); + try engram.encodeCommit(e, &p, view(ids), view(map), view(mask), view(history), 3); + e.end(); + var blit = cb.blit(); + engram.encodeSnapshot(blit, view(history), view(saved)); + blit.end(); + e = cb.compute(.serial); + try engram.encodeHash(e, &p, hash.layers[0], view(ids).shifted(12), view(map), view(mask).shifted(3), view(history), view(out).shifted(3 * 24 * 4), 4); + try engram.encodeCommit(e, &p, view(ids).shifted(12), view(map), view(mask).shifted(3), view(history), 4); + e.end(); + blit = cb.blit(); + engram.encodeRollback(blit, view(saved), view(history)); + blit.end(); + e = cb.compute(.serial); + try engram.encodeHash(e, &p, hash.layers[1], view(ids).shifted(16), view(map), null, view(history), view(replay), 1); + e.end(); + try completed(cb); + var cpu: hash.History = .{}; + for (ids.slice(u32, 7), mask.slice(u8, 7), 0..) |id, alive, i| { + const want = try cpu.push(if (alive == 0) hash.dead else @intCast(id), hash.layers[0]); + try std.testing.expectEqualSlices(u32, &want, out.slice(u32, 7 * 24)[i * 24 ..][0..24]); + } + cpu = .{ .tail = .{ 11, 9, 7 }, .position = 3 }; + const want = try cpu.push(13, hash.layers[1]); + try std.testing.expectEqualSlices(u32, &want, replay.slice(u32, 24)); + try std.testing.expectEqualSlices(i32, &.{ 11, 9, 7 }, history.slice(i32, 4)[0..3]); +} + +test "Metal gathers only selected FP8 rows with rowwise scales and gates each mHC copy independently" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + var p = try engram.Pipelines.load(device); + defer p.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + const table = try device.buffer(3 * 272, 0); + defer table.deinit(); + @memset(table.slice(u8, 3 * 272), 0x7f); + const scales = try device.buffer(3 * 12, 0); + defer scales.deinit(); + @memset(scales.slice(u8, 3 * 12), 255); + for (0..3) |r| { + @memset(table.slice(u8, 3 * 272)[r * 272 ..][0..256], if (r == 0) 0x38 else if (r == 1) 0xb8 else 0x7e); + for (0..8) |g| scales.slice(u8, 3 * 12)[r * 12 + g] = @intCast(127 + g); + } + table.slice(u8, 3 * 272)[2 * 272] = 0x01; + table.slice(u8, 3 * 272)[2 * 272 + 1] = 0x80; + scales.slice(u8, 3 * 12)[12 + 7] = 255; + const ids = try device.buffer(24 * 4, 0); + defer ids.deinit(); + for (ids.slice(u32, 24), 0..) |*id, i| id.* = @intCast((i * 2 + 1) % 3); + const gathered = try device.buffer((engram.gather_dim + 16) * 2, 0); + defer gathered.deinit(); + const hidden = try device.buffer(2 * 4 * 2, 0); + defer hidden.deinit(); + for (hidden.slice(u16, 8), 0..) |*v, i| v.* = bf(if (i < 4) 1 else 2); + const kv = try device.buffer(12 * 2, 0); + defer kv.deinit(); + for (kv.slice(u16, 12), 0..) |*v, i| v.* = bf(if (i < 4) 1 else if (i < 8) -2 else 3); + const q = try device.buffer(8 * 4, 0); + defer q.deinit(); + for (q.slice(f32, 8), 0..) |*v, i| v.* = if (i < 4) 1 else 0.5; + const k = try device.buffer(8 * 4, 0); + defer k.deinit(); + for (k.slice(f32, 8), 0..) |*v, i| v.* = if (i < 4) 1 else 2; + const output = try device.buffer(8 * 2, 0); + defer output.deinit(); + const dead_mask = try device.buffer(1, 0); + defer dead_mask.deinit(); + dead_mask.slice(u8, 1)[0] = 0; + const masked_output = try device.buffer(8 * 2, 0); + defer masked_output.deinit(); + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + try engram.encodeGather(e, &p, .{ .weight = view(table), .scale = view(scales), .rows = 3, .weight_stride_bytes = 272, .scale_stride_bytes = 12 }, view(ids), view(gathered), 1, engram.gather_dim + 16); + try engram.encodeGate(e, &p, .{ .q_weight = view(q), .k_weight = view(k), .dim = 4, .copies = 2, .hidden_stride = 8, .kv_stride = 12, .output_stride = 8, .eps = 1e-20 }, view(hidden), view(kv), null, view(output), 1); + try engram.encodeGate(e, &p, .{ .q_weight = view(q), .k_weight = view(k), .dim = 4, .copies = 2, .hidden_stride = 8, .kv_stride = 12, .output_stride = 8, .eps = 1e-20 }, view(hidden), view(kv), view(dead_mask), view(masked_output), 1); + e.end(); + try completed(cb); + for (ids.slice(u32, 24), 0..) |r, col| for (0..256) |d| { + const got = gathered.slice(u16, engram.gather_dim)[col * 256 + d]; + if (r == 1 and d >= 224) { + try std.testing.expect(std.math.isNan(float(got))); + continue; + } + const base: f32 = if (r == 2 and d == 0) 0x1p-9 else if (r == 2 and d == 1) -0.0 else if (r == 0) 1 else if (r == 1) -1 else 448; + const scale: f32 = @floatFromInt(@as(u32, 1) << @as(u5, @intCast(d / 32))); + try std.testing.expectEqual(bf(base * scale), got); + }; + const gate_positive = 1.0 / (1.0 + @exp(-@sqrt(@as(f32, 2)))); + const gate_negative = 1.0 / (1.0 + @exp(@sqrt(@as(f32, 2)))); + for (output.slice(u16, 8), 0..) |v, i| { + const want = bf(if (i < 4) 1 + 3 * gate_positive else 2 + 3 * gate_negative); + try std.testing.expectApproxEqAbs(float(want), float(v), 0.015625); + } + try std.testing.expectEqualSlices(u16, hidden.slice(u16, 8), masked_output.slice(u16, 8)); +} diff --git a/zig/src/families/deepseek_v41/fixtures/hc_attention_rms.json b/zig/src/families/deepseek_v41/fixtures/hc_attention_rms.json new file mode 100644 index 0000000000..d1f1c2b61d --- /dev/null +++ b/zig/src/families/deepseek_v41/fixtures/hc_attention_rms.json @@ -0,0 +1,25 @@ +{ + "embedding_bf16": "WT1bPAO857tZvTS9Cr2HvDY9MT2nvUI867vvPIG9Xr3wPMs8GLwLvCG91b1CPf87mTzuvC69rTzOvM28x7w+PSC6VL0YPOG8Fb1TvRE84zwNuw48pTwrvF89OTxEveU8SDybPBA9Yz0XPKU8Nj33PJK8jjtTvNW8C731PDk8Rz0SvAK9l7xnPB69ojvNuxM9y7yMvDC97Dx3PA89Ib2TO0k8j7orvDc7xbwwvCM9lLydvDS9OD1JPQC9JL2yvBu8ebyCvUE9Qz3+vDm8Bb0XPGG8gryUuAi8ZT1vvN+8wbyQPI47orz/vCE93jzVPHU90DwgvVe8rLzPvDe9Xr1tPZK8ojtCvcG84byJOzq9QD2APGE9Xzu3OwY90LzkPNY9iLsZPao8kz2Ivbs85LxKvZu9AjuiPQO9BL6RvAk9hL37u+08gjoavBw97rxDPXq5kz3vO5e7arp3vX+9hj2dO6m8B7sBPHS9jDywPGI907zYvAy8P7y9PDw8u7zrO1O8rjzLvD28Yz3yu5u8lLz0vCa9kjxuPE+9ED1XvJW7RbwfPDe9Mz1yvRy9Br0CPNc8OL2zPLI8tjwdPQw8Fb03vIM8kTzrvPU8DTwgvdM8YbwJPKY877vVPLo8Y7x6PAw9IbyOPEu9+TxuPfo8E7zVPO48ybzNPOe8EbtlvYi8Kb3VvCI8Wz2rvb88Wz1HvYe9oLzFPMY8pDyCPXO8k7yQPQq9IrxbvS89JjwKvQo7Jb0QvVa9oDxWPLM8rrgMvI46mjwWPdi8nzwevUG9Dz3eumy8K70bPZS9Jz2/vIO9xrxLvZi9W73GOsY9mL2OvCe8BL1DvOE8D70tvbU9mzxPvT082zwnvCy9Ar0+O1s9qTwbvIY9Sb2qvTY9KzvvvCA93TxAPeG83TtbPN07s7vXu2M8RjzyvAe9wb1fvUk9YrwwvE+8nbywPVG8WzsDPZI9XD2RvG09+DybPAo81zx8PdI8kr0JvJI7Xj1GPTS9irycPFS8gTzcvES9wLwvPY+9Bju2vRs9IDziPF09GD3dvHA7ML0MvEa9nroCPbo8yrtcvZg7QT3WvC47aLxmPKa7zbwVvGq7Dz30Oyc9Bz2FPBo9Ob14vVW8ELsDPUc8qL2kvCA8DbzlOz49EbwsvS8907wQO5w6sDzcuo085jt3Pdu6Hr0UPBm9OLzVvOq8LT0wO8y7Hz2/vFa9rLwePSC91LyXPa87rr0xveK8jDuMu8C54rx6uzU7HT3AvLE8Jj37O047/zz1PB+7Ab0QPDa9Oz2JvZW8Lb3guwQ9njtivey8vTyhOuE8Brx5vXC80jxdOw08i70OvcS8MD2qvbu8XbyZPLm9mz1aPV66iTx3vMQ7oTyyPIw9PbuOuzM9HjzgO4c7ubzivS69PDzTvLi8H712u767ZL2dPFk9gTzvvK48BTwOvYA9TbtlPIy8Vj0wPSu8D7x2vZQ88ruvPLy8tjwyPSk99rycOxk91DzgvL87pjyGPUi8Uz1ivBS9hb28OjM9izxevFq7Rb39vK07fL2QvcU8uD1vvPq6TLz2PCk9hLzhvIC7gzyBO2Q65DySux6967zbvA09Iz2cPVe8rrwmvbC8djwtPVe8Pz1qvUi9e71ZvYU9R7okPS49jbpIvaY8QbwiPee8rLwKvbE6WT2MPFa8YbxJvLo7Hr0iPVM94DzMu4s8Ojw5PRs9N730O568Fz27vFK8tL2uvEa8irrtO0K89TtAPVk8vbgmvTk9pzqDvLs7KTxtO6W8GjoWPCc9DjxBPZi9oryoPJS8O72jvBu9Fz3lPKA8bj1lPAS9UD0iPQ48KbxFPJy9J703Pbi8/rwqPVs7Er3Iu3q9qDsvvR28VrxzPVE8Y72UPB+9WbsfvS+8KL2BOu08B71ZPVK6Nbo0PZg9iT1pPYE8RD0IPB89Mr3NPEG8oz2wvLU8QjzavLK7qLwgve88ybxCPV68obxbvbu8f71xPGa87LwRPa08Pj1vvG89mz1WPdk8E711O5C89zwXPOy8hDzxPAc9ELwqvGI9I70SO7Q7Ab3AOzg8abtfvTe8zbzAvLS8ibyVPTu9tTyYPKy857wnPZi9JDx5uz07E70MPVq8ZLysOy67BT00Pda8bTx0PN082T0nvdS8Lztyuzm95DyoO1I9vLv5PBO9FD2kPWQ9kLuguQY8A73LPD27pryUvPq8Rb2Pvf68kjxzvGS9+LyDvOC8TDyRPAG9HDy0vLW89DyGPQg78LqMOza9ULt+POM44zzRvIU9xTyLPQc8zbxtvaY9bLuoPKw79zvMO7U8h72pPAa9ML0BvUK907xDvZC9Fbx3PfA8VDxsvW88Bz0TvDK91DwrPIk9ab2uu0a8iTtBvck8sjz7PCQ9D70JvQe94DrHOhG7hLzvvJ894DwxO2q7mb1jPDY8qTzPvEM9N7zZPHE9IboPPUM84zxbO0y9q7ylOh68JD3/u5c7ujwKPIu7yTzLvOO8Ej0YPdE9r726PR28C732PBW8wztrvZY8eLyVPLU8T7xhPea9i7qvPES7qbsvPI08jj2uPOM8T71bvOC8ijwrPY28G707PeO8JD0BvC85sbwPPA491bzNO5Y6J7wbPX88n71hPA+9wbyzPBi7kj2JPM+7j7sROgG8Zj2nPJ88uzwGvJE9j70PPay7yLz+vAC9Sj19vYe8hz3OvBW9DrswvYM8Eb2dvGy9/TuKPeI7JTwIvUk9mLuLvI68rrx2vUg9izxfPf08ez33vB+8L70lvdk8Sb0lvYY8Pry0POk7yDy/PCA8cDwNPQY8dT2DvYu6FT2dPWy8C7wcPUw9nzxhvKA9t7sWPci8Tr25vL47Vb02vBQ9j71vva+8tzyNPDe9lrm3PAK82jq1O2M7WbuXvCw9krsmvag7tDoIPXM86zzJPAU9aj2bvRG8uD3SOqa7XrxNuy09gDvHPF+93DwxPAu9Ej1FPU69hTuyPOu8L70avQ89zTqzvUk9tjvvPBw7u7zEO7k8Ez0hO0U7NL01PQM83zzJO3y8oDz8PO0797yOPIG9nLx9PLs8sbx2PZa7kT2NPCe8k7vUPJU6Gb0fPL48cDzDOzQ8RLxKPU89Wr36PJY8rDy+uya7E7wMPau6TryIO6O9kDyyvc87Er3hvAg8BLzWPEi8yzxqO5I9vLwWvRs9sryUPCa8trw6PA09EDuNvaa7QDxXvSq8U71fvHQ9M71QvU88TLuYvV89xrxFPXS8/jxAu2E9cLyZPL29+DvHPOq8QL0XvCi9TT0JPX+8prxPPSe9wr7FvG09OD2mPJM8g72NvLS80jytPLo8F73cOz+9CD2LO1Q9Mrx1vA+9WT2UO4K9MT1uPIk96L1Qu9i8cL1oPU69v7rePNg7Kz2ePLA7Dj0oPEU9Mj3/vDY9hL2SO488J7sGvIE8Kz1Xuxs9KDwCPpa8O725vGA8SrxAvSI8A73HvF47n7x7PEA9ejyWPUw8NT0TPXo9DT2ePGu9yjzXu4I9tTsAPee97rsEvSO9GDw/PEq96DvOPQS8Yz1dvSw9cz2vOxq71boZvVM8AD1vvYQ5UT08POm8zjuEvZy8JT0Puw466LyfPfg87zqFvMw6Ujzqulm6fz2muwM9t72pPc87+LzIPci8xLzzO2S95LtoPXu9bDx0vBG8/TzPPHC97zzWPDe9k7wXPJQ78rwavQW9gj2NOFw8/rxgvX28gjwRvei8F7ywvDW9z7s0PUE90rsePV2987yZO8o8Rj2TO8e7mT3oPM47zDs3vBS90jykPBa9zTyNvOM8kTwTvT87Wjz2PLi8rDs8ugS9CT11u3e8Wr0rvNs7ITz3vA09T73bvIu8qjyIvYC9oz21vIC9Z7uAPEs9Gr0MvYe84ztXPeq86TykvAW9FD0uvII8ELxAvK67Sj3GPIe9HrxcPAW9yjxdPSc9RL3kvC48gbysPBg9BL0SvUE83jzCPMI7Cryyu448+rr0PEM8Wj3mvAY8mLyGuyw8Mjp4vcS9AjwlPXw8Rz1BPq89HT2yu9i8Zbz4Ois9S701Oym9FrwIvaI9lz1nPMO8nb0OutA7rT2IPFa96ryrvTG9e73xvKS8ETzsvGs95zuzPC28dz2dPIc9/DxPvGG8tDw+PGo8h70HPCq93TwqPJG6erznO8C8qbxfPBK8iD0EvWw68bwpO2a9lbvSPHE73DoEPGo9YD3vOzY9Q7ouPCo9I7wjPeu65TysvBA7gD2VO5S9BD07vbo8hL2QPG28AD0vvWG9JL3EPFw9q72guw68qzwRPRy9rDybPPG8BL3svGK9nj17PWi7ETwEO+s7Czz8u/E80DyPvFA9fLyHPEI9vT1ovJe7Gb0aPRQ8br0IPVe8Wr04u8c7BTw8vT+9Kb3ZPJW9+DxQPFC8UrzBPDc9Jb2Dvbq8Sz2zPIa9aT17POO8lbzEPL88Ar66O4U8Zj0Xu5W8Vz0NPSQ9tr2dvIM8PL0VvMs8n7wBvXq9sTuCPbu9ezwBvQO9Ubw4PSA9ITyruwm70bwSPUA8hDwPvZ89F73tvOa7Ij3OuZU8AT1svb28P7xIOzu8jz1oPQo9HDwePcG8ozy+vSm9vjyJPEW9ezyOPO28tTzLu8E8Er1mvdK8TD25vCO9fz1VPJu9BbxxvGc8rbwJvYO8bTzKPIo9Rb0yPJo82LzAu9I8Bz1XvGI9RrwMvSE9IjvAvA08FzoUvbQ8PLxbvfm8Zz0zvIK7zTv/PF88Mz1PvJa9IT3JPKW8Sb0QvQW8cjyjPLs737z5O4+807w0vQs9ZD0LvUk9Gj20PPe8I7suvOm8l7xVvca8KrtzvQ89QzyHPWU8ejzUPA+9Aj2aPBY9ebwmPXy7bj1mPS09Z7sLPZ+8KL28PCS9z7xQvW47QTw3vdu7/DvhvPq8X7ySvTq6O72nPUY80LwJvR496TxXPIU7LrywPfq87zz5uwW9/DsEvUI8Qr2wPPE7pz1SvbS8Xjx5vTe8Qj0LPcM9JjwzPZa8HT0zvBw9KjyIPZQ9WLwTPIg8cTutvHm8+Lweveo8k72ivOM8Sz17vYO9VD0Zvek8wjzAvdg8cr0IPFo9mLygPZe9GTz+OYc9mD1NvBs97TyHvTq96bwovXE8zTtIPK08qz2XPCw8MT2/O0m9ezwEPE07lT1uPU29s7zHPAO9CD3Ru2g8K71yPUA8nD3Ku0I8YT0ePU48TbzYu1s8sDwtPY29BD0UPPI74joFPUs9rr0uPD88Cb3CvOa8rjx1vVe8xbzlO6S8yTyBPJq7YLpOvLe8IbyzPDW9C7vkO8s8izwtvQQ9gbwbvIW8HT5PPXo8ID0oPAu8dbstvRq7pzxkvMM87jymPcE6BDyOvSI9w7tiPXW8vbwJvAi8pryJvHa6oz3APPS8QD3wvIE94Ty8vNc8Qz2RPGs9LD3YO4+9r7yhvEO8nzutvDe88bxVPHc8tTwuvV68Ab0NO5w9ijz5O6W8S71UPQG87TyIvAA9KLyWuys9/buVPHG8dL3oPMg8oTy7vNO86DlLvQs94jwUPHK8rTsTvQ49SzwyvXo8XTzbut28aTwQvOS5KbzkvJs7Pbkcvdm8P7x9vek8V70KPb489zx8PVo8gz3lPAE9RD04vSs6Trx6u9M8uDwOPLq7OT0rvSm8Jz2svI08t7yiO5A9X73Au1i9zjxvPES9Qj3zvGw8Tb0+vSw9Vb1pOxu9zL00vYu9QL0fPcs8djy1vS88vD1PPcI8NzyFPIK97zxXvOE8hz3iOg49Gz3XvBA9sr0bvJ884jzkvNy8kbyzO++8U7s/vXo8hj2cvDQ9R72DPYI9P7p/u2m9z7w2vDK8GD02vbc97bwqPP28Yb0xvWY8RjsQvX47eDySvAU9nLwVPVG9Hz3SPIa8f723PEo8Gb28PIW7Nr1cO5M9+rxsPDA9ODz5vIE8Az3QPFu9jTw7PQs9Yj2IvBk9kL2GvAS9zLx6PI+8oLzMuwu9Jj3pvGu8Vr1SvOw7V7yJPLc8G72ovGA8dT1JvKu8KT2CPHU9ET2ZPKM84rn5ug49mD0sPfo8Uz1yu8u8T7kPvXs9qDyrPe27Lb2ePKo8/rswvPS7wb2pvLc7n72VvFe8BzyrPKC8CD2ovHA8oD0ivF07+ryLvTA8mTxtPBS9xTvXO/K8Qb2fPDu8ID2sPLO7Rj0Kve68ibzrus68jrxMOye9ybyFPF88TD2KvfU84Ty1vKM86Lw4vFm9iTzcPCq997wIPBo9TD3MPUg8W7xbPVA93jiwvGA9J70uPbu8Rz0hPM883LxNPR48Jb0HPCu9sDyNuwu8ED3RPP28gr0dPUo8Ej0pvM29tT3dvLy85rw6vZc9k7ynPM+7Q7zeu0u9Rr3tPLK8Fz2FvGs7kbxWvOS7LD2FvDC90jwjvIE8Kb2mvH48PDzlvNC5G7z7PBW9Kb1vPG48oTwkO6e9HT26OzW8QL0APEg9t7uRvA68l73zOaG87LwwvVy7ZLy+vNg7HT0sOyM8vDyevcg7xrx0vNE9wryNvYw7Nz0wvIy9W701vCi8qD2DPba83zx7PSU9+ju5PD69Lj2BvCU9hjlJPQO9sjwdPQS7Mb2ROzm9OjwYvWQ9Ur13O0I9gT3dPKU8xjuVPIk5gjyku1+97rwcvIq9dTsKvVQ88jvLPB894Tv0vAu8yL1XuqC9ibwivSY8RDw1vYw8oTwEvZI8C71xvEM9HztYPXY9FLwdu5G8dDxavDu8ir3ZvPo8tbyWvVu8Kr0pvZA8ZT3Euxe9e7youQ28ZjwHvVk9Dr1OPEQ8/ryfPD+9kb0RPFS91jurvL08pDsFPRS957wLPSa9QLw9PVu8J73+uFK89DxXve68mD2XO4Q9Bb1FvIg8Uz1pPWg9qbtsvI494TwPuzI9hLysu4k9QDvKvPM8bbsGPea7Jb30vBs8yzxavRa9STyWvOc8Sj2KvHy9ZTwqO4y8mLwhvWW8GTxZOzo9wzx+vVK8Srx6vJi8EL2nvCg9AjxwvSM8ZD0uuK88x7zuvDa9UL1SvIe8CD0hvZ088Lymu4I81D2TPIi6OD3aPGk857yOPaA8yTzdvJG8uTsmvcs8ijz8PNE8J70WvG68OL3lu9q8l732vY49Ab1sPG28bbw6vRm9zbxEvUA9+Lw7vH07Yr0QvAo8orxmvHi6FLyavCu9gDyNvBy8zjxAvQY9aT38ux49bTynvAY9ojycvQ080zy9uhi9Zz3bOxC9jT3tu0m8iLuRvYY8s7zQu5g9jT3nvM67LD0nPcG8fTwkuxc6tr11utk8l7uvPDq9Dr2kPVW9Lj2hPCO9Qz1TvY68cL21PD69mLw/O6q84T2JPYO9cz2mPZA8v7xeuXy8hz14vJ08ljxyPZY9Uz16Paq65TmKvGc7OLrou928Ib1mPcC8ub0tPSg7LTz3O6W8/LyIPUM7Trxkvbw9Dr3cvOO8C71aPeS8lrw4vDe8db07PXq5CLzDPMI96DywvAA9KrzrPJY8Dz2jPHG8k7xBvGi9OTyxPMi8QbyIPJu8k7t/POc6rbrWPCi9NTxOPZ49VT0UPSy9Sj1YPCO9JztUPQ68gjwwvLi8jDuRPEw9U70xvcW73btcPbE9/jw/PfW8ND0Kvay8W72wPFU7Iz2bPeG7OTyCPao8nzwUPTI8fLvdO/G6trt1vfy897tBvIa8JzwWPS49uzytPKM8k71VPN+8KT1bvdM8zrxaPD07VT2kPBg9TLyPPJa9xLsjPSe7GT1hvO88QjxSvds8bz2mPGw9xLx8PW89pb0Kuww9jzzAPW+7E7wxvLC8WDxnPAq8sLzNu0C9KD3jPNy8/jvqPKq9I70WPDs8UD27PNE8vjyAvWM9Xrx0vIM8Zjw3vAi9Bj5XvT09j7xvvDm9Rr15PJS8/DuXPKU9vrw0PYm8CzuYPOe8LT0zvQG9aL36uxI8RjxKu667Uj0fO6w8Lb29uqM8hrzmPMW4Xr2OPBm9/zyNPZ+7Lbw4PVi8j72/u4q7tjtJvDO9ED0QPHk9wzybPKI8i727PIe8RjwNPVo9ADwgu1U8ybxJPey8VztTOwc9KD3lO0+9sbwpvTM8gLwQvSQ8SL2ZPNC8ST2pPNC8aT0CPXe9dLuCPGw9YD1fPAY81TxIOye8wrwcPRe9SjwePdw8BT12vQe8Vb0JvYQ96zuTPS09PrzdPJE8rDzmvKe7/Dz9u/K8Zj2IPZ28Ez1SvCc8T7wEvb67R73mO/O8sTvXO3W8FL1Avfs7xTwavVY757x/PcW7GL30vBc91rz0uiY9i71kvIG8WzzQvMc9QrzHPDu9ULvwPGQ9XT26vPy6q7uGvQQ9DT07PCk9mbyHO0S7eD0BPSE9Aj3bOsQ8Fr1dvbO8IT0QPsG81L2UPN28JT0GPYG8lzwsPWy7kLyyPeE8Wru0PBq86btYu/68ibxsPGk8Jj13vMe8jLxqPGg90btGPIO8gT27PJK9P737vF28tro6vZO8Jr20O3A96zw3vZe8eT1yPYw8vDsqPRk7obxnPd87j73lPNs8Hzo2PGW82jxOvYo8vbxPPWU8HD1SPZm8Br3GvKG8ZTylPG89kD3xvH69Ir1+Ozm95DoEPIO8Pj1BPQo9hb05PHW96jxevaC8TT2BuoO807w8vNq8E71JPQy8gz0DvKS8lTyQvTC9vjyUvcG7Qr0WvK+8CL27PcK8Hj25PEo9KD2KvBo9ybv5u628abouvAW96bwpPVG8tzoOO+a7WrxKvcE8xLtQvWs95bqDPa889bsePaY8STkyPbI7FD2HO6W8u7yJvT89FjwOPcG84Tyhvey8arzGOtc81ryyvNs8fT0avYq7Kz26PNK9Szk5vfI8PT0ZvQc8vD1IvaK8Bj2mPAe94bvhPEA9TzyEPZE9zDsKvSI9s71APXS73TyBPKQ8Hr0APQs8bz0zu1e9qDx5vZq9fzybPXa9qr2OvKg9STz1vIs9cLzvPBQ96Tsmvam9/rsMvUo957o6PbA8tbslO3W7az0MPaw8Cr1gPMa8ab1ku828Gb23OwI9b7yYvCQ9aT0WvbG847wevFA7BjwbPRc8hLx5u4i8gDy4PCA9dD2MvFC8iD30PFc9SDyNPHi7xrwyvXW9kT2LPFy9+Tt+PKw8qLwUPe+55DtzvRo9HD0NupG9rjsIPCc7pjpDOwk7Nr2UvMO86zyHvTK8H70lvas9MTwvPZG8hL5UPXg7FzxrvCu9d71LvS69vbwKvVG9tzy2vZe7ED1kvFy8nDvgPIo9BL2DPQ89LL3avEA9Tj1TPYY99r0ZPR27w7tru7Q8O7sBPd+827wqPYM8WDxkPaO8Pjx1Pc26mrzXOyW9mj0Tvc29sjzWuzY9Arx4vUM81Twduhs9hjsCvS89qDudPDM6lj3NuxW7Gb1EPcK8xjzwPAi9Xz0SOw094rkXPdq8ab0/PfI7bbv7uh28GL1YPfw8K7xJPQS9mbxbvMa8xjt0PBK9a70qPSS9HDwHPTO9yTxNPMs8DbxkPCW9lL38O029nz1QvL08hb2wvJS8Zjx6vMC8JTs8PQ28aDwUu1A8Bb0GO8c8Dj1NPFi8G72EPZ48DzxrPbI7ljwYu9e5Nj0TvaC65zyLPU886j0OvZo907u1vCo9Jb0au1E7szwdPSc98Ls0Pdc8Bjz1PLK8FT0Jvmm8l7yzO1E8Uj1BPfu7MD2lvHK8i7wWvTu8TzzlvIm98zvZPBi5gb3ZO4w9xzz7O/C84Tv1vCE8PL1GvRS8Y7z+PMs8nbx3OxG9XrwsvRA927uQvJ+9U71zvDo9oTyVPTS8wTwRvbG8FD3CPYy9vb2QPUC9XD0iPC06PL2VvLa8C72VOz89ZL33vHi8Sr0KPQI9nrxUPQq9xDpOvaO7sjxWPUc8+jxbvco8jDwAPXs8TLyWvPe7vLz7u7y6Wz1Ju7Y8BD0uu2C8j7zbutM7IDz/Oki9YbsoPCC9F7ytvBO85Ls2vb28UDuWO5w8Br0TPSu9ZbwUvfe8dj2gvUA9pT2nPFC9bzy9vLa8+TqCPPY8ibx/PeY8kLw6vfK7vzzzOw88u7wnvcs6mzwwvbu8PLx2PVA6Zj2SvQu9G72rPIy93bxiPQA9Iz3KvJQ9t7zPPX49oTy3vDS7XDzZORY83rwRPeM8/zvVvC+9CLy2vBM9SD3gvB07qzzgvIS8qz2KvOs6iLtIPS+8hT1dvP88lTzUPJq6WL1KvY69WD3HvD+93rzbvAY8HLx3vXK8ujtMPNk5yrxqvKS8nLtevYO9or3lu5C97rwOOma8lT09Pci8iT2KvJI8Hr0Kvcw8Jz0OvPY8kT33O6m9mzubvOq8JD2XOje9Mj1FPZc8pbvYvCK93Tsxu9U8ELomPZo8wbyHPL87+LyHPJ+6c7x2va68i7wqvTa8mj1zvaS8lj23PDC9sryhPSi9sTy9u6C8t7ktvCI9VT6+O7o8vDvfPGG8ozyJPH68tDuQvE69AzwrvPA8YD3MveI74bwDPBO9YD0SO1e9Fj0qO2W8IbvEPAQ9WjwCvb+8s72uOQo6nL2YPKq8Zb2zPJk8Cr0WPGi9Jj2tuxW9izzGPTY81rwEvM28pbnjPbS8gDyIPQQ96bzwPFS9SrqWPJe90zugO+q7G714OkI8vDyeOzW9aLzCPcS8cj1NPci8/rycPLc8Qb2IPPg6irw1PQi9irzkPIk9brzuvAo9cD2pvA87vjyeunK8kL1RvUY8ebrEO/u83DwLvMM8jz2OPVI8FT3evB09PzwgvUs9JT3ovG+8yTvDuGq8urxfvJg9uT0uPY69YzxHvIo8AL0cPSw8trz9PMG9xL0QPdo8CL1bPNm9UjyzO4K907zKu5w8ID0yPBo8s7wwvYu8jrynu328TztyvAQ9YzygvAo9hDxMN9Y8DT2yPc0837yVuhU8kbw3PV07h7wPOw69XD2rPDk7azxXPRQ9Hz0LvVm8zzxkvX+9Gb1GvJE7oL0/uwS8WDxAvYO9JD3xu887Fj3LvEo94zwlPSi74rwnu/Y8ZD2/PAy9/zymOhQ9uDuJvEm8STwLvZS8jj1xPSK9oDygPTk8VT3bPFq7rDsWPJQ8iTwFPQa9ezztvCE9bLs1u/s8bz0fvQe4Q7sfvea8Db0dPca8MD2bPYM8Ir0OPaY6hjwkPSQ6fDsBu1C9zzxNPd65LLuQvVk9ibtMvDS81jxnPWK9SjzTvKi8Sjx3vSg9KT0NPV68Jr0Rvau8DL0IvIk83DxePYK8iz2RO9u8tb3/u6y8LT23PGY9x7wru5E9ojzlPJ28hLyDOx28JLzwvCs9zTzzPNO6gT1xO4m8nr0uu3I8FT1vPFu8Dry+vBG9tbxYvWu8Az1tPKK8HT3CPI49pzxUPQO8sTxmvXK9STxAPLQ9bTy2PMA8tj2GvKu8eLwWvSy9Kjz5uzQ8F733PLK8q7zuO0O9rDx5PYC9BL0kvRu827z8u5S9izz4O5A86Dv3PBE8Gz0dvKc7ozxWvHG9EL1DvZC9KTwNPci83jqevYE8mDzxPNq7Dr3bunA9Tr01vZc7UD2AOhU9dTp0PIE66LxUPFU8GLv4vA89YjxjvaQ8w7y+PAa8eDw1O049Ajw9vMS9R73ovB481zuTvPQ5az1lPcA8urzVu3c95zrSvA47ejzIvIO9WTuSu7Y8JD1sOza9FL2EPMO8nz3yuys8Yz2EOQG9f7zHPEO9LD3QPLA8Xr1MPU88+bxZPKK7dTwNPfS7kTyYu1Y8Sb3zPEA8NL0UPdc8nzqBvJ28gzwTPaY8y7z8vBM8+ryeOiI9vDwzPfc7zbsVvNu8QjxlPfw6ZLwwvJ28FbsCvWW9LT2rveE8HT2FPUg8v7xKPO29VL1dve08Qry3u1E8IbwOPaw86jyuO5g8nz3mvOO8jzy7vAO9BT1CPQq8H72hvNO7ijwvuz+8pL2QvVg9d7sCvtK8U70XPa07IL1rvea8PLugPRS9hDxqvP+8G7ywvHY7OztIvdc627sJvKg80DxFPEQ9dj0Zuwo9uzx2PG87kD1aPac7oj1xvHE7dDyfPMW8Ejucu5e7Ub3UOwc8qDwMPXu6uz3VvF+8j7zEPIk6tLyUPWY7LbypvPW8NbxGuuy6qD1TvWq707wWvfO8Q71yPTQ8Ej1sPU+7wzqdu9c4X7xKvQ88SrzTuxK9qjtZvQW9vjuJvMQ8Ir1gvRo9ojxfPSg9yjyFPIA8A71EO7a8ebzUvIi8FDwUPRe9yr1xPeK6JrsOveC8hzzmO3+8IDySPRa9Jz1MPKE8Dz2SvfQ8xjyWvMC85LyavB28lDwVvQC7TjuGvAK96zyxPIG8VDzkuzq6ZzqsPHY7Az2EvTU9B7zDvNC8mzzsPJE8Kj2JvCK8lDz/PLc8l7xJvFw9Bz1uPIA8sDzXvHA9Zz1Cvam90zyjvAM9qDmGvHo7nDsQPd28bLzZvKg9Cr0bvTE9qLdAPc86Hr2oPKu8drwNvVm8Hr32vKu8cb0ivZU6870HPJk9hbxsPaE8CD2AvF09sLyOPDG9tzz5vK+9nbpBPRc9jT1CPNW7i7xaPai8I72cvB07j7zOO0m9p7xvPYC7pLu3PEy93rwzPQ09EDzEPPM8XbwLO4w9JD1Hu2O9qDm1OxK9yTx8u7y8qryZvNA8FT0Ku5m9TT0LvOO8sbyMPTa9VTy2vKi7Cz2RPJK8JrsrvQ69iL3MPL68F70mPai9NT39OyI8xbyBPVo8yLuBvdm7gTxSO9E8JrwhPBE9RL2fvNe7Hzw+PJy9wTqzPGq7ET0mvYY8NL2TvHU9FD2WPO+88Dw/vWU9VrwavRo9h70RPbS7bz1HvQe727w7PQe7J70ZO0G9Kb4hPK681TswPDO8ozyBPB49Qj33vP47Pr2WvcW8I70UPfw8Vz0qPRW7vrwFOzu9YLmWPEG98bwUPS69Or1cvPc8lTz4OrG7Qb2IPIk9mjwVPLk8zryGvII8U7zdPIW8jLwxvHM8BbyOvPA6Dr3OOS+81rwNvF06tzzRvM08Vj3TPHU7jjsPPXC75rwlPQG88jzaPJS7Ij3ivKw9qjxTPeY8DT0OPTk9Lj1BvEq9RT1TPIy8lroSPdo7ez1AvYy8xzsbPYc9ND25vJQ9qz3mvBA99LtivMG8WLyePLc8xDyrvPC8AjzYPI49V7yuPJo9nT0mvUq8P70pvSi9hbwEvNS7Mz0pPQu7mbxWPIm8u7xrvXK7LzxkPDG9IL0aPS69g72jPD09ELwIPWw9ETytvEA8NTyYPM08xjwPPYe8hjyjO8G8bryGOhw8MTyIvPA7TD06vZy8Br3zPM48Dz1SvbM87b0wuwC9hrxvvTY9VroZPUq8WLyBuxw9sLyWPYi8Eb1SvRc9uzxWu628BL1GvHy9pDuMvAy9A7xPvIU8dbxVuwM9X7wevTC8jL2yvK29ObyYPbM82ryrPLQ6yrwVu068yD2evAu9Bz0qvYm7UjwNPTQ97TuhvFG9pLsHvl+6Yj00PQG817wsvf48jjwiPai7H72JvDM73DyovDI8o7yUvG69qLy0Pbg8VrzOPAs9qzw5vuM78jwRPTA9JLtZPAm8sL0PPRg9HjzoO2g9C7wJvea8VL3IvMa8Oz0Uvby89L28POa8Qz1KvZq8zbuPPDk97jk2vM88/zwnPRu9lLxCPTe9szyuPLQ6mz1vvGG9mb26PA==", + "mixes_f32": "5ertwO6atMABjNfAmaqHQE2ySsG204rBMOp7wJFAr8AAOxE/S0CVuhU2ssD8mqBA2IAxv1nkMT83yZu6uaNIuUHJwbqG2oG/SCKCPzAy3LloCZa7EL8+uv22bLroEMs7", + "scale_f32": "UyCBPWQ7RDxMEXw9", + "base_f32": "ChY2QHj3VcDr8U7AWOUNwHV/hcDK9HzApJWcwXxptb9nNFU+IZzLwZlLo8AJrcU9GSQBP0av4b4WAsrBt7OswV3ty8HInUA+BOflvQtYsMFBCZzB416twTi2s8G+BFdB", + "branch_bf16_bits": [ + 16053, + 16067 + ], + "residual_bf16_bits": [ + 48359, + 48377, + 48359, + 48377, + 48359, + 48377, + 48359, + 48377 + ], + "expected_source_tree_target_bf16_bits": [ + 14092, + 46834 + ], + "provenance": "Token 671, layer-0 attention HC, official model revision 2cba9e42aa026125f3ed06c6d98c1db82f7ca027. Four copies of the actual BF16 embedding; mixes are the independently executed native FP32 projection boundary. Target columns 3093 and 4687 of copy 3 distinguish native old RMS from pinned Torch CUDA single-row mean. Base64 payloads are little-endian BF16 or FP32." +} diff --git a/zig/src/families/deepseek_v41/index_precision_test.zig b/zig/src/families/deepseek_v41/index_precision_test.zig new file mode 100644 index 0000000000..1822f7ad0f --- /dev/null +++ b/zig/src/families/deepseek_v41/index_precision_test.zig @@ -0,0 +1,57 @@ +//! BF16 tensor boundaries are observable through which compressed positions enter sparse attention. +const std = @import("std"); +const mtl = @import("metal"); +const index = @import("csa2_index.zig"); +const kv = @import("kv.zig"); + +// The last key's extra 2^-9 dot term disappears at the source BF16 boundary; all 513 scores must tie. +test "index BF16 dot product and reduction boundaries preserve top-512 tie selection" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var ip = try index.Pipelines.load(device); + defer ip.deinit(); + var kp = try kv.Pipelines.load(device); + defer kp.deinit(); + var scratch = try index.Scratch.init(device, 1); + defer scratch.deinit(); + const q = try device.buffer(4096 * 2, mtl.ResourceOptions.shared); + defer q.deinit(); + const w = try device.buffer(32 * 2, mtl.ResourceOptions.shared); + defer w.deinit(); + const raw = try device.buffer(513 * 128 * 2, mtl.ResourceOptions.shared); + defer raw.deinit(); + const storage = try device.buffer(513 * 68, mtl.ResourceOptions.shared); + defer storage.deinit(); + @memset(q.slice(u16, 4096), 0); + @memset(w.slice(u16, 32), 0); + @memset(raw.slice(u16, 513 * 128), 0); + q.slice(u16, 4096)[0] = 0x3f80; // 1 + q.slice(u16, 4096)[32] = 0x3b00; // 2^-9, in a separate FP4 scale block + q.slice(u16, 4096)[128] = 0x3f80; + w.slice(u16, 32)[0] = 0x3f40; // 3/4 + w.slice(u16, 32)[1] = 0x3b00; + for (0..513) |row| raw.slice(u16, 513 * 128)[row * 128] = 0x3f80; + raw.slice(u16, 513 * 128)[512 * 128 + 32] = 0x3f80; + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try kp.encodePack(enc, .{ .buffer = raw }, .{ .buffer = storage }, .{ .rows = 513, .width = 128, .stride = 128, .packed_stride = 68, .block = 32, .kind = .index_fp4 }); + try ip.encode(enc, .{ .buffer = q }, .{ .buffer = w }, .{ .buffer = storage }, scratch, .{ .rows = 1, .q_stride = 4096, .weight_stride = 32, .position = 512, .ratio = 1, .keys = 513 }); + enc.end(); + cb.commit(); + cb.wait(); + if (cb.failure()) |message| { + std.log.err("index precision qualification: {s}", .{message}); + return error.GpuExecutionFailed; + } + const positions = scratch.positions(); + const ids: [*]const i32 = @ptrCast(@alignCast(positions.buffer.contents() + positions.offset)); + for (0..512) |i| try std.testing.expectEqual(@as(i32, @intCast(i)), ids[i]); + const scores = scratch.scores(); + const values: [*]const f32 = @ptrCast(@alignCast(scores.buffer.contents() + scores.offset)); + try std.testing.expectEqual(@as(f32, 0.01171875), values[0]); + try std.testing.expectEqual(values[0], values[512]); +} diff --git a/zig/src/families/deepseek_v41/kv.zig b/zig/src/families/deepseek_v41/kv.zig new file mode 100644 index 0000000000..ac9c4839b9 --- /dev/null +++ b/zig/src/families/deepseek_v41/kv.zig @@ -0,0 +1,162 @@ +//! Cache operators accept projected inputs; no source projection or hidden allocations live here. +const std = @import("std"); +const mtl = @import("metal"); +const sources = @import("kernel_sources"); +const View = @import("view.zig").View; +pub const cache = @import("kv_state.zig"); +pub const Kind = enum(u32) { window_fp8 = 0, main_fp4 = 1, index_fp4 = 2 }; +pub const Pack = extern struct { + rows: u32, + width: u32, + stride: u32, + packed_stride: u32, + block: u32, + kind: Kind, + first: u32 = 0, + ring: u32 = 0, + inplace: u32 = 0, + pub fn validate(p: Pack) !void { + const block: u32 = if (p.kind == .main_fp4) 16 else 32; + const bytes = if (p.kind == .window_fp8) p.width else p.width / 2; + if (p.rows == 0 or p.width == 0 or p.width % block != 0 or p.block != block or + p.stride < p.width or @as(u64, p.packed_stride) < @as(u64, bytes) + p.width / block or + p.inplace > 1 or @as(u64, p.first) + p.rows > std.math.maxInt(u32) or + @as(u64, p.rows -| 1) * p.stride + p.width > std.math.maxInt(u32) or + @as(u64, if (p.ring != 0) p.ring else p.first +| p.rows) * p.packed_stride > + std.math.maxInt(u32) or (p.ring != 0 and p.rows > p.ring)) + return error.InvalidCachePackShape; + } +}; +pub const Pool = extern struct { + rows: u32, + width: u32 = 512, + ratio: u32, + position: u32, + stride: u32, + input_f32: u32, + eps: f32 = 1e-20, + pub fn completed(p: Pool) u32 { + return (p.position + p.rows) / p.ratio - p.position / p.ratio; + } + pub fn validate(p: Pool) !void { + if (p.rows == 0 or p.width == 0 or p.width > 512 or (p.ratio != 1 and p.ratio != 2) or + p.stride < p.width or p.input_f32 != @intFromBool(p.ratio > 1) or + !std.math.isFinite(p.eps) or p.eps <= 0 or + @as(u64, p.position) + p.rows > cache.max_context or + @as(u64, p.rows -| 1) * p.stride + p.width > std.math.maxInt(u32)) + return error.InvalidCompressorShape; + } +}; +pub const Pipelines = struct { + pack: mtl.Pipeline, + pool: mtl.Pipeline, + tail: mtl.Pipeline, + copy: mtl.Pipeline, + publish: mtl.Pipeline, + unpack: mtl.Pipeline, + pub fn load(device: mtl.Device) !Pipelines { + var options = mtl.CompileOptions.mlx(); + options.math_functions = .precise; + const lib = try mtl.Library.fromSource(device, sources.deepseek_v41_kv, options); + defer lib.deinit(); + var p: Pipelines = undefined; + p.pack = try mtl.Pipeline.init(device, lib, "dsv41_pack", false); + errdefer p.pack.deinit(); + p.pool = try mtl.Pipeline.init(device, lib, "dsv41_pool", false); + errdefer p.pool.deinit(); + p.tail = try mtl.Pipeline.init(device, lib, "dsv41_pool_tail", false); + errdefer p.tail.deinit(); + p.copy = try mtl.Pipeline.init(device, lib, "dsv41_copy", false); + errdefer p.copy.deinit(); + p.publish = try mtl.Pipeline.init(device, lib, "dsv41_publish_window", false); + errdefer p.publish.deinit(); + p.unpack = try mtl.Pipeline.init(device, lib, "dsv41_unpack_rows", false); + return p; + } + pub fn deinit(p: *Pipelines) void { + p.pack.deinit(); + p.pool.deinit(); + p.tail.deinit(); + p.copy.deinit(); + p.publish.deinit(); + p.unpack.deinit(); + } + pub fn encodePack(p: *const Pipelines, enc: mtl.ComputeEncoder, x: View, storage: View, args: Pack) !void { + try args.validate(); + try x.requireRows(args.rows, args.stride, args.width, 2); + try storage.requireRows(if (args.ring != 0) args.ring else args.first + args.rows, args.packed_stride, args.width / (if (args.kind == .window_fp8) @as(u32, 1) else 2) + + args.width / args.block, 1); + enc.setPipeline(p.pack); + enc.setBuffer(x.buffer, x.offset, 0); + enc.setBuffer(storage.buffer, storage.offset, 1); + enc.setValue(args, 2); + enc.dispatchGroups(mtl.Size.of(args.width / args.block, args.rows, 1), mtl.Size.of(32, 1, 1)); + enc.barrier(); + } + pub fn encodeUnpack(p: *const Pipelines, enc: mtl.ComputeEncoder, storage: View, out: View, args: Pack) !void { + try args.validate(); + try out.requireRows(args.rows, args.stride, args.width, 2); + try storage.requireRows(if (args.ring != 0) args.ring else args.first + args.rows, args.packed_stride, args.width / (if (args.kind == .window_fp8) @as(u32, 1) else 2) + + args.width / args.block, 1); + enc.setPipeline(p.unpack); + enc.setBuffer(storage.buffer, storage.offset, 0); + enc.setBuffer(out.buffer, out.offset, 1); + enc.setValue(args, 2); + enc.dispatchThreads(mtl.Size.of(args.width, args.rows, 1), mtl.Size.of(32, 1, 1)); + enc.barrier(); + } + /// ratio2 values/gates are FP32; ratio1 values and RMS weight are BF16. Latent output is BF16. + pub fn encodePool(p: *const Pipelines, enc: mtl.ComputeEncoder, values: View, gates: View, partial: View, norm_weight: View, latent: View, args: Pool) !void { + try args.validate(); + try values.requireRows(args.rows, args.stride, args.width, if (args.input_f32 != 0) 4 else 2); + if (args.ratio > 1) { + try gates.requireRows(args.rows, args.stride, args.width, 4); + try partial.require(@as(usize, 2) * args.ratio * args.width * 4, 4); + } + if (args.completed() != 0) { + try norm_weight.require(@as(usize, args.width) * 2, 2); + try latent.requireRows(args.completed(), args.width, args.width, 2); + enc.setPipeline(p.pool); + for ([_]View{ values, gates, partial, norm_weight, latent }, 0..) |v, i| + enc.setBuffer(v.buffer, v.offset, i); + enc.setValue(args, 5); + enc.dispatchGroups(mtl.Size.of(args.completed(), 1, 1), mtl.Size.of(128, 1, 1)); + enc.barrier(); + } + if (args.ratio > 1) { + enc.setPipeline(p.tail); + for ([_]View{ values, gates, partial }, 0..) |v, i| enc.setBuffer(v.buffer, v.offset, i); + enc.setValue(args, 3); + enc.dispatchThreads(mtl.Size.of(args.width, 1, 1), mtl.Size.of(128, 1, 1)); + enc.barrier(); + } + } + pub fn encodeCopy(p: *const Pipelines, enc: mtl.ComputeEncoder, src: View, dst: View, bytes: u32) void { + enc.setPipeline(p.copy); + enc.setBuffer(src.buffer, src.offset, 0); + enc.setBuffer(dst.buffer, dst.offset, 1); + enc.setValue(bytes, 2); + enc.dispatchThreads(mtl.Size.of(bytes, 1, 1), mtl.Size.of(256, 1, 1)); + enc.barrier(); + } + /// Copy only the final window; each destination slot has one writer even for large prefill chunks. + pub fn publishWindow(p: *const Pipelines, enc: mtl.ComputeEncoder, chunk: View, ring: View, rows: u32, position: u32) !void { + if (rows == 0 or @as(u64, position) + rows > cache.max_context) return error.NoCacheRows; + try chunk.requireRows(rows, 528, 528, 1); + try ring.require(128 * 528, 1); + enc.setPipeline(p.publish); + enc.setBuffer(chunk.buffer, chunk.offset, 0); + enc.setBuffer(ring.buffer, ring.offset, 1); + enc.setValue([4]u32{ rows, position, 528, 528 }, 2); + enc.dispatchThreads(mtl.Size.of(528, @min(rows, 128), 1), mtl.Size.of(32, 1, 1)); + enc.barrier(); + } + pub fn snapshot(p: *const Pipelines, enc: mtl.ComputeEncoder, state: *cache.State) !void { + try state.begin(); + p.encodeCopy(enc, .{ .buffer = state.live }, .{ .buffer = state.backup }, @intCast(state.plan.mutableBytes())); + } + pub fn rollback(p: *const Pipelines, enc: mtl.ComputeEncoder, state: *cache.State) !void { + try state.rollback(); + p.encodeCopy(enc, .{ .buffer = state.backup }, .{ .buffer = state.live }, @intCast(state.plan.mutableBytes())); + } +}; diff --git a/zig/src/families/deepseek_v41/kv_test.zig b/zig/src/families/deepseek_v41/kv_test.zig new file mode 100644 index 0000000000..0d1ae738bc --- /dev/null +++ b/zig/src/families/deepseek_v41/kv_test.zig @@ -0,0 +1,234 @@ +const std = @import("std"); +const mtl = @import("metal"); +const kv = @import("kv.zig"); +const cache = @import("kv_state.zig"); +const View = @import("view.zig").View; +fn bf(x: f32) u16 { + const bits: u32 = @bitCast(x); + return @intCast((bits +% 0x7fff +% ((bits >> 16) & 1)) >> 16); +} +fn view(b: mtl.Buffer) View { + return .{ .buffer = b }; +} +fn submit(cb: mtl.CommandBuffer, enc: mtl.ComputeEncoder) !void { + enc.end(); + cb.commit(); + cb.wait(); + try std.testing.expect(cb.failure() == null); +} + +test "cache owners and one million token storage match the released layout" { + try std.testing.expectEqual(@as(usize, 0), try cache.ownerFor(7, 2)); + try std.testing.expectEqual(@as(usize, 1), try cache.ownerFor(13, 2)); + try std.testing.expectEqual(@as(usize, 2), try cache.ownerFor(19, 2)); + try std.testing.expectEqual(@as(usize, 3), try cache.ownerFor(39, 1)); + try std.testing.expectError(error.NoCompressedCache, cache.ownerFor(0, 0)); + const plan = cache.Plan{ .capacity = 1048576, .layers = 40 }; + try std.testing.expectEqual(@as(usize, 1048576 * 890), plan.packedBytes()); + try std.testing.expectEqual(@as(usize, 2732032), plan.mutableBytes()); + try plan.validate(); + try std.testing.expectError(error.InvalidCacheCapacity, (cache.Plan{ .capacity = 1, .layers = 41 }).validate()); + try std.testing.expectError(error.InvalidCacheLayer, cache.ownerFor(40, 1)); + try std.testing.expectError(error.InvalidCacheOwner, cache.ownerFor(39, 2)); +} + +test "FP4 E2M1 ties, signs, zero scales and BF16 in-place values" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var pipes = try kv.Pipelines.load(device); + defer pipes.deinit(); + const x = try device.buffer(64, mtl.ResourceOptions.shared); + defer x.deinit(); + const storage = try device.buffer(36, mtl.ResourceOptions.shared); + defer storage.deinit(); + const values = [_]f32{ 0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.5, -1, -1.5, -2, 0.25, 0.75, 1.25, 1.75 }; + for (values, 0..) |v, i| x.slice(u16, 32)[i] = bf(v); + @memset(x.slice(u16, 32)[16..32], 0); + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try pipes.encodePack(enc, view(x), view(storage), .{ .rows = 2, .width = 16, .stride = 16, .packed_stride = 9, .block = 16, .kind = .main_fp4, .inplace = 1 }); + try submit(cb, enc); + const expected = [_]u8{ 0x10, 0x32, 0x54, 0x76, 0xa9, 0xcb, 0x20, 0x42, 0x38 }; + try std.testing.expectEqualSlices(u8, &expected, storage.contents()[0..9]); + try std.testing.expectEqual(@as(u8, 1), storage.contents()[17]); + const rounded = [_]f32{ 0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.5, -1, -1.5, -2, 0, 1, 1, 2 }; + for (rounded, 0..) |v, i| try std.testing.expectEqual(bf(v), x.slice(u16, 32)[i]); +} + +test "compressor carries a partial group across chunks without changing the pooled latent" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var pipes = try kv.Pipelines.load(device); + defer pipes.deinit(); + const values = try device.buffer(3 * 512 * 4, mtl.ResourceOptions.shared); + defer values.deinit(); + const gates = try device.buffer(3 * 512 * 4, mtl.ResourceOptions.shared); + defer gates.deinit(); + const partial = try device.buffer(4 * 512 * 4, mtl.ResourceOptions.shared); + defer partial.deinit(); + const one_shot = try device.buffer(4 * 512 * 4, mtl.ResourceOptions.shared); + defer one_shot.deinit(); + const weight = try device.buffer(512 * 2, mtl.ResourceOptions.shared); + defer weight.deinit(); + const out = try device.buffer(2 * 512 * 2, mtl.ResourceOptions.shared); + defer out.deinit(); + @memset(partial.contents()[0 .. 4 * 512 * 4], 0); + @memset(one_shot.contents()[0 .. 4 * 512 * 4], 0); + @memset(weight.slice(u16, 512), bf(1)); + for (0..3 * 512) |i| { + values.slice(f32, 3 * 512)[i] = @as(f32, @floatFromInt((i % 512) % 11)) - 5 + @as(f32, @floatFromInt(i / 512)); + gates.slice(f32, 3 * 512)[i] = @as(f32, @floatFromInt(i % 7)) * 0.25; + } + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try pipes.encodePool(enc, view(values), view(gates), view(partial), view(weight), view(out), .{ .rows = 1, .ratio = 2, .position = 0, .stride = 512, .input_f32 = 1 }); + try pipes.encodePool(enc, view(values).shifted(512 * 4), view(gates).shifted(512 * 4), view(partial), view(weight), view(out), .{ .rows = 2, .ratio = 2, .position = 1, .stride = 512, .input_f32 = 1 }); + try pipes.encodePool(enc, view(values), view(gates), view(one_shot), view(weight), view(out).shifted(512 * 2), .{ .rows = 3, .ratio = 2, .position = 0, .stride = 512, .input_f32 = 1 }); + try submit(cb, enc); + try std.testing.expectEqualSlices(u16, out.slice(u16, 1024)[0..512], out.slice(u16, 1024)[512..1024]); + try std.testing.expectEqualSlices(f32, partial.slice(f32, 2048)[0..512], one_shot.slice(f32, 2048)[0..512]); + try std.testing.expectEqualSlices(f32, partial.slice(f32, 2048)[1024..1536], one_shot.slice(f32, 2048)[1024..1536]); +} + +test "compressor rounds the pooled FP32 latent to BF16 before RMS normalization" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var pipes = try kv.Pipelines.load(device); + defer pipes.deinit(); + const values = try device.buffer(4 * 4, mtl.ResourceOptions.shared); + defer values.deinit(); + const gates = try device.buffer(4 * 4, mtl.ResourceOptions.shared); + defer gates.deinit(); + const partial = try device.buffer(4 * 2 * 4, mtl.ResourceOptions.shared); + defer partial.deinit(); + const weight = try device.buffer(2 * 2, mtl.ResourceOptions.shared); + defer weight.deinit(); + const out = try device.buffer(2 * 2, mtl.ResourceOptions.shared); + defer out.deinit(); + @memcpy(values.slice(f32, 4), &[_]f32{ 1.00390625, 0.5, 1.00390625, 0.5 }); + @memset(gates.slice(f32, 4), 0); + @memset(partial.contents()[0..32], 0); + @memset(weight.slice(u16, 2), bf(1)); + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try pipes.encodePool(enc, view(values), view(gates), view(partial), view(weight), view(out), .{ .rows = 2, .width = 2, .ratio = 2, .position = 0, .stride = 2, .input_f32 = 1 }); + try submit(cb, enc); + // 1.00390625 ties to 1 in BF16; normalizing the unrounded value changes the second result. + try std.testing.expectEqual(bf(1.265625), out.slice(u16, 2)[0]); + try std.testing.expectEqual(bf(0.6328125), out.slice(u16, 2)[1]); +} + +test "ring publish handles chunks beyond the window and transaction rollback restores every mutable byte" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var pipes = try kv.Pipelines.load(device); + defer pipes.deinit(); + var state = try cache.State.init(device, .{ .capacity = 1024, .layers = 40 }); + defer state.deinit(); + const chunk = try device.buffer(131 * 528, mtl.ResourceOptions.shared); + defer chunk.deinit(); + for (0..131) |r| @memset(chunk.contents()[r * 528 .. (r + 1) * 528], @intCast(r)); + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try pipes.publishWindow(enc, view(chunk), try state.ring(0), 131, 0); + try state.append(131); + try pipes.snapshot(enc, &state); + try pipes.publishWindow(enc, view(chunk), try state.ring(0), 131, 131); + try state.append(131); + try pipes.rollback(enc, &state); + try submit(cb, enc); + try std.testing.expectEqual(@as(u32, 131), state.position); + try std.testing.expectEqualSlices(u8, state.backup.contents()[0..state.plan.mutableBytes()], state.live.contents()[0..state.plan.mutableBytes()]); + for (3..131) |r| { + const slot = r % 128; + for (state.live.contents()[slot * 528 .. (slot + 1) * 528]) |byte| + try std.testing.expectEqual(@as(u8, @intCast(r)), byte); + } + const cb2 = queue.commandBuffer(); + const enc2 = cb2.compute(.serial); + try pipes.snapshot(enc2, &state); + try pipes.publishWindow(enc2, view(chunk), try state.ring(0), 131, 131); + try state.append(131); + try state.commit(); + try submit(cb2, enc2); + try std.testing.expectEqual(@as(u32, 262), state.position); + try std.testing.expect(state.saved_position == null); +} + +test "window FP8 quantization includes the RoPE tail and index FP4 uses E8M0 scale bytes" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var pipes = try kv.Pipelines.load(device); + defer pipes.deinit(); + const x = try device.buffer(64 * 2, mtl.ResourceOptions.shared); + defer x.deinit(); + const storage = try device.buffer(66, mtl.ResourceOptions.shared); + defer storage.deinit(); + const decoded = try device.buffer(64 * 2, mtl.ResourceOptions.shared); + defer decoded.deinit(); + @memset(x.slice(u16, 64), 0); + @memset(x.slice(u16, 64)[32..64], bf(0.3)); + x.slice(u16, 64)[63] = bf(448); + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + const args = kv.Pack{ .rows = 1, .width = 64, .stride = 64, .packed_stride = 66, .block = 32, .kind = .window_fp8, .inplace = 1 }; + try pipes.encodePack(enc, view(x), view(storage), args); + try pipes.encodeUnpack(enc, view(storage), view(decoded), args); + try submit(cb, enc); + try std.testing.expectEqual(@as(u8, 127), storage.contents()[65]); + try std.testing.expectEqual(@as(u8, 126), storage.contents()[63]); + try std.testing.expectEqual(bf(0.3125), decoded.slice(u16, 64)[62]); + try std.testing.expectEqualSlices(u16, x.slice(u16, 64), decoded.slice(u16, 64)); + @memset(x.slice(u16, 64)[0..32], bf(6)); + const cb2 = queue.commandBuffer(); + const enc2 = cb2.compute(.serial); + try pipes.encodePack(enc2, view(x), view(storage), .{ .rows = 1, .width = 32, .stride = 32, .packed_stride = 17, .block = 32, .kind = .index_fp4, .inplace = 1 }); + try submit(cb2, enc2); + for (storage.contents()[0..16]) |byte| try std.testing.expectEqual(@as(u8, 0x77), byte); + try std.testing.expectEqual(@as(u8, 127), storage.contents()[16]); +} + +test { + _ = @import("csa2_test.zig"); +} + +test "cache pack rejects undersized borrowed input and output before encoding" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const small = try device.buffer(16, mtl.ResourceOptions.shared); + defer small.deinit(); + const input = try device.buffer(64, mtl.ResourceOptions.shared); + defer input.deinit(); + const args = kv.Pack{ .rows = 1, .width = 32, .stride = 32, .packed_stride = 33, .block = 32, .kind = .window_fp8 }; + const pipelines: *const kv.Pipelines = undefined; + try std.testing.expectError(error.InvalidTensorView, pipelines.encodePack(undefined, view(small), view(input), args)); + try std.testing.expectError(error.InvalidTensorView, pipelines.encodePack(undefined, view(input), view(small), args)); + try std.testing.expectError(error.InvalidTensorView, pipelines.encodePack(undefined, view(input).shifted(1), view(input), args)); +} + +test "cache shapes reject overflow and nonfinite compressor epsilon" { + try std.testing.expectError(error.InvalidCachePackShape, (kv.Pack{ .rows = 2, .width = 32, .stride = 32, .packed_stride = 33, .block = 32, .kind = .window_fp8, .first = std.math.maxInt(u32) }).validate()); + try std.testing.expectError(error.InvalidCompressorShape, (kv.Pool{ .rows = 1, .ratio = 1, .position = 0, .stride = 512, .input_f32 = 0, .eps = std.math.nan(f32) }).validate()); +} diff --git a/zig/src/families/deepseek_v41/mhc.zig b/zig/src/families/deepseek_v41/mhc.zig new file mode 100644 index 0000000000..3ac499423f --- /dev/null +++ b/zig/src/families/deepseek_v41/mhc.zig @@ -0,0 +1,95 @@ +//! Four-copy mHC; callers validate shapes/storage and own previous-pre coefficients. +const mtl = @import("metal"); +const sources = @import("kernel_sources"); +pub const View = @import("view.zig").View; +pub const Pipelines = struct { + rms: mtl.Pipeline, + split: mtl.Pipeline, + collapse: mtl.Pipeline, + post: mtl.Pipeline, + expand: mtl.Pipeline, + pub fn load(device: mtl.Device) !Pipelines { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + var options = mtl.CompileOptions.mlx(); + options.math_functions = .precise; + const lib = try mtl.Library.fromSource(device, sources.deepseek_v41_mhc, options); + defer lib.deinit(); + const rms = try mtl.Pipeline.init(device, lib, "dsv41_hc_rms", false); + errdefer rms.deinit(); + const split = try mtl.Pipeline.init(device, lib, "dsv41_hc_split", false); + errdefer split.deinit(); + const collapse = try mtl.Pipeline.init(device, lib, "dsv41_hc_collapse", false); + errdefer collapse.deinit(); + const post = try mtl.Pipeline.init(device, lib, "dsv41_hc_post", false); + errdefer post.deinit(); + const expand = try mtl.Pipeline.init(device, lib, "dsv41_hc_expand", false); + return .{ .rms = rms, .split = split, .collapse = collapse, .post = post, .expand = expand }; + } + pub fn deinit(p: *Pipelines) void { + p.rms.deinit(); + p.split.deinit(); + p.collapse.deinit(); + p.post.deinit(); + p.expand.deinit(); + } +}; +fn bind(e: mtl.ComputeEncoder, refs: anytype) void { + inline for (refs, 0..) |r, i| e.setBuffer(r.buffer, r.offset, i); +} +pub const Rms = extern struct { rows: u32, width: u32, stream_stride: u32, eps: f32 }; +/// BF16 flattened [rows,4*width] -> FP32 inv[rows]; fixed CUDA single-row tree, not every batched CUDA grouping. +pub fn encodeRms(e: mtl.ComputeEncoder, p: *const Pipelines, stream: View, inv: View, d: Rms) void { + e.setPipeline(p.rms); + bind(e, .{ stream, inv }); + e.setValue(d, 2); + e.dispatchGroups(mtl.Size.of(d.rows, 1, 1), mtl.Size.of(512, 1, 1)); +} +pub const Split = extern struct { + rows: u32, + mix_stride: u32 = 24, + coefficient_stride: u32 = 4, + comb_stride: u32 = 16, + eps: f32 = 1e-6, +}; +/// FP32 mixes[rows,24], inv, scale[3], base[24] -> pre/post[rows,4], comb[rows,4,4]; 20 iterations. +pub fn encodeSplit(e: mtl.ComputeEncoder, p: *const Pipelines, mixes: View, inv: View, scale: View, base: View, pre: View, post: View, comb: View, d: Split) void { + e.setPipeline(p.split); + bind(e, .{ mixes, inv, scale, base, pre, post, comb }); + e.setValue(d, 7); + e.dispatchThreads(mtl.Size.of(d.rows, 1, 1), mtl.Size.of(32, 1, 1)); +} +pub const Collapse = extern struct { rows: u32, width: u32, stream_stride: u32, y_stride: u32, pre_stride: u32 = 4 }; +/// Collapse BF16 [rows,4,width] with previous sublayer pre[rows,4]; head uses final FFN pre, initially one-hot. +pub fn encodeCollapse(e: mtl.ComputeEncoder, p: *const Pipelines, stream: View, previous_pre: View, y: View, d: Collapse) void { + e.setPipeline(p.collapse); + bind(e, .{ stream, previous_pre, y }); + e.setValue(d, 3); + e.dispatchThreads(mtl.Size.of(d.width, d.rows, 1), mtl.Size.of(256, 1, 1)); +} +pub const Post = extern struct { + rows: u32, + width: u32, + branch_stride: u32, + residual_stride: u32, + y_stride: u32, + coefficient_stride: u32 = 4, + comb_stride: u32 = 16, +}; +/// y[k,d]=post[k]*branch[d]+sum_j comb[j,k]*residual[j,d]; BF16 inputs/output, FP32 coefficients/math. + +// Element pitches; exact residual alias is safe, but output must not alias branch/post/comb. +pub fn encodePost(e: mtl.ComputeEncoder, p: *const Pipelines, branch: View, residual: View, post: View, comb: View, y: View, d: Post) void { + e.setPipeline(p.post); + bind(e, .{ branch, residual, post, comb, y }); + e.setValue(d, 5); + e.dispatchThreads(mtl.Size.of(d.width, d.rows, 1), mtl.Size.of(256, 1, 1)); +} +pub const Expand = extern struct { rows: u32, width: u32, x_stride: u32, stream_stride: u32, pre_stride: u32 = 4 }; +/// BF16 x[rows,width] copied into four streams and FP32 initial pre[rows,4] set to [1,0,0,0]. +pub fn encodeExpand(e: mtl.ComputeEncoder, p: *const Pipelines, x: View, stream: View, pre: View, d: Expand) void { + e.setPipeline(p.expand); + bind(e, .{ x, stream, pre }); + e.setValue(d, 3); + e.dispatchThreads(mtl.Size.of(d.width, d.rows, 1), mtl.Size.of(256, 1, 1)); +} diff --git a/zig/src/families/deepseek_v41/mhc_test.zig b/zig/src/families/deepseek_v41/mhc_test.zig new file mode 100644 index 0000000000..ee0d888e28 --- /dev/null +++ b/zig/src/families/deepseek_v41/mhc_test.zig @@ -0,0 +1,209 @@ +//! Consumer-visible mHC coefficient, reduction and BF16 boundary regressions. +const std = @import("std"); +const mtl = @import("metal"); +const mhc = @import("mhc.zig"); +const support = @import("test_support.zig"); +const bf = support.bf; +const fp = support.fp; +const view = support.view; +const Buffers = support.Buffers; +const done = support.done; +const close = support.close; +fn splitReference(mixes: []const f32, inv: f64, scales: []const f32, base: []const f32) [24]f64 { + var out: [24]f64 = undefined; + for (0..4) |j| { + out[j] = 1 / (1 + @exp(-(@as(f64, mixes[j]) * inv * scales[0] + base[j]))) + 1e-6; + out[4 + j] = 2 / (1 + @exp(-(@as(f64, mixes[4 + j]) * inv * scales[1] + base[4 + j]))); + } + for (0..4) |j| { + var max: f64 = -std.math.inf(f64); + for (0..4) |k| { + const q = 8 + 4 * j + k; + out[q] = @as(f64, mixes[q]) * inv * scales[2] + base[q]; + max = @max(max, out[q]); + } + var sum: f64 = 0; + for (0..4) |k| { + const q = 8 + 4 * j + k; + out[q] = @exp(out[q] - max); + sum += out[q]; + } + for (0..4) |k| out[8 + 4 * j + k] = out[8 + 4 * j + k] / sum + 1e-6; + } + for (0..20) |it| { + if (it > 0) for (0..4) |j| { + var s: f64 = 0; + for (0..4) |k| s += out[8 + 4 * j + k]; + for (0..4) |k| out[8 + 4 * j + k] /= s + 1e-6; + }; + for (0..4) |k| { + var s: f64 = 0; + for (0..4) |j| s += out[8 + 4 * j + k]; + for (0..4) |j| out[8 + 4 * j + k] /= s + 1e-6; + } + } + return out; +} + +test "V4.1 flattened mHC RMS, 20 Sinkhorn iterations, previous collapse, source-destination post and identity expand" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try mhc.Pipelines.load(device); + defer p.deinit(); + var b = Buffers{ .device = device }; + defer b.deinit(); + const stream = try b.get(u16, 16); + for (stream.slice(u16, 16), 0..) |*v, i| v.* = bf(@as(f32, @floatFromInt(i + 1)) * 0.25); + const inv = try b.get(f32, 1); + const mixes = try b.get(f32, 24); + for (mixes.slice(f32, 24), 0..) |*v, i| v.* = (@as(f32, @floatFromInt(i % 9)) - 4) * 3; + const scale = try b.get(f32, 3); + @memcpy(scale.slice(f32, 3), &[_]f32{ 0.3, -0.7, 1.2 }); + const base = try b.get(f32, 24); + for (base.slice(f32, 24), 0..) |*v, i| v.* = (@as(f32, @floatFromInt(i % 5)) - 2) * 0.3; + const pre = try b.get(f32, 4); + const post = try b.get(f32, 4); + const comb = try b.get(f32, 16); + const previous = try b.get(f32, 4); + @memcpy(previous.slice(f32, 4), &[_]f32{ 0.1, 0.2, 0.3, 0.4 }); + const collapse = try b.get(u16, 4); + const branch = try b.get(u16, 4); + @memset(branch.slice(u16, 4), bf(2)); + const y = try b.get(u16, 16); + const expanded = try b.get(u16, 16); + const initial = try b.get(f32, 4); + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + mhc.encodeRms(e, &p, view(stream), view(inv), .{ .rows = 1, .width = 4, .stream_stride = 16, .eps = 1e-20 }); + mhc.encodeSplit(e, &p, view(mixes), view(inv), view(scale), view(base), view(pre), view(post), view(comb), .{ .rows = 1 }); + mhc.encodeCollapse(e, &p, view(stream), view(previous), view(collapse), .{ .rows = 1, .width = 4, .stream_stride = 16, .y_stride = 4 }); + mhc.encodePost(e, &p, view(branch), view(stream), view(post), view(comb), view(y), .{ .rows = 1, .width = 4, .branch_stride = 4, .residual_stride = 16, .y_stride = 16 }); + mhc.encodeExpand(e, &p, view(branch), view(expanded), view(initial), .{ .rows = 1, .width = 4, .x_stride = 4, .stream_stride = 16 }); + try done(cb, e); + var squares: f64 = 0; + for (stream.slice(u16, 16)) |v| squares += @as(f64, fp(v)) * fp(v); + const ir = 1 / @sqrt(squares / 16 + 1e-20); + try close(ir, inv.slice(f32, 1)[0], 1e-6); + const ref = splitReference(mixes.slice(f32, 24), ir, scale.slice(f32, 3), base.slice(f32, 24)); + for (0..4) |j| { + try close(ref[j], pre.slice(f32, 4)[j], 2e-6); + try close(ref[4 + j], post.slice(f32, 4)[j], 2e-6); + } + for (0..16) |q| try close(ref[8 + q], comb.slice(f32, 16)[q], 3e-6); + for (0..4) |i| { + var collapsed: f64 = 0; + for (0..4) |j| collapsed += @as(f64, previous.slice(f32, 4)[j]) * fp(stream.slice(u16, 16)[j * 4 + i]); + try close(collapsed, fp(collapse.slice(u16, 4)[i]), 0.02); + for (0..4) |k| { + var expected: f64 = 2 * ref[4 + k]; + for (0..4) |j| expected += ref[8 + 4 * j + k] * fp(stream.slice(u16, 16)[j * 4 + i]); + try close(expected, fp(y.slice(u16, 16)[k * 4 + i]), 0.04); + try std.testing.expectEqual(bf(2), expanded.slice(u16, 16)[k * 4 + i]); + } + try std.testing.expectEqual(@as(f32, if (i == 0) 1 else 0), initial.slice(f32, 4)[i]); + } +} + +test "V4.1 source single-row HC RMS reaches the retained attention BF16 consumer" { + const Fixture = struct { + embedding_bf16: []const u8, + mixes_f32: []const u8, + scale_f32: []const u8, + base_f32: []const u8, + branch_bf16_bits: [2]u16, + residual_bf16_bits: [8]u16, + expected_source_tree_target_bf16_bits: [2]u16, + }; + const fixture = try std.json.parseFromSlice(Fixture, std.testing.allocator, @embedFile("fixtures/hc_attention_rms.json"), .{ .ignore_unknown_fields = true }); + defer fixture.deinit(); + const f = fixture.value; + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try mhc.Pipelines.load(device); + defer p.deinit(); + var b = Buffers{ .device = device }; + defer b.deinit(); + const width = 5120; + const stream = try b.get(u16, 4 * width); + const xs = stream.slice(u16, 4 * width); + try std.base64.standard.Decoder.decode(std.mem.sliceAsBytes(xs[0..width]), f.embedding_bf16); + for (1..4) |copy| @memcpy(xs[copy * width ..][0..width], xs[0..width]); + const inv = try b.get(f32, 1); + const mixes = try b.get(f32, 24); + const scale = try b.get(f32, 3); + const base = try b.get(f32, 24); + try std.base64.standard.Decoder.decode(std.mem.sliceAsBytes(mixes.slice(f32, 24)), f.mixes_f32); + try std.base64.standard.Decoder.decode(std.mem.sliceAsBytes(scale.slice(f32, 3)), f.scale_f32); + try std.base64.standard.Decoder.decode(std.mem.sliceAsBytes(base.slice(f32, 24)), f.base_f32); + const pre = try b.get(f32, 4); + const post = try b.get(f32, 4); + const comb = try b.get(f32, 16); + const branch = try b.get(u16, 2); + const residual = try b.get(u16, 8); + const output = try b.get(u16, 8); + @memcpy(branch.slice(u16, 2), &f.branch_bf16_bits); + @memcpy(residual.slice(u16, 8), &f.residual_bf16_bits); + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + // Recompute statistic, coefficients and post products from the retained projection; never inject an oracle inverse. + mhc.encodeRms(e, &p, view(stream), view(inv), .{ .rows = 1, .width = width, .stream_stride = 4 * width, .eps = 1e-20 }); + mhc.encodeSplit(e, &p, view(mixes), view(inv), view(scale), view(base), view(pre), view(post), view(comb), .{ .rows = 1 }); + mhc.encodePost(e, &p, view(branch), view(residual), view(post), view(comb), view(output), .{ .rows = 1, .width = 2, .branch_stride = 2, .residual_stride = 8, .y_stride = 8 }); + try done(cb, e); + // The former reduction produces [14091,46835], despite unchanged branch BF16 bits. + for (0..2) |i| try std.testing.expectEqual(f.expected_source_tree_target_bf16_bits[i], output.slice(u16, 8)[6 + i]); +} + +test "V4.1 hyperconnections preserve FP32 products and residual boundaries under cancellation" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try mhc.Pipelines.load(device); + defer p.deinit(); + var b = Buffers{ .device = device }; + defer b.deinit(); + const stream = try b.get(u16, 8); + @memcpy(stream.slice(u16, 8), &[_]u16{ bf(3), bf(3), bf(0), bf(0), bf(1), bf(1), bf(1), bf(1) }); + const previous = try b.get(f32, 8); + @memcpy(previous.slice(f32, 8), &[_]f32{ -1, 1 + 0x1p-23, 0, 0, 1, 0x1p-24, 0x1p-24, -1 }); + const collapsed = try b.get(u16, 2); + const branch = try b.get(u16, 2); + @memcpy(branch.slice(u16, 2), &[_]u16{ bf(0), bf(1) }); + const post = try b.get(f32, 8); + post.slice(f32, 8)[4] = -1; + const comb = try b.get(f32, 32); + comb.slice(f32, 32)[0] = -1; + comb.slice(f32, 32)[4] = 1 + 0x1p-23; + comb.slice(f32, 32)[16] = 1; + comb.slice(f32, 32)[20] = 0x1p-24; + comb.slice(f32, 32)[24] = 0x1p-24; + const output = try b.get(u16, 8); + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + mhc.encodeCollapse(e, &p, view(stream), view(previous), view(collapsed), .{ .rows = 2, .width = 1, .stream_stride = 4, .y_stride = 1 }); + mhc.encodePost(e, &p, view(branch), view(stream), view(post), view(comb), view(output), .{ .rows = 2, .width = 1, .branch_stride = 1, .residual_stride = 4, .y_stride = 4 }); + try done(cb, e); + try std.testing.expectEqual(bf(0x1p-21), collapsed.slice(u16, 2)[0]); + try std.testing.expectEqual(bf(0), collapsed.slice(u16, 2)[1]); + try std.testing.expectEqual(bf(0x1p-21), output.slice(u16, 8)[0]); + try std.testing.expectEqual(bf(0), output.slice(u16, 8)[4]); + branch.slice(u16, 2)[0] = bf(3); + post.slice(f32, 8)[0] = 1 + 0x1p-23; + comb.slice(f32, 32)[4] = 0; + const branch_cb = queue.commandBuffer(); + const branch_enc = branch_cb.compute(.serial); + mhc.encodePost(branch_enc, &p, view(branch), view(stream), view(post), view(comb), view(output), .{ .rows = 1, .width = 1, .branch_stride = 1, .residual_stride = 4, .y_stride = 4 }); + try done(branch_cb, branch_enc); + try std.testing.expectEqual(bf(0x1p-21), output.slice(u16, 8)[0]); +} diff --git a/zig/src/families/deepseek_v41/ops.zig b/zig/src/families/deepseek_v41/ops.zig new file mode 100644 index 0000000000..68fd169857 --- /dev/null +++ b/zig/src/families/deepseek_v41/ops.zig @@ -0,0 +1,110 @@ +//! Trusted plaintext encoders: callers validate nonzero shapes/storage; strides count elements, EXL3 is separate. +const mtl = @import("metal"); +const sources = @import("kernel_sources"); +pub const View = @import("view.zig").View; + +pub const Pipelines = struct { + rms: mtl.Pipeline, + dense: mtl.Pipeline, + embedding: mtl.Pipeline, + swiglu: mtl.Pipeline, + combine: mtl.Pipeline, + + pub fn load(device: mtl.Device) !Pipelines { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + var options = mtl.CompileOptions.mlx(); + options.math_functions = .precise; + const lib = try mtl.Library.fromSource(device, sources.deepseek_v41_ops, options); + defer lib.deinit(); + const rms = try mtl.Pipeline.init(device, lib, "dsv41_rms", false); + errdefer rms.deinit(); + const dense = try mtl.Pipeline.init(device, lib, "dsv41_dense", false); + errdefer dense.deinit(); + const embedding = try mtl.Pipeline.init(device, lib, "dsv41_embedding", false); + errdefer embedding.deinit(); + const swiglu = try mtl.Pipeline.init(device, lib, "dsv41_swiglu", false); + errdefer swiglu.deinit(); + const combine = try mtl.Pipeline.init(device, lib, "dsv41_combine", false); + return .{ .rms = rms, .dense = dense, .embedding = embedding, .swiglu = swiglu, .combine = combine }; + } + pub fn deinit(p: *Pipelines) void { + p.rms.deinit(); + p.dense.deinit(); + p.embedding.deinit(); + p.swiglu.deinit(); + p.combine.deinit(); + } +}; +fn bind(e: mtl.ComputeEncoder, refs: anytype) void { + inline for (refs, 0..) |r, i| e.setBuffer(r.buffer, r.offset, i); +} +pub const Rms = struct { rows: u32, width: u32, x_stride: u32, y_stride: u32, eps: f32, weight_f32: bool = false }; +/// BF16 x/y [rows,width], BF16 or FP32 weight [width]; sums, mean and affine product are FP32. +pub fn encodeRms(e: mtl.ComputeEncoder, p: *const Pipelines, x: View, weight: View, y: View, d: Rms) void { + e.setPipeline(p.rms); + bind(e, .{ x, weight, y }); + const D = extern struct { rows: u32, width: u32, x_stride: u32, y_stride: u32, eps: f32, weight_f32: u32 }; + e.setValue(D{ .rows = d.rows, .width = d.width, .x_stride = d.x_stride, .y_stride = d.y_stride, .eps = d.eps, .weight_f32 = @intFromBool(d.weight_f32) }, 3); + e.dispatchGroups(mtl.Size.of(d.rows, 1, 1), mtl.Size.of(256, 1, 1)); +} +pub const Dtype = enum(u32) { bf16 = 0, f32 = 1, f16 = 2 }; +pub const Dense = struct { + rows: u32, + k: u32, + n: u32, + groups: u32 = 1, + x_stride: u32, + w_stride: u32, + y_stride: u32, + x_dtype: Dtype = .bf16, + w_dtype: Dtype = .bf16, + y_dtype: Dtype = .bf16, +}; +/// x[rows,groups,k], w[groups,n,k], y[rows,groups,n]; packed groups, padded token pitches, FP32 sums, final BF16. +pub fn encodeDense(e: mtl.ComputeEncoder, p: *const Pipelines, x: View, w: View, y: View, d: Dense) void { + e.setPipeline(p.dense); + bind(e, .{ x, w, y }); + e.setValue([10]u32{ d.rows, d.k, d.n, d.groups, d.x_stride, d.w_stride, d.y_stride, @backingInt(d.x_dtype), @backingInt(d.w_dtype), @backingInt(d.y_dtype) }, 3); + e.dispatchGroups(mtl.Size.of((d.n + 15) / 16, d.rows, d.groups), mtl.Size.of(128, 1, 1)); +} +pub const Embedding = extern struct { rows: u32, width: u32, table_stride: u32, y_stride: u32, copies: u32 = 4 }; +/// ids[rows] u32 must be in vocabulary; table[vocab,width] BF16 -> y[rows,copies,width] BF16. +pub fn encodeEmbedding(e: mtl.ComputeEncoder, p: *const Pipelines, ids: View, table: View, y: View, d: Embedding) void { + e.setPipeline(p.embedding); + bind(e, .{ ids, table, y }); + e.setValue(d, 3); + e.dispatchThreads(mtl.Size.of(d.width, d.rows, 1), mtl.Size.of(256, 1, 1)); +} +pub const Activation = struct { + rows: u32, + width: u32, + input_stride: u32, + output_stride: u32, + weighted: bool = false, + input_f32: bool = false, +}; +/// Separate gate/up; apply FP32 weights before BF16 cast/down. Unweighted calls borrow but never read weights. +pub fn encodeSwiGLU(e: mtl.ComputeEncoder, p: *const Pipelines, gate: View, up: View, weights: View, y: View, d: Activation) void { + e.setPipeline(p.swiglu); + bind(e, .{ gate, up, weights, y }); + e.setValue([6]u32{ d.rows, d.width, d.input_stride, d.output_stride, @intFromBool(d.weighted), @intFromBool(d.input_f32) }, 4); + e.dispatchThreads(mtl.Size.of(d.width, d.rows, 1), mtl.Size.of(256, 1, 1)); +} +pub const Combine = struct { + rows: u32, + width: u32, + routed_row_stride: u32, + slot_stride: u32, + shared_stride: u32, + y_stride: u32, + input_f32: bool = false, + id_stride: u32 = 6, +}; +/// Sum six weighted down slots by ascending expert ID in FP32, then shared; final BF16, invalid IDs yield NaN. +pub fn encodeCombine(e: mtl.ComputeEncoder, p: *const Pipelines, routed: View, shared: View, ids: View, y: View, d: Combine) void { + e.setPipeline(p.combine); + bind(e, .{ routed, shared, ids, y }); + e.setValue([8]u32{ d.rows, d.width, d.routed_row_stride, d.slot_stride, d.shared_stride, d.y_stride, @intFromBool(d.input_f32), d.id_stride }, 4); + e.dispatchThreads(mtl.Size.of(d.width, d.rows, 1), mtl.Size.of(256, 1, 1)); +} diff --git a/zig/src/families/deepseek_v41/ops_check.zig b/zig/src/families/deepseek_v41/ops_check.zig new file mode 100644 index 0000000000..bfe08f4dd8 --- /dev/null +++ b/zig/src/families/deepseek_v41/ops_check.zig @@ -0,0 +1,191 @@ +//! Independent GPU consumer fixtures; pinned official inference equations and provenance in THIRD_PARTY_NOTICES.md. +const std = @import("std"); +const mtl = @import("metal"); +const ops = @import("ops.zig"); +const support = @import("test_support.zig"); +const bf = support.bf; +const fp = support.fp; +const view = support.view; +const Buffers = support.Buffers; +const done = support.done; +const close = support.close; + +test "V4.1 BF16 RMS, F16 dense row independence, grouped wo_a, embedding, clamped weighted SwiGLU and shared combine" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try ops.Pipelines.load(device); + defer p.deinit(); + var b = Buffers{ .device = device }; + defer b.deinit(); + const x = try b.get(u16, 16); + const w = try b.get(u16, 4); + const norm = try b.get(u16, 16); + const xs = x.slice(u16, 16); + const ws = w.slice(u16, 4); + for (0..2) |r| for (0..4) |i| { + xs[r * 8 + i] = bf(@as(f32, @floatFromInt(1 + i + r))); + }; + for (ws, 0..) |*v, i| v.* = bf(@as(f32, @floatFromInt(i + 1)) * 0.5); + const f16w = try b.get(f16, 8); + const fws = f16w.slice(f16, 8); + for (fws, 0..) |*v, i| v.* = @floatCast(@as(f32, @floatFromInt(i + 1)) * 0.125 + 0.0009765625); + const dense = try b.get(f32, 8); + const dense_one = try b.get(f32, 2); + const grouped = try b.get(u16, 8); + const gw = try b.get(u16, 8); + for (gw.slice(u16, 8), 0..) |*v, i| v.* = bf(@as(f32, @floatFromInt(i + 1)) * 0.25); + const ids = try b.get(u32, 2); + ids.slice(u32, 2)[0] = 1; + ids.slice(u32, 2)[1] = 0; + const embed = try b.get(u16, 32); + const gate = try b.get(f32, 8); + const up = try b.get(f32, 8); + @memcpy(gate.slice(f32, 8), &[_]f32{ -100, -12, 1, 100, -2, 0, 8, 12 }); + @memcpy(up.slice(f32, 8), &[_]f32{ -100, -12, 1, 100, 12, -20, 0.5, 1 }); + const weights = try b.get(f32, 2); + @memcpy(weights.slice(f32, 2), &[_]f32{ 0.25, 1.25 }); + const act = try b.get(u16, 8); + const routed = try b.get(u16, 48); + const combine_ids = try b.get(u32, 12); + @memcpy(combine_ids.slice(u32, 12), &[_]u32{ 5, 3, 1, 4, 2, 0, 5, 3, 1, 4, 2, 0 }); + const shared = try b.get(u16, 8); + const combined = try b.get(u16, 8); + const values = [_]f32{ 1, 33554432, -33554432, 2, 4, 8 }; + for (routed.slice(u16, 48), 0..) |*v, i| v.* = bf(values[(i % 24) / 4]); + @memset(shared.slice(u16, 8), bf(2.5)); + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + ops.encodeRms(e, &p, view(x), view(w), view(norm), .{ .rows = 2, .width = 4, .x_stride = 8, .y_stride = 8, .eps = 1e-20 }); + ops.encodeDense(e, &p, view(x), view(f16w), view(dense), .{ .rows = 2, .k = 4, .n = 2, .x_stride = 8, .w_stride = 4, .y_stride = 4, .w_dtype = .f16, .y_dtype = .f32 }); + ops.encodeDense(e, &p, view(x).shifted(16), view(f16w), view(dense_one), .{ .rows = 1, .k = 4, .n = 2, .x_stride = 8, .w_stride = 4, .y_stride = 2, .w_dtype = .f16, .y_dtype = .f32 }); + ops.encodeDense(e, &p, view(x), view(gw), view(grouped), .{ .rows = 2, .k = 2, .n = 2, .groups = 2, .x_stride = 8, .w_stride = 2, .y_stride = 4 }); + ops.encodeEmbedding(e, &p, view(ids), view(x), view(embed), .{ .rows = 2, .width = 4, .table_stride = 8, .y_stride = 16 }); + ops.encodeSwiGLU(e, &p, view(gate), view(up), view(weights), view(act), .{ .rows = 2, .width = 4, .input_stride = 4, .output_stride = 4, .weighted = true, .input_f32 = true }); + ops.encodeCombine(e, &p, view(routed), view(shared), view(combine_ids), view(combined), .{ .rows = 2, .width = 4, .routed_row_stride = 24, .slot_stride = 4, .shared_stride = 4, .y_stride = 4 }); + try done(cb, e); + for (0..2) |r| { + var sum: f64 = 0; + for (0..4) |i| sum += @as(f64, fp(xs[r * 8 + i])) * fp(xs[r * 8 + i]); + for (0..4) |i| try close(@as(f64, fp(xs[r * 8 + i])) / @sqrt(sum / 4 + 1e-20) * fp(ws[i]), fp(norm.slice(u16, 16)[r * 8 + i]), 0.04); + for (0..2) |j| { + var ref: f64 = 0; + for (0..4) |k| ref += @as(f64, fp(xs[r * 8 + k])) * @as(f64, fws[j * 4 + k]); + try close(ref, dense.slice(f32, 8)[r * 4 + j], 1e-6); + } + for (0..2) |g| for (0..2) |j| { + var ref: f32 = 0; + for (0..2) |k| ref += fp(xs[r * 8 + g * 2 + k]) * fp(gw.slice(u16, 8)[g * 4 + j * 2 + k]); + try std.testing.expectEqual(bf(ref), grouped.slice(u16, 8)[r * 4 + g * 2 + j]); + }; + for (0..4) |copy| for (0..4) |i| try std.testing.expectEqual(xs[(1 - r) * 8 + i], embed.slice(u16, 32)[r * 16 + copy * 4 + i]); + for (0..4) |i| { + const k = r * 4 + i; + const a: f64 = @min(gate.slice(f32, 8)[k], 10); + const u: f64 = @max(-10, @min(up.slice(f32, 8)[k], 10)); + const activated = a / (1 + @exp(-a)) * u * weights.slice(f32, 2)[r]; + try close(activated, fp(act.slice(u16, 8)[k]), @abs(activated) * 0.004 + 1e-35); + var ref: f32 = 0; + for ([_]usize{ 5, 2, 4, 1, 3, 0 }) |slot| ref += fp(routed.slice(u16, 48)[r * 24 + slot * 4 + i]); + try std.testing.expectEqual(bf(ref + 2.5), combined.slice(u16, 8)[k]); + } + } + for (0..2) |j| try std.testing.expectEqual(dense.slice(f32, 8)[4 + j], dense_one.slice(f32, 2)[j]); +} + +test "V4.1 CUDA RMS mean projection preserves the BF16 affine tie" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try ops.Pipelines.load(device); + defer p.deinit(); + var buffers = Buffers{ .device = device }; + defer buffers.deinit(); + const width = 5120; + const x = try buffers.get(u16, width); + const weight = try buffers.get(f32, width); + const output = try buffers.get(u16, width); + x.slice(u16, width)[0] = bf(3); + @memset(weight.slice(f32, width), @bitCast(@as(u32, 0x3c65de29))); + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + ops.encodeRms(e, &p, view(x), view(weight), view(output), .{ .rows = 1, .width = width, .x_stride = width, .y_stride = width, .eps = 1e-20, .weight_f32 = true }); + try done(cb, e); + // Pinned CUDA RMSNorm.forward returns 1; sum / width crosses to the next BF16 value. + try std.testing.expectEqual(bf(1), output.slice(u16, width)[0]); + for (output.slice(u16, width)[1..]) |v| try std.testing.expectEqual(@as(u16, 0), v); +} + +test "V4.1 FP32 dense retains product and reduction cancellation" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try ops.Pipelines.load(device); + defer p.deinit(); + var buffers = Buffers{ .device = device }; + defer buffers.deinit(); + const k = 96; + const n = 3; + const x = try buffers.get(u16, k); + const w = try buffers.get(f32, k * n); + const y = try buffers.get(f32, n); + @memset(x.slice(u16, k), bf(3)); + const weights = w.slice(f32, k * n); + weights[0] = 0x1p25; + weights[32] = 1; + weights[64] = -0x1p25; + weights[k] = 1 + 0x1p-23; + weights[k + 32] = -1; + weights[2 * k] = 0x1p25; + weights[2 * k + 1] = 1; + weights[2 * k + 2] = -0x1p25; + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + ops.encodeDense(enc, &p, view(x), view(w), view(y), .{ .rows = 1, .k = k, .n = n, .x_stride = k, .w_stride = k, .y_stride = n, .w_dtype = .f32, .y_dtype = .f32 }); + try done(cb, enc); + for (0..n) |column| { + var expected: f64 = 0; + for (0..k) |index| expected += @as(f64, fp(x.slice(u16, k)[index])) * weights[column * k + index]; + try std.testing.expectEqual(@as(f32, @floatCast(expected)), y.slice(f32, n)[column]); + } +} + +test "V4.1 weighted SiLU preserves division rounding before BF16 activation" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try ops.Pipelines.load(device); + defer p.deinit(); + var buffers = Buffers{ .device = device }; + defer buffers.deinit(); + const gate = try buffers.get(u16, 1); + const up = try buffers.get(u16, 1); + const weights = try buffers.get(f32, 1); + const output = try buffers.get(u16, 1); + gate.slice(u16, 1)[0] = bf(0.78515625); + up.slice(u16, 1)[0] = bf(-6.96875); + weights.slice(f32, 1)[0] = 0.25; + const cb = queue.commandBuffer(); + const encoder = cb.compute(.serial); + ops.encodeSwiGLU(encoder, &p, view(gate), view(up), view(weights), view(output), .{ .rows = 1, .width = 1, .input_stride = 1, .output_stride = 1, .weighted = true }); + try done(cb, encoder); + // Source SiLU divides before up/route multiplication; reciprocal multiplication crosses this BF16 tie. + try std.testing.expectEqual(bf(-0.94140625), output.slice(u16, 1)[0]); +} + +test { + _ = @import("mhc_test.zig"); + _ = @import("route_test.zig"); +} diff --git a/zig/src/families/deepseek_v41/profiler.zig b/zig/src/families/deepseek_v41/profiler.zig new file mode 100644 index 0000000000..fe3a85b5fc --- /dev/null +++ b/zig/src/families/deepseek_v41/profiler.zig @@ -0,0 +1,93 @@ +//! Family stage labels over the generic same-command-buffer counter recorder; no extra storage or GPU waits. +const std = @import("std"); +const mtl = @import("metal"); +const counters = @import("../../core/gpu_counters.zig"); + +pub const Stage = enum { engram, mhc, attention, moe, head, other }; +pub const stage_count = @typeInfo(Stage).@"enum".field_names.len; +pub const Options = counters.Options; +pub const Aggregate = counters.Aggregate; +pub const Span = counters.Span; + +pub fn class(stage: Stage) counters.Class { + return switch (stage) { + .engram => .embedding, + .mhc => .norm, + .attention => .attention, + .moe, .head => .dense, + .other => .copy, + }; +} +pub fn stageOf(span: Span) !Stage { + if (span.metadata >= stage_count) return error.InvalidProfileStage; + return @fromBackingInt(@intCast(span.metadata)); +} + +/// Unique owner of the recorder; stage metadata and class totals share the recorder's bounded span storage. +pub const Recorder = struct { + recorder: counters.Recorder, + totals: [stage_count]Aggregate = @splat(.{}), + + pub fn init(allocator: std.mem.Allocator, device: mtl.Device, options: Options) !Recorder { + return .{ .recorder = try counters.Recorder.init(allocator, device, options) }; + } + pub fn enabled(self: *const Recorder) bool { + return self.recorder.enabled(); + } + pub fn capability(self: *const Recorder) mtl.counters.Capability { + return self.recorder.capability(); + } + pub fn activeSpan(self: *const Recorder) bool { + return self.recorder.active_span != null; + } + pub fn beginCommand(self: *Recorder, cb: mtl.CommandBuffer) !void { + try self.recorder.beginCommand(cb); + self.totals = @splat(.{}); + } + pub fn beginSpan(self: *Recorder, encoder: *mtl.ComputeEncoder, stage: Stage) !void { + try self.recorder.beginSpan(encoder, class(stage), @intCast(@backingInt(stage))); + } + pub fn endSpan(self: *Recorder, encoder: *mtl.ComputeEncoder) !void { + try self.recorder.endSpan(encoder); + } + pub fn finishEncoding(self: *Recorder, encoder: *mtl.ComputeEncoder) !void { + try self.recorder.finishEncoding(encoder); + } + pub fn cancelEncoding(self: *Recorder, encoder: *mtl.ComputeEncoder) !void { + try self.recorder.cancelEncoding(encoder); + self.totals = @splat(.{}); + } + pub fn discardAfterCompletion(self: *Recorder) !void { + try self.recorder.discardAfterCompletion(); + self.totals = @splat(.{}); + } + pub fn readAfterCompletion(self: *Recorder) ![]const Span { + const spans = try self.recorder.readAfterCompletion(); + var totals: [stage_count]Aggregate = @splat(.{}); + for (spans) |span| { + const total = &totals[@backingInt(try stageOf(span))]; + total.calls += 1; + total.gpu_seconds += span.gpu_seconds; + } + self.totals = totals; + return spans; + } + pub fn aggregates(self: *const Recorder) [stage_count]Aggregate { + return self.totals; + } + pub fn deinit(self: *Recorder) !void { + try self.recorder.deinit(); + self.totals = @splat(.{}); + } +}; + +test "disabled family profiling allocates nothing and leaves no stage receipts" { + var recorder = try Recorder.init(std.testing.failing_allocator, undefined, .{}); + defer recorder.deinit() catch unreachable; + var encoder: mtl.ComputeEncoder = undefined; + try recorder.beginSpan(&encoder, .mhc); + try recorder.endSpan(&encoder); + try std.testing.expect(!recorder.activeSpan()); + try std.testing.expectEqual(@as(usize, 0), (try recorder.readAfterCompletion()).len); + for (recorder.aggregates()) |total| try std.testing.expectEqual(@as(u64, 0), total.calls); +} diff --git a/zig/src/families/deepseek_v41/rope.zig b/zig/src/families/deepseek_v41/rope.zig new file mode 100644 index 0000000000..cc3419f7c3 --- /dev/null +++ b/zig/src/families/deepseek_v41/rope.zig @@ -0,0 +1,134 @@ +//! Adjacent-pair BF16 RoPE using resident, source-style complex rotation factors. +const std = @import("std"); +const mtl = @import("metal"); +const sources = @import("kernel_sources"); +const View = @import("view.zig").View; + +pub const Spec = struct { + theta: f32, + factor: f32 = 16, + original: u32 = 0, + beta_fast: f32 = 32, + beta_slow: f32 = 1, + + fn validate(s: Spec) !void { + if (!std.math.isFinite(s.theta) or s.theta <= 1 or + !std.math.isFinite(s.factor) or s.factor <= 0 or + !std.math.isFinite(s.beta_fast) or s.beta_fast <= 0 or + !std.math.isFinite(s.beta_slow) or s.beta_slow <= 0) + return error.InvalidRotarySpec; + } +}; +pub const Spectrum = enum(u32) { normal, compressed }; +pub const Params = extern struct { + rows: u32, + heads: u32, + width: u32, + rotary: u32, + stride: u32, + position: u32, + step: u32 = 1, + inverse: u32 = 0, + spectrum: Spectrum = .normal, + + pub fn validate(p: Params) !void { + if (p.rows == 0 or p.heads == 0 or p.rotary == 0 or p.rotary % 2 != 0 or + p.rotary > p.width or @as(u64, p.stride) < @as(u64, p.heads) * p.width or + p.step == 0 or p.inverse > 1 or + @as(u64, p.rows - 1) * p.stride + @as(u64, p.heads) * p.width > std.math.maxInt(u32)) + return error.InvalidRotaryShape; + } +}; + +// Keep FP32 tensor/angle boundaries, FP64 corrected_dim logs, and FP32-rounded polar unaries; never per-head recompute. +fn fillFactors(out: []f32, capacity: u32, rotary: u32, spec: Spec) void { + @setFloatMode(.strict); + const dim: f32 = @floatFromInt(rotary); + const pairs: usize = rotary / 2; + var low: f32 = 0; + var high: f32 = 0; + if (spec.original > 0) { + const original: f64 = @floatFromInt(spec.original); + const dim64: f64 = @floatFromInt(rotary); + const denominator = 2 * @log(@as(f64, spec.theta)); + low = @floatCast(@max(@floor(dim64 * @log(original / + (@as(f64, spec.beta_fast) * 2 * std.math.pi)) / denominator), 0)); + high = @floatCast(@min(@ceil(dim64 * @log(original / + (@as(f64, spec.beta_slow) * 2 * std.math.pi)) / denominator), dim64 - 1)); + } + for (0..pairs) |pair| { + const exponent: f32 = @as(f32, @floatFromInt(pair * 2)) / dim; + const power: f32 = @floatCast(std.math.pow(f64, spec.theta, exponent)); + var frequency: f32 = 1 / power; + if (spec.original > 0) { + const ramp: f32 = std.math.clamp((@as(f32, @floatFromInt(pair)) - low) / + @max(high - low, @as(f32, 1e-3)), 0, 1); + const smooth: f32 = 1 - ramp; + const scaled: f32 = frequency / spec.factor; + const slow: f32 = scaled * (1 - smooth); + const fast: f32 = frequency * smooth; + frequency = slow + fast; + } + for (0..capacity) |position| { + const angle: f32 = @as(f32, @floatFromInt(position)) * frequency; + const at = (position * pairs + pair) * 2; + out[at] = @floatCast(@cos(@as(f64, angle))); + out[at + 1] = @floatCast(@sin(@as(f64, angle))); + } + } +} + +pub const Pipelines = struct { + rotate: mtl.Pipeline, + factors: mtl.Buffer, + capacity: u32, + rotary: u32, + + pub fn residentBytes(capacity: u32, rotary: u32) usize { + return 2 * @as(usize, capacity) * (rotary / 2) * 2 * @sizeOf(f32); + } + + pub fn load(device: mtl.Device, capacity: u32, rotary: u32, normal: Spec, compressed: Spec) !Pipelines { + if (capacity == 0 or rotary == 0 or rotary % 2 != 0 or + @as(u64, capacity) * rotary > std.math.maxInt(usize) / (2 * @sizeOf(f32))) + return error.InvalidRotaryShape; + try normal.validate(); + try compressed.validate(); + var options = mtl.CompileOptions.mlx(); + options.math_functions = .precise; + const lib = try mtl.Library.fromSource(device, sources.deepseek_v41_rope, options); + defer lib.deinit(); + const rotate = try mtl.Pipeline.init(device, lib, "dsv41_rope", false); + errdefer rotate.deinit(); + const factors = try device.buffer(residentBytes(capacity, rotary), mtl.ResourceOptions.shared | mtl.ResourceOptions.untracked); + errdefer factors.deinit(); + const count = @as(usize, capacity) * rotary; + const values = factors.slice(f32, 2 * count); + fillFactors(values[0..count], capacity, rotary, normal); + fillFactors(values[count..], capacity, rotary, compressed); + return .{ .rotate = rotate, .factors = factors, .capacity = capacity, .rotary = rotary }; + } + pub fn deinit(p: *Pipelines) void { + p.factors.deinit(); + p.rotate.deinit(); + } + pub fn registerResidency(p: *const Pipelines, set: mtl.ResidencySet) void { + set.add(p.factors); + } + pub fn encode(p: *const Pipelines, enc: mtl.ComputeEncoder, x: View, args: Params) !void { + try args.validate(); + const last = @as(u64, args.position) + @as(u64, args.rows - 1) * args.step; + if (args.rotary != p.rotary or last >= p.capacity) return error.InvalidRotaryShape; + try x.requireRows(args.rows, args.stride, @as(usize, args.heads) * args.width, 2); + const offset = switch (args.spectrum) { + .normal => 0, + .compressed => @as(usize, p.capacity) * p.rotary * @sizeOf(f32), + }; + enc.setPipeline(p.rotate); + enc.setBuffer(x.buffer, x.offset, 0); + enc.setValue(args, 1); + enc.setBuffer(p.factors, offset, 2); + enc.dispatchThreads(mtl.Size.of(args.rotary / 2, args.heads, args.rows), mtl.Size.of(32, 1, 1)); + enc.barrier(); + } +}; diff --git a/zig/src/families/deepseek_v41/route.zig b/zig/src/families/deepseek_v41/route.zig new file mode 100644 index 0000000000..00c1a1ff10 --- /dev/null +++ b/zig/src/families/deepseek_v41/route.zig @@ -0,0 +1,39 @@ +//! Exhaustive 384-expert sqrt-softplus top-six routing. Bias affects selection, never returned weights. +const mtl = @import("metal"); +const sources = @import("kernel_sources"); +pub const View = @import("view.zig").View; +pub const Dtype = @import("ops.zig").Dtype; +pub const Pipelines = struct { + select: mtl.Pipeline, + pub fn load(device: mtl.Device) !Pipelines { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + var options = mtl.CompileOptions.mlx(); + options.math_functions = .precise; + const lib = try mtl.Library.fromSource(device, sources.deepseek_v41_route, options); + defer lib.deinit(); + return .{ .select = try mtl.Pipeline.init(device, lib, "dsv41_route", false) }; + } + pub fn deinit(p: *Pipelines) void { + p.select.deinit(); + } +}; +pub const Shape = struct { + rows: u32, + logit_stride: u32 = 384, + output_stride: u32 = 6, + temperature: f32 = 1, + bias_dtype: Dtype = .f32, +}; +/// FP32 logits[rows,384], bias F16/BF16/FP32 -> u32 IDs and FP32 weights[rows,6]; lower ID wins biased-score ties. + +// Nonzero rows, logit/output pitches >=384/6, finite positive temperature, complete storage are caller invariants. + +// All six execute; weights use unbiased scores / (sum+1e-20) * 1.5. Invalid comparisons emit invalid IDs/NaNs. +pub fn encode(e: mtl.ComputeEncoder, p: *const Pipelines, logits: View, bias: View, ids: View, weights: View, d: Shape) void { + e.setPipeline(p.select); + inline for (.{ logits, bias, ids, weights }, 0..) |r, i| e.setBuffer(r.buffer, r.offset, i); + const D = extern struct { rows: u32, logit_stride: u32, output_stride: u32, temperature: f32, bias_dtype: u32 }; + e.setValue(D{ .rows = d.rows, .logit_stride = d.logit_stride, .output_stride = d.output_stride, .temperature = d.temperature, .bias_dtype = @backingInt(d.bias_dtype) }, 4); + e.dispatchGroups(mtl.Size.of(d.rows, 1, 1), mtl.Size.of(128, 1, 1)); +} diff --git a/zig/src/families/deepseek_v41/route_test.zig b/zig/src/families/deepseek_v41/route_test.zig new file mode 100644 index 0000000000..f653fa9241 --- /dev/null +++ b/zig/src/families/deepseek_v41/route_test.zig @@ -0,0 +1,152 @@ +//! Consumer-visible expert IDs, weights and nonfinite-input routing boundaries. +const std = @import("std"); +const mtl = @import("metal"); +const route = @import("route.zig"); +const ops = @import("ops.zig"); +const support = @import("test_support.zig"); +const view = support.view; +const Buffers = support.Buffers; +const done = support.done; +const close = support.close; +fn scoreReference(x: f64) f64 { + return @sqrt(if (x > 20) x else std.math.log1p(@exp(x))); +} +test "V4.1 exhaustive router F16 correction bias, unbiased normalization, extreme logits and row independence" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try route.Pipelines.load(device); + defer p.deinit(); + var b = Buffers{ .device = device }; + defer b.deinit(); + const logits = try b.get(f32, 2 * 389); + const bias = try b.get(f16, 384); + for (0..384) |i| { + logits.slice(f32, 778)[i] = (@as(f32, @floatFromInt(i)) - 190) * 0.5; + logits.slice(f32, 778)[389 + i] = 0; + bias.slice(f16, 384)[i] = if (i >= 17 and i < 23) 100 else 0; + } + const ids = try b.get(u32, 16); + const weights = try b.get(f32, 16); + const ids_one = try b.get(u32, 6); + const weights_one = try b.get(f32, 6); + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + route.encode(e, &p, view(logits), view(bias), view(ids), view(weights), .{ .rows = 2, .logit_stride = 389, .output_stride = 8, .bias_dtype = .f16 }); + route.encode(e, &p, view(logits).shifted(389 * 4), view(bias), view(ids_one), view(weights_one), .{ .rows = 1, .bias_dtype = .f16 }); + try done(cb, e); + for (0..2) |r| { + var used: [384]bool = @splat(false); + var picks: [6]usize = undefined; + var raw: [6]f64 = undefined; + var sum: f64 = 0; + for (0..6) |slot| { + var best: f64 = -std.math.inf(f64); + var pick: usize = 0; + for (0..384) |i| { + const sc = scoreReference(logits.slice(f32, 778)[r * 389 + i]) + @as(f64, bias.slice(f16, 384)[i]); + if (!used[i] and sc > best) { + best = sc; + pick = i; + } + } + used[pick] = true; + picks[slot] = pick; + raw[slot] = scoreReference(logits.slice(f32, 778)[r * 389 + pick]); + sum += raw[slot]; + } + for (0..6) |slot| { + try std.testing.expectEqual(@as(u32, @intCast(picks[slot])), ids.slice(u32, 16)[r * 8 + slot]); + try close(raw[slot] / (sum + 1e-20) * 1.5, weights.slice(f32, 16)[r * 8 + slot], 3e-6); + } + } + for (0..6) |i| { + try std.testing.expectEqual(ids.slice(u32, 16)[8 + i], ids_one.slice(u32, 6)[i]); + try std.testing.expectEqual(weights.slice(f32, 16)[8 + i], weights_one.slice(f32, 6)[i]); + } +} + +test "V4.1 router normalization matches source CUDA six-score FP32 reduction boundaries" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try route.Pipelines.load(device); + defer p.deinit(); + var b = Buffers{ .device = device }; + defer b.deinit(); + // Power-of-two sqrt(softplus(score^2)) with logits >20 eliminates unary-rounding ambiguity. + const scores = [2][6]f32{ + .{ 268435456, 16, 16, 16, 16, 16 }, + .{ 2097152, 8, 16, 134217728, 8388608, 8192 }, + }; + // Independent pinned Torch CUDA Gate expected bits; the second row also distinguishes wide normalization. + const expected_bits = [2][6]u32{ + .{ 1069547517, 868220925, 868220925, 868220925, 868220925, 868220925 }, + .{ 1018303405, 867308461, 875697069, 1068635053, 1035080621, 951194541 }, + }; + const logits = try b.get(f32, scores.len * 384); + const bias = try b.get(f32, 384); + const ids = try b.get(u32, scores.len * 6); + const weights = try b.get(f32, scores.len * 6); + @memset(logits.slice(f32, scores.len * 384), -100); + for (scores, 0..) |row, r| { + for (row, 0..) |s, slot| logits.slice(f32, scores.len * 384)[r * 384 + slot] = s * s; + } + // Distinct biases force slot order without topk ties; weights still use unbiased scores. + for (0..6) |slot| bias.slice(f32, 384)[slot] = 1099511627776 - @as(f32, @floatFromInt(slot)) * 68719476736; + const cb = queue.commandBuffer(); + const encoder = cb.compute(.serial); + route.encode(encoder, &p, view(logits), view(bias), view(ids), view(weights), .{ .rows = scores.len, .bias_dtype = .f32 }); + try done(cb, encoder); + for (expected_bits, 0..) |row, r| { + for (row, 0..) |expected, slot| { + try std.testing.expectEqual(@as(u32, @intCast(slot)), ids.slice(u32, scores.len * 6)[r * 6 + slot]); + try std.testing.expectEqual(expected, @as(u32, @bitCast(weights.slice(f32, scores.len * 6)[r * 6 + slot]))); + } + } +} + +test "nonfinite router inputs propagate invalid routes without reading outside the expert arrays" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try route.Pipelines.load(device); + defer p.deinit(); + var buffers = Buffers{ .device = device }; + defer buffers.deinit(); + const logits = try buffers.get(f32, 384); + const bias = try buffers.get(f32, 384); + const ids = try buffers.get(u32, 6); + const weights = try buffers.get(f32, 6); + @memset(logits.slice(f32, 384), std.math.nan(f32)); + const cb = queue.commandBuffer(); + const encoder = cb.compute(.serial); + route.encode(encoder, &p, view(logits), view(bias), view(ids), view(weights), .{ .rows = 1 }); + try done(cb, encoder); + for (ids.slice(u32, 6), weights.slice(f32, 6)) |id, weight| { + try std.testing.expectEqual(std.math.maxInt(u32), id); + try std.testing.expect(std.math.isNan(weight)); + } + var combine = try ops.Pipelines.load(device); + defer combine.deinit(); + const routed = try buffers.get(u16, 6 * 4); + const shared = try buffers.get(u16, 4); + const output = try buffers.get(u16, 4); + // Invalid routes skip expert writes; finite slots from a prior chunk must not hide that failure. + @memset(routed.slice(u16, 24), support.bf(2)); + @memset(shared.slice(u16, 4), support.bf(3)); + const combine_cb = queue.commandBuffer(); + const combine_encoder = combine_cb.compute(.serial); + ops.encodeCombine(combine_encoder, &combine, view(routed), view(shared), view(ids), view(output), .{ .rows = 1, .width = 4, .routed_row_stride = 24, .slot_stride = 4, .shared_stride = 4, .y_stride = 4 }); + try done(combine_cb, combine_encoder); + for (output.slice(u16, 4)) |value| try std.testing.expect(std.math.isNan(support.fp(value))); +} diff --git a/zig/src/families/deepseek_v41/test_support.zig b/zig/src/families/deepseek_v41/test_support.zig new file mode 100644 index 0000000000..80bf87e8d2 --- /dev/null +++ b/zig/src/families/deepseek_v41/test_support.zig @@ -0,0 +1,40 @@ +const std = @import("std"); +const mtl = @import("metal"); +const View = @import("view.zig").View; + +pub fn bf(x: f32) u16 { + const u: u32 = @bitCast(x); + return @intCast((u +% 0x7fff +% ((u >> 16) & 1)) >> 16); +} +pub fn fp(x: u16) f32 { + return @bitCast(@as(u32, x) << 16); +} +pub fn view(b: mtl.Buffer) View { + return .{ .buffer = b }; +} +pub const Buffers = struct { + device: mtl.Device, + items: [32]mtl.Buffer = undefined, + count: usize = 0, + + pub fn get(b: *Buffers, comptime T: type, n: usize) !mtl.Buffer { + if (b.count == b.items.len) return error.TooManyTestBuffers; + const x = try b.device.buffer(try std.math.mul(usize, @sizeOf(T), n), mtl.ResourceOptions.shared); + b.items[b.count] = x; + b.count += 1; + @memset(x.slice(T, n), 0); + return x; + } + pub fn deinit(b: *Buffers) void { + for (b.items[0..b.count]) |x| x.deinit(); + } +}; +pub fn done(cb: mtl.CommandBuffer, e: mtl.ComputeEncoder) !void { + e.end(); + cb.commit(); + cb.wait(); + try std.testing.expect(cb.failure() == null); +} +pub fn close(expected: f64, actual: f32, tolerance: f64) !void { + try std.testing.expect(@abs(expected - @as(f64, actual)) <= tolerance); +} diff --git a/zig/src/families/deepseek_v41/workspace.zig b/zig/src/families/deepseek_v41/workspace.zig new file mode 100644 index 0000000000..fcef57cd6e --- /dev/null +++ b/zig/src/families/deepseek_v41/workspace.zig @@ -0,0 +1,107 @@ +//! Fixed chunk-sized activations: no layer history or context-by-query score matrices. +const std = @import("std"); +const mtl = @import("metal"); +const exl3 = @import("../../core/exl3_projection.zig"); +const View = @import("view.zig").View; +const hash = @import("engram_hash.zig"); +const groups = @import("../../core/expert_groups.zig"); + +pub const Field = enum { + ids, + token_map, + history, + stream, + collapsed, + normalized, + branch, + previous_pre, + attention_pre, + post, + comb, + mixes, + inv, + engram_ids_0, + engram_ids_1, + engram_gather, + engram_kv, + router_logits, + route_ids, + route_weights, + gate, + up, + activation, + routed, + shared_gate, + shared_up, + shared_activation, + shared_down, + logits, + projection, + grouping, +}; +const count = std.enums.values(Field).len; + +pub const Plan = struct { + rows: u32, + offsets: [count]usize, + bytes: usize, + + pub fn init(rows: u32) !Plan { + if (rows == 0 or rows > 4096) return error.InvalidPrefillChunk; + const t: usize = rows; + const grouped = if (rows > 1) try groups.Plan.init(t * 6, 384) else null; + const projection_bytes = @max(try exl3.scratchBytes(5120, 129280, t), if (grouped) |g| @max(try exl3.scratchBytes(5120, 2304, g.padded), try exl3.scratchBytes(2304, 5120, g.padded)) else try exl3.scratchBytes(2304, 5120, t * 6)); + const sizes = [_]usize{ + t * 4, hash.raw_vocab * 4, 16, t * 20480 * 2, t * 5120 * 2, t * 5120 * 2, t * 5120 * 2, + t * 4 * 4, t * 4 * 4, t * 4 * 4, t * 16 * 4, t * 24 * 4, t * 4, t * 24 * 4, + t * 24 * 4, t * 6144 * 2, t * 25600 * 2, t * 384 * 4, t * 6 * 4, t * 6 * 4, t * 6 * 2304 * 2, + t * 6 * 2304 * 2, t * 6 * 2304 * 2, t * 6 * 5120 * 2, t * 2304 * 2, t * 2304 * 2, t * 2304 * 2, t * 5120 * 2, + t * 129280 * 4, projection_bytes, if (grouped) |g| g.bytes else 0, + }; + var out = Plan{ .rows = rows, .offsets = undefined, .bytes = 0 }; + for (sizes, 0..) |size, i| { + out.offsets[i] = std.mem.alignForward(usize, out.bytes, 16); + out.bytes = try std.math.add(usize, out.offsets[i], size); + } + out.bytes = std.mem.alignForward(usize, out.bytes, 16384); + return out; + } +}; + +pub const Workspace = struct { + buffer: mtl.Buffer, + plan: Plan, + + pub fn init(device: mtl.Device, plan: Plan, token_map: []const u8) !Workspace { + if (token_map.len != hash.raw_vocab * 4) return error.InvalidTokenMap; + const buffer = try device.buffer(plan.bytes, mtl.ResourceOptions.shared | mtl.ResourceOptions.untracked); + const w = Workspace{ .buffer = buffer, .plan = plan }; + @memcpy(buffer.contents()[plan.offsets[@backingInt(Field.token_map)]..][0..token_map.len], token_map); + @memset(buffer.contents()[plan.offsets[@backingInt(Field.history)]..][0..16], 0); + return w; + } + + pub fn deinit(w: *Workspace) void { + w.buffer.deinit(); + } + pub fn view(w: *const Workspace, field: Field) View { + return .{ .buffer = w.buffer, .offset = w.plan.offsets[@backingInt(field)] }; + } + pub fn projection(w: *const Workspace) exl3.View { + const v = w.view(.projection); + return .{ .buffer = v.buffer, .offset = v.offset }; + } + pub fn grouping(w: *const Workspace, rows: u32) !groups.Scratch { + if (rows < 2 or rows > w.plan.rows) return error.InvalidPrefillChunk; + const v = w.view(.grouping); + return .{ .buffer = v.buffer, .offset = v.offset, .plan = try groups.Plan.init(@as(usize, rows) * 6, 384) }; + } + pub fn tokenIds(w: *const Workspace, rows: usize) []u32 { + const v = w.view(.ids); + return @as([*]u32, @ptrCast(@alignCast(v.buffer.contents() + v.offset)))[0..rows]; + } + pub fn logits(w: *const Workspace, rows: usize) []const f32 { + const v = w.view(.logits); + return @as([*]const f32, @ptrCast(@alignCast(v.buffer.contents() + v.offset)))[0 .. rows * 129280]; + } +}; diff --git a/zig/src/metal/counters.zig b/zig/src/metal/counters.zig index 610e22086b..0bbd65f186 100644 --- a/zig/src/metal/counters.zig +++ b/zig/src/metal/counters.zig @@ -1,4 +1,4 @@ -//! Metal counter sampling; owned resources must outlive every command buffer using them. +//! Metal counter sampling; owned descriptors, samples and fences must outlive their command buffers. const std = @import("std"); const objc = @import("objc.zig"); const types = @import("types.zig"); @@ -107,7 +107,7 @@ pub const SampleBuffer = struct { objc.msg(void, encoder.id, "sampleCountersInBuffer:atSampleIndex:withBarrier:", .{ self.id, index, true }); } - /// Caller must ensure GPU completion; borrows a bounded NSData result without a GPU blit or second copy. + /// Borrow bounded NSData bytes after completion; same-command blit resolution is unsupported, no result copy. pub fn resolveCompleted(self: SampleBuffer, count: usize) !Resolved { if (count == 0 or count > self.count) return error.InvalidSampleCount; const data = objc.msg(?objc.Id, self.id, "resolveCounterRange:", .{ diff --git a/zig/src/metal/metal.zig b/zig/src/metal/metal.zig index 6404c4227d..71bed247d2 100644 --- a/zig/src/metal/metal.zig +++ b/zig/src/metal/metal.zig @@ -29,6 +29,7 @@ pub const keepalive = @import("keepalive.zig"); pub const clock = @import("clock.zig"); pub const counters = @import("counters.zig"); pub const ane = @import("ane.zig"); +pub const counters = @import("counters.zig"); test { std.testing.refAllDecls(@This()); diff --git a/zig/src/tensorfold.zig b/zig/src/tensorfold.zig index 9c660ed577..adaeb80099 100644 --- a/zig/src/tensorfold.zig +++ b/zig/src/tensorfold.zig @@ -26,6 +26,10 @@ pub const deepseek_v41_config = @import("families/deepseek_v41/config.zig"); pub const deepseek_v41_load = @import("families/deepseek_v41/load.zig"); pub const deepseek_v41_weights = @import("families/deepseek_v41/weights.zig"); pub const deepseek_v41_view = @import("families/deepseek_v41/view.zig"); +pub const deepseek_v41 = @import("families/deepseek_v41/model.zig"); +pub const deepseek_v41_profiler = @import("families/deepseek_v41/profiler.zig"); +pub const gpu_counters = @import("core/gpu_counters.zig"); +pub const expert_groups = @import("core/expert_groups.zig"); pub const deepseek_v41_engram_hash = @import("families/deepseek_v41/engram_hash.zig"); pub const exl3_format = @import("core/exl3_format.zig"); pub const exl3_projection = @import("core/exl3_projection.zig"); diff --git a/zig/tests/deepseek_v41_reference/attention_math.zig b/zig/tests/deepseek_v41_reference/attention_math.zig new file mode 100644 index 0000000000..d3df4d0ac7 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/attention_math.zig @@ -0,0 +1,298 @@ +//! CPU CSA2 from DeepSeek AI inference/{model,kernel}.py, rev 2cba9e42aa026125f3ed06c6d98c1db82f7ca027. +const std = @import("std"); +const tf = @import("tensorfold"); +const m = @import("math.zig"); +const projection = @import("projection.zig"); +const Config = tf.deepseek_v41_config.Text; +const Attention = tf.deepseek_v41_weights.Attention; + +pub const max_capacity = 4096; +pub const State = struct { + gpa: std.mem.Allocator, + capacity: usize, + window: []u16, + sources: []u16, + scores: []f32, + candidates: []bool, + blocks: []f32, + active_owner: usize = 0, + selected: [512]u32 = undefined, + selected_len: usize = 0, + partial_kv: [4][2][512]f32 = undefined, + partial_gate: [4][2][512]f32 = undefined, + qr: [1280]u16 = undefined, + q: [32768]u16 = undefined, + index_q: [4096]u16 = undefined, + index_weights: [32]f32 = undefined, + latent: [512]u16 = undefined, + index_key: [128]u16 = undefined, + kv: [512]u16 = undefined, + o: [32768]u16 = undefined, + grouped: [8192]u16 = undefined, + observer: ?Observer = null, + sparse_snapshot: ?[]u16 = null, + pub const Observer = struct { + ctx: *anyopaque, + capture: *const fn (*anyopaque, tf.deepseek_v41.attention.DiagnosticStage, []const u16) void, + }; + fn observe(self: *State, stage: tf.deepseek_v41.attention.DiagnosticStage, values: []const u16) void { + if (self.observer) |observer| observer.capture(observer.ctx, stage, values); + } + + pub fn init(gpa: std.mem.Allocator, capacity: usize) !State { + if (capacity == 0 or capacity > max_capacity) return error.InvalidAttentionReferenceCapacity; + const window = try gpa.alloc(u16, 40 * 128 * 512); + errdefer gpa.free(window); + const sources = try gpa.alloc(u16, 4 * capacity * 640); + errdefer gpa.free(sources); + const scores = try gpa.alloc(f32, capacity); + errdefer gpa.free(scores); + const candidates = try gpa.alloc(bool, capacity); + errdefer gpa.free(candidates); + const blocks = try gpa.alloc(f32, (capacity + 7) / 8); + @memset(window, 0); + return .{ .gpa = gpa, .capacity = capacity, .window = window, .sources = sources, .scores = scores, .candidates = candidates, .blocks = blocks }; + } + pub fn deinit(self: *State) void { + self.gpa.free(self.window); + self.gpa.free(self.sources); + self.gpa.free(self.scores); + self.gpa.free(self.candidates); + self.gpa.free(self.blocks); + self.* = undefined; + } + fn mainRow(self: *State, owner: usize, row: usize) []u16 { + return self.sources[owner * self.capacity * 640 + row * 640 ..][0..512]; + } + fn indexRow(self: *State, owner: usize, row: usize) []u16 { + return self.sources[owner * self.capacity * 640 + row * 640 + 512 ..][0..128]; + } + fn publish(self: *State, a: Attention, layer: usize, x: []const u16, position: usize, c: Config, scratch: *projection.Scratch) !bool { + @setFloatMode(.strict); + const compressor = a.compressor orelse return false; + const owner = (layer - 2) / 6; + self.active_owner = owner; + const ratio: usize = c.compress_ratios[layer]; + if (ratio == 1) { + try projection.apply(compressor.wkv, x, &self.latent, scratch); + } else { + const slot = position % ratio; + try projection.applyF32(compressor.wkv, x, &self.partial_kv[owner][slot], scratch); + try projection.applyF32(compressor.wgate.?, x, &self.partial_gate[owner][slot], scratch); + if ((position + 1) % ratio != 0) return false; + for (0..512) |d| { + const high = @max(self.partial_gate[owner][0][d], self.partial_gate[owner][1][d]); + const p0 = @exp(self.partial_gate[owner][0][d] - high); + const p1 = @exp(self.partial_gate[owner][1][d] - high); + const denominator = p0 + p1; + const weighted0 = self.partial_kv[owner][0][d] * (p0 / denominator); + const weighted1 = self.partial_kv[owner][1][d] * (p1 / denominator); + const pooled = weighted0 + weighted1; + self.latent[d] = m.b(pooled); + } + } + m.rms(&self.latent, compressor.norm, c.rms_norm_eps, &self.latent); + const indexer = a.indexer.?; + try projection.apply(indexer.wk.?, &self.latent, &self.index_key, scratch); + m.rms(&self.index_key, indexer.k_norm.?, c.rms_norm_eps, &self.index_key); + const group_position = position + 1 - ratio; + m.rope(&self.index_key, group_position, true, false, c); + m.quantize(&self.index_key, .index); + const row = position / ratio; + @memcpy(self.indexRow(owner, row), &self.index_key); + m.rope(&self.latent, group_position, true, false, c); + m.quantize(&self.latent, .main); + @memcpy(self.mainRow(owner, row), &self.latent); + return true; + } + fn chooseCandidates(self: *State, count: usize, c: Config) void { + @memset(self.candidates[0..count], false); + const nblocks = (count + 7) / 8; + for (0..nblocks) |block| { + var score: f32 = -std.math.inf(f32); + for (block * 8..@min(block * 8 + 8, count)) |row| score = @max(score, self.scores[row]); + self.blocks[block] = score; + } + if (nblocks == 0) return; + self.blocks[nblocks - 1] = std.math.inf(f32); + for (0..@min(nblocks, c.candidate_topk_blocks)) |_| { + var best: usize = 0; + for (0..nblocks) |i| if (self.blocks[i] > self.blocks[best]) { + best = i; + }; + if (self.blocks[best] == -std.math.inf(f32)) break; + for (best * 8..@min(best * 8 + 8, count)) |row| self.candidates[row] = true; + self.blocks[best] = -std.math.inf(f32); + } + } + fn selectPositions(self: *State, a: Attention, layer: usize, x: []const u16, position: usize, count: usize, c: Config, scratch: *projection.Scratch) !void { + const indexer = a.indexer orelse return; + self.selected_len = @min(count, c.index_topk); + if (count == 0) return; + try projection.apply(indexer.wq_b, &self.qr, &self.index_q, scratch); + for (0..32) |head| { + const q = self.index_q[head * 128 ..][0..128]; + m.rope(q, position, true, false, c); + m.quantize(q, .index); + } + m.dense(indexer.weights_proj, x, &self.index_weights); + const scale: f32 = 1 / @sqrt(@as(f32, 128 * 32)); + // Python scalar multiplication preserves the source weights projection's BF16 dtype. + for (&self.index_weights) |*v| v.* = m.f(m.b(m.f(m.b(v.*)) * scale)); + for (0..count) |row| { + const key = self.indexRow(self.active_owner, row); + var score: f64 = 0; + for (0..32) |head| { + var dot: f64 = 0; + for (key, 0..) |v, d| dot += @as(f64, m.f(v)) * m.f(self.index_q[head * 128 + d]); + // einsum produces BF16, then BF16 multiply and head sum, in the pinned source. + const rectified = @max(m.f(m.b(@floatCast(dot))), 0); + const weighted = m.f(m.b(rectified * self.index_weights[head])); + score += weighted; + } + self.scores[row] = m.f(m.b(@floatCast(score))); + } + if (layer == c.candidate_source_layer_id) self.chooseCandidates(count, c); + if (layer > c.candidate_source_layer_id) for (0..count) |row| { + if (!self.candidates[row]) self.scores[row] = -std.math.inf(f32); + }; + for (0..self.selected_len) |pick| { + var best: usize = 0; + for (0..count) |i| if (self.scores[i] > self.scores[best]) { + best = i; + }; + self.selected[pick] = @intCast(best); + self.scores[best] = -std.math.inf(f32); + } + std.mem.sort(u32, self.selected[0..self.selected_len], {}, std.sort.asc(u32)); + } + pub fn apply(self: *State, a: Attention, layer: usize, x: []const u16, position: usize, c: Config, out: []u16, scratch: *projection.Scratch) !void { + const compressed = c.compress_ratios[layer] != 0; + try projection.apply(a.wq_a, x, &self.qr, scratch); + self.observe(.qr_projection, &self.qr); + m.rms(&self.qr, a.q_norm, c.rms_norm_eps, &self.qr); + self.observe(.qr_norm, &self.qr); + try projection.apply(a.wq_b, &self.qr, &self.q, scratch); + self.observe(.q_projection, &self.q); + for (0..64) |head| m.rope(self.q[head * 512 ..][0..512], position, compressed, false, c); + self.observe(.q, &self.q); + try projection.apply(a.wkv, x, &self.kv, scratch); + self.observe(.kv_projection, &self.kv); + m.rms(&self.kv, a.kv_norm, c.rms_norm_eps, &self.kv); + self.observe(.kv_norm, &self.kv); + m.rope(&self.kv, position, compressed, false, c); + self.observe(.kv_rope, &self.kv); + m.quantize(&self.kv, .window); + self.observe(.kv_quantized, &self.kv); + const window = self.window[layer * 128 * 512 ..][0 .. 128 * 512]; + @memcpy(window[(position % 128) * 512 ..][0..512], &self.kv); + var count: usize = 0; + if (compressed) { + _ = try self.publish(a, layer, x, position, c, scratch); + count = (position + 1) / c.compress_ratios[layer]; + try self.selectPositions(a, layer, x, position, count, c, scratch); + } + var indices: [640]i32 = undefined; + var slots: usize = 0; + if (position == 0) { + indices[0] = 0; + slots = 1; + } else { + const oldest = position % 128 + 1; + for (0..128) |i| { + const at = (oldest + i) % 128; + indices[i] = if (at > position) -1 else @intCast(at); + } + slots = 128; + } + if (compressed) for (self.selected[0..self.selected_len]) |row| { + indices[slots] = @intCast(128 + row); + slots += 1; + }; + for (0..64) |head| { + self.sparseHead(self.q[head * 512 ..][0..512], window, indices[0..slots], m.value(a.sink, head), self.o[head * 512 ..][0..512]); + self.observeHead(head); + m.rope(self.o[head * 512 ..][0..512], position, compressed, true, c); + } + if (self.sparse_snapshot) |snapshot| self.observe(.sparse_o, snapshot); + self.observe(.o, &self.o); + for (0..8) |group| try projection.apply(a.wo_a[group], self.o[group * 4096 ..][0..4096], self.grouped[group * 1024 ..][0..1024], scratch); + self.observe(.grouped, &self.grouped); + try projection.apply(a.wo_b, &self.grouped, out, scratch); + self.observe(.wo_b, out); + } + fn observeHead(self: *State, head: usize) void { + if (self.sparse_snapshot) |snapshot| + @memcpy(snapshot[head * 512 ..][0..512], self.o[head * 512 ..][0..512]); + } + fn keyRow(self: *State, window: []const u16, key_id: i32) []const u16 { + const at: usize = @intCast(key_id); + return if (at < 128) window[at * 512 ..][0..512] else self.mainRow(self.active_owner, at - 128); + } + pub fn sparseHead(self: *State, q: []const u16, window: []const u16, indices: []const i32, sink: f32, out: []u16) void { + @setFloatMode(.strict); + var acc: [512]f32 = @splat(0); + var high: f32 = -1e30; + var denominator: f32 = 0; + var base: usize = 0; + while (base < indices.len) : (base += 64) { + var probabilities: [64]f32 = undefined; + const width = @min(64, indices.len - base); + var next_high = high; + for (0..width) |j| { + const index = indices[base + j]; + var dot: f64 = 0; + if (index >= 0) { + const key = self.keyRow(window, index); + for (q, key) |a, v| dot += @as(f64, m.f(a)) * m.f(v); + } + probabilities[j] = if (index < 0) -std.math.inf(f32) else @as(f32, @floatCast(dot)) * (1 / @sqrt(@as(f32, 512))); + next_high = @max(next_high, probabilities[j]); + } + const correction = @exp(high - next_high); + var sum: f64 = 0; + for (0..width) |j| { + probabilities[j] = @exp(probabilities[j] - next_high); + sum += probabilities[j]; + } + denominator = denominator * correction + @as(f32, @floatCast(sum)); + for (0..512) |d| { + var accumulated: f64 = acc[d] * correction; + for (0..width) |j| if (indices[base + j] >= 0) { + // Online 64-column source kernel rounds unnormalized probabilities to BF16 before P*V. + accumulated += @as(f64, m.f(m.b(probabilities[j]))) * m.f(self.keyRow(window, indices[base + j])[d]); + }; + acc[d] = @floatCast(accumulated); + } + high = next_high; + } + denominator += @exp(sink - high); + for (out, acc) |*v, a| v.* = m.b(a / denominator); + } +}; + +test "online 64-column sparse attention rounds probabilities before value GEMM" { + var state = try State.init(std.testing.allocator, 2); + defer state.deinit(); + var q: [512]u16 = @splat(0); + q[0] = m.b(8); + state.window[1] = m.b(1); + state.window[512] = m.b(1); + var indices: [65]i32 = @splat(-1); + indices[0] = 0; + indices[1] = 1; + indices[64] = 0; + var out: [512]u16 = undefined; + state.sparseHead(&q, state.window[0..1024], &indices, 0, &out); + const probability = @exp(-@as(f32, 8) * (1 / @sqrt(@as(f32, 512)))); + var denominator = @as(f32, @floatCast(@as(f64, probability) + 1)); + denominator += probability; + denominator += probability; + const expected = m.b((2 * m.f(m.b(probability))) / denominator); + try std.testing.expectEqual(expected, out[1]); +} + +test "attention reference capacity rejects unbounded history before allocation" { + try std.testing.expectError(error.InvalidAttentionReferenceCapacity, State.init(std.testing.allocator, 0)); + try std.testing.expectError(error.InvalidAttentionReferenceCapacity, State.init(std.testing.allocator, max_capacity + 1)); +} diff --git a/zig/tests/deepseek_v41_reference/engram_math.zig b/zig/tests/deepseek_v41_reference/engram_math.zig new file mode 100644 index 0000000000..d57567957f --- /dev/null +++ b/zig/tests/deepseek_v41_reference/engram_math.zig @@ -0,0 +1,106 @@ +//! CPU Engram from DeepSeek AI inference/engram.py and model.py, rev 2cba9e42aa026125f3ed06c6d98c1db82f7ca027. +const std = @import("std"); +const tf = @import("tensorfold"); +const m = @import("math.zig"); +const projection = @import("projection.zig"); +const Engram = tf.deepseek_v41_weights.Engram; +const Config = tf.deepseek_v41_config.Text; +const multipliers = [2][4]u64{ + .{ 76632096046245, 4839876093313, 35959672319349, 73987337458391 }, + .{ 67716810739261, 51510806800915, 30921347202721, 82619226485591 }, +}; +const primes = [2][24]u32{ + .{ 16000057, 16000079, 16000081, 16000097, 16000121, 16000129, 16000133, 16000183, 16000189, 16000207, 16000211, 16000253, 16000277, 16000289, 16000307, 16000321, 16000339, 16000381, 16000393, 16000399, 16000403, 16000409, 16000447, 16000463 }, + .{ 16000477, 16000487, 16000499, 16000507, 16000511, 16000573, 16000609, 16000627, 16000667, 16000669, 16000693, 16000697, 16000711, 16000729, 16000759, 16000769, 16000781, 16000799, 16000813, 16000819, 16000841, 16000877, 16000879, 16000889 }, +}; +fn word(raw: []const u8, at: usize) u32 { + return std.mem.readInt(u32, raw[at..][0..4], .little); +} +pub const State = struct { + map: []const u8, + history: [3]u32 = .{ 2, 2, 2 }, + gathered: [6144]u16 = undefined, + kv: [25600]u16 = undefined, + + pub fn init(blob: []const u8) !State { + const at = 96 + 2 * 232; + if (blob.len != at + 129280 * 4 or !std.mem.eql(u8, blob[0..8], "DSENGR01")) + return error.InvalidEngramMetadata; + for ([_]u32{ 129280, 99092, 2, 4, 8, 2 }, 0..) |v, i| + if (word(blob, 8 + i * 4) != v) return error.InvalidEngramMetadata; + for (0..2) |layer| { + const base = 96 + layer * 232; + if (word(blob, base) != (if (layer == 0) @as(u32, 1) else @as(u32, 14))) + return error.InvalidEngramMetadata; + var offset: u32 = 0; + for (multipliers[layer], 0..) |v, j| + if (std.mem.readInt(u64, blob[base + 8 + 8 * j ..][0..8], .little) != v) + return error.InvalidEngramMetadata; + for (primes[layer], 0..) |v, j| { + if (word(blob, base + 40 + 4 * j) != v or word(blob, base + 136 + 4 * j) != offset) + return error.InvalidEngramMetadata; + offset += v; + } + if (word(blob, base + 4) != offset) return error.InvalidEngramMetadata; + } + const map = blob[at..]; + const expected = [_]u8{ 0xc6, 0x0a, 0x86, 0x32, 0x2e, 0xc1, 0x7b, 0x41, 0x42, 0xbf, 0xef, 0x3c, 0x57, 0xa8, 0xd8, 0x1f, 0xb4, 0x28, 0x55, 0x0c, 0xdf, 0x48, 0x7f, 0x88, 0xa4, 0x32, 0x0c, 0xb5, 0x9f, 0xe4, 0x64, 0x81 }; + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(map, &digest, .{}); + if (!std.mem.eql(u8, &expected, &digest)) return error.InvalidEngramTokenMap; + return .{ .map = map }; + } + pub fn compressed(self: *const State, id: u32) !u32 { + if (id >= 129280) return error.InvalidToken; + return word(self.map, @as(usize, id) * 4); + } + pub fn advance(self: *State, id: u32) void { + self.history[2] = self.history[1]; + self.history[1] = self.history[0]; + self.history[0] = id; + } + pub fn apply(self: *State, e: Engram, layer: usize, id: u32, h: []u16, c: Config, scratch: *projection.Scratch) !void { + @setFloatMode(.strict); + const tokens = [4]u32{ id, self.history[0], self.history[1], self.history[2] }; + var rolling = @as(u64, tokens[0]) * multipliers[layer][0]; + var offset: usize = 0; + for (1..4) |lookback| { + rolling ^= @as(u64, tokens[lookback]) * multipliers[layer][lookback]; + for (0..8) |head| { + const column = (lookback - 1) * 8 + head; + const row = offset + @as(usize, @intCast(rolling % primes[layer][column])); + for (0..256) |d| self.gathered[column * 256 + d] = + m.b(m.value(e.table, row * 256 + d) * m.value(e.scales, row * 8 + d / 32)); + offset += primes[layer][column]; + } + } + try projection.apply(e.wkv, &self.gathered, &self.kv, scratch); + for (0..4) |copy| { + var hh: f64 = 0; + var kk: f64 = 0; + var dot: f64 = 0; + for (0..5120) |d| { + const at = copy * 5120 + d; + const x = m.f(h[at]); + const k = m.f(self.kv[at]); + const weight = m.value(e.q_weight, at) * m.value(e.k_weight, at); + hh += @as(f64, x) * x; + kk += @as(f64, k) * k; + const weighted: f32 = (x * weight) * k; + dot += weighted; + } + const h_mean = m.meanFromWide(hh, 5120); + const k_mean = m.meanFromWide(kk, 5120); + const h_inverse = m.idealRsqrt(h_mean + c.rms_norm_eps); + const k_inverse = m.idealRsqrt(k_mean + c.rms_norm_eps); + const rstd: f32 = h_inverse * k_inverse; + const dot_f32: f32 = @floatCast(dot); + const normalized: f32 = (dot_f32 * rstd) * (1 / @sqrt(@as(f32, 5120))); + const gate = m.sigmoid(std.math.copysign(@sqrt(@max(@abs(normalized), 1e-6)), normalized)); + for (0..5120) |d| { + const at = copy * 5120 + d; + h[at] = m.b(m.f(h[at]) + gate * m.f(self.kv[20480 + d])); + } + } + } +}; diff --git a/zig/tests/deepseek_v41_reference/math.zig b/zig/tests/deepseek_v41_reference/math.zig new file mode 100644 index 0000000000..a6dccb2b93 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/math.zig @@ -0,0 +1,289 @@ +//! Independent CPU equations: DeepSeek AI inference/{model,kernel}.py, rev 2cba9e42aa026125f3ed06c6d98c1db82f7ca027. +const std = @import("std"); +const tf = @import("tensorfold"); +const Tensor = tf.checkpoint_metal.Tensor; +const Hc = tf.deepseek_v41_weights.Hc; +const Config = tf.deepseek_v41_config.Text; + +pub fn f(x: u16) f32 { + return @bitCast(@as(u32, x) << 16); +} +pub fn b(x: f32) u16 { + const bits: u32 = @bitCast(x); + if ((bits & 0x7fffffff) > 0x7f800000) return @truncate((bits >> 16) | 0x40); + return @truncate((bits +% 0x7fff +% ((bits >> 16) & 1)) >> 16); +} +pub fn bytes(t: Tensor) []const u8 { + return t.buffer.contents()[t.offset..][0..t.bytes]; +} +pub fn value(t: Tensor, index: usize) f32 { + const raw = bytes(t); + return switch (t.dtype) { + .bf16 => f(std.mem.readInt(u16, raw[index * 2 ..][0..2], .little)), + .f16 => @floatCast(@as(f16, @bitCast(std.mem.readInt(u16, raw[index * 2 ..][0..2], .little)))), + .f32 => @bitCast(std.mem.readInt(u32, raw[index * 4 ..][0..4], .little)), + .f8_e4m3 => e4(raw[index]), + .f8_e8m0 => e8(raw[index]), + else => unreachable, + }; +} +pub fn e4(code: u8) f32 { + const magnitude = code & 127; + if (magnitude == 127) return std.math.nan(f32); + const exponent = magnitude >> 3; + const mantissa = magnitude & 7; + const power: f32 = @bitCast((@as(u32, exponent) + 120) << 23); + const v: f32 = if (exponent == 0) @as(f32, @floatFromInt(mantissa)) / 512 else power * (1 + @as(f32, @floatFromInt(mantissa)) / 8); + return if ((code & 128) != 0) -v else v; +} +pub fn e8(code: u8) f32 { + if (code == 255) return std.math.nan(f32); + if (code == 0) return 0x1p-127; + return @bitCast(@as(u32, code) << 23); +} +pub fn roundE4(x: f32) f32 { + const a = @min(@abs(x), 448); + var low: u8 = 0; + var high: u8 = 126; + while (low < high) { + const mid = low + (high - low + 1) / 2; + if (e4(mid) <= a) low = mid else high = mid - 1; + } + var best = low; + if (low < 126) { + const below = a - e4(low); + const above = e4(low + 1) - a; + if (above < below or (above == below and (low + 1) % 2 == 0)) best += 1; + } + return std.math.copysign(e4(best), x); +} +pub fn roundE2(x: f32) f32 { + const levels = [_]f32{ 0, 0.5, 1, 1.5, 2, 3, 4, 6 }; + var best: usize = 0; + var distance: f32 = std.math.inf(f32); + for (levels, 0..) |v, i| { + const d = @abs(@abs(x) - v); + if (d < distance or (d == distance and i % 2 == 0)) { + best = i; + distance = d; + } + } + return std.math.copysign(levels[best], x); +} +fn powerScale(x: f32) f32 { + const bits: u32 = @bitCast(x); + const exponent = (bits >> 23) & 255; + const rounded = exponent + @as(u32, @intFromBool((bits & 0x7fffff) != 0)); + return @bitCast(rounded << 23); +} +pub const Quant = enum { window, main, index }; +pub fn quantize(x: []u16, kind: Quant) void { + const group: usize = if (kind == .main) 16 else 32; + var start: usize = 0; + while (start < x.len) : (start += group) { + var a: f32 = if (kind == .window) 1e-4 else if (kind == .main) 6 * 0x1p-9 else 6 * 0x1p-126; + for (x[start..][0..group]) |v| a = @max(a, @abs(f(v))); + const scale = if (kind == .main) roundE4(a / 6) else powerScale(a * (if (kind == .window) @as(f32, 1.0 / 448.0) else @as(f32, 1.0 / 6.0))); + for (x[start..][0..group]) |*v| { + const z = f(v.*) / scale; + v.* = b((if (kind == .window) roundE4(z) else roundE2(@max(-6, @min(6, z)))) * scale); + } + } +} +pub fn dense(t: Tensor, x: []const u16, out: []f32) void { + for (out, 0..) |*y, row| { + var sum: f64 = 0; + for (x, 0..) |v, k| sum += @as(f64, f(v)) * value(t, row * x.len + k); + y.* = @floatCast(sum); + } +} +/// CUDA MeanOps scalar projection; the internal reduction remains independently wide. +pub fn meanFromWide(sum: f64, count: usize) f32 { + @setFloatMode(.strict); + const accumulator: f32 = @floatCast(sum); + const factor: f32 = 1 / @as(f32, @floatFromInt(count)); + return accumulator * factor; +} +/// Independent ideal FP32 unary result, not an emulation of CUDA's rsqrt approximation. +pub fn idealRsqrt(x: f32) f32 { + @setFloatMode(.strict); + return @floatCast(1 / @sqrt(@as(f64, x))); +} +pub fn rms(x: []const u16, weight: Tensor, eps: f32, out: []u16) void { + @setFloatMode(.strict); + var sum: f64 = 0; + for (x) |v| sum += @as(f64, f(v)) * f(v); + const mean = meanFromWide(sum, x.len); + const r = idealRsqrt(mean + eps); + for (x, out, 0..) |v, *y, i| y.* = b(f(v) * r * value(weight, i)); +} +pub fn sigmoid(x: f32) f32 { + return 1 / (1 + @exp(-x)); +} +pub const Mix = struct { pre: [4]f32, post: [4]f32, comb: [16]f32 }; +pub fn mixes(x: []const u16, hc: Hc, c: Config) Mix { + @setFloatMode(.strict); + var projected: [24]f32 = undefined; + dense(hc.projection, x, &projected); + var squares: f64 = 0; + for (x) |v| squares += @as(f64, f(v)) * f(v); + const mean = meanFromWide(squares, x.len); + const r = idealRsqrt(mean + c.rms_norm_eps); + var m: Mix = undefined; + for (0..4) |i| { + m.pre[i] = sigmoid(@mulAdd(f32, projected[i] * r, value(hc.scale, 0), value(hc.base, i))) + c.hc_eps; + m.post[i] = 2 * sigmoid(@mulAdd(f32, projected[i + 4] * r, value(hc.scale, 1), value(hc.base, i + 4))); + } + for (0..4) |i| { + var high: f32 = -std.math.inf(f32); + for (0..4) |j| { + const at = i * 4 + j; + m.comb[at] = @mulAdd(f32, projected[at + 8] * r, value(hc.scale, 2), value(hc.base, at + 8)); + high = @max(high, m.comb[at]); + } + for (0..4) |j| { + m.comb[i * 4 + j] = @exp(m.comb[i * 4 + j] - high); + } + // TileLang 0.1.8 emits FP32 XOR reductions with offsets 2, then 1. + const denominator = (m.comb[i * 4] + m.comb[i * 4 + 2]) + + (m.comb[i * 4 + 1] + m.comb[i * 4 + 3]); + for (0..4) |j| m.comb[i * 4 + j] = m.comb[i * 4 + j] / denominator + c.hc_eps; + } + for (0..c.hc_sinkhorn_iters) |iteration| { + if (iteration != 0) for (0..4) |i| { + const denominator = ((m.comb[i * 4] + m.comb[i * 4 + 2]) + + (m.comb[i * 4 + 1] + m.comb[i * 4 + 3])) + c.hc_eps; + for (0..4) |j| m.comb[i * 4 + j] /= denominator; + }; + for (0..4) |j| { + const denominator = ((m.comb[j] + m.comb[8 + j]) + + (m.comb[4 + j] + m.comb[12 + j])) + c.hc_eps; + for (0..4) |i| m.comb[i * 4 + j] /= denominator; + } + } + return m; +} +pub fn collapse(h: []const u16, pre: [4]f32, out: []u16) void { + @setFloatMode(.strict); + for (out, 0..) |*v, d| { + var sum: f32 = 0; + for (0..4) |i| { + const product: f32 = pre[i] * f(h[i * out.len + d]); + sum += product; + } + v.* = b(sum); + } +} +pub fn expand(h: []u16, x: []const u16, m: Mix) void { + @setFloatMode(.strict); + for (0..x.len) |d| { + var old: [4]f32 = undefined; + for (0..4) |i| old[i] = f(h[i * x.len + d]); + for (0..4) |i| { + var residual: f32 = 0; + for (0..4) |j| { + const product: f32 = m.comb[j * 4 + i] * old[j]; + residual += product; + } + h[i * x.len + d] = b(m.post[i] * f(x[d]) + residual); + } + } +} +pub fn ropeFrequency(index: usize, theta: f32, original: u32, factor: f32, beta_fast: f32, beta_slow: f32) f32 { + @setFloatMode(.strict); + const exponent: f32 = @as(f32, @floatFromInt(index * 2)) / 64; + const power: f32 = @floatCast(std.math.pow(f64, theta, exponent)); + var frequency: f32 = 1 / power; + if (original > 0) { + const original_f64: f64 = @floatFromInt(original); + const theta_f64: f64 = theta; + const low: f32 = @floatCast(@max(@floor(64 * @log(original_f64 / + (@as(f64, beta_fast) * 2 * std.math.pi)) / (2 * @log(theta_f64))), 0)); + const high: f32 = @floatCast(@min(@ceil(64 * @log(original_f64 / + (@as(f64, beta_slow) * 2 * std.math.pi)) / (2 * @log(theta_f64))), 63)); + const ramp: f32 = @max(0, @min(1, (@as(f32, @floatFromInt(index)) - low) / @max(high - low, 1e-3))); + const smooth: f32 = 1 - ramp; + frequency = frequency / factor * (1 - smooth) + frequency * smooth; + } + return frequency; +} +pub fn rope(x: []u16, position: usize, compressed: bool, inverse: bool, c: Config) void { + for (0..32) |i| { + const frequency = ropeFrequency(i, if (compressed) c.compress_rope_theta else c.rope_theta, if (compressed) c.rope_scaling.original_max_position_embeddings else 0, c.rope_scaling.factor, c.rope_scaling.beta_fast, c.rope_scaling.beta_slow); + const angle: f32 = @as(f32, @floatFromInt(position)) * frequency; + const cosine = @cos(angle); + const sine = @sin(angle) * (if (inverse) @as(f32, -1) else @as(f32, 1)); + const at = x.len - 64 + 2 * i; + const a = f(x[at]); + const z = f(x[at + 1]); + x[at] = b(a * cosine - z * sine); + x[at + 1] = b(a * sine + z * cosine); + } +} + +test "BF16 probability ties and FP4 ties retain source boundaries" { + try std.testing.expectEqual(@as(u16, 0x3f80), b(@bitCast(@as(u32, 0x3f808000)))); + try std.testing.expectEqual(@as(f32, 2), roundE2(2.5)); + try std.testing.expectEqual(@as(f32, 4), roundE2(5)); + try std.testing.expectEqual(@as(f32, 448), roundE4(500)); +} + +test "delayed hyperconnection residual uses comb columns, not rows" { + var h = [_]u16{ b(1), b(2), b(3), b(4) }; + const mix = Mix{ .pre = .{ 1, 0, 0, 0 }, .post = .{ 0, 0, 0, 0 }, .comb = .{ 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0 } }; + expand(&h, &.{b(0)}, mix); + try std.testing.expectEqualSlices(u16, &.{ b(4), b(1), b(2), b(3) }, &h); +} + +test "hyperconnection FP32 accumulation preserves collapse and branch cancellation" { + var h = [_]u16{ b(1), b(1), b(1), b(1) }; + var collapsed: [1]u16 = undefined; + collapse(&h, .{ 1, 0x1p-24, 0x1p-24, -1 }, &collapsed); + try std.testing.expectEqual(@as(u16, 0), collapsed[0]); + const mix = Mix{ .pre = .{ 1, 0, 0, 0 }, .post = .{ -1, 0, 0, 0 }, .comb = .{ 1, 0, 0, 0, 0x1p-24, 0, 0, 0, 0x1p-24, 0, 0, 0, 0, 0, 0, 0 } }; + expand(&h, &.{b(1)}, mix); + try std.testing.expectEqual(@as(u16, 0), h[0]); +} + +test "hyperconnection rounds elementwise products before cancelling reductions" { + var h = [_]u16{ b(3), b(3), b(0), b(0) }; + const coefficient: f32 = 1 + 0x1p-23; + var collapsed: [1]u16 = undefined; + collapse(&h, .{ -1, coefficient, 0, 0 }, &collapsed); + try std.testing.expectEqual(b(0x1p-21), collapsed[0]); + const mix = Mix{ .pre = .{ 1, 0, 0, 0 }, .post = .{ 0, 0, 0, 0 }, .comb = .{ -1, 0, 0, 0, coefficient, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; + expand(&h, &.{b(0)}, mix); + try std.testing.expectEqual(b(0x1p-21), h[0]); + h = .{ b(3), b(0), b(0), b(0) }; + const branch_mix = Mix{ .pre = .{ 1, 0, 0, 0 }, .post = .{ coefficient, 0, 0, 0 }, .comb = .{ -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; + expand(&h, &.{b(3)}, branch_mix); + try std.testing.expectEqual(b(0x1p-21), h[0]); +} + +test "RoPE and YaRN frequency vectors retain official FP32 elementwise boundaries" { + const parsed = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, @embedFile("rope_frequencies.json"), .{}); + defer parsed.deinit(); + for (parsed.value.object.get("vectors").?.array.items) |vector| { + const theta: f32 = @floatFromInt(vector.object.get("theta").?.integer); + const original: u32 = @intCast(vector.object.get("original").?.integer); + for (vector.object.get("bits").?.array.items, 0..) |expected, index| { + const actual = ropeFrequency(index, theta, original, 16, 32, 1); + const bits: u32 = @intCast(expected.integer); + const source: f32 = @bitCast(bits); + try std.testing.expectApproxEqAbs(source, actual, source * std.math.floatEps(f32)); + } + } +} + +test "ideal FP32 inverse square root avoids an intermediate square-root rounding" { + // Actual CUDA agrees on this HC RMS argument; CPU/MPS's two-round result is one ULP higher. + try std.testing.expectEqual(@as(f32, 0.3651483356952667), idealRsqrt(7.500000953674316)); +} + +test "CUDA mean projection preserves the BF16 RMS affine tie" { + @setFloatMode(.strict); + // Constructed exact-sum row isolates canonical CUDA RMSNorm mean projection through the BF16 affine tie. + const gamma: f32 = @bitCast(@as(u32, 0x3c65de29)); + const inverse = idealRsqrt(meanFromWide(9, 5120) + @as(f32, 1e-20)); + try std.testing.expectEqual(b(1), b((@as(f32, 3) * inverse) * gamma)); +} diff --git a/zig/tests/deepseek_v41_reference/operator_attention.zig b/zig/tests/deepseek_v41_reference/operator_attention.zig new file mode 100644 index 0000000000..73e94641c3 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_attention.zig @@ -0,0 +1,155 @@ +//! Single-position retained-input attention isolation. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const projection = @import("projection.zig"); +const math = @import("math.zig"); +const attention = tf.deepseek_v41.attention; +const Model = tf.deepseek_v41.Model; +const diagnostic = @import("operator_diagnostic.zig"); +const diagnosticRow = diagnostic.diagnosticRow; +const attentionMetric = diagnostic.attentionMetric; +const attention_math = @import("attention_math.zig"); +const AttentionStage = diagnostic.AttentionStage; +const AttentionCapture = diagnostic.AttentionCapture; +const attention_elements = diagnostic.attention_elements; +const attentionProjection = diagnostic.attentionProjection; +const hyperconnections = @import("operator_hc.zig").hyperconnections; + +/// Local isolation receipt, not a replacement for the full eight-token oracle or its 1% gate. +pub fn runAttention(gpa: std.mem.Allocator, io: std.Io, model: *Model, snapshot: []const u8, dir: []const u8, layer: usize) !void { + const blob = try std.Io.Dir.cwd().readFileAlloc(io, snapshot, gpa, .limited(8 << 20)); + defer gpa.free(blob); + const cfg = model.cfg.text_config; + if (layer >= model.weights.layers.len) return error.InvalidAttentionLayer; + const snapshot_offset = layer * 2 * 61488; + if (blob.len < snapshot_offset + 40960 + 10240) return error.InvalidAttentionSnapshot; + try model.reset(); + const input = model.workspace.view(.normalized); + @memcpy(input.buffer.contents()[input.offset..][0..10240], blob[snapshot_offset + 40960 ..][0..10240]); + const input_row = @as([*]const u16, @ptrCast(@alignCast(input.buffer.contents() + input.offset)))[0..5120]; + const native = try model.device.buffer(attention_elements * 2, mtl.ResourceOptions.shared); + defer native.deinit(); + const reference_rows = try gpa.alloc(u16, attention_elements); + defer gpa.free(reference_rows); + var capture = AttentionCapture{ .model = model, .native = native, .reference = reference_rows }; + model.residency.add(native); + model.residency.commit(); + model.residency.requestResidency(); + defer { + model.residency.remove(native); + model.residency.commit(); + } + var scratch = try projection.Scratch.init(gpa, 8192, 32768); + defer scratch.deinit(); + var state = try attention_math.State.init(gpa, 1); + defer state.deinit(); + var output: [5120]u16 = undefined; + // Replay prerequisite normalized layers; complete each GPU command before reusing the CPU input buffer. + for (0..layer) |prior| { + const prior_offset = prior * 2 * 61488 + 40960; + @memcpy(input.buffer.contents()[input.offset..][0..10240], blob[prior_offset..][0..10240]); + const warm_cb = model.queue.commandBufferUnretained(); + const warm_enc = warm_cb.compute(.serial); + var warm_encoding = true; + errdefer if (warm_encoding) warm_enc.end(); + for (model.checkpoint.shards.items) |shard| warm_enc.useResource(shard.buffer, mtl.ResourceUsage.read); + try attention.encode(&model.pipelines.attention, warm_enc, &model.weights.layers[prior].attn, &model.cache, &model.attention_workspace, input, model.workspace.view(.branch), &model.pipelines.projection, model.workspace.projection(), @intCast(prior), 1, cfg); + warm_enc.end(); + warm_encoding = false; + warm_cb.commit(); + warm_cb.wait(); + if (warm_cb.failure() != null) return error.OperatorGpuFailure; + try state.apply(model.weights.layers[prior].attn, prior, input_row, 0, cfg, &output, &scratch); + } + @memcpy(input.buffer.contents()[input.offset..][0..10240], blob[snapshot_offset + 40960 ..][0..10240]); + const weights = &model.weights.layers[layer].attn; + const cb = model.queue.commandBufferUnretained(); + const enc = cb.compute(.serial); + var encoding = true; + errdefer if (encoding) enc.end(); + for (model.checkpoint.shards.items) |shard| enc.useResource(shard.buffer, mtl.ResourceUsage.read); + try attention.encodeDiagnostic(&model.pipelines.attention, enc, weights, &model.cache, &model.attention_workspace, input, model.workspace.view(.branch), &model.pipelines.projection, model.workspace.projection(), @intCast(layer), 1, cfg, true, .{ .ctx = &capture, .capture = AttentionCapture.gpu }); + enc.end(); + encoding = false; + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.OperatorGpuFailure; + const sparse_snapshot = try gpa.alloc(u16, 32768); + defer gpa.free(sparse_snapshot); + state.sparse_snapshot = sparse_snapshot; + state.observer = .{ .ctx = &capture, .capture = AttentionCapture.cpu }; + try state.apply(weights.*, layer, input_row, 0, cfg, &output, &scratch); + state.observer = null; + state.sparse_snapshot = null; + + var report: std.ArrayList(u8) = .empty; + defer report.deinit(gpa); + var count: usize = 0; + var header: [512]u8 = undefined; + try report.appendSlice(gpa, try std.fmt.bufPrint(&header, "{{\"mode\":\"attention-sameinput\",\"token\":0,\"layer\":{d},\"input_offset_bytes\":{d},\"input_length_bytes\":10240,\"identical_normalized_input\":true,\"retained_normalized_prefix_layers\":{d},\"compression_ratio\":{d},\"reference_dot\":\"independent_EXL3_FP64\",\"qualification_gate_applied\":false,\"cases\":[\n", .{ layer, snapshot_offset + 40960, layer, cfg.compress_ratios[layer] })); + inline for (std.enums.values(AttentionStage)) |stage| + try attentionMetric(u16, gpa, &report, &count, @tagName(stage), "same_normalized_input_chain", capture.row(stage), capture.cpuRow(stage)); + + const expected = try gpa.alloc(f32, 32768); + defer gpa.free(expected); + try attentionProjection(gpa, model, weights.wq_a, input, "wq_a", &scratch, expected, &report, &count); + try attentionProjection(gpa, model, weights.wq_b, capture.view(.qr_norm), "wq_b", &scratch, expected, &report, &count); + try attentionProjection(gpa, model, weights.wkv, input, "wkv", &scratch, expected, &report, &count); + for (weights.wo_a, 0..) |matrix, group| { + var name: [32]u8 = undefined; + try attentionProjection(gpa, model, matrix, capture.view(.o).shifted(group * 4096 * 2), try std.fmt.bufPrint(&name, "wo_a_group{d}", .{group}), &scratch, expected, &report, &count); + } + try attentionProjection(gpa, model, weights.wo_b, capture.view(.grouped), "wo_b", &scratch, expected, &report, &count); + + var same_input: [32768]u16 = undefined; + math.rms(capture.row(.qr_projection), weights.q_norm, cfg.rms_norm_eps, same_input[0..1280]); + try attentionMetric(u16, gpa, &report, &count, "qr_norm", "same_native_input_reference_rms", capture.row(.qr_norm), same_input[0..1280]); + math.rms(capture.row(.kv_projection), weights.kv_norm, cfg.rms_norm_eps, same_input[0..512]); + try attentionMetric(u16, gpa, &report, &count, "kv_norm", "same_native_input_reference_rms", capture.row(.kv_norm), same_input[0..512]); + @memcpy(same_input[0..512], capture.row(.kv_rope)); + math.quantize(same_input[0..512], .window); + try attentionMetric(u16, gpa, &report, &count, "kv_quantized", "same_native_input_reference_quantize", capture.row(.kv_quantized), same_input[0..512]); + if (cfg.compress_ratios[layer] == 1) { + const source = try model.cache.source(@intCast(layer), 1); + const compressed = model.workspace.view(.collapsed); + const unpack_cb = model.queue.commandBufferUnretained(); + const unpack_enc = unpack_cb.compute(.serial); + var unpack_encoding = true; + errdefer if (unpack_encoding) unpack_enc.end(); + try model.pipelines.attention.cache.encodeUnpack(unpack_enc, source.main, compressed, .{ .rows = 1, .width = 512, .stride = 512, .packed_stride = 288, .block = 16, .kind = .main_fp4 }); + unpack_enc.end(); + unpack_encoding = false; + unpack_cb.commit(); + unpack_cb.wait(); + if (unpack_cb.failure() != null) return error.OperatorGpuFailure; + const actual_compressed = diagnosticRow(u16, compressed, 512); + const reference_compressed = state.sources[state.active_owner * state.capacity * 640 ..][0..512]; + try attentionMetric(u16, gpa, &report, &count, "compressed_kv", "same_retained_normalized_prefix", actual_compressed, reference_compressed); + var sinks: [64]f32 = undefined; + for (&sinks, 0..) |*v, head| v.* = math.value(weights.sink, head); + inline for (.{ "attention-compressed-native.bf16", "attention-compressed-reference.bf16", "attention-sinks.f32" }, .{ std.mem.sliceAsBytes(actual_compressed), std.mem.sliceAsBytes(reference_compressed), std.mem.sliceAsBytes(&sinks) }) |name, data| { + const path = try std.fs.path.join(gpa, &.{ dir, name }); + defer gpa.free(path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = data }); + } + @memcpy(reference_compressed, actual_compressed); + const indices = [_]i32{ 0, 128 }; + for (0..64) |head| + state.sparseHead(capture.row(.q)[head * 512 ..][0..512], capture.row(.kv_quantized), &indices, sinks[head], same_input[head * 512 ..][0..512]); + try attentionMetric(u16, gpa, &report, &count, "sparse_o", "same_native_q_window_and_compressed_kv_reference_sparse", capture.row(.sparse_o), &same_input); + } + if (cfg.compress_ratios[layer] == 0) { + const indices = [_]i32{0}; + for (0..64) |head| + state.sparseHead(capture.row(.q)[head * 512 ..][0..512], capture.row(.kv_quantized), &indices, math.value(weights.sink, head), same_input[head * 512 ..][0..512]); + try attentionMetric(u16, gpa, &report, &count, "sparse_o", "same_native_q_and_quantized_kv_reference_sparse", capture.row(.sparse_o), &same_input); + } + try hyperconnections(gpa, model, blob[snapshot_offset..], capture.view(.wo_b), &report, &count, layer); + try report.appendSlice(gpa, "\n]}\n"); + inline for (.{ "attention-sameinput.json", "attention-native.bf16", "attention-reference.bf16", "attention-input.bf16" }, .{ report.items, std.mem.sliceAsBytes(native.slice(u16, attention_elements)), std.mem.sliceAsBytes(reference_rows), std.mem.sliceAsBytes(input_row) }) |name, data| { + const path = try std.fs.path.join(gpa, &.{ dir, name }); + defer gpa.free(path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = data }); + } +} diff --git a/zig/tests/deepseek_v41_reference/operator_diagnostic.zig b/zig/tests/deepseek_v41_reference/operator_diagnostic.zig new file mode 100644 index 0000000000..c37682bf8e --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_diagnostic.zig @@ -0,0 +1,124 @@ +//! Shared retained-input diagnostic capture and metrics. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const projection = @import("projection.zig"); +const math = @import("math.zig"); +const attention = tf.deepseek_v41.attention; +const Model = tf.deepseek_v41.Model; +const ex = tf.exl3_projection; + +pub const AttentionStage = attention.DiagnosticStage; +pub const attention_widths = [_]usize{ 1280, 1280, 32768, 32768, 512, 512, 512, 512, 32768, 32768, 8192, 5120 }; +pub const attention_elements = blk: { + var total: usize = 0; + for (attention_widths) |width| total += width; + break :blk total; +}; +pub fn attentionOffset(stage: AttentionStage) usize { + var offset: usize = 0; + for (attention_widths[0..@backingInt(stage)]) |width| offset += width; + return offset; +} +pub const AttentionCapture = struct { + model: *Model, + native: mtl.Buffer, + reference: []u16, + + pub fn view(self: *const AttentionCapture, stage: AttentionStage) attention.View { + return .{ .buffer = self.native, .offset = attentionOffset(stage) * 2 }; + } + pub fn row(self: *const AttentionCapture, stage: AttentionStage) []u16 { + return self.native.slice(u16, attention_elements)[attentionOffset(stage)..][0..attention_widths[@backingInt(stage)]]; + } + pub fn cpuRow(self: *const AttentionCapture, stage: AttentionStage) []u16 { + return self.reference[attentionOffset(stage)..][0..attention_widths[@backingInt(stage)]]; + } + pub fn gpu(ctx: *anyopaque, enc: mtl.ComputeEncoder, stage: AttentionStage, source: attention.View) !void { + const self: *AttentionCapture = @ptrCast(@alignCast(ctx)); + const cache = &self.model.pipelines.attention.cache; + if (stage == .kv_quantized) { + try cache.encodeUnpack(enc, source, self.view(stage), .{ .rows = 1, .width = 512, .stride = 512, .packed_stride = 528, .block = 32, .kind = .window_fp8 }); + } else { + cache.encodeCopy(enc, source, self.view(stage), @intCast(attention_widths[@backingInt(stage)] * 2)); + } + } + pub fn cpu(ctx: *anyopaque, stage: AttentionStage, values: []const u16) void { + const self: *AttentionCapture = @ptrCast(@alignCast(ctx)); + @memcpy(self.cpuRow(stage), values); + } +}; + +pub fn attentionMetric(comptime T: type, gpa: std.mem.Allocator, report: *std.ArrayList(u8), count: *usize, name: []const u8, comparison: []const u8, actual: []const T, expected: []const T) !void { + if (actual.len == 0 or actual.len != expected.len) return error.InvalidAttentionMetricShape; + var squared: f64 = 0; + var magnitude: f64 = 0; + var max_abs: f64 = 0; + var worst: usize = 0; + var unequal: usize = 0; + var first: ?usize = null; + for (actual, expected, 0..) |a, r, i| { + const x = if (T == u16) math.f(a) else a; + const y = if (T == u16) math.f(r) else r; + if (!std.math.isFinite(x) or !std.math.isFinite(y)) return error.NonfiniteAttentionDiagnostic; + const delta = @as(f64, x) - y; + squared += delta * delta; + magnitude += @as(f64, y) * y; + if (@abs(delta) > max_abs) { + max_abs = @abs(delta); + worst = i; + } + const ab = if (T == u16) a else math.b(a); + const rb = if (T == u16) r else math.b(r); + if (ab != rb) { + unequal += 1; + if (first == null) first = i; + } + } + const line = try std.json.Stringify.valueAlloc(gpa, .{ + .name = name, + .comparison = comparison, + .elements = actual.len, + .output_dtype = if (T == u16) "bf16" else "f32", + .normalized_rms_error = if (magnitude == 0) @sqrt(squared) else @sqrt(squared / magnitude), + .max_absolute_error = max_abs, + .worst_index = worst, + .bf16_unequal_count = unequal, + .bf16_first_unequal_index = first, + .worst_native = if (T == u16) math.f(actual[worst]) else actual[worst], + .worst_reference = if (T == u16) math.f(expected[worst]) else expected[worst], + }, .{}); + defer gpa.free(line); + if (count.* != 0) try report.appendSlice(gpa, ",\n"); + try report.appendSlice(gpa, line); + count.* += 1; + std.debug.print("{s}\n", .{line}); +} + +pub fn attentionProjection(gpa: std.mem.Allocator, model: *Model, matrix: ex.Matrix, input: attention.View, name: []const u8, scratch: *projection.Scratch, expected: []f32, report: *std.ArrayList(u8), count: *usize) !void { + const input_row = @as([*]const u16, @ptrCast(@alignCast(input.buffer.contents() + input.offset)))[0..matrix.k]; + try projection.applyF32(matrix, input_row, expected[0..matrix.n], scratch); + const output = model.workspace.view(.logits); + const cb = model.queue.commandBufferUnretained(); + const enc = cb.compute(.serial); + var encoding = true; + errdefer if (encoding) enc.end(); + ex.useMatrix(enc, matrix); + try ex.encode(&model.pipelines.projection, enc, matrix, .{ .buffer = input.buffer, .offset = input.offset }, .{ .buffer = output.buffer, .offset = output.offset }, model.workspace.projection(), .{ .count = 1, .x_stride = matrix.k, .y_stride = matrix.n, .output = .f32 }); + enc.end(); + encoding = false; + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.OperatorGpuFailure; + const actual = @as([*]const f32, @ptrCast(@alignCast(output.buffer.contents() + output.offset)))[0..matrix.n]; + try attentionMetric(f32, gpa, report, count, name, "same_native_input_fp64_dot", actual, expected[0..matrix.n]); +} + +pub fn diagnosticRow(comptime T: type, view: attention.View, len: usize) []T { + return @as([*]T, @ptrCast(@alignCast(view.buffer.contents() + view.offset)))[0..len]; +} +pub fn attentionWrite(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, name: []const u8, data: []const u8) !void { + const path = try std.fs.path.join(gpa, &.{ dir, name }); + defer gpa.free(path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = data }); +} diff --git a/zig/tests/deepseek_v41_reference/operator_hc.zig b/zig/tests/deepseek_v41_reference/operator_hc.zig new file mode 100644 index 0000000000..4aec23362a --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_hc.zig @@ -0,0 +1,98 @@ +//! Retained-input hyperconnection isolation, separate from qualification. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const math = @import("math.zig"); +const attention = tf.deepseek_v41.attention; +const Model = tf.deepseek_v41.Model; +const diagnostic = @import("operator_diagnostic.zig"); +const diagnosticRow = diagnostic.diagnosticRow; +const attentionMetric = diagnostic.attentionMetric; + +pub fn hyperconnections(gpa: std.mem.Allocator, model: *Model, stream_bytes: []const u8, branch: attention.View, report: *std.ArrayList(u8), count: *usize, layer: usize) !void { + const ws = &model.workspace; + const hc = model.weights.layers[layer].hc_ffn; + const cfg = model.cfg.text_config; + const stream = ws.view(.stream); + @memcpy(stream.buffer.contents()[stream.offset..][0..40960], stream_bytes[0..40960]); + var input: [20480]u16 = undefined; + @memcpy(&input, diagnosticRow(u16, stream, input.len)); + var projected: [24]f32 = undefined; + math.dense(hc.projection, &input, &projected); + const expected_mix = math.mixes(&input, hc, cfg); + var squares: f64 = 0; + for (input) |v| squares += @as(f64, math.f(v)) * math.f(v); + const mean = math.meanFromWide(squares, input.len); + const expected_inv = [_]f32{math.idealRsqrt(mean + cfg.rms_norm_eps)}; + const cb = model.queue.commandBufferUnretained(); + const enc = cb.compute(.serial); + for (model.checkpoint.shards.items) |shard| enc.useResource(shard.buffer, mtl.ResourceUsage.read); + tf.deepseek_v41.mhc.encodeRms(enc, &model.pipelines.mhc, stream, ws.view(.inv), .{ .rows = 1, .width = 5120, .stream_stride = 20480, .eps = cfg.rms_norm_eps }); + tf.deepseek_v41.ops.encodeDense(enc, &model.pipelines.ops, stream, .{ .buffer = hc.projection.buffer, .offset = hc.projection.offset }, ws.view(.mixes), .{ .rows = 1, .k = 20480, .n = 24, .x_stride = 20480, .w_stride = 20480, .y_stride = 24, .w_dtype = .f32, .y_dtype = .f32 }); + enc.barrier(); + tf.deepseek_v41.mhc.encodeSplit(enc, &model.pipelines.mhc, ws.view(.mixes), ws.view(.inv), .{ .buffer = hc.scale.buffer, .offset = hc.scale.offset }, .{ .buffer = hc.base.buffer, .offset = hc.base.offset }, ws.view(.attention_pre), ws.view(.post), ws.view(.comb), .{ .rows = 1, .eps = cfg.hc_eps }); + enc.barrier(); + tf.deepseek_v41.mhc.encodeCollapse(enc, &model.pipelines.mhc, stream, ws.view(.attention_pre), ws.view(.collapsed), .{ .rows = 1, .width = 5120, .stream_stride = 20480, .y_stride = 5120 }); + enc.barrier(); + tf.deepseek_v41.mhc.encodePost(enc, &model.pipelines.mhc, branch, stream, ws.view(.post), ws.view(.comb), stream, .{ .rows = 1, .width = 5120, .branch_stride = 5120, .residual_stride = 20480, .y_stride = 20480 }); + enc.end(); + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.OperatorGpuFailure; + try attentionMetric(f32, gpa, report, count, "hc_projection", "same_retained_stream_fp64_dot", diagnosticRow(f32, ws.view(.mixes), 24), &projected); + try attentionMetric(f32, gpa, report, count, "hc_inverse", "same_retained_stream_reference_statistic", diagnosticRow(f32, ws.view(.inv), 1), &expected_inv); + try attentionMetric(f32, gpa, report, count, "hc_pre", "same_retained_stream_chain", diagnosticRow(f32, ws.view(.attention_pre), 4), &expected_mix.pre); + try attentionMetric(f32, gpa, report, count, "hc_post", "same_retained_stream_chain", diagnosticRow(f32, ws.view(.post), 4), &expected_mix.post); + try attentionMetric(f32, gpa, report, count, "hc_comb", "same_retained_stream_chain", diagnosticRow(f32, ws.view(.comb), 16), &expected_mix.comb); + var native_mix: math.Mix = undefined; + @memcpy(&native_mix.pre, diagnosticRow(f32, ws.view(.attention_pre), 4)); + @memcpy(&native_mix.post, diagnosticRow(f32, ws.view(.post), 4)); + @memcpy(&native_mix.comb, diagnosticRow(f32, ws.view(.comb), 16)); + var collapsed: [5120]u16 = undefined; + math.collapse(&input, native_mix.pre, &collapsed); + try attentionMetric(u16, gpa, report, count, "hc_collapse", "same_retained_stream_and_native_coefficients", diagnosticRow(u16, ws.view(.collapsed), 5120), &collapsed); + math.expand(&input, diagnosticRow(u16, branch, 5120), native_mix); + try attentionMetric(u16, gpa, report, count, "hc_expand", "same_retained_stream_branch_and_native_coefficients", diagnosticRow(u16, stream, 20480), &input); + if (layer + 1 < model.weights.layers.len and stream_bytes.len >= 61488 + 40960) { + @memcpy(stream.buffer.contents()[stream.offset..][0..40960], stream_bytes[61488..][0..40960]); + @memcpy(&input, diagnosticRow(u16, stream, input.len)); + const norm = model.weights.layers[layer + 1].attn_norm; + const next_cb = model.queue.commandBufferUnretained(); + const next_enc = next_cb.compute(.serial); + next_enc.useResource(norm.buffer, mtl.ResourceUsage.read); + tf.deepseek_v41.mhc.encodeCollapse(next_enc, &model.pipelines.mhc, stream, ws.view(.attention_pre), ws.view(.collapsed), .{ .rows = 1, .width = 5120, .stream_stride = 20480, .y_stride = 5120 }); + next_enc.barrier(); + tf.deepseek_v41.ops.encodeRms(next_enc, &model.pipelines.ops, ws.view(.collapsed), .{ .buffer = norm.buffer, .offset = norm.offset }, ws.view(.branch), .{ .rows = 1, .width = 5120, .x_stride = 5120, .y_stride = 5120, .eps = cfg.rms_norm_eps }); + next_enc.end(); + next_cb.commit(); + next_cb.wait(); + if (next_cb.failure() != null) return error.OperatorGpuFailure; + math.collapse(&input, native_mix.pre, &collapsed); + try attentionMetric(u16, gpa, report, count, "next_collapse", "same_retained_ffn_stream_and_native_pre", diagnosticRow(u16, ws.view(.collapsed), 5120), &collapsed); + math.collapse(&input, expected_mix.pre, &collapsed); + try attentionMetric(u16, gpa, report, count, "next_collapse", "same_retained_ffn_stream_reference_pre", diagnosticRow(u16, ws.view(.collapsed), 5120), &collapsed); + math.rms(&collapsed, norm, cfg.rms_norm_eps, &collapsed); + try attentionMetric(u16, gpa, report, count, "next_normalized", "same_retained_ffn_stream_reference_pre_and_norm", diagnosticRow(u16, ws.view(.branch), 5120), &collapsed); + } +} + +pub fn runHyperconnections(gpa: std.mem.Allocator, io: std.Io, model: *Model, snapshot: []const u8, dir: []const u8, layer: usize) !void { + if (layer >= model.weights.layers.len) return error.InvalidHyperconnectionLayer; + const blob = try std.Io.Dir.cwd().readFileAlloc(io, snapshot, gpa, .limited(8 << 20)); + defer gpa.free(blob); + const offset = layer * 2 * 61488; + if (blob.len < offset + 2 * 61488) return error.InvalidHyperconnectionSnapshot; + try model.reset(); + const branch = model.workspace.view(.branch); + @memcpy(branch.buffer.contents()[branch.offset..][0..10240], blob[offset + 61488 + 51200 ..][0..10240]); + var report: std.ArrayList(u8) = .empty; + defer report.deinit(gpa); + var count: usize = 0; + var header: [256]u8 = undefined; + try report.appendSlice(gpa, try std.fmt.bufPrint(&header, "{{\"mode\":\"hc-sameinput\",\"token\":null,\"token_identity_source\":\"unknown_headerless_snapshot\",\"layer\":{d},\"qualification_gate_applied\":false,\"cases\":[\n", .{layer})); + try hyperconnections(gpa, model, blob[offset..], branch, &report, &count, layer); + try report.appendSlice(gpa, "\n]}\n"); + const path = try std.fs.path.join(gpa, &.{ dir, "hc-sameinput.json" }); + defer gpa.free(path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = report.items }); +} diff --git a/zig/tests/deepseek_v41_reference/operator_history.zig b/zig/tests/deepseek_v41_reference/operator_history.zig new file mode 100644 index 0000000000..499a1cd463 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_history.zig @@ -0,0 +1,298 @@ +//! Retained-input attention history and source-layout receipts. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const projection = @import("projection.zig"); +const math = @import("math.zig"); +const attention = tf.deepseek_v41.attention; +const Model = tf.deepseek_v41.Model; +const diagnostic = @import("operator_diagnostic.zig"); +const diagnosticRow = diagnostic.diagnosticRow; +const attentionMetric = diagnostic.attentionMetric; +const attention_math = @import("attention_math.zig"); +const AttentionStage = diagnostic.AttentionStage; +const AttentionCapture = diagnostic.AttentionCapture; +const attention_widths = diagnostic.attention_widths; +const attention_elements = diagnostic.attention_elements; +const attentionOffset = diagnostic.attentionOffset; +const attentionProjection = diagnostic.attentionProjection; +const attentionWrite = diagnostic.attentionWrite; + +// Source indices address physical ring slots then compressed rows; reference reserves 128 window rows even at zero. +fn attentionHistoryIndices(position: usize, selected: []const i32, compressed_offset: usize, out: *[640]i32) []const i32 { + const window_columns: usize = if (position == 0) 1 else 128; + if (position == 0) { + out[0] = 0; + } else { + for (0..128) |slot| { + const at = (position % 128 + 1 + slot) % 128; + out[slot] = if (at > position) -1 else @intCast(at); + } + } + for (selected, 0..) |row, slot| + out[window_columns + slot] = if (row < 0) -1 else @intCast(compressed_offset + @as(usize, @intCast(row))); + return out[0 .. window_columns + selected.len]; +} + +/// Replay prerequisite layers at each position; native/reference cursors advance without borrowed intermediates. +pub fn runAttentionHistory(gpa: std.mem.Allocator, io: std.Io, model: *Model, trace_dir: []const u8, dir: []const u8, token: usize, layer: usize) !void { + if (layer >= model.weights.layers.len) return error.InvalidAttentionLayer; + if (token >= model.cache.plan.capacity) return error.InvalidAttentionHistory; + if (token >= attention_math.max_capacity) return error.DiagnosticHistoryTooLarge; + const prefix_elements = try std.math.mul(usize, token + 1, (layer + 1) * 5120); + if (prefix_elements > (256 << 20) / @sizeOf(u16)) return error.DiagnosticHistoryTooLarge; + const cfg = model.cfg.text_config; + const history_count = token + 1; + const ratio = cfg.compress_ratios[layer]; + const compressed_rows = if (ratio == 0) 0 else history_count / ratio; + const topk = @min(compressed_rows, cfg.index_topk); + const window_rows: usize = if (token == 0) 1 else 128; + const snapshot_bytes = 61488; + const snapshot_offset = layer * 2 * snapshot_bytes; + const weights = &model.weights.layers[layer].attn; + const input = model.workspace.view(.normalized); + const native = try model.device.buffer(attention_elements * 2, mtl.ResourceOptions.shared); + defer native.deinit(); + const decoded = try model.device.buffer((128 * 512 + compressed_rows * 640) * 2, mtl.ResourceOptions.shared); + defer decoded.deinit(); + const reference_rows = try gpa.alloc(u16, attention_elements); + defer gpa.free(reference_rows); + const sparse_snapshot = try gpa.alloc(u16, 32768); + defer gpa.free(sparse_snapshot); + const normalized_prefix = try gpa.alloc(u16, prefix_elements); + defer gpa.free(normalized_prefix); + var retained_branch: [5120]u16 = undefined; + var capture = AttentionCapture{ .model = model, .native = native, .reference = reference_rows }; + for ([_]mtl.Buffer{ native, decoded }) |buffer| model.residency.add(buffer); + model.residency.commit(); + model.residency.requestResidency(); + defer { + for ([_]mtl.Buffer{ native, decoded }) |buffer| model.residency.remove(buffer); + model.residency.commit(); + } + var scratch = try projection.Scratch.init(gpa, 8192, 32768); + defer scratch.deinit(); + var state = try attention_math.State.init(gpa, history_count); + defer state.deinit(); + var output: [5120]u16 = undefined; + try model.reset(); + for (0..history_count) |position| { + var name: [64]u8 = undefined; + const trace_name = try std.fmt.bufPrint(&name, "trace-{d}-gpu.bin", .{position}); + const path = try std.fs.path.join(gpa, &.{ trace_dir, trace_name }); + defer gpa.free(path); + const blob = try std.Io.Dir.cwd().readFileAlloc(io, path, gpa, .limited(8 << 20)); + defer gpa.free(blob); + if (blob.len != model.weights.layers.len * 2 * snapshot_bytes) return error.InvalidAttentionSnapshot; + if (model.cache.position != position) return error.InvalidAttentionHistoryCursor; + for (0..layer + 1) |prior| { + const offset = prior * 2 * snapshot_bytes + 40960; + const row = normalized_prefix[(position * (layer + 1) + prior) * 5120 ..][0..5120]; + @memcpy(std.mem.sliceAsBytes(row), blob[offset..][0..10240]); + @memcpy(input.buffer.contents()[input.offset..][0..10240], std.mem.sliceAsBytes(row)); + const target = position == token and prior == layer; + const cb = model.queue.commandBufferUnretained(); + const enc = cb.compute(.serial); + var encoding = true; + errdefer if (encoding) enc.end(); + for (model.checkpoint.shards.items) |shard| enc.useResource(shard.buffer, mtl.ResourceUsage.read); + if (target) { + try attention.encodeDiagnostic(&model.pipelines.attention, enc, &model.weights.layers[prior].attn, &model.cache, &model.attention_workspace, input, model.workspace.view(.branch), &model.pipelines.projection, model.workspace.projection(), @intCast(prior), 1, cfg, true, .{ .ctx = &capture, .capture = AttentionCapture.gpu }); + @memcpy(std.mem.sliceAsBytes(&retained_branch), blob[snapshot_offset + 51200 ..][0..10240]); + } else { + try attention.encode(&model.pipelines.attention, enc, &model.weights.layers[prior].attn, &model.cache, &model.attention_workspace, input, model.workspace.view(.branch), &model.pipelines.projection, model.workspace.projection(), @intCast(prior), 1, cfg); + } + enc.end(); + encoding = false; + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.OperatorGpuFailure; + // Wait before the next retained row replaces the native workspace input. + if (target) { + state.observer = .{ .ctx = &capture, .capture = AttentionCapture.cpu }; + state.sparse_snapshot = sparse_snapshot; + } + try state.apply(model.weights.layers[prior].attn, prior, row, position, cfg, &output, &scratch); + state.observer = null; + state.sparse_snapshot = null; + } + try model.cache.append(1); + } + + // Export post-insertion physical ring slots, masking never-written slots in source indices. + const decoded_view = attention.View{ .buffer = decoded }; + const native_window = decoded.slice(u16, 128 * 512 + compressed_rows * 640)[0 .. 128 * 512]; + const native_compressed = decoded.slice(u16, 128 * 512 + compressed_rows * 640)[128 * 512 ..][0 .. compressed_rows * 512]; + const native_index_keys = decoded.slice(u16, 128 * 512 + compressed_rows * 640)[128 * 512 + compressed_rows * 512 ..][0 .. compressed_rows * 128]; + const unpack_cb = model.queue.commandBufferUnretained(); + const unpack_enc = unpack_cb.compute(.serial); + var unpack_encoding = true; + errdefer if (unpack_encoding) unpack_enc.end(); + const ring = try model.cache.ring(@intCast(layer)); + try model.pipelines.attention.cache.encodeUnpack(unpack_enc, ring, decoded_view, .{ .rows = 128, .width = 512, .stride = 512, .packed_stride = 528, .block = 32, .kind = .window_fp8 }); + if (compressed_rows != 0) { + const source = try model.cache.source(@intCast(layer), ratio); + try model.pipelines.attention.cache.encodeUnpack(unpack_enc, source.main, decoded_view.shifted(128 * 512 * 2), .{ .rows = @intCast(compressed_rows), .width = 512, .stride = 512, .packed_stride = 288, .block = 16, .kind = .main_fp4 }); + try model.pipelines.attention.cache.encodeUnpack(unpack_enc, source.index, decoded_view.shifted((128 * 512 + compressed_rows * 512) * 2), .{ .rows = @intCast(compressed_rows), .width = 128, .stride = 128, .packed_stride = 68, .block = 32, .kind = .index_fp4 }); + } + unpack_enc.end(); + unpack_encoding = false; + unpack_cb.commit(); + unpack_cb.wait(); + if (unpack_cb.failure() != null) return error.OperatorGpuFailure; + const reference_window = state.window[layer * 128 * 512 ..][0 .. 128 * 512]; + const reference_compressed = try gpa.alloc(u16, compressed_rows * 640); + defer gpa.free(reference_compressed); + const reference_main = reference_compressed[0 .. compressed_rows * 512]; + const reference_index_keys = reference_compressed[compressed_rows * 512 ..]; + for (0..compressed_rows) |row| { + const source_row = state.sources[state.active_owner * state.capacity * 640 + row * 640 ..][0..640]; + @memcpy(reference_main[row * 512 ..][0..512], source_row[0..512]); + @memcpy(reference_index_keys[row * 128 ..][0..128], source_row[512..640]); + } + const native_selected = diagnosticRow(i32, model.attention_workspace.selection.positions(), 512)[0..topk]; + var reference_selected: [512]i32 = undefined; + for (state.selected[0..topk], 0..) |row, i| reference_selected[i] = @intCast(row); + var native_indices_storage: [640]i32 = undefined; + var reference_indices_storage: [640]i32 = undefined; + const native_indices = attentionHistoryIndices(token, native_selected, window_rows, &native_indices_storage); + const reference_indices = attentionHistoryIndices(token, reference_selected[0..topk], window_rows, &reference_indices_storage); + var sinks: [64]f32 = undefined; + for (&sinks, 0..) |*value, head| value.* = math.value(weights.sink, head); + + var first_stage: ?[]const u8 = null; + inline for (std.enums.values(AttentionStage)) |stage| + if (first_stage == null and !std.mem.eql(u16, capture.row(stage), capture.cpuRow(stage))) { + first_stage = @tagName(stage); + }; + const header = try std.json.Stringify.valueAlloc(gpa, .{ + .mode = "attention-history-sameinput", + .token = token, + .layer = layer, + .history_count = history_count, + .token_identity_source = "caller_supplied_TOKEN_and_trace_POSITION_gpu_filenames_raw_snapshots_have_no_token_header", + .normalized_prefix_replay = .{ .file_pattern = "trace-{position}-gpu.bin", .origin = "retained_native_GPU_normalized_attention_inputs", .first_position = 0, .last_position = token, .first_layer = 0, .last_layer = layer, .order = "position_major_then_layer_ascending_inclusive", .snapshot_count_per_file = model.weights.layers.len * 2, .snapshot_length_bytes = snapshot_bytes, .attention_snapshot_index = "2*layer", .normalized_input_offset_within_snapshot_bytes = 40960, .normalized_input_length_bytes = 10240, .native_cache_reset = true, .native_cache_cursor_matches_each_position = true, .native_cache_append_rows_per_position = 1, .reference_position_passed_explicitly = true, .reference_capacity = state.capacity }, + .identical_retained_normalized_inputs = true, + .independent_reference_history = true, + .native_history_replay_branch_matches_retained_trace = std.mem.eql(u16, capture.row(.wo_b), &retained_branch), + .first_differing_same_history_input_stage = first_stage, + .window_history_equal = std.mem.eql(u16, native_window, reference_window), + .compressed_main_equal = std.mem.eql(u16, native_compressed, reference_main), + .compressed_index_keys_equal = std.mem.eql(u16, native_index_keys, reference_index_keys), + .selection_equal = std.mem.eql(i32, native_selected, reference_selected[0..topk]), + .compression_ratio = ratio, + .compressed_rows = compressed_rows, + .selection_count = topk, + .reference_dot = "independent_EXL3_FP64", + .reference_sparse_dot = "independent_FP64", + .qualification_gate_applied = false, + .qualification_oracle_modified_to_reuse_native_values = false, + .same_native_query_cache_diagnostic = "separate_reference_State_with_captured_native_window_main_selection_and_q_not_used_in_independent_chain", + .official_revision = "2cba9e42aa026125f3ed06c6d98c1db82f7ca027", + .receipt_dtype_endianness = "little_endian", + .receipt_strides = "contiguous_element_strides", + .files = .{ + .stage_chain = .{ .native = "attention-native.bf16", .reference = "attention-reference.bf16", .dtype = "bf16", .shape = .{attention_elements}, .strides = .{1}, .layout = "stages_table_below" }, + .normalized_prefix = .{ .file = "attention-normalized-prefix.bf16", .dtype = "bf16", .shape = .{ history_count, layer + 1, 5120 }, .strides = .{ (layer + 1) * 5120, 5120, 1 } }, + .target_input = .{ .file = "attention-input.bf16", .dtype = "bf16", .shape = .{5120}, .strides = .{1} }, + .retained_branch = .{ .file = "attention-retained-branch.bf16", .dtype = "bf16", .shape = .{5120}, .strides = .{1} }, + .query = .{ .native = "attention-query-native.bf16", .reference = "attention-query-reference.bf16", .dtype = "bf16", .shape = .{ 1, 1, 64, 512 }, .strides = .{ 32768, 32768, 512, 1 } }, + .window = .{ .native = "attention-window-native.bf16", .reference = "attention-window-reference.bf16", .dtype = "bf16", .shape = .{ 1, 128, 512 }, .strides = .{ 65536, 512, 1 }, .layout = "physical_ring_slot_position_modulo_128_includes_current_quantized_KV" }, + .compressed_main = .{ .native = "attention-compressed-native.bf16", .reference = "attention-compressed-reference.bf16", .dtype = "bf16", .shape = .{ 1, compressed_rows, 512 }, .strides = .{ compressed_rows * 512, 512, 1 } }, + .compressed_index_keys = .{ .native = "attention-index-keys-native.bf16", .reference = "attention-index-keys-reference.bf16", .dtype = "bf16", .shape = .{ 1, compressed_rows, 128 }, .strides = .{ compressed_rows * 128, 128, 1 } }, + .selection = .{ .native = "attention-selection-native.i32", .reference = "attention-selection-reference.i32", .dtype = "i32", .shape = .{ 1, topk }, .strides = .{ topk, 1 }, .layout = "actual_compressed_row_ids_compact_export_native_workspace_row_stride_is_512" }, + .source_kv = .{ .native = "attention-source-kv-native.bf16", .reference = "attention-source-kv-reference.bf16", .dtype = "bf16", .shape = .{ 1, window_rows + compressed_rows, 512 }, .strides = .{ (window_rows + compressed_rows) * 512, 512, 1 }, .layout = "source_sparse_attn_KV_window_then_compressed_rows_no_requantization" }, + .source_indices = .{ .native = "attention-source-indices-native.i32", .reference = "attention-source-indices-reference.i32", .dtype = "i32", .shape = .{ 1, 1, native_indices.len }, .strides = .{ native_indices.len, native_indices.len, 1 }, .layout = "source_get_window_topk_idxs_then_actual_selection_plus_window_rows_minus_one_remains_masked" }, + .sinks = .{ .file = "attention-sinks.f32", .dtype = "f32", .shape = .{64}, .strides = .{1} }, + .sparse_output = .{ .native = "attention-sparse-native.bf16", .reference = "attention-sparse-reference.bf16", .same_native_input_reference = "attention-sparse-same-native-input-reference.bf16", .dtype = "bf16", .shape = .{ 1, 1, 64, 512 }, .strides = .{ 32768, 32768, 512, 1 } }, + }, + .source_sparse_call = "sparse_attn(query,source_kv,sinks,source_indices,1/sqrt(512)); output_before_inverse_RoPE", + .source_sparse_replay_is_operator_evidence_only = true, + }, .{}); + defer gpa.free(header); + var report: std.ArrayList(u8) = .empty; + defer report.deinit(gpa); + try report.appendSlice(gpa, header[0 .. header.len - 1]); + try report.appendSlice(gpa, ",\"cases\":[\n"); + var count: usize = 0; + inline for (std.enums.values(AttentionStage)) |stage| + try attentionMetric(u16, gpa, &report, &count, @tagName(stage), "same_retained_normalized_history_input_chain", capture.row(stage), capture.cpuRow(stage)); + try attentionMetric(u16, gpa, &report, &count, "retained_branch", "native_history_replay_vs_retained_native_trace", capture.row(.wo_b), &retained_branch); + try attentionMetric(u16, gpa, &report, &count, "window_history", "same_retained_normalized_history_input_chain", native_window, reference_window); + if (compressed_rows != 0) { + try attentionMetric(u16, gpa, &report, &count, "compressed_kv", "same_retained_normalized_history_input_chain", native_compressed, reference_main); + try attentionMetric(u16, gpa, &report, &count, "compressed_index_keys", "same_retained_normalized_history_input_chain", native_index_keys, reference_index_keys); + } + + // This separate diagnostic State never overwrites independent retained-history reference data. + var native_input_state = try attention_math.State.init(gpa, history_count); + defer native_input_state.deinit(); + for (0..compressed_rows) |row| + @memcpy(native_input_state.sources[row * 640 ..][0..512], native_compressed[row * 512 ..][0..512]); + var helper_indices_storage: [640]i32 = undefined; + const helper_indices = attentionHistoryIndices(token, native_selected, 128, &helper_indices_storage); + var same_input: [32768]u16 = undefined; + for (0..64) |head| + native_input_state.sparseHead(capture.row(.q)[head * 512 ..][0..512], native_window, helper_indices, sinks[head], same_input[head * 512 ..][0..512]); + try attentionMetric(u16, gpa, &report, &count, "sparse_o", "same_native_query_window_compressed_cache_selection_reference_sparse_diagnostic", capture.row(.sparse_o), &same_input); + try attentionWrite(gpa, io, dir, "attention-sparse-same-native-input-reference.bf16", std.mem.sliceAsBytes(&same_input)); + math.rms(capture.row(.qr_projection), weights.q_norm, cfg.rms_norm_eps, same_input[0..1280]); + try attentionMetric(u16, gpa, &report, &count, "qr_norm", "same_native_input_reference_rms_diagnostic", capture.row(.qr_norm), same_input[0..1280]); + math.rms(capture.row(.kv_projection), weights.kv_norm, cfg.rms_norm_eps, same_input[0..512]); + try attentionMetric(u16, gpa, &report, &count, "kv_norm", "same_native_input_reference_rms_diagnostic", capture.row(.kv_norm), same_input[0..512]); + @memcpy(&same_input, capture.row(.q_projection)); + for (0..64) |head| math.rope(same_input[head * 512 ..][0..512], token, ratio != 0, false, cfg); + try attentionMetric(u16, gpa, &report, &count, "q", "same_native_input_reference_rope_diagnostic", capture.row(.q), &same_input); + @memcpy(same_input[0..512], capture.row(.kv_norm)); + math.rope(same_input[0..512], token, ratio != 0, false, cfg); + try attentionMetric(u16, gpa, &report, &count, "kv_rope", "same_native_input_reference_rope_diagnostic", capture.row(.kv_rope), same_input[0..512]); + @memcpy(same_input[0..512], capture.row(.kv_rope)); + math.quantize(same_input[0..512], .window); + try attentionMetric(u16, gpa, &report, &count, "kv_quantized", "same_native_input_reference_quantize_diagnostic", capture.row(.kv_quantized), same_input[0..512]); + const expected = try gpa.alloc(f32, 32768); + defer gpa.free(expected); + try attentionProjection(gpa, model, weights.wq_a, input, "wq_a", &scratch, expected, &report, &count); + try attentionProjection(gpa, model, weights.wq_b, capture.view(.qr_norm), "wq_b", &scratch, expected, &report, &count); + try attentionProjection(gpa, model, weights.wkv, input, "wkv", &scratch, expected, &report, &count); + for (weights.wo_a, 0..) |matrix, group| { + var name: [32]u8 = undefined; + try attentionProjection(gpa, model, matrix, capture.view(.o).shifted(group * 4096 * 2), try std.fmt.bufPrint(&name, "wo_a_group{d}", .{group}), &scratch, expected, &report, &count); + } + try attentionProjection(gpa, model, weights.wo_b, capture.view(.grouped), "wo_b", &scratch, expected, &report, &count); + try report.appendSlice(gpa, "\n],\"stages\":["); + inline for (std.enums.values(AttentionStage), 0..) |stage, i| { + const descriptor = try std.json.Stringify.valueAlloc(gpa, .{ .name = @tagName(stage), .offset_bytes = attentionOffset(stage) * 2, .elements = attention_widths[i] }, .{}); + defer gpa.free(descriptor); + if (i != 0) try report.appendSlice(gpa, ","); + try report.appendSlice(gpa, descriptor); + } + try report.appendSlice(gpa, "]}\n"); + const source_kv = try gpa.alloc(u16, 2 * (window_rows + compressed_rows) * 512); + defer gpa.free(source_kv); + const source_native = source_kv[0 .. (window_rows + compressed_rows) * 512]; + const source_reference = source_kv[(window_rows + compressed_rows) * 512 ..]; + @memcpy(source_native[0 .. window_rows * 512], native_window[0 .. window_rows * 512]); + @memcpy(source_native[window_rows * 512 ..], native_compressed); + @memcpy(source_reference[0 .. window_rows * 512], reference_window[0 .. window_rows * 512]); + @memcpy(source_reference[window_rows * 512 ..], reference_main); + const input_row = normalized_prefix[(history_count * (layer + 1) - 1) * 5120 ..][0..5120]; + inline for (.{ + "attention-history-sameinput.json", "attention-native.bf16", "attention-reference.bf16", + "attention-normalized-prefix.bf16", "attention-input.bf16", "attention-retained-branch.bf16", + "attention-query-native.bf16", "attention-query-reference.bf16", "attention-window-native.bf16", + "attention-window-reference.bf16", "attention-compressed-native.bf16", "attention-compressed-reference.bf16", + "attention-index-keys-native.bf16", "attention-index-keys-reference.bf16", "attention-selection-native.i32", + "attention-selection-reference.i32", "attention-source-kv-native.bf16", "attention-source-kv-reference.bf16", + "attention-source-indices-native.i32", "attention-source-indices-reference.i32", "attention-sinks.f32", + "attention-sparse-native.bf16", "attention-sparse-reference.bf16", + }, .{ + report.items, std.mem.sliceAsBytes(native.slice(u16, attention_elements)), std.mem.sliceAsBytes(reference_rows), + std.mem.sliceAsBytes(normalized_prefix), std.mem.sliceAsBytes(input_row), std.mem.sliceAsBytes(&retained_branch), + std.mem.sliceAsBytes(capture.row(.q)), std.mem.sliceAsBytes(capture.cpuRow(.q)), std.mem.sliceAsBytes(native_window), + std.mem.sliceAsBytes(reference_window), std.mem.sliceAsBytes(native_compressed), std.mem.sliceAsBytes(reference_main), + std.mem.sliceAsBytes(native_index_keys), std.mem.sliceAsBytes(reference_index_keys), std.mem.sliceAsBytes(native_selected), + std.mem.sliceAsBytes(reference_selected[0..topk]), std.mem.sliceAsBytes(source_native), std.mem.sliceAsBytes(source_reference), + std.mem.sliceAsBytes(native_indices), std.mem.sliceAsBytes(reference_indices), std.mem.sliceAsBytes(&sinks), + std.mem.sliceAsBytes(capture.row(.sparse_o)), std.mem.sliceAsBytes(capture.cpuRow(.sparse_o)), + }) |name, data| try attentionWrite(gpa, io, dir, name, data); +} diff --git a/zig/tests/deepseek_v41_reference/operator_moe.zig b/zig/tests/deepseek_v41_reference/operator_moe.zig new file mode 100644 index 0000000000..eb9780db96 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_moe.zig @@ -0,0 +1,409 @@ +//! Retained-input MoE routing and expert isolation. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const projection = @import("projection.zig"); +const math = @import("math.zig"); +const attention = tf.deepseek_v41.attention; +const Model = tf.deepseek_v41.Model; +const ex = tf.exl3_projection; +const diagnosticRow = @import("operator_diagnostic.zig").diagnosticRow; + +const moe_reference = @import("reference.zig"); +const MoeRows = struct { + gate: [7][2304]u16 = undefined, + up: [7][2304]u16 = undefined, + activation: [7][2304]u16 = undefined, + down: [7][5120]u16 = undefined, + branch: [5120]u16 = undefined, +}; +const MoeFirst = struct { + source_chain: ?[]const u8 = null, + source_expert_stage: ?[]const u8 = null, + native_input_diagnostic: ?[]const u8 = null, + native_weights_reference_chain_diagnostic: ?[]const u8 = null, + retained_weights_reference_chain_diagnostic: ?[]const u8 = null, + native_down_reference_combine_diagnostic_differs: bool = false, +}; + +fn moeMetric(comptime T: type, gpa: std.mem.Allocator, report: *std.ArrayList(u8), count: *usize, name: []const u8, comparison: []const u8, diagnostic_reuse: bool, actual: []const T, expected: []const T) !bool { + if (actual.len == 0 or actual.len != expected.len) return error.InvalidMoeMetricShape; + var squared: f64 = 0; + var magnitude: f64 = 0; + var max_abs: f64 = 0; + var worst: usize = 0; + var unequal: usize = 0; + var first: ?usize = null; + for (actual, expected, 0..) |a, r, i| { + const x = if (T == u16) math.f(a) else a; + const y = if (T == u16) math.f(r) else r; + if (!std.math.isFinite(x) or !std.math.isFinite(y)) return error.NonfiniteMoeDiagnostic; + const delta = @as(f64, x) - y; + squared += delta * delta; + magnitude += @as(f64, y) * y; + if (@abs(delta) > max_abs) { + max_abs = @abs(delta); + worst = i; + } + const ab = if (T == u16) a else @as(u32, @bitCast(a)); + const rb = if (T == u16) r else @as(u32, @bitCast(r)); + if (ab != rb) { + unequal += 1; + if (first == null) first = i; + } + } + const line = try std.json.Stringify.valueAlloc(gpa, .{ + .name = name, + .comparison = comparison, + .diagnostic_native_value_reuse = diagnostic_reuse, + .qualification_gate_applied = false, + .elements = actual.len, + .output_dtype = if (T == u16) "bf16" else "f32", + .normalized_rms_error = if (magnitude == 0) @sqrt(squared) else @sqrt(squared / magnitude), + .max_absolute_error = max_abs, + .worst_index = worst, + .bit_unequal_count = unequal, + .first_bit_unequal_index = first, + .worst_native = if (T == u16) math.f(actual[worst]) else actual[worst], + .worst_reference = if (T == u16) math.f(expected[worst]) else expected[worst], + }, .{}); + defer gpa.free(line); + if (count.* != 0) try report.appendSlice(gpa, ",\n"); + try report.appendSlice(gpa, line); + count.* += 1; + std.debug.print("{s}\n", .{line}); + return unequal != 0; +} + +fn moeWrite(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, name: []const u8, bytes: []const u8) !void { + const path = try std.fs.path.join(gpa, &.{ dir, name }); + defer gpa.free(path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = bytes }); +} + +fn moeWriteRows(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, prefix: []const u8, rows: *const MoeRows) !void { + inline for (.{ "gate", "up", "activation", "down", "branch" }) |stage| { + const name = try std.fmt.allocPrint(gpa, "{s}-{s}.bf16", .{ prefix, stage }); + defer gpa.free(name); + try moeWrite(gpa, io, dir, name, std.mem.asBytes(&@field(rows.*, stage))); + } +} + +fn moeMatrices(model: *Model, layer: usize, expert: ?u32) ![3]ex.Matrix { + if (expert == null) return model.weights.layers[layer].moe.shared; + var matrices: [3]ex.Matrix = undefined; + var name: [128]u8 = undefined; + inline for (.{ "w1", "w3", "w2" }, 0..) |part, i| { + const key = try std.fmt.bufPrint(&name, "layers.{d}.ffn.experts.{d}.{s}", .{ layer, expert.?, part }); + matrices[i] = try tf.deepseek_v41_weights.lookup(&model.checkpoint, key, if (i == 2) 2304 else 5120, if (i == 2) 5120 else 2304); + } + return matrices; +} + +fn moeCombineDown(ids: *const [6]u32, down: *const [7][5120]u16, branch: *[5120]u16) void { + @setFloatMode(.strict); + var sum: [5120]f32 = @splat(0); + // Keep slots in top-six order in receipts, but match the official ascending expert traversal. + for (0..384) |expert| { + for (ids, 0..) |id, slot| { + if (id == expert) + for (&sum, down[slot]) |*s, value| { + s.* += math.f(value); + }; + } + } + for (branch, sum, down[6]) |*v, routed, shared| v.* = math.b(routed + math.f(shared)); +} + +fn moeCombine(ids: *const [6]u32, rows: *MoeRows) void { + moeCombineDown(ids, &rows.down, &rows.branch); +} + +fn moeReferenceRows(model: *Model, layer: usize, input: []const u16, ids: *const [6]u32, routing: *const [6]f32, rows: *MoeRows, scratch: *projection.Scratch, previous: ?*const MoeRows, previous_ids: ?*const [6]u32) !void { + for (0..7) |slot| { + const matrices = try moeMatrices(model, layer, if (slot == 6) null else ids[slot]); + var reuse: ?usize = if (slot == 6 and previous != null) 6 else null; + if (slot != 6 and previous_ids != null) + for (previous_ids.?, 0..) |id, prior| { + if (id == ids[slot]) { + reuse = prior; + break; + } + }; + if (reuse) |prior| { + rows.gate[slot] = previous.?.gate[prior]; + rows.up[slot] = previous.?.up[prior]; + } else { + try projection.apply(matrices[0], input, &rows.gate[slot], scratch); + try projection.apply(matrices[1], input, &rows.up[slot], scratch); + } + moe_reference.moeActivation(&rows.gate[slot], &rows.up[slot], if (slot == 6) 1 else routing[slot], model.cfg.text_config, &rows.activation[slot]); + // A shared or identical-weight expert result can be reused only when the BF16 input is identical. + if (reuse != null and std.mem.eql(u16, &rows.activation[slot], &previous.?.activation[reuse.?])) { + rows.down[slot] = previous.?.down[reuse.?]; + } else { + try projection.apply(matrices[2], &rows.activation[slot], &rows.down[slot], scratch); + } + } + moeCombine(ids, rows); +} + +fn moeEncodeNative(model: *Model, layer: usize, select_routes: bool) !void { + const ws = &model.workspace; + const p = &model.pipelines; + const w = &model.weights.layers[layer].moe; + const cb = model.queue.commandBufferUnretained(); + const enc = cb.compute(.serial); + var encoding = true; + errdefer if (encoding) enc.end(); + for (model.checkpoint.shards.items) |shard| enc.useResource(shard.buffer, mtl.ResourceUsage.read); + for (w.descriptors) |buffer| enc.useResource(buffer, mtl.ResourceUsage.read); + if (select_routes) { + tf.deepseek_v41.ops.encodeDense(enc, &p.ops, ws.view(.normalized), .{ .buffer = w.gate.buffer, .offset = w.gate.offset }, ws.view(.router_logits), .{ .rows = 1, .k = 5120, .n = 384, .x_stride = 5120, .w_stride = 5120, .y_stride = 384, .w_dtype = .f16, .y_dtype = .f32 }); + enc.barrier(); + // Bind the existing production router; no new native interface or diagnostic math kernel. + enc.setPipeline(p.route.select); + inline for (.{ ws.view(.router_logits), attention.View{ .buffer = w.bias.buffer, .offset = w.bias.offset }, ws.view(.route_ids), ws.view(.route_weights) }, 0..) |view, i| + enc.setBuffer(view.buffer, view.offset, i); + const Shape = extern struct { rows: u32, logit_stride: u32, output_stride: u32, temperature: f32, bias_dtype: u32 }; + enc.setValue(Shape{ .rows = 1, .logit_stride = 384, .output_stride = 6, .temperature = 1, .bias_dtype = 2 }, 4); + enc.dispatchGroups(mtl.Size.of(1, 1, 1), mtl.Size.of(128, 1, 1)); + enc.barrier(); + } + var routed_rows = ex.Rows{ .count = 6, .x_stride = 5120, .y_stride = 2304, .routes = .{ .ids = .{ .buffer = ws.view(.route_ids).buffer, .offset = ws.view(.route_ids).offset }, .repeat = 6 } }; + for (0..2) |i| { + const target = ws.view(if (i == 0) .gate else .up); + const input = ws.view(.normalized); + try ex.encodeIndexed(&p.projection, enc, .{ .descriptors = .{ .buffer = w.descriptors[i] }, .count = 384, .k = 5120, .n = 2304 }, .{ .buffer = input.buffer, .offset = input.offset }, .{ .buffer = target.buffer, .offset = target.offset }, ws.projection(), routed_rows); + enc.barrier(); + } + tf.deepseek_v41.ops.encodeSwiGLU(enc, &p.ops, ws.view(.gate), ws.view(.up), ws.view(.route_weights), ws.view(.activation), .{ .rows = 6, .width = 2304, .input_stride = 2304, .output_stride = 2304, .weighted = true }); + enc.barrier(); + routed_rows.x_stride = 2304; + routed_rows.y_stride = 5120; + routed_rows.routes.?.repeat = 1; + const activation = ws.view(.activation); + const down = ws.view(.routed); + try ex.encodeIndexed(&p.projection, enc, .{ .descriptors = .{ .buffer = w.descriptors[2] }, .count = 384, .k = 2304, .n = 5120 }, .{ .buffer = activation.buffer, .offset = activation.offset }, .{ .buffer = down.buffer, .offset = down.offset }, ws.projection(), routed_rows); + enc.barrier(); + for (0..2) |i| { + const target = ws.view(if (i == 0) .shared_gate else .shared_up); + const input = ws.view(.normalized); + ex.useMatrix(enc, w.shared[i]); + try ex.encode(&p.projection, enc, w.shared[i], .{ .buffer = input.buffer, .offset = input.offset }, .{ .buffer = target.buffer, .offset = target.offset }, ws.projection(), .{ .count = 1, .x_stride = 5120, .y_stride = 2304 }); + enc.barrier(); + } + tf.deepseek_v41.ops.encodeSwiGLU(enc, &p.ops, ws.view(.shared_gate), ws.view(.shared_up), ws.view(.route_weights), ws.view(.shared_activation), .{ .rows = 1, .width = 2304, .input_stride = 2304, .output_stride = 2304 }); + enc.barrier(); + const shared_activation = ws.view(.shared_activation); + const shared_down = ws.view(.shared_down); + ex.useMatrix(enc, w.shared[2]); + try ex.encode(&p.projection, enc, w.shared[2], .{ .buffer = shared_activation.buffer, .offset = shared_activation.offset }, .{ .buffer = shared_down.buffer, .offset = shared_down.offset }, ws.projection(), .{ .count = 1, .x_stride = 2304, .y_stride = 5120 }); + enc.barrier(); + tf.deepseek_v41.ops.encodeCombine(enc, &p.ops, ws.view(.routed), ws.view(.shared_down), ws.view(.route_ids), ws.view(.branch), .{ .rows = 1, .width = 5120, .routed_row_stride = 6 * 5120, .slot_stride = 5120, .shared_stride = 5120, .y_stride = 5120 }); + enc.end(); + encoding = false; + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.OperatorGpuFailure; +} + +fn moeCaptureNative(model: *Model, rows: *MoeRows) void { + const ws = &model.workspace; + inline for (.{ "gate", "up", "activation", "down" }, .{ .gate, .up, .activation, .routed }, .{ .shared_gate, .shared_up, .shared_activation, .shared_down }) |stage, routed, shared| { + const width = if (comptime std.mem.eql(u8, stage, "down")) 5120 else 2304; + const source = diagnosticRow(u16, ws.view(routed), 6 * width); + for (0..6) |slot| @memcpy(&@field(rows.*, stage)[slot], source[slot * width ..][0..width]); + @memcpy(&@field(rows.*, stage)[6], diagnosticRow(u16, ws.view(shared), width)); + } + @memcpy(&rows.branch, diagnosticRow(u16, ws.view(.branch), 5120)); +} + +fn moeCompareRows(gpa: std.mem.Allocator, report: *std.ArrayList(u8), count: *usize, comparison: []const u8, reuse: bool, actual: *const MoeRows, actual_ids: *const [6]u32, expected: *const MoeRows, expected_ids: *const [6]u32, first: *?[]const u8) !void { + // Compare like expert IDs rather than assuming independent top-six orders are identical. + inline for (.{ "gate", "up", "activation", "down" }) |stage| { + var differs = false; + for (0..7) |slot| { + var reference_slot: ?usize = if (slot == 6) 6 else null; + if (slot != 6) + for (expected_ids, 0..) |id, i| { + if (id == actual_ids[slot]) { + reference_slot = i; + break; + } + }; + if (reference_slot == null) continue; + var name: [64]u8 = undefined; + const label = if (slot == 6) try std.fmt.bufPrint(&name, "shared_{s}", .{stage}) else try std.fmt.bufPrint(&name, "expert_{d}_{s}", .{ actual_ids[slot], stage }); + differs = (try moeMetric(u16, gpa, report, count, label, comparison, reuse, &@field(actual.*, stage)[slot], &@field(expected.*, stage)[reference_slot.?])) or differs; + } + if (differs and first.* == null) first.* = stage; + } + if (try moeMetric(u16, gpa, report, count, "combined_branch", comparison, reuse, &actual.branch, &expected.branch)) + if (first.* == null) { + first.* = "combined_branch"; + }; +} + +/// Retained local snapshot isolation. MoE has no history; qualification computes every value independently. +pub fn runMoe(gpa: std.mem.Allocator, io: std.Io, model: *Model, snapshot: []const u8, dir: []const u8, layer: usize) !void { + if (layer >= model.weights.layers.len) return error.InvalidMoeLayer; + const blob = try std.Io.Dir.cwd().readFileAlloc(io, snapshot, gpa, .limited(8 << 20)); + defer gpa.free(blob); + const snapshot_offset = (layer * 2 + 1) * 61488; + if (blob.len != 80 * 61488 or blob.len < snapshot_offset + 61488) return error.InvalidMoeSnapshot; + try model.reset(); + const input = model.workspace.view(.normalized); + @memcpy(input.buffer.contents()[input.offset..][0..10240], blob[snapshot_offset + 40960 ..][0..10240]); + const input_row = diagnosticRow(u16, input, 5120); + var retained_ids: [6]u32 = undefined; + var retained_weights: [6]f32 = undefined; + var retained_branch: [5120]u16 = undefined; + for (&retained_ids, 0..) |*id, i| id.* = std.mem.readInt(u32, blob[snapshot_offset + 61440 + 4 * i ..][0..4], .little); + for (&retained_weights, 0..) |*weight, i| + weight.* = @bitCast(std.mem.readInt(u32, blob[snapshot_offset + 61464 + 4 * i ..][0..4], .little)); + @memcpy(std.mem.asBytes(&retained_branch), blob[snapshot_offset + 51200 ..][0..10240]); + for (retained_ids, retained_weights, 0..) |id, weight, i| { + if (id >= 384 or !std.math.isFinite(weight)) return error.InvalidMoeRetainedRouting; + for (retained_ids[0..i]) |prior| if (id == prior) return error.InvalidMoeRetainedRouting; + } + const native = try gpa.create(MoeRows); + defer gpa.destroy(native); + const source = try gpa.create(MoeRows); + defer gpa.destroy(source); + const native_weights_reference = try gpa.create(MoeRows); + defer gpa.destroy(native_weights_reference); + const retained_reference = try gpa.create(MoeRows); + defer gpa.destroy(retained_reference); + const native_inputs_reference = try gpa.create(MoeRows); + defer gpa.destroy(native_inputs_reference); + const retained_native = try gpa.create(MoeRows); + defer gpa.destroy(retained_native); + var scratch = try projection.Scratch.init(gpa, 5120, 5120); + defer scratch.deinit(); + + try moeEncodeNative(model, layer, true); + moeCaptureNative(model, native); + var native_ids: [6]u32 = undefined; + var native_weights: [6]f32 = undefined; + var native_logits_routing: moe_reference.MoeRouting = .{}; + @memcpy(&native_ids, diagnosticRow(u32, model.workspace.view(.route_ids), 6)); + @memcpy(&native_weights, diagnosticRow(f32, model.workspace.view(.route_weights), 6)); + @memcpy(&native_logits_routing.logits, diagnosticRow(f32, model.workspace.view(.router_logits), 384)); + moe_reference.moeSelect(model.weights.layers[layer].moe, model.cfg.text_config, &native_logits_routing.logits, &native_logits_routing.scores, &native_logits_routing.ids, &native_logits_routing.weights); + var source_routing: moe_reference.MoeRouting = .{}; + math.dense(model.weights.layers[layer].moe.gate, input_row, &source_routing.logits); + moe_reference.moeSelect(model.weights.layers[layer].moe, model.cfg.text_config, &source_routing.logits, &source_routing.scores, &source_routing.ids, &source_routing.weights); + try moeReferenceRows(model, layer, input_row, &source_routing.ids, &source_routing.weights, source, &scratch, null, null); + try moeReferenceRows(model, layer, input_row, &native_ids, &native_weights, native_weights_reference, &scratch, source, &source_routing.ids); + try moeReferenceRows(model, layer, input_row, &retained_ids, &retained_weights, retained_reference, &scratch, native_weights_reference, &native_ids); + for (0..7) |slot| { + native_inputs_reference.gate[slot] = native.gate[slot]; + native_inputs_reference.up[slot] = native.up[slot]; + moe_reference.moeActivation(&native.gate[slot], &native.up[slot], if (slot == 6) 1 else native_weights[slot], model.cfg.text_config, &native_inputs_reference.activation[slot]); + // Deliberately use the captured native activation, not the preceding reference activation. + if (std.mem.eql(u16, &native.activation[slot], &native_weights_reference.activation[slot])) { + native_inputs_reference.down[slot] = native_weights_reference.down[slot]; + } else { + const matrices = try moeMatrices(model, layer, if (slot == 6) null else native_ids[slot]); + try projection.apply(matrices[2], &native.activation[slot], &native_inputs_reference.down[slot], &scratch); + } + } + moeCombine(&native_ids, native_inputs_reference); + @memcpy(diagnosticRow(u32, model.workspace.view(.route_ids), 6), &retained_ids); + @memcpy(diagnosticRow(f32, model.workspace.view(.route_weights), 6), &retained_weights); + try moeEncodeNative(model, layer, false); + moeCaptureNative(model, retained_native); + + var report: std.ArrayList(u8) = .empty; + defer report.deinit(gpa); + const header = try std.json.Stringify.valueAlloc(gpa, .{ + .mode = "moe-sameinput", + .token = @as(?usize, null), + .layer = layer, + .token_identity_source = "caller_supplied_snapshot_argument_raw_snapshot_has_no_token_header", + .retained_route_origin = "native_GPU_snapshot_command_contract", + .snapshot_count = 80, + .snapshot_length_bytes = 61488, + .snapshot_index = layer * 2 + 1, + .snapshot_stage = "moe", + .input_offset_bytes = snapshot_offset + 40960, + .input_length_bytes = 10240, + .retained_ids_offset_bytes = snapshot_offset + 61440, + .retained_weights_offset_bytes = snapshot_offset + 61464, + .identical_normalized_input = true, + .reference_dot = "independent_EXL3_FP64", + .qualification_gate_applied = false, + .qualification_oracle_modified_to_reuse_native_values = false, + .official_revision = "2cba9e42aa026125f3ed06c6d98c1db82f7ca027", + .source_ids = source_routing.ids, + .source_weights = source_routing.weights, + .native_ids = native_ids, + .native_weights = native_weights, + .retained_ids = retained_ids, + .retained_weights = retained_weights, + .source_ids_equal_native_order = std.mem.eql(u32, &source_routing.ids, &native_ids), + .native_ids_equal_retained_order = std.mem.eql(u32, &native_ids, &retained_ids), + .source_select_on_native_logits_ids = native_logits_routing.ids, + .source_select_on_native_logits_weights = native_logits_routing.weights, + .score_receipt_origin = "source_sqrt_softplus_applied_to_captured_native_logits_not_GPU_score_capture", + .expert_receipt_layout = "slot_major_top6_ids_order_then_shared_slot6", + .expert_sum_order = "ascending_expert_id_FP32_shared_last_final_BF16", + .receipt_prefixes = .{ "moe-native", "moe-source", "moe-native-weights-reference-diagnostic", "moe-native-inputs-reference-diagnostic", "moe-retained-weights-reference-diagnostic", "moe-retained-route-native-diagnostic" }, + .receipt_shapes = .{ .gate = .{ 7, 2304 }, .up = .{ 7, 2304 }, .activation = .{ 7, 2304 }, .down = .{ 7, 5120 }, .branch = .{5120} }, + .native_inputs_reference_contract = "activation:reference_on_native_gate_up_and_weight;down:FP64_on_native_activation;branch:reference_combine_of_these_down_outputs", + }, .{}); + defer gpa.free(header); + try report.appendSlice(gpa, header[0 .. header.len - 1]); + try report.appendSlice(gpa, ",\"cases\":[\n"); + var count: usize = 0; + var first: MoeFirst = .{}; + if (try moeMetric(f32, gpa, &report, &count, "route_logits", "source_chain", false, &native_logits_routing.logits, &source_routing.logits)) first.source_chain = "route_logits"; + if (try moeMetric(f32, gpa, &report, &count, "route_scores_source_transform", "source_score_transform_on_captured_native_logits_diagnostic", true, &native_logits_routing.scores, &source_routing.scores)) + if (first.source_chain == null) { + first.source_chain = "route_scores_source_transform"; + }; + if (!std.mem.eql(u32, &source_routing.ids, &native_ids) and first.source_chain == null) first.source_chain = "route_ids"; + if (std.mem.eql(u32, &source_routing.ids, &native_ids)) { + if (try moeMetric(f32, gpa, &report, &count, "route_weights", "source_chain", false, &native_weights, &source_routing.weights)) + if (first.source_chain == null) { + first.source_chain = "route_weights"; + }; + } + if (std.mem.eql(u32, &native_logits_routing.ids, &native_ids)) + _ = try moeMetric(f32, gpa, &report, &count, "route_weights", "source_select_on_same_native_logits_diagnostic", true, &native_weights, &native_logits_routing.weights); + if (std.mem.eql(u32, &retained_ids, &native_ids)) + _ = try moeMetric(f32, gpa, &report, &count, "route_weights", "retained_native_weights_vs_reexecuted_native", true, &native_weights, &retained_weights); + try moeCompareRows(gpa, &report, &count, "source_chain", false, native, &native_ids, source, &source_routing.ids, &first.source_expert_stage); + if (first.source_chain == null) first.source_chain = first.source_expert_stage; + try moeCompareRows(gpa, &report, &count, "same_native_ids_weights_reference_chain_diagnostic", true, native, &native_ids, native_weights_reference, &native_ids, &first.native_weights_reference_chain_diagnostic); + inline for (.{ "activation", "down" }) |stage| { + var differs = false; + for (0..7) |slot| { + var name: [64]u8 = undefined; + const label = if (slot == 6) try std.fmt.bufPrint(&name, "shared_{s}", .{stage}) else try std.fmt.bufPrint(&name, "expert_{d}_{s}", .{ native_ids[slot], stage }); + differs = (try moeMetric(u16, gpa, &report, &count, label, if (comptime std.mem.eql(u8, stage, "activation")) "same_native_gate_up_weight_reference_activation_diagnostic" else "same_native_activation_FP64_down_diagnostic", true, &@field(native.*, stage)[slot], &@field(native_inputs_reference.*, stage)[slot])) or differs; + } + if (differs and first.native_input_diagnostic == null) first.native_input_diagnostic = stage; + } + _ = try moeMetric(u16, gpa, &report, &count, "combined_branch", "same_native_activation_FP64_down_then_reference_combine_diagnostic", true, &native.branch, &native_inputs_reference.branch); + // Isolate combination arithmetic independently of every projection and route-weight difference. + var native_down_reference_branch: [5120]u16 = undefined; + moeCombineDown(&native_ids, &native.down, &native_down_reference_branch); + first.native_down_reference_combine_diagnostic_differs = + try moeMetric(u16, gpa, &report, &count, "combined_branch", "same_native_down_reference_combine_diagnostic", true, &native.branch, &native_down_reference_branch); + try moeWrite(gpa, io, dir, "moe-native-down-reference-combine-diagnostic.bf16", std.mem.asBytes(&native_down_reference_branch)); + try moeCompareRows(gpa, &report, &count, "same_retained_native_ids_weights_reference_chain_diagnostic", true, retained_native, &retained_ids, retained_reference, &retained_ids, &first.retained_weights_reference_chain_diagnostic); + _ = try moeMetric(u16, gpa, &report, &count, "combined_branch", "retained_native_branch_vs_reexecuted_native_routing", true, &native.branch, &retained_branch); + _ = try moeMetric(u16, gpa, &report, &count, "combined_branch", "retained_native_branch_vs_reexecuted_retained_routes_diagnostic", true, &retained_native.branch, &retained_branch); + const finding = try std.json.Stringify.valueAlloc(gpa, first, .{}); + defer gpa.free(finding); + try report.appendSlice(gpa, "\n],\"first_differing_intermediate\":"); + try report.appendSlice(gpa, finding); + try report.appendSlice(gpa, "}\n"); + try moeWrite(gpa, io, dir, "moe-sameinput.json", report.items); + inline for (.{ "moe-native", "moe-source", "moe-native-weights-reference-diagnostic", "moe-native-inputs-reference-diagnostic", "moe-retained-weights-reference-diagnostic", "moe-retained-route-native-diagnostic" }, .{ native, source, native_weights_reference, native_inputs_reference, retained_reference, retained_native }) |prefix, rows| + try moeWriteRows(gpa, io, dir, prefix, rows); + inline for (.{ "moe-input.bf16", "moe-retained-branch.bf16", "moe-native-logits.f32", "moe-source-logits.f32", "moe-native-logits-source-scores.f32", "moe-source-scores.f32", "moe-native-ids.u32", "moe-source-ids.u32", "moe-retained-ids.u32", "moe-native-weights.f32", "moe-source-weights.f32", "moe-retained-weights.f32", "moe-native-logits-source-select-ids.u32", "moe-native-logits-source-select-weights.f32" }, .{ std.mem.sliceAsBytes(input_row), std.mem.asBytes(&retained_branch), std.mem.asBytes(&native_logits_routing.logits), std.mem.asBytes(&source_routing.logits), std.mem.asBytes(&native_logits_routing.scores), std.mem.asBytes(&source_routing.scores), std.mem.asBytes(&native_ids), std.mem.asBytes(&source_routing.ids), std.mem.asBytes(&retained_ids), std.mem.asBytes(&native_weights), std.mem.asBytes(&source_routing.weights), std.mem.asBytes(&retained_weights), std.mem.asBytes(&native_logits_routing.ids), std.mem.asBytes(&native_logits_routing.weights) }) |name, bytes| + try moeWrite(gpa, io, dir, name, bytes); +} diff --git a/zig/tests/deepseek_v41_reference/operators.zig b/zig/tests/deepseek_v41_reference/operators.zig new file mode 100644 index 0000000000..8d050300e2 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operators.zig @@ -0,0 +1,92 @@ +//! Actual checkpoint projections on identical deterministic BF16 rows, against independent FP64 dot products. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const projection = @import("projection.zig"); +const math = @import("math.zig"); +const Model = tf.deepseek_v41.Model; +const ex = tf.exl3_projection; +const Case = struct { name: []const u8, matrix: ex.Matrix }; + +pub fn run(gpa: std.mem.Allocator, io: std.Io, model: *Model, dir: []const u8) !void { + const first = &model.weights.layers[0]; + const cases = [_]Case{ + .{ .name = "wq_a", .matrix = first.attn.wq_a }, + .{ .name = "wq_b", .matrix = first.attn.wq_b }, + .{ .name = "wkv", .matrix = first.attn.wkv }, + .{ .name = "wo_a_group0", .matrix = first.attn.wo_a[0] }, + .{ .name = "wo_b", .matrix = first.attn.wo_b }, + .{ .name = "engram_wkv", .matrix = model.weights.layers[1].engram.?.wkv }, + .{ .name = "routed_gate", .matrix = try tf.deepseek_v41_weights.lookup(&model.checkpoint, "layers.0.ffn.experts.0.w1", 5120, 2304) }, + .{ .name = "routed_up", .matrix = try tf.deepseek_v41_weights.lookup(&model.checkpoint, "layers.0.ffn.experts.0.w3", 5120, 2304) }, + .{ .name = "routed_down", .matrix = try tf.deepseek_v41_weights.lookup(&model.checkpoint, "layers.0.ffn.experts.0.w2", 2304, 5120) }, + .{ .name = "head", .matrix = model.weights.head }, + }; + const x = try model.device.buffer(8192 * 2, mtl.ResourceOptions.shared); + defer x.deinit(); + const y = try model.device.buffer(2 * 129280 * 4, mtl.ResourceOptions.shared); + defer y.deinit(); + const scratch = try model.device.buffer(try ex.scratchBytes(8192, 129280, 1), mtl.ResourceOptions.shared); + defer scratch.deinit(); + for (x.slice(u16, 8192), 0..) |*value, i| + value.* = math.b(@as(f32, @floatFromInt(@as(i32, @intCast((i * 17) % 97)) - 48)) / 32); + for ([_]mtl.Buffer{ x, y, scratch }) |buffer| model.residency.add(buffer); + model.residency.commit(); + model.residency.requestResidency(); + defer { + for ([_]mtl.Buffer{ x, y, scratch }) |buffer| model.residency.remove(buffer); + model.residency.commit(); + } + var reference = try projection.Scratch.init(gpa, 8192, 129280); + defer reference.deinit(); + const expected = try gpa.alloc(f32, 129280); + defer gpa.free(expected); + var report: std.ArrayList(u8) = .empty; + defer report.deinit(gpa); + try report.appendSlice(gpa, "{\"input\":\"bf16((17*i%97-48)/32)\",\"reference_dot\":\"FP64\",\"cases\":["); + var all_pass = true; + for (cases, 0..) |case, index| { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const matrix = case.matrix; + const cb = model.queue.commandBufferUnretained(); + const enc = cb.compute(.serial); + var encoding = true; + errdefer if (encoding) enc.end(); + ex.useMatrix(enc, matrix); + try ex.encode(&model.pipelines.projection, enc, matrix, .{ .buffer = x }, .{ .buffer = y }, .{ .buffer = scratch }, .{ .count = 1, .x_stride = matrix.k, .y_stride = matrix.n, .output = .f32 }); + enc.end(); + encoding = false; + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.OperatorGpuFailure; + const actual = y.slice(f32, 129280); + try projection.applyF32(matrix, x.slice(u16, matrix.k), expected[0..matrix.n], &reference); + var max_abs: f64 = 0; + var worst: usize = 0; + var squared: f64 = 0; + var magnitude: f64 = 0; + for (actual[0..matrix.n], expected[0..matrix.n], 0..) |a, b, column| { + if (!std.math.isFinite(a) or !std.math.isFinite(b)) return error.NonfiniteOperatorOutput; + const delta: f64 = @as(f64, a) - b; + if (@abs(delta) > max_abs) { + max_abs = @abs(delta); + worst = column; + } + squared += delta * delta; + magnitude += @as(f64, b) * b; + } + const nrms = if (magnitude == 0) @sqrt(squared) else @sqrt(squared / magnitude); + all_pass = all_pass and nrms <= 1e-4; + const line = try std.fmt.allocPrint(gpa, "{{\"name\":\"{s}\",\"k\":{d},\"n\":{d},\"normalized_rms_error\":{d},\"max_absolute_error\":{d},\"worst_column\":{d},\"worst_gpu\":{d},\"worst_reference\":{d}}}", .{ case.name, matrix.k, matrix.n, nrms, max_abs, worst, actual[worst], expected[worst] }); + defer gpa.free(line); + if (index > 0) try report.appendSlice(gpa, ","); + try report.appendSlice(gpa, line); + std.debug.print("{s}\n", .{line}); + } + try report.appendSlice(gpa, "]}\n"); + const path = try std.fs.path.join(gpa, &.{ dir, "operators.json" }); + defer gpa.free(path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = report.items }); + if (!all_pass) return error.OperatorReferenceMismatch; +} diff --git a/zig/tests/deepseek_v41_reference/profile.zig b/zig/tests/deepseek_v41_reference/profile.zig new file mode 100644 index 0000000000..0fb15a8b90 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/profile.zig @@ -0,0 +1,52 @@ +//! Real-checkpoint calibrated GPU stage timings, with profile-on/off logits checked before reporting. +const std = @import("std"); +const tf = @import("tensorfold"); +const counters = tf.deepseek_v41_profiler; +const Model = tf.deepseek_v41.Model; + +pub fn run(gpa: std.mem.Allocator, io: std.Io, model: *Model, ids: []const u32, dir: []const u8) !void { + if (ids.len == 0 or ids.len > model.workspace.plan.rows) return error.InvalidProfileWindow; + const previous_grouping = model.grouped_prefill; + defer model.grouped_prefill = previous_grouping; + model.grouped_prefill = false; + try model.reset(); + const baseline = try model.forward(ids, .all); + const baseline_gpu_seconds = model.last_gpu_seconds; + const expected = try gpa.dupe(f32, baseline); + defer gpa.free(expected); + model.grouped_prefill = true; + try model.reset(); + const candidate = try model.forward(ids, .all); + const candidate_gpu_seconds = model.last_gpu_seconds; + try check(expected, candidate); + var recorder = try counters.Recorder.init(gpa, model.device, .{ .enabled = true, .max_spans = 512 }); + defer recorder.deinit() catch unreachable; + const previous_profile = model.profile; + model.profile = &recorder; + defer model.profile = previous_profile; + try model.reset(); + const actual = try model.forward(ids, .last); + try check(expected[expected.len - 129280 ..], actual); + const totals = recorder.aggregates(); + var sampled_gpu_seconds: f64 = 0; + for (totals) |value| sampled_gpu_seconds += value.gpu_seconds; + var text: std.ArrayList(u8) = .empty; + defer text.deinit(gpa); + try text.print(gpa, "{{\"tokens\":{d},\"comparison_logits\":\"all\",\"candidate_grouping\":true,\"same_logits\":true,\"baseline_command_gpu_seconds\":{d},\"candidate_command_gpu_seconds\":{d},\"profiled_logits\":\"last\",\"sampled_span_gpu_seconds\":{d},\"stages\":[", .{ ids.len, baseline_gpu_seconds, candidate_gpu_seconds, sampled_gpu_seconds }); + for (std.enums.values(counters.Stage), totals, 0..) |stage, value, i| { + try text.print(gpa, "{s}{{\"stage\":\"{s}\",\"calls\":{d},\"gpu_seconds\":{d}}}", .{ if (i == 0) "" else ",", @tagName(stage), value.calls, value.gpu_seconds }); + } + try text.appendSlice(gpa, "]}\n"); + const path = try std.fs.path.join(gpa, &.{ dir, "gpu-profile.json" }); + defer gpa.free(path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = text.items }); + std.debug.print("{s}", .{text.items}); +} + +fn check(expected: []const f32, actual: []const f32) !void { + if (expected.len != actual.len) return error.ProfileLogitShapeMismatch; + for (actual, expected) |a, b| { + if (!std.math.isFinite(a) or !std.math.isFinite(b)) return error.NonfiniteProfileLogits; + if (@as(u32, @bitCast(a)) != @as(u32, @bitCast(b))) return error.ProfileChangedLogits; + } +} diff --git a/zig/tests/deepseek_v41_reference/projection.zig b/zig/tests/deepseek_v41_reference/projection.zig new file mode 100644 index 0000000000..ca0f932c22 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/projection.zig @@ -0,0 +1,261 @@ +//! Independent scalar EXL3 oracle over resident bytes; FP64 dots and FP32 rotation/output boundaries, no GPU work. + +// EXL3 attribution: docs/recipes/exl3.md; pinned checkpoint provenance in THIRD_PARTY_NOTICES.md. + +const std = @import("std"); +const exl3 = @import("tensorfold").exl3_projection; +const format = exl3.format; +const block_size = format.hadamard_dim; +const inverse: f32 = 0.08838834764831845; +const states_count = 65536; + +/// Reuse bounded vector/codebook storage for heterogeneous matrices; never allocate decoded W[K,N]. +pub const Scratch = struct { + gpa: std.mem.Allocator, + rotated: []f64, + sums: []f64, + codebooks: []f16, + + pub fn init(gpa: std.mem.Allocator, max_k: usize, max_n: usize) !Scratch { + if (max_k == 0 or max_n == 0) return error.InvalidReferenceProjectionCapacity; + const rotated = try gpa.alloc(f64, max_k); + errdefer gpa.free(rotated); + const sums = try gpa.alloc(f64, max_n); + errdefer gpa.free(sums); + const codebooks = try gpa.alloc(f16, 3 * states_count); + errdefer gpa.free(codebooks); + for ([_]format.Codebook{ .inst3, .mcg, .mul1 }, 0..) |book, bi| { + for (0..states_count) |state| { + codebooks[bi * states_count + state] = format.codebookValue(book, @intCast(state)); + } + } + return .{ .gpa = gpa, .rotated = rotated, .sums = sums, .codebooks = codebooks }; + } + + pub fn deinit(s: *Scratch) void { + s.gpa.free(s.rotated); + s.gpa.free(s.sums); + s.gpa.free(s.codebooks); + s.* = undefined; + } +}; + +/// One BF16 input row and BF16 output row; caller must finish GPU writes before borrowing shared CPU bytes. +pub fn apply(matrix: exl3.Matrix, x: []const u16, y: []u16, scratch: *Scratch) !void { + try project(matrix, x, y.len, scratch); + try finish(u16, matrix, y, scratch.sums[0..matrix.n]); +} + +/// The same arithmetic with an FP32 output boundary, for compressor projections and full-vocabulary logits. +pub fn applyF32(matrix: exl3.Matrix, x: []const u16, y: []f32, scratch: *Scratch) !void { + try project(matrix, x, y.len, scratch); + try finish(f32, matrix, y, scratch.sums[0..matrix.n]); +} + +fn borrowed(view: exl3.View, bytes: usize) ![]const u8 { + const len = view.buffer.length(); + if (view.offset % 2 != 0 or view.offset > len or bytes > len - view.offset) + return error.InvalidReferenceProjectionView; + const address = view.buffer.contents(); + if (@intFromPtr(address) == 0) return error.NonCpuAccessibleExl3Buffer; + return address[view.offset..][0..bytes]; +} + +fn scaleBytes(count: usize, kind: exl3.ScaleFormat) !usize { + return std.math.mul(usize, if (kind == .packed_signs) count / 16 else count, 2); +} + +fn half(bytes: []const u8, i: usize) f32 { + const value: f16 = @bitCast(std.mem.readInt(u16, bytes[2 * i ..][0..2], .little)); + return @floatCast(value); +} + +fn scale(bytes: []const u8, i: usize, kind: exl3.ScaleFormat) f32 { + if (kind == .f16) return half(bytes, i); + const word = std.mem.readInt(u16, bytes[2 * (i / 16) ..][0..2], .little); + return if (((word >> @as(u4, @intCast(i % 16))) & 1) != 0) -1.0 else 1.0; +} + +fn bf16(value: f32) u16 { + const bits: u32 = @bitCast(value); + // Preserve a quiet NaN instead of letting rounding turn a tiny NaN payload into infinity. + if ((bits & 0x7fffffff) > 0x7f800000) return @as(u16, @truncate(bits >> 16)) | 0x0040; + return @truncate((bits +% (0x7fff + ((bits >> 16) & 1))) >> 16); +} + +/// Seven ordinary Sylvester butterflies, with explicit FP32 stage boundaries and normalization afterward. +fn hadamard(values: *[block_size]f32) void { + @setFloatMode(.strict); + var stride: usize = 1; + while (stride < block_size) : (stride *= 2) { + var base: usize = 0; + while (base < block_size) : (base += 2 * stride) { + for (0..stride) |j| { + const a = values[base + j]; + const b = values[base + stride + j]; + values[base + j] = a + b; + values[base + stride + j] = a - b; + } + } + } +} + +fn prepare(x: []const u16, su: []const u8, kind: exl3.ScaleFormat, rotated: []f64) void { + @setFloatMode(.strict); + var base: usize = 0; + while (base < x.len) : (base += block_size) { + var values: [block_size]f32 = undefined; + for (&values, 0..) |*value, i| { + const input: f32 = @bitCast(@as(u32, x[base + i]) << 16); + value.* = input * scale(su, base + i, kind); + } + hadamard(&values); + for (values, 0..) |value, i| { + const rounded: f16 = @floatCast(value * inverse); + rotated[base + i] = @floatCast(rounded); + } + } +} + +/// CPU stream reader consumes MSB-first bits from little-endian int16 pairs, not the GPU indexed decoder. +const Stream = struct { + bytes: []const u8, + cursor: usize = 0, + pending: u32 = 0, + available: u6 = 0, + + fn take(s: *Stream, count: u5) u16 { + if (s.available == 0) { + s.pending = std.mem.readInt(u32, s.bytes[s.cursor..][0..4], .little); + s.cursor += 4; + s.available = 32; + } + const first: u5 = @intCast(@min(s.available, count)); + const head: u32 = s.pending >> @as(u5, @intCast(32 - @as(u6, first))); + s.pending <<= first; + s.available -= first; + if (first == count) return @intCast(head); + const rest = count - first; + s.pending = std.mem.readInt(u32, s.bytes[s.cursor..][0..4], .little); + s.cursor += 4; + const tail = s.pending >> @as(u5, @intCast(32 - @as(u6, rest))); + s.pending <<= rest; + s.available = 32 - @as(u6, rest); + return @intCast((head << rest) | tail); + } +}; + +/// Circular initial state uses final 16 stream bits; decode only a sixteen-column accumulator and live stream. +fn accumulateTile(comptime halves: u16, tile: []const u8, table: []const f16, x: []const f64, acc: *[16]f64) void { + @setFloatMode(.strict); + var stream: Stream = .{ .bytes = tile }; + var state: u16 = @truncate(std.mem.readInt(u32, tile[tile.len - 4 ..][0..4], .little)); + var p: usize = 0; + while (p < 256) : (p += 1) { + // Odd half-widths alternate floor(bits),ceil(bits), exactly as format.streamEnds specifies. + const advance: u4 = @intCast(halves / 2 + (if (halves % 2 == 1) p % 2 else 0)); + state = (state << advance) | stream.take(advance); + const lane = p / 8; + const j = p % 8; + const row = 2 * (lane % 4) + (j & 1) + 8 * ((j >> 1) & 1); + const col = lane / 4 + 8 * (j >> 2); + const weight: f64 = @floatCast(table[state]); + acc[col] += x[row] * weight; + } +} + +fn product(comptime halves: u16, matrix: exl3.Matrix, trellis: []const u8, table: []const f16, rotated: []const f64, sums: []f64) !void { + const ks = try matrix.kStride(); + const ns = matrix.nStride(); + const tile_bytes = @as(usize, halves) * 16; + @memset(sums, 0); + for (0..matrix.k / 16) |tk| { + const input = rotated[tk * 16 ..][0..16]; + for (0..matrix.n / 16) |tn| { + const at = tk * ks + tn * ns; + accumulateTile(halves, trellis[at..][0..tile_bytes], table, input, sums[tn * 16 ..][0..16]); + } + } +} + +fn project(matrix: exl3.Matrix, x: []const u16, output_len: usize, scratch: *Scratch) !void { + try matrix.validate(); + if (x.len != matrix.k or output_len != matrix.n) return error.InvalidReferenceProjectionLength; + if (matrix.k > scratch.rotated.len or matrix.n > scratch.sums.len) + return error.ReferenceProjectionScratchTooSmall; + const trellis = try borrowed(matrix.trellis, try matrix.storageBytes()); + const su = try borrowed(matrix.suh, try scaleBytes(matrix.k, matrix.suh_format)); + // Check every borrowed view before doing arithmetic, including the output-only scale and optional bias. + _ = try borrowed(matrix.svh, try scaleBytes(matrix.n, matrix.svh_format)); + if (matrix.bias) |bias| _ = try borrowed(bias, try std.math.mul(usize, matrix.n, 2)); + const rotated = scratch.rotated[0..matrix.k]; + const sums = scratch.sums[0..matrix.n]; + prepare(x, su, matrix.suh_format, rotated); + const book_index: usize = @intCast(@backingInt(matrix.codebook)); + const table = scratch.codebooks[book_index * states_count ..][0..states_count]; + switch (matrix.width.halves) { + inline 2...16 => |halves| try product(halves, matrix, trellis, table, rotated, sums), + else => unreachable, + } +} + +fn finish(comptime T: type, matrix: exl3.Matrix, y: []T, sums: []const f64) !void { + @setFloatMode(.strict); + const sv = try borrowed(matrix.svh, try scaleBytes(matrix.n, matrix.svh_format)); + const bias = if (matrix.bias) |view| try borrowed(view, try std.math.mul(usize, matrix.n, 2)) else null; + var base: usize = 0; + while (base < matrix.n) : (base += block_size) { + var values: [block_size]f32 = undefined; + // One FP64->FP32 boundary replaces GPU FP32 dot-product accumulation, not the output rotation. + for (&values, 0..) |*value, i| value.* = @floatCast(sums[base + i]); + hadamard(&values); + for (values, 0..) |value, i| { + var out: f32 = (value * inverse) * scale(sv, base + i, matrix.svh_format); + if (bias) |bytes| out += half(bytes, base + i); + y[base + i] = if (T == u16) bf16(out) else out; + } + } +} + +test "CPU circular streaming matches scalar format for every supported width and codebook" { + var scratch = try Scratch.init(std.testing.allocator, 128, 128); + defer scratch.deinit(); + var words: [128]i16 = undefined; + for (&words, 0..) |*word, i| word.* = @bitCast(@as(u16, @truncate(i * 40503 + 17329))); + var x: [16]f64 = undefined; + for (&x, 0..) |*value, i| value.* = @as(f64, @floatFromInt(i + 1)) * 0.0625; + var placement: [256]format.Placement = undefined; + format.tilePlacement(&placement); + inline for (2..17) |halves| { + const width: format.Width = .{ .halves = halves }; + const tile = words[0..width.tileWords()]; + var states: [256]u32 = undefined; + format.tileStates(width, tile, &states); + for ([_]format.Codebook{ .inst3, .mcg, .mul1 }, 0..) |book, bi| { + if (width.isHalf() and (book != .mul1 or halves > 7)) continue; + const table = scratch.codebooks[bi * states_count ..][0..states_count]; + var expected: [16]f64 = @splat(0); + for (states, placement) |state, where| { + const weight: f64 = @floatCast(format.codebookValue(book, @intCast(state))); + expected[where.col] += x[where.row] * weight; + } + var actual: [16]f64 = @splat(0); + accumulateTile(halves, std.mem.sliceAsBytes(tile), table, &x, &actual); + try std.testing.expectEqualSlices(f64, &expected, &actual); + } + } +} + +test "FP32 Hadamard then FP16 and BF16 boundaries are deterministic" { + var x: [128]u16 = @splat(0); + x[0] = 0x3f80; + var signs: [16]u8 = @splat(0); + signs[0] = 1; + var rotated: [128]f64 = undefined; + prepare(&x, &signs, .packed_signs, &rotated); + const rounded: f16 = @floatCast(-inverse); + for (rotated) |value| try std.testing.expectEqual(@as(f64, @floatCast(rounded)), value); + try std.testing.expectEqual(@as(u16, 0x3f80), bf16(@bitCast(@as(u32, 0x3f808000)))); + try std.testing.expectEqual(@as(u16, 0x3f82), bf16(@bitCast(@as(u32, 0x3f818000)))); + try std.testing.expectEqual(@as(u16, 0x7fc0), bf16(@bitCast(@as(u32, 0x7f800001)))); +} diff --git a/zig/tests/deepseek_v41_reference/qualify.zig b/zig/tests/deepseek_v41_reference/qualify.zig new file mode 100644 index 0000000000..e5190ee546 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/qualify.zig @@ -0,0 +1,97 @@ +//! Actual checkpoint teacher comparison: independent CPU arithmetic versus the resident GPU decode path. +const std = @import("std"); +const mtl = @import("metal"); +const Model = @import("tensorfold").deepseek_v41.Model; +const Reference = @import("reference.zig").Reference; +const nrms_limit: f64 = 0.01; + +pub const Result = struct { + tokens: usize, + max_absolute_error: f64, + rms_error: f64, + normalized_rms_error: f64, + greedy_disagreements: usize, + gpu_seconds: f64, + cpu_seconds: f64, + normalized_rms_limit: f64 = nrms_limit, + numeric_gate_passed: bool = false, + greedy_gate_passed: bool = false, +}; + +fn maximum(row: []const f32) !u32 { + var id: u32 = 0; + var best = -std.math.inf(f32); + for (row, 0..) |v, i| { + if (!std.math.isFinite(v)) return error.NonfiniteTeacherLogits; + if (v > best) { + best = v; + id = @intCast(i); + } + } + return id; +} + +pub fn run(gpa: std.mem.Allocator, io: std.Io, model: *Model, ids: []const u32, token_map: []const u8, dir: []const u8) !Result { + if (ids.len == 0 or ids.len > 16) return error.InvalidOracleWindow; + var reference = try Reference.init(gpa, model.cfg.text_config, &model.weights, &model.checkpoint, token_map, @intCast(ids.len)); + defer reference.deinit(); + const cpu = try gpa.alloc(f32, ids.len * 129280); + defer gpa.free(cpu); + const gpu = try gpa.alloc(f32, cpu.len); + defer gpa.free(gpu); + try model.reset(); + var result = Result{ .tokens = ids.len, .max_absolute_error = 0, .rms_error = 0, .normalized_rms_error = 0, .greedy_disagreements = 0, .gpu_seconds = 0, .cpu_seconds = 0 }; + var squared_error: f64 = 0; + var squared_reference: f64 = 0; + for (ids, 0..) |id, i| { + const actual = try model.forward(&.{id}, .all); + result.gpu_seconds += model.last_gpu_seconds; + @memcpy(gpu[i * 129280 ..][0..129280], actual); + const start = mtl.clock.seconds(); + const expected = try reference.step(id); + result.cpu_seconds += mtl.clock.seconds() - start; + @memcpy(cpu[i * 129280 ..][0..129280], expected); + if (try maximum(actual) != try maximum(expected)) result.greedy_disagreements += 1; + for (actual, expected) |a, b| { + if (!std.math.isFinite(a) or !std.math.isFinite(b)) return error.NonfiniteTeacherLogits; + const error_: f64 = @as(f64, a) - b; + result.max_absolute_error = @max(result.max_absolute_error, @abs(error_)); + squared_error += error_ * error_; + squared_reference += @as(f64, b) * b; + } + std.debug.print("oracle teacher {d}/{d}: GPU token={d}, CPU token={d}\n", .{ i + 1, ids.len, try maximum(actual), try maximum(expected) }); + } + result.rms_error = @sqrt(squared_error / @as(f64, @floatFromInt(cpu.len))); + result.normalized_rms_error = if (squared_reference == 0) result.rms_error else @sqrt(squared_error / squared_reference); + result.numeric_gate_passed = result.normalized_rms_error <= result.normalized_rms_limit; + result.greedy_gate_passed = result.greedy_disagreements == 0; + const cp = try std.fs.path.join(gpa, &.{ dir, "oracle-logits.f32" }); + defer gpa.free(cp); + const gp = try std.fs.path.join(gpa, &.{ dir, "gpu-logits.f32" }); + defer gpa.free(gp); + const report = try std.fs.path.join(gpa, &.{ dir, "oracle.json" }); + defer gpa.free(report); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = cp, .data = std.mem.sliceAsBytes(cpu) }); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = gp, .data = std.mem.sliceAsBytes(gpu) }); + const json = try std.json.Stringify.valueAlloc(gpa, result, .{}); + defer gpa.free(json); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = report, .data = json }); + std.debug.print("{s}\n", .{json}); + try enforceGate(result.normalized_rms_error, result.greedy_disagreements); + return result; +} + +fn enforceGate(normalized_rms_error: f64, greedy_disagreements: usize) !void { + if (!std.math.isFinite(normalized_rms_error)) return error.NonfiniteTeacherLogits; + if (normalized_rms_error > nrms_limit) return error.TeacherReferenceMismatch; + if (greedy_disagreements != 0) return error.TeacherGreedyMismatch; +} + +test "teacher numeric gate remains independent of greedy agreement" { + try std.testing.expectError(error.TeacherReferenceMismatch, enforceGate(0.04, 0)); + try std.testing.expectError(error.TeacherReferenceMismatch, enforceGate(0.010000001, 1)); + try std.testing.expectError(error.TeacherGreedyMismatch, enforceGate(0.01, 1)); + try enforceGate(0.01, 0); + try std.testing.expectError(error.NonfiniteTeacherLogits, enforceGate(std.math.nan(f64), 0)); + try std.testing.expectError(error.NonfiniteTeacherLogits, enforceGate(std.math.inf(f64), 0)); +} diff --git a/zig/tests/deepseek_v41_reference/reference.zig b/zig/tests/deepseek_v41_reference/reference.zig new file mode 100644 index 0000000000..23d811a532 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/reference.zig @@ -0,0 +1,176 @@ +//! Independent wide-sum qualification with explicit FP32/BF16 boundaries; not GPU self-reference or CUDA emulation. + +// Equations: pinned DeepSeek AI inference/{model,engram,kernel}.py; provenance in THIRD_PARTY_NOTICES.md. +const std = @import("std"); +const tf = @import("tensorfold"); +const weights = tf.deepseek_v41_weights; +const config = tf.deepseek_v41_config; +const cp = tf.checkpoint_metal; +const exl3 = tf.exl3_projection; +const projection = @import("projection.zig"); +const math = @import("math.zig"); +const attention = @import("attention_math.zig"); +const engram = @import("engram_math.zig"); +const Stage = tf.deepseek_v41_profiler.Stage; + +/// Local routing isolation also uses these source equations; no native values enter qualification. +pub const MoeRouting = struct { + logits: [384]f32 = undefined, + scores: [384]f32 = undefined, + ids: [6]u32 = undefined, + weights: [6]f32 = undefined, +}; + +pub fn moeSelect(moe_weights: weights.Moe, c: config.Text, logits: *const [384]f32, scores: *[384]f32, ids: *[6]u32, routing_weights: *[6]f32) void { + @setFloatMode(.strict); + for (logits, scores) |z, *score| { + const softplus = if (z > 20) z else std.math.log1p(@exp(z)); + score.* = @sqrt(softplus); + } + var used: [384]bool = @splat(false); + for (0..6) |pick| { + var best: usize = 0; + var high: f32 = -std.math.inf(f32); + for (0..384) |expert| { + const biased = scores[expert] + math.value(moe_weights.bias, expert); + if (!used[expert] and biased > high) { + best = expert; + high = biased; + } + } + ids[pick] = @intCast(best); + routing_weights[pick] = scores[best]; + used[best] = true; + } + var normalization: f64 = 0; + for (routing_weights) |v| normalization += v; + const denominator = @as(f32, @floatCast(normalization)) + 1e-20; + for (routing_weights) |*v| v.* = (v.* / denominator) * c.routed_scaling_factor; +} + +pub fn moeActivation(gate: []const u16, up: []const u16, route: f32, c: config.Text, activation: []u16) void { + @setFloatMode(.strict); + for (activation, gate, up) |*v, g_value, u_value| { + const g = @min(math.f(g_value), c.swiglu_limit); + const u = @max(-c.swiglu_limit, @min(math.f(u_value), c.swiglu_limit)); + v.* = math.b(route * ((g / (1 + @exp(-g))) * u)); + } +} + +pub const Reference = struct { + gpa: std.mem.Allocator, + c: config.Text, + w: *const weights.Model, + checkpoint: *const cp.Checkpoint, + capacity: u32, + position: u32 = 0, + projection_scratch: projection.Scratch, + attention_state: attention.State, + engram_state: engram.State, + logits: []f32, + h: [20480]u16 = undefined, + input: [5120]u16 = undefined, + output: [5120]u16 = undefined, + gate: [2304]u16 = undefined, + up: [2304]u16 = undefined, + activation: [2304]u16 = undefined, + expert_output: [5120]u16 = undefined, + moe_sum: [5120]f32 = undefined, + route_logits: [384]f32 = undefined, + route_scores: [384]f32 = undefined, + last_routes: [6]u32 = undefined, + last_routing: [6]f32 = undefined, + observer: ?Observer = null, + pub const Observer = struct { + ctx: *anyopaque, + read: *const fn (*anyopaque, usize, Stage, *const Reference) void, + }; + + /// Borrowed model/checkpoint/config/map must outlive this instance; capacity is the teacher sequence length. + pub fn init(gpa: std.mem.Allocator, c: config.Text, w: *const weights.Model, checkpoint: *const cp.Checkpoint, token_map: []const u8, capacity: u32) !Reference { + if (capacity == 0 or capacity > c.max_position_embeddings or capacity > attention.max_capacity) + return error.InvalidReferenceCapacity; + if (c.hidden_size != 5120 or c.num_hidden_layers != 40 or c.vocab_size != 129280 or + c.num_experts_per_tok != 6 or c.n_routed_experts != 384 or c.hc_mult != 4) + return error.UnsupportedReferenceArchitecture; + const engram_state = try engram.State.init(token_map); + var projection_scratch = try projection.Scratch.init(gpa, 8192, 129280); + errdefer projection_scratch.deinit(); + var attention_state = try attention.State.init(gpa, capacity); + errdefer attention_state.deinit(); + const logits = try gpa.alloc(f32, 129280); + return .{ .gpa = gpa, .c = c, .w = w, .checkpoint = checkpoint, .capacity = capacity, .projection_scratch = projection_scratch, .attention_state = attention_state, .engram_state = engram_state, .logits = logits }; + } + pub fn deinit(self: *Reference) void { + self.projection_scratch.deinit(); + self.attention_state.deinit(); + self.gpa.free(self.logits); + self.* = undefined; + } + /// Consume exactly one teacher token and return all actual checkpoint logits, borrowed until the next step. + pub fn step(self: *Reference, id: u32) ![]const f32 { + if (self.position >= self.capacity) return error.ContextFull; + const compressed = try self.engram_state.compressed(id); + for (0..5120) |d| { + const v = math.b(math.value(self.w.embed, @as(usize, id) * 5120 + d)); + for (0..4) |copy| self.h[copy * 5120 + d] = v; + } + var pre = [4]f32{ 1, 0, 0, 0 }; + for (&self.w.layers, 0..) |*layer, layer_id| { + if (layer.engram) |e| try self.engram_state.apply(e, if (layer_id == 1) 0 else 1, compressed, &self.h, self.c, &self.projection_scratch); + const attn_mix = math.mixes(&self.h, layer.hc_attn, self.c); + math.collapse(&self.h, pre, &self.input); + math.rms(&self.input, layer.attn_norm, self.c.rms_norm_eps, &self.input); + try self.attention_state.apply(layer.attn, layer_id, &self.input, self.position, self.c, &self.output, &self.projection_scratch); + math.expand(&self.h, &self.output, attn_mix); + if (self.observer) |observer| observer.read(observer.ctx, layer_id, .attention, self); + const ffn_mix = math.mixes(&self.h, layer.hc_ffn, self.c); + math.collapse(&self.h, attn_mix.pre, &self.input); + math.rms(&self.input, layer.ffn_norm, self.c.rms_norm_eps, &self.input); + try self.moe(layer.moe, layer_id); + math.expand(&self.h, &self.output, ffn_mix); + if (self.observer) |observer| observer.read(observer.ctx, layer_id, .moe, self); + pre = ffn_mix.pre; + } + math.collapse(&self.h, pre, &self.input); + math.rms(&self.input, self.w.norm, self.c.rms_norm_eps, &self.input); + try projection.applyF32(self.w.head, &self.input, self.logits, &self.projection_scratch); + for (self.logits) |v| if (!std.math.isFinite(v)) return error.NonFiniteReferenceLogits; + self.engram_state.advance(compressed); + self.position += 1; + return self.logits; + } + fn moe(self: *Reference, moe_weights: weights.Moe, layer: usize) !void { + @setFloatMode(.strict); + math.dense(moe_weights.gate, &self.input, &self.route_logits); + moeSelect(moe_weights, self.c, &self.route_logits, &self.route_scores, &self.last_routes, &self.last_routing); + @memset(&self.moe_sum, 0); + // Official MoE visits expert ids in ascending order, accumulates FP32, then adds shared expert last. + for (0..384) |expert| { + var route: ?f32 = null; + for (self.last_routes, self.last_routing) |selected, weight| + if (selected == expert) { + route = weight; + }; + const expert_route = route orelse continue; + var matrices: [3]exl3.Matrix = undefined; + var name: [128]u8 = undefined; + inline for (.{ "w1", "w3", "w2" }, 0..) |part, i| { + const prefix = try std.fmt.bufPrint(&name, "layers.{d}.ffn.experts.{d}.{s}", .{ layer, expert, part }); + matrices[i] = try weights.lookup(self.checkpoint, prefix, if (i == 2) 2304 else 5120, if (i == 2) 5120 else 2304); + } + try self.applyExpert(matrices, expert_route); + for (&self.moe_sum, self.expert_output) |*sum, v| sum.* += math.f(v); + } + try self.applyExpert(moe_weights.shared, 1); + for (&self.output, self.moe_sum, self.expert_output) |*v, routed, shared| + v.* = math.b(routed + math.f(shared)); + } + fn applyExpert(self: *Reference, matrices: [3]exl3.Matrix, route: f32) !void { + @setFloatMode(.strict); + try projection.apply(matrices[0], &self.input, &self.gate, &self.projection_scratch); + try projection.apply(matrices[1], &self.input, &self.up, &self.projection_scratch); + moeActivation(&self.gate, &self.up, route, self.c, &self.activation); + try projection.apply(matrices[2], &self.activation, &self.expert_output, &self.projection_scratch); + } +}; diff --git a/zig/tests/deepseek_v41_reference/rope_frequencies.json b/zig/tests/deepseek_v41_reference/rope_frequencies.json new file mode 100644 index 0000000000..2ffe1d1556 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/rope_frequencies.json @@ -0,0 +1,14 @@ +{ + "reference": "deepseek-ai/DeepSeek-V4.1-Flash", + "revision": "2cba9e42aa026125f3ed06c6d98c1db82f7ca027", + "source_sha256": "4e9ae23620edc8028ccc5d5fef552ab7fdc7dcd6f79608754fe9f67644056f65", + "torch_version": "2.14.1", + "extraction": "precompute_freqs_cis FP32 frequency vector before outer/polar; CPU", + "factor": 16, + "beta_fast": 32, + "beta_slow": 1, + "vectors": [ + {"theta": 10000, "original": 0, "bits": [1065353216,1061157137,1058010522,1054337179,1050798235,1047712803,1043732615,1040747895,1036831949,1033475085,1030116802,1026341397,1023510243,1019364175,1016180025,1012562930,1008981770,1005959600,1001931932,998911607,995049016,991652109,988353832,984533390,981668463,977573004,974350870,970790693,967166816,964208661,960132947,957076594]}, + {"theta": 160000, "original": 65536, "bits": [1065353216,1060112954,1056054303,1051098369,1046804782,1042117749,1037601972,1033169251,1028443341,1024251131,1019326489,1015361742,1010249151,1006366088,1001209182,997081655,991197921,985076960,979287478,973418702,966505391,959469944,952012877,943627103,933484300,917783504,913828774,908753896,904558591,899759068,895336240,890797133]} + ] +} diff --git a/zig/tests/deepseek_v41_reference/trace.zig b/zig/tests/deepseek_v41_reference/trace.zig new file mode 100644 index 0000000000..79cdef0fa4 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/trace.zig @@ -0,0 +1,123 @@ +//! Bounded investigation snapshots; file I/O and independent CPU work are not performance measurements. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const Model = tf.deepseek_v41.Model; +const Reference = @import("reference.zig").Reference; +const Stage = tf.deepseek_v41_profiler.Stage; +const widths = [_]usize{ 20480 * 2, 5120 * 2, 5120 * 2, 6 * 4, 6 * 4 }; +const stride = 61488; +const bytes = stride * 80; + +const Capture = struct { + buffer: mtl.Buffer, + pipeline: mtl.Pipeline, + cpu: []u8, + + fn gpu(ctx: *anyopaque, enc: mtl.ComputeEncoder, layer: u32, stage: Stage, model: *const Model) void { + const self: *Capture = @ptrCast(@alignCast(ctx)); + var destination: usize = (@as(usize, layer) * 2 + @intFromBool(stage == .moe)) * stride; + const ws = &model.workspace; + inline for (.{ ws.view(.stream), ws.view(.normalized), ws.view(.branch), ws.view(.route_ids), ws.view(.route_weights) }, widths, 0..) |source, count, field| { + if (stage == .moe or field < 3) { + enc.setPipeline(self.pipeline); + enc.setBuffer(source.buffer, source.offset, 0); + enc.setBuffer(self.buffer, destination, 1); + enc.setValue(@as(u32, @intCast(count / 4)), 2); + enc.dispatchThreads(.{ .width = count / 4 }, .{ .width = 256 }); + destination += count; + } + } + enc.barrier(); + } + fn reference(ctx: *anyopaque, layer: usize, stage: Stage, ref: *const Reference) void { + const self: *Capture = @ptrCast(@alignCast(ctx)); + const start = (layer * 2 + @intFromBool(stage == .moe)) * stride; + var destination: []u8 = self.cpu[start..][0..stride]; + inline for (.{ std.mem.sliceAsBytes(&ref.h), std.mem.sliceAsBytes(&ref.input), std.mem.sliceAsBytes(&ref.output), std.mem.sliceAsBytes(&ref.last_routes), std.mem.sliceAsBytes(&ref.last_routing) }, 0..) |source, field| { + if (stage == .moe or field < 3) { + @memcpy(destination[0..source.len], source); + destination = destination[source.len..]; + } + } + } +}; +fn normalizedError(a: []const u8, b: []const u8) !f64 { + var squared: f64 = 0; + var denominator: f64 = 0; + var i: usize = 0; + while (i < a.len) : (i += 2) { + const x: f32 = @bitCast(@as(u32, std.mem.readInt(u16, a[i..][0..2], .little)) << 16); + const y: f32 = @bitCast(@as(u32, std.mem.readInt(u16, b[i..][0..2], .little)) << 16); + if (!std.math.isFinite(x) or !std.math.isFinite(y)) return error.NonfiniteTraceValues; + squared += (@as(f64, x) - y) * (@as(f64, x) - y); + denominator += @as(f64, y) * y; + } + return if (denominator == 0) @sqrt(squared) else @sqrt(squared / denominator); +} + +pub fn run(gpa: std.mem.Allocator, io: std.Io, model: *Model, ids: []const u32, token_map: []const u8, dir: []const u8) !void { + if (ids.len == 0 or ids.len > 16) return error.InvalidOracleWindow; + const lib = try mtl.Library.fromSource(model.device, "#include \nusing namespace metal;\nkernel void trace_copy(const device uint* x [[buffer(0)]], device uint* y [[buffer(1)]], constant uint& n [[buffer(2)]], uint i [[thread_position_in_grid]]) { if(i 0.01 or !routes_match) std.debug.print("trace token={d} layer={d} stage={s} stream={d} input={d} branch={d} routes_match={}\n", .{ token, point / 2, if (point % 2 == 0) "attention" else "moe", stream_error, input_error, branch_error, routes_match }); + } + inline for (.{ "gpu", "cpu" }, .{ gpu, cpu }) |name, data| { + const path = try std.fmt.allocPrint(gpa, "{s}/trace-{d}-{s}.bin", .{ dir, token, name }); + defer gpa.free(path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = data }); + } + std.debug.print("trace captured token {d}/{d}, same_logits=true\n", .{ token + 1, ids.len }); + } + try report.appendSlice(gpa, "\n]\n"); + const path = try std.fs.path.join(gpa, &.{ dir, "layer-trace.json" }); + defer gpa.free(path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = report.items }); +} From 4e682615224fa3b4d1d42a38861b581795c3af65 Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Mon, 12 Oct 2026 21:13:58 -0700 Subject: [PATCH 03/33] THIRD_PARTY_NOTICES: the native DS4.1 reference follows DeepSeek-V4.1-Flash's MIT inference --- THIRD_PARTY_NOTICES.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 018947342d..5b34839762 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -217,6 +217,13 @@ TensorFold ships no model weights. The `z-lab/Qwen3.8-27B-DFlash2` model card st The optional `incoai/GLM-5.3-Flash-DFlash2` model card states CC BY-NC-ND 4.0, for non-commercial use without derivatives; only the Python engine reads that drafter. Each checkpoint keeps its own license. +The native `zig/src/families/deepseek_v41/` implementation and `zig/tests/deepseek_v41_reference/` +follow [DeepSeek-V4.1-Flash's official inference](https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/tree/2cba9e42aa026125f3ed06c6d98c1db82f7ca027/inference), +MIT License, Copyright (c) 2023 DeepSeek: the reference's equations, the Engram normalization, +PCG64 multipliers and prime bucket layout, and the pinned chat template come from that revision. +The EXL3 checkpoint reads through the ExLlamaV3 format covered above; no captured CUDA/Triton +kernel assets, full model shards or binary traces are bundled. + ## Structured output: libtfgrammar `lib/libtfgrammar` in the release archives is `zig/src/grammar/tf_grammar.cc` compiled with the C++ sources of From 74b7541169d2fc8c46cd241b10874de6186cb39f Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Tue, 13 Oct 2026 07:47:38 -0700 Subject: [PATCH 04/33] DeepSeek V4.1 load: the router gate and bias ride the pack's dtypes The released checkpoints store the gate f16 and its bias f16; rank-split repacks store the weight bf16 and the bias f32. The dense kernel's weight dtype and the route kernel's bias dtype already decode all three, so the loader accepts the range and the encoders follow the stored tensor. --- zig/src/families/deepseek_v41/model.zig | 376 +++++++++++++++++++++- zig/src/families/deepseek_v41/weights.zig | 28 +- 2 files changed, 396 insertions(+), 8 deletions(-) diff --git a/zig/src/families/deepseek_v41/model.zig b/zig/src/families/deepseek_v41/model.zig index 130f288657..2a23801927 100644 --- a/zig/src/families/deepseek_v41/model.zig +++ b/zig/src/families/deepseek_v41/model.zig @@ -1,7 +1,371 @@ -//! Backend-neutral checkpoint owners. CUDA rank slicing is an explicit capability adapter. -pub const Config = @import("config.zig").Config; -pub const Plan = @import("plan.zig").CheckpointPlan; -pub const CudaRankPlan = @import("plan.zig").Plan; -pub const Image = @import("named.zig").Named; +//! Plain V4.1 execution over resident EXL3 weights and source-owned packed caches. +const std = @import("std"); +const mtl = @import("metal"); +const checkpoint = @import("../../core/checkpoint_metal.zig"); +const exl3 = @import("../../core/exl3_projection.zig"); +const config = @import("config.zig"); +const load = @import("load.zig"); +const weights = @import("weights.zig"); +const work = @import("workspace.zig"); +pub const attention = @import("attention.zig"); +const kv = @import("kv.zig"); +pub const ops = @import("ops.zig"); +pub const mhc = @import("mhc.zig"); +const route = @import("route.zig"); +const engram = @import("engram.zig"); +const View = @import("view.zig").View; +const gpu_counters = @import("profiler.zig"); +const expert_groups = @import("../../core/expert_groups.zig"); -pub const KvState = @import("kv_state.zig").State; +pub const prefill_step = 256; +pub const Logits = enum { none, last, all }; +pub const Options = struct { + dir: []const u8, + table_dir: ?[]const u8 = null, + token_map: []const u8, + context: u32 = 0, + chunk_rows: u32 = prefill_step, +}; + +const Pipelines = struct { + projection: exl3.Pipelines, + ops: ops.Pipelines, + mhc: mhc.Pipelines, + route: route.Pipelines, + engram: engram.Pipelines, + attention: attention.Pipelines, + groups: expert_groups.Pipelines, + + fn init(device: mtl.Device, capacity: u32, c: config.Text) !Pipelines { + var ep = try exl3.Pipelines.load(device); + errdefer ep.deinit(); + var op = try ops.Pipelines.load(device); + errdefer op.deinit(); + var hp = try mhc.Pipelines.load(device); + errdefer hp.deinit(); + var rp = try route.Pipelines.load(device); + errdefer rp.deinit(); + var gp = try engram.Pipelines.load(device); + errdefer gp.deinit(); + var groups = try expert_groups.Pipelines.load(device); + errdefer groups.deinit(); + const ap = try attention.Pipelines.load(device, capacity, c); + return .{ .projection = ep, .ops = op, .mhc = hp, .route = rp, .engram = gp, .attention = ap, .groups = groups }; + } + + fn deinit(p: *Pipelines) void { + p.attention.deinit(); + p.engram.deinit(); + p.route.deinit(); + p.groups.deinit(); + p.mhc.deinit(); + p.ops.deinit(); + p.projection.deinit(); + } +}; + +/// Serialized unique owner; submissions finish before return/release, cancellation is between bounded chunks. +pub const Model = struct { + gpa: std.mem.Allocator, + device: mtl.Device, + queue: mtl.Queue, + residency: mtl.ResidencySet, + checkpoint: checkpoint.Checkpoint, + weights: weights.Model, + cfg: config.Config, + cfg_arena: std.heap.ArenaAllocator, + cache: kv.cache.State, + workspace: work.Workspace, + attention_workspace: attention.Workspace, + pipelines: Pipelines, + admitted_bytes: usize, + last_gpu_seconds: f64 = 0, + logits_rows: usize = 0, + valid: bool = true, + profile: ?*gpu_counters.Recorder = null, + grouped_prefill: bool = true, + observer: ?Observer = null, + pub const Observer = struct { + ctx: *anyopaque, + encode: *const fn (*anyopaque, mtl.ComputeEncoder, u32, gpu_counters.Stage, *const Model) void, + }; + + pub fn open(gpa: std.mem.Allocator, io: std.Io, o: Options) !Model { + var arena = std.heap.ArenaAllocator.init(gpa); + errdefer arena.deinit(); + const c = try config.Config.load(arena.allocator(), io, o.dir); + const capacity = try c.context(o.context); + const table_dir = o.table_dir orelse c.engram_table_dir orelse return error.MissingEngramTables; + var plan = try load.Plan.open(gpa, io, o.dir, table_dir); + defer plan.deinit(); + try weights.validate(&plan); + const map_blob = try std.Io.Dir.cwd().readFileAlloc(io, o.token_map, arena.allocator(), .limited(2 << 20)); + const map = try engram.hash.TokenMap.parse(map_blob); + const wp = try work.Plan.init(o.chunk_rows); + const kp = kv.cache.Plan{ .capacity = capacity, .layers = 40 }; + const workspace_bytes = try std.math.add(usize, wp.bytes, attention.Workspace.bytes(o.chunk_rows)); + const cache_bytes = try std.math.add(usize, kp.residentBytes(), weights.sizeofDescriptors()); + const execution_bytes = try std.math.add(usize, workspace_bytes, cache_bytes); + const rotary_bytes = attention.Pipelines.residentBytes(capacity, c.text_config); + const needed = try std.math.add(usize, plan.bytes, try std.math.add(usize, execution_bytes, rotary_bytes)); + const device = try mtl.Device.init(); + errdefer device.deinit(); + const margin: u64 = 8 << 30; + const available = device.maxWorkingSet() -| device.allocated() -| margin; + if (needed > available) { + std.log.err("full residency needs {d} bytes plus {d} margin; Metal admission allows {d}", .{ needed, margin, available }); + return error.InsufficientResidentMemory; + } + const max_buffer = mtl.objc.msg(u64, device.id, "maxBufferLength", .{}); + for (plan.shards.items) |shard| if (shard.end - shard.begin > max_buffer) return error.ShardExceedsMetalBufferLimit; + std.log.info("full resident admission: weights={d}, total={d}, context={d}, headroom={d}", .{ plan.bytes, needed, capacity, available - needed }); + var pipelines = try Pipelines.init(device, capacity, c.text_config); + errdefer pipelines.deinit(); + const queue = try device.queue(); + errdefer queue.deinit(); + const residency = try device.residencySet(plan.shards.items.len + 128); + errdefer residency.deinit(); + var cp = try plan.materialize(gpa, device, residency, queue); + errdefer cp.deinit(); + var bound = try weights.bind(device, &cp); + errdefer bound.deinit(); + var cache = try kv.cache.State.init(device, kp); + errdefer cache.deinit(); + var workspace = try work.Workspace.init(device, wp, map.bytes); + errdefer workspace.deinit(); + var aw = try attention.Workspace.init(device, o.chunk_rows); + errdefer aw.deinit(); + bound.registerResidency(residency); + cache.registerResidency(residency); + residency.add(workspace.buffer); + aw.registerResidency(residency); + pipelines.attention.registerResidency(residency); + residency.commit(); + residency.requestResidency(); + queue.addResidencySet(residency); + return .{ .gpa = gpa, .device = device, .queue = queue, .residency = residency, .checkpoint = cp, .weights = bound, .cfg = c, .cfg_arena = arena, .cache = cache, .workspace = workspace, .attention_workspace = aw, .pipelines = pipelines, .admitted_bytes = needed }; + } + + pub fn deinit(m: *Model) void { + m.queue.removeResidencySet(m.residency); + m.residency.deinit(); + m.attention_workspace.deinit(); + m.workspace.deinit(); + m.cache.deinit(); + m.weights.deinit(); + m.checkpoint.deinit(); + m.pipelines.deinit(); + m.queue.deinit(); + m.device.deinit(); + m.cfg_arena.deinit(); + } + + pub fn reset(m: *Model) !void { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + m.valid = false; + m.logits_rows = 0; + m.cache.position = 0; + m.attention_workspace.selection_layer = null; + m.attention_workspace.selection_rows = 0; + m.attention_workspace.candidate_rows = 0; + m.last_gpu_seconds = 0; + m.cache.saved_position = null; + @memset(m.cache.live.contents()[0..m.cache.plan.mutableBytes()], 0); + const cb = m.queue.commandBufferUnretained(); + const enc = cb.compute(.serial); + engram.encodeReset(enc, &m.pipelines.engram, m.workspace.view(.history)); + enc.end(); + cb.commit(); + cb.wait(); + try completed(cb); + m.valid = true; + m.logits_rows = 0; + } + + /// One command buffer per chunk; no operator waits or reads routes back to the CPU. + pub fn forward(m: *Model, ids: []const u32, logits: Logits) ![]const f32 { + if (!m.valid) return error.InvalidModelState; + if (ids.len == 0 or ids.len > m.workspace.plan.rows) return error.InvalidPrefillChunk; + if (m.cache.position > m.cache.plan.capacity or + ids.len > m.cache.plan.capacity - m.cache.position) return error.ContextFull; + for (ids) |id| if (id >= 129280) return error.UnsupportedToken; + m.logits_rows = 0; + @memcpy(m.workspace.tokenIds(ids.len), ids); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const rows: u32 = @intCast(ids.len); + const e = m.queue.commandBufferUnretained(); + var enc = e.compute(.serial); + for (m.checkpoint.shards.items) |shard| enc.useResource(shard.buffer, mtl.ResourceUsage.read); + var encoding = true; + var record_started = false; + errdefer { + if (m.profile) |recorder| { + if (record_started) { + if (encoding) recorder.cancelEncoding(&enc) catch {} else recorder.discardAfterCompletion() catch {}; + } else if (encoding) enc.end(); + } else if (encoding) enc.end(); + } + if (m.profile) |recorder| { + try recorder.beginCommand(e); + record_started = true; + } + try m.profileSpan(&enc, .other); + const p = &m.pipelines; + const ws = &m.workspace; + for (engram.hash.layers, 0..) |meta, index| { + try engram.encodeHash(enc, &p.engram, meta, ws.view(.ids), ws.view(.token_map), null, ws.view(.history), ws.view(if (index == 0) .engram_ids_0 else .engram_ids_1), rows); + } + try engram.encodeCommit(enc, &p.engram, ws.view(.ids), ws.view(.token_map), null, ws.view(.history), rows); + ops.encodeEmbedding(enc, &p.ops, ws.view(.ids), tensor(m.weights.embed), ws.view(.collapsed), .{ .rows = rows, .width = 5120, .table_stride = 5120, .y_stride = 5120, .copies = 1 }); + enc.barrier(); + mhc.encodeExpand(enc, &p.mhc, ws.view(.collapsed), ws.view(.stream), ws.view(.previous_pre), .{ .rows = rows, .width = 5120, .x_stride = 5120, .stream_stride = 20480 }); + enc.barrier(); + for (&m.weights.layers, 0..) |*layer, index| { + if (layer.engram) |g| { + try m.profileSpan(&enc, .engram); + try engram.encodeGather(enc, &p.engram, .{ .weight = tensor(g.table), .scale = tensor(g.scales), .rows = @intCast(g.table.shape[0]) }, ws.view(if (index == 1) .engram_ids_0 else .engram_ids_1), ws.view(.engram_gather), rows, 6144); + try m.project(enc, g.wkv, ws.view(.engram_gather), ws.view(.engram_kv), rows, 6144, 25600, .bf16); + try engram.encodeGate(enc, &p.engram, .{ .q_weight = tensor(g.q_weight), .k_weight = tensor(g.k_weight), .dim = 5120, .copies = 4, .hidden_stride = 20480, .kv_stride = 25600, .output_stride = 20480, .eps = m.cfg.text_config.rms_norm_eps }, ws.view(.stream), ws.view(.engram_kv), null, ws.view(.stream), rows); + } + try m.profileSpan(&enc, .mhc); + m.mixes(enc, layer.hc_attn, ws.view(.attention_pre), rows); + mhc.encodeCollapse(enc, &p.mhc, ws.view(.stream), ws.view(.previous_pre), ws.view(.collapsed), .{ .rows = rows, .width = 5120, .stream_stride = 20480, .y_stride = 5120 }); + enc.barrier(); + m.normalize(enc, ws.view(.collapsed), layer.attn_norm, ws.view(.normalized), rows); + try m.profileSpan(&enc, .attention); + try attention.encode(&p.attention, enc, &layer.attn, &m.cache, &m.attention_workspace, ws.view(.normalized), ws.view(.branch), &p.projection, ws.projection(), @intCast(index), rows, m.cfg.text_config); + enc.barrier(); + try m.profileSpan(&enc, .mhc); + m.post(enc, rows); + if (m.observer) |observer| observer.encode(observer.ctx, enc, @intCast(index), .attention, m); + m.mixes(enc, layer.hc_ffn, ws.view(.previous_pre), rows); + mhc.encodeCollapse(enc, &p.mhc, ws.view(.stream), ws.view(.attention_pre), ws.view(.collapsed), .{ .rows = rows, .width = 5120, .stream_stride = 20480, .y_stride = 5120 }); + enc.barrier(); + m.normalize(enc, ws.view(.collapsed), layer.ffn_norm, ws.view(.normalized), rows); + try m.profileSpan(&enc, .moe); + try m.moe(enc, &layer.moe, rows); + try m.profileSpan(&enc, .mhc); + m.post(enc, rows); + if (m.observer) |observer| observer.encode(observer.ctx, enc, @intCast(index), .moe, m); + } + if (logits != .none) { + try m.profileSpan(&enc, .head); + const output_rows: u32 = if (logits == .all) rows else 1; + const first: usize = if (logits == .all) 0 else rows - 1; + mhc.encodeCollapse(enc, &p.mhc, ws.view(.stream).shifted(first * 20480 * 2), ws.view(.previous_pre).shifted(first * 4 * 4), ws.view(.collapsed), .{ .rows = output_rows, .width = 5120, .stream_stride = 20480, .y_stride = 5120 }); + enc.barrier(); + m.normalize(enc, ws.view(.collapsed), m.weights.norm, ws.view(.normalized), output_rows); + try m.project(enc, m.weights.head, ws.view(.normalized), ws.view(.logits), output_rows, 5120, 129280, .f32); + } + if (m.profile) |recorder| { + if (recorder.activeSpan()) try recorder.endSpan(&enc); + try recorder.finishEncoding(&enc); + } else enc.end(); + encoding = false; + m.valid = false; + e.commit(); + e.wait(); + try completed(e); + if (m.profile) |recorder| _ = try recorder.readAfterCompletion(); + try m.cache.append(rows); + m.last_gpu_seconds = e.gpuSeconds(); + m.valid = true; + m.logits_rows = if (logits == .all) rows else if (logits == .last) 1 else 0; + return ws.logits(m.logits_rows); + } + + fn profileSpan(m: *Model, encoder: *mtl.ComputeEncoder, stage: gpu_counters.Stage) !void { + const recorder = m.profile orelse return; + if (!recorder.enabled()) return; + if (recorder.activeSpan()) try recorder.endSpan(encoder); + try recorder.beginSpan(encoder, stage); + for (m.checkpoint.shards.items) |shard| encoder.useResource(shard.buffer, mtl.ResourceUsage.read); + } + + fn normalize(m: *Model, enc: mtl.ComputeEncoder, x: View, w: checkpoint.Tensor, y: View, rows: u32) void { + ops.encodeRms(enc, &m.pipelines.ops, x, tensor(w), y, .{ .rows = rows, .width = 5120, .x_stride = 5120, .y_stride = 5120, .eps = m.cfg.text_config.rms_norm_eps }); + enc.barrier(); + } + + fn mixes(m: *Model, enc: mtl.ComputeEncoder, w: weights.Hc, pre: View, rows: u32) void { + const ws = &m.workspace; + mhc.encodeRms(enc, &m.pipelines.mhc, ws.view(.stream), ws.view(.inv), .{ .rows = rows, .width = 5120, .stream_stride = 20480, .eps = m.cfg.text_config.rms_norm_eps }); + ops.encodeDense(enc, &m.pipelines.ops, ws.view(.stream), tensor(w.projection), ws.view(.mixes), .{ .rows = rows, .k = 20480, .n = 24, .x_stride = 20480, .w_stride = 20480, .y_stride = 24, .w_dtype = .f32, .y_dtype = .f32 }); + enc.barrier(); + mhc.encodeSplit(enc, &m.pipelines.mhc, ws.view(.mixes), ws.view(.inv), tensor(w.scale), tensor(w.base), pre, ws.view(.post), ws.view(.comb), .{ .rows = rows, .eps = m.cfg.text_config.hc_eps }); + enc.barrier(); + } + + fn post(m: *Model, enc: mtl.ComputeEncoder, rows: u32) void { + const ws = &m.workspace; + mhc.encodePost(enc, &m.pipelines.mhc, ws.view(.branch), ws.view(.stream), ws.view(.post), ws.view(.comb), ws.view(.stream), .{ .rows = rows, .width = 5120, .branch_stride = 5120, .residual_stride = 20480, .y_stride = 20480 }); + enc.barrier(); + } + + fn project(m: *Model, enc: mtl.ComputeEncoder, matrix: exl3.Matrix, x: View, y: View, rows: u32, xs: usize, ys: usize, output: exl3.Output) !void { + try exl3.encode(&m.pipelines.projection, enc, matrix, convert(x), convert(y), m.workspace.projection(), .{ .count = rows, .x_stride = xs, .y_stride = ys, .output = output }); + enc.barrier(); + } + + fn moe(m: *Model, enc: mtl.ComputeEncoder, w: *const weights.Moe, rows: u32) !void { + const ws = &m.workspace; + const p = &m.pipelines; + ops.encodeDense(enc, &p.ops, ws.view(.normalized), tensor(w.gate), ws.view(.router_logits), .{ .rows = rows, .k = 5120, .n = 384, .x_stride = 5120, .w_stride = 5120, .y_stride = 384, .w_dtype = switch (w.gate.dtype) { + .bf16 => .bf16, + else => .f16, + }, .y_dtype = .f32 }); + enc.barrier(); + route.encode(enc, &p.route, ws.view(.router_logits), tensor(w.bias), ws.view(.route_ids), ws.view(.route_weights), .{ .rows = rows, .bias_dtype = switch (w.bias.dtype) { + .f32 => .f32, + .f16 => .f16, + .bf16 => .bf16, + else => .f16, + } }); + enc.barrier(); + const grouped: ?expert_groups.Scratch = if (m.grouped_prefill and rows >= 8) try ws.grouping(rows) else null; + var routed_rows = exl3.Rows{ .count = @as(usize, rows) * 6, .x_stride = 5120, .y_stride = 2304, .input_rows = rows, .output_rows = @as(usize, rows) * 6, .routes = .{ .ids = convert(ws.view(.route_ids)), .repeat = 6 } }; + if (grouped) |g| { + try expert_groups.encode(enc, &p.groups, convert(ws.view(.route_ids)), g, 6); + routed_rows.count = g.plan.padded; + routed_rows.input_ids = g.view(.token_ids); + routed_rows.output_ids = g.view(.slot_ids); + routed_rows.routes = .{ .ids = g.view(.expert_ids) }; + routed_rows.grouped = true; + } + for (0..2) |i| { + try exl3.encodeIndexed(&p.projection, enc, .{ .descriptors = .{ .buffer = w.descriptors[i] }, .count = 384, .k = 5120, .n = 2304 }, convert(ws.view(.normalized)), convert(ws.view(if (i == 0) .gate else .up)), ws.projection(), routed_rows); + enc.barrier(); + } + ops.encodeSwiGLU(enc, &p.ops, ws.view(.gate), ws.view(.up), ws.view(.route_weights), ws.view(.activation), .{ .rows = rows * 6, .width = 2304, .input_stride = 2304, .output_stride = 2304, .weighted = true }); + enc.barrier(); + routed_rows.x_stride = 2304; + routed_rows.y_stride = 5120; + routed_rows.routes.?.repeat = 1; + routed_rows.input_rows = @as(usize, rows) * 6; + if (grouped) |g| routed_rows.input_ids = g.view(.slot_ids); + try exl3.encodeIndexed(&p.projection, enc, .{ .descriptors = .{ .buffer = w.descriptors[2] }, .count = 384, .k = 2304, .n = 5120 }, convert(ws.view(.activation)), convert(ws.view(.routed)), ws.projection(), routed_rows); + enc.barrier(); + try m.project(enc, w.shared[0], ws.view(.normalized), ws.view(.shared_gate), rows, 5120, 2304, .bf16); + try m.project(enc, w.shared[1], ws.view(.normalized), ws.view(.shared_up), rows, 5120, 2304, .bf16); + ops.encodeSwiGLU(enc, &p.ops, ws.view(.shared_gate), ws.view(.shared_up), ws.view(.route_weights), ws.view(.shared_activation), .{ .rows = rows, .width = 2304, .input_stride = 2304, .output_stride = 2304 }); + enc.barrier(); + try m.project(enc, w.shared[2], ws.view(.shared_activation), ws.view(.shared_down), rows, 2304, 5120, .bf16); + ops.encodeCombine(enc, &p.ops, ws.view(.routed), ws.view(.shared_down), ws.view(.route_ids), ws.view(.branch), .{ .rows = rows, .width = 5120, .routed_row_stride = 6 * 5120, .slot_stride = 5120, .shared_stride = 5120, .y_stride = 5120 }); + enc.barrier(); + } +}; + +fn tensor(t: checkpoint.Tensor) View { + return .{ .buffer = t.buffer, .offset = t.offset }; +} +fn convert(v: View) exl3.View { + return .{ .buffer = v.buffer, .offset = v.offset }; +} +fn completed(cb: mtl.CommandBuffer) !void { + if (cb.failure()) |message| { + std.log.err("DeepSeek V4.1 GPU execution failed: {s}", .{message}); + return error.GpuExecutionFailed; + } +} diff --git a/zig/src/families/deepseek_v41/weights.zig b/zig/src/families/deepseek_v41/weights.zig index 10afd22356..c2670815d3 100644 --- a/zig/src/families/deepseek_v41/weights.zig +++ b/zig/src/families/deepseek_v41/weights.zig @@ -250,8 +250,32 @@ fn populate(comptime binding: bool, source: anytype, model: ?*Model) !void { } } const moe: ?*Moe = if (binding) &out.?.moe else null; - try plain(binding, source, try named(&name, "layers.{d}.ffn.gate.weight", .{layer}), .f16, &.{ 384, 5120 }, if (binding) &moe.?.gate else null); - try plain(binding, source, try named(&name, "layers.{d}.ffn.gate.bias", .{layer}), .f16, &.{384}, if (binding) &moe.?.bias else null); + // The released checkpoints store the router gate f16; rank-split repacks store it bf16. + if (binding) { + const tensor = try source.get(try named(&name, "layers.{d}.ffn.gate.weight", .{layer})); + if ((tensor.dtype != .f16 and tensor.dtype != .bf16) or tensor.rank != 2 or + tensor.shape[0] != 384 or tensor.shape[1] != 5120) + { + std.log.err("unsupported router gate {s}: {s} {any}", .{ name, @tagName(tensor.dtype), tensor.shape[0..tensor.rank] }); + return error.UnsupportedTensor; + } + moe.?.gate = tensor; + } else { + const tensor = try source.get(try named(&name, "layers.{d}.ffn.gate.weight", .{layer})); + if ((tensor.dtype != .f16 and tensor.dtype != .bf16) or tensor.rank != 2 or + tensor.shape[0] != 384 or tensor.shape[1] != 5120) return error.UnsupportedTensor; + } + // The bias rides the same three dtypes the route kernel decodes (f32, f16, bf16). + if (binding) { + const tensor = try source.get(try named(&name, "layers.{d}.ffn.gate.bias", .{layer})); + if ((tensor.dtype != .f16 and tensor.dtype != .bf16 and tensor.dtype != .f32) or tensor.rank != 1 or tensor.shape[0] != 384) + return error.UnsupportedTensor; + moe.?.bias = tensor; + } else { + const tensor = try source.get(try named(&name, "layers.{d}.ffn.gate.bias", .{layer})); + if ((tensor.dtype != .f16 and tensor.dtype != .bf16 and tensor.dtype != .f32) or tensor.rank != 1 or tensor.shape[0] != 384) + return error.UnsupportedTensor; + } inline for (.{ "w1", "w3", "w2" }, 0..) |part, projection| { const k: usize = if (projection == 2) 2304 else 5120; const n: usize = if (projection == 2) 5120 else 2304; From 86b6eec6db91da39530f64b1f2f5ca9d5adb2bce Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Tue, 13 Oct 2026 07:51:07 -0700 Subject: [PATCH 05/33] DeepSeek V4.1 load: the Engram gate weights ride f32 or bf16 The gate kernel decodes its q/k weights by dtype, the Gate descriptor carries it, and the loader accepts the rank-split packs' bf16 projections beside the released f32 ones. --- zig/src/families/deepseek_v41/engram.zig | 6 +++++- zig/src/families/deepseek_v41/model.zig | 5 ++++- zig/src/families/deepseek_v41/weights.zig | 23 +++++++++++++++++++++-- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/zig/src/families/deepseek_v41/engram.zig b/zig/src/families/deepseek_v41/engram.zig index 0f5a0287c2..867cc78d39 100644 --- a/zig/src/families/deepseek_v41/engram.zig +++ b/zig/src/families/deepseek_v41/engram.zig @@ -3,6 +3,7 @@ const std = @import("std"); const mtl = @import("metal"); const sources = @import("kernel_sources"); pub const View = @import("view.zig").View; +const Dtype = @import("ops.zig").Dtype; pub const hash = @import("engram_hash.zig"); pub const head_dim = 256; @@ -29,6 +30,8 @@ pub const Table = struct { pub const Gate = struct { q_weight: View, k_weight: View, + /// The released checkpoints store these f32; rank-split repacks store bf16. + weight_dtype: Dtype = .f32, dim: u32, copies: u32, hidden_stride: usize, @@ -97,6 +100,7 @@ const GateArgs = extern struct { dim: u32, copies: u32, masked: u32, + weight_dtype: u32, eps: f32, }; @@ -193,7 +197,7 @@ pub fn encodeGate(e: mtl.ComputeEncoder, p: *const Pipelines, gate: Gate, hidden bind(e, gate.k_weight, 3); bind(e, mask orelse hidden, 4); bind(e, out, 5); - e.setValue(GateArgs{ .hidden_stride = gate.hidden_stride, .kv_stride = gate.kv_stride, .output_stride = gate.output_stride, .dim = gate.dim, .copies = gate.copies, .masked = @intFromBool(mask != null), .eps = gate.eps }, 6); + e.setValue(GateArgs{ .hidden_stride = gate.hidden_stride, .kv_stride = gate.kv_stride, .output_stride = gate.output_stride, .dim = gate.dim, .copies = gate.copies, .masked = @intFromBool(mask != null), .weight_dtype = @backingInt(gate.weight_dtype), .eps = gate.eps }, 6); e.dispatchGroups(.{ .width = tokens, .height = gate.copies }, .{ .width = 256 }); e.barrier(); } diff --git a/zig/src/families/deepseek_v41/model.zig b/zig/src/families/deepseek_v41/model.zig index 2a23801927..4ad847bc45 100644 --- a/zig/src/families/deepseek_v41/model.zig +++ b/zig/src/families/deepseek_v41/model.zig @@ -227,7 +227,10 @@ pub const Model = struct { try m.profileSpan(&enc, .engram); try engram.encodeGather(enc, &p.engram, .{ .weight = tensor(g.table), .scale = tensor(g.scales), .rows = @intCast(g.table.shape[0]) }, ws.view(if (index == 1) .engram_ids_0 else .engram_ids_1), ws.view(.engram_gather), rows, 6144); try m.project(enc, g.wkv, ws.view(.engram_gather), ws.view(.engram_kv), rows, 6144, 25600, .bf16); - try engram.encodeGate(enc, &p.engram, .{ .q_weight = tensor(g.q_weight), .k_weight = tensor(g.k_weight), .dim = 5120, .copies = 4, .hidden_stride = 20480, .kv_stride = 25600, .output_stride = 20480, .eps = m.cfg.text_config.rms_norm_eps }, ws.view(.stream), ws.view(.engram_kv), null, ws.view(.stream), rows); + try engram.encodeGate(enc, &p.engram, .{ .q_weight = tensor(g.q_weight), .k_weight = tensor(g.k_weight), .dim = 5120, .copies = 4, .weight_dtype = switch (g.q_weight.dtype) { + .bf16 => .bf16, + else => .f32, + }, .hidden_stride = 20480, .kv_stride = 25600, .output_stride = 20480, .eps = m.cfg.text_config.rms_norm_eps }, ws.view(.stream), ws.view(.engram_kv), null, ws.view(.stream), rows); } try m.profileSpan(&enc, .mhc); m.mixes(enc, layer.hc_attn, ws.view(.attention_pre), rows); diff --git a/zig/src/families/deepseek_v41/weights.zig b/zig/src/families/deepseek_v41/weights.zig index c2670815d3..dc8d508404 100644 --- a/zig/src/families/deepseek_v41/weights.zig +++ b/zig/src/families/deepseek_v41/weights.zig @@ -106,6 +106,25 @@ fn named(buffer: []u8, comptime fmt: []const u8, args: anytype) ![]const u8 { return std.fmt.bufPrint(buffer, fmt, args); } +/// A plain tensor the encoders decode as f32, f16 or bf16; the shape must match exactly. +fn plainEither(comptime binding: bool, source: anytype, name: []const u8, shape: []const usize, destination: ?*Tensor) !void { + const tensor = try source.get(name); + if ((tensor.dtype != .f32 and tensor.dtype != .f16 and tensor.dtype != .bf16) or tensor.rank != shape.len or + !std.mem.eql(usize, tensor.shape[0..tensor.rank], shape)) + { + std.log.err("unsupported tensor {s}: {s} {any}", .{ name, @tagName(tensor.dtype), tensor.shape[0..tensor.rank] }); + return error.UnsupportedTensor; + } + if (binding) { + var count: usize = 1; + for (shape) |dim| count = try std.math.mul(usize, count, dim); + if (tensor.bytes != try std.math.mul(usize, count, tensor.dtype.size())) return error.UnsupportedTensor; + if (tensor.offset % tensor.dtype.size() != 0 or tensor.offset > tensor.buffer.length() or + tensor.bytes > tensor.buffer.length() - tensor.offset) return error.InvalidTensorView; + destination.?.* = tensor; + } else if (tensor.begin % 4 != 0) return error.InvalidTensorView; +} + fn plain(comptime binding: bool, source: anytype, name: []const u8, dtype: st.DType, shape: []const usize, destination: ?*Tensor) !void { const tensor = try source.get(name); if (tensor.dtype != dtype or tensor.rank != shape.len or @@ -289,8 +308,8 @@ fn populate(comptime binding: bool, source: anytype, model: ?*Model) !void { const rows: usize = if (layer == 1) 384006168 else 384016682; try plain(binding, source, try named(&name, "layers.{d}.engram.embed.weight", .{layer}), .f8_e4m3, &.{ rows, 256 }, if (binding) &e.?.table else null); try plain(binding, source, try named(&name, "layers.{d}.engram.embed.scale", .{layer}), .f8_e8m0, &.{ rows, 8 }, if (binding) &e.?.scales else null); - try plain(binding, source, try named(&name, "layers.{d}.engram.q_weight", .{layer}), .f32, &.{ 4, 5120 }, if (binding) &e.?.q_weight else null); - try plain(binding, source, try named(&name, "layers.{d}.engram.k_weight", .{layer}), .f32, &.{ 4, 5120 }, if (binding) &e.?.k_weight else null); + try plainEither(binding, source, try named(&name, "layers.{d}.engram.q_weight", .{layer}), &.{ 4, 5120 }, if (binding) &e.?.q_weight else null); + try plainEither(binding, source, try named(&name, "layers.{d}.engram.k_weight", .{layer}), &.{ 4, 5120 }, if (binding) &e.?.k_weight else null); try matrix(binding, source, try named(&prefix, "layers.{d}.engram.wkv", .{layer}), 6144, 25600, if (binding) &e.?.wkv else null); } } From 611a130db6703822b901a8d39ab2669975a56a8a Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Tue, 13 Oct 2026 08:15:39 -0700 Subject: [PATCH 06/33] DeepSeek V4.1: the kept-dense EXL3 form, for rank-split repacks Jayleaton's 2.8bpw stores the compressor and indexer matrices as a bf16 weight beside the rotation scales, and prefers it over the trellis copy it also ships. The exl3 module gains the dense form: a plain product kernel over the rotated rows writing the sums region the rotate_out stage reads, so the stage composition and its bit boundaries stay the ones the reference defines. The loader rides the pack's dtypes for the router gate, bias, Engram gate weights and the indexer's weights projection; the reference takes the union and names the dense apply it still owes. The 2.8bpw plan passes: 406.6 GB, 44 shards. --- zig/kernels/metal/core/exl3_projection.metal | 15 +++++ zig/kernels/metal/deepseek_v41/engram.metal | 13 ++-- zig/src/core/exl3_projection.zig | 64 ++++++++++++++++++- zig/src/families/deepseek_v41/attention.zig | 29 ++++++--- zig/src/families/deepseek_v41/weights.zig | 55 ++++++++++++---- .../deepseek_v41_reference/attention_math.zig | 10 +-- .../deepseek_v41_reference/engram_math.zig | 2 +- .../operator_diagnostic.zig | 2 +- .../deepseek_v41_reference/operator_moe.zig | 8 +-- .../deepseek_v41_reference/operators.zig | 2 +- .../deepseek_v41_reference/projection.zig | 22 ++++++- .../deepseek_v41_reference/reference.zig | 8 +-- 12 files changed, 187 insertions(+), 43 deletions(-) diff --git a/zig/kernels/metal/core/exl3_projection.metal b/zig/kernels/metal/core/exl3_projection.metal index 0eca0d80c5..51777a82a8 100644 --- a/zig/kernels/metal/core/exl3_projection.metal +++ b/zig/kernels/metal/core/exl3_projection.metal @@ -239,6 +239,21 @@ kernel void tf_exl3_state_window_check(const device uint4* inputs [[buffer(0)]], const device ushort* tile = reinterpret_cast(inputs + tid); const uint off = inputs[tid].z; outputs[tid] = tf_exl3_mapped_state(tile, (off << 6) | (1u << 11)); + +// A kept-dense matrix's plain product: the rotated rows times the bf16 weight, accumulated +// in f32 in the sums region the rotate_out stage reads. Sequential k, one thread an output +// element — the fp64 reference mirrors this order. +kernel void tf_exl3_dense(const device half* rotated [[buffer(0)]], + const device bfloat* weight [[buffer(1)]], + device float* sums [[buffer(2)]], + constant TfExl3Args& a [[buffer(3)]], + const device uint* input_ids [[buffer(4)]], + uint2 g [[thread_position_in_grid]]) { + const uint row = a.input_ids ? input_ids[g.y] : g.y; + float acc = 0; + for (uint k = 0; k < a.k; ++k) + acc += float(rotated[ulong(g.y) * a.k + k]) * float(weight[ulong(g.x) * a.k + k]); + sums[ulong(g.y) * a.n + g.x] = acc; } // Four SIMDgroups share decoded weights while each retains its original eight-row compensated dots. diff --git a/zig/kernels/metal/deepseek_v41/engram.metal b/zig/kernels/metal/deepseek_v41/engram.metal index 532bbd01ee..b3d454cf59 100644 --- a/zig/kernels/metal/deepseek_v41/engram.metal +++ b/zig/kernels/metal/deepseek_v41/engram.metal @@ -1,6 +1,11 @@ // Math/layout attribution: DeepSeek AI DeepSeek-V4.1-Flash inference/{engram,model}.py (MIT). #include using namespace metal; +inline float rd(const device uchar* p, ulong i, uint dtype) { + if (dtype == 1u) return ((const device float*)p)[i]; + if (dtype == 2u) return float(((const device half*)p)[i]); + return as_type(uint(((const device ushort*)p)[i]) << 16); +} struct EngramLayer { uint layer_id, rows; @@ -96,15 +101,15 @@ kernel void tf_v41_engram_gather(const device uint* ids [[buffer(0)]], struct EngramGateArgs { ulong hidden_stride, kv_stride, output_stride; - uint dim, copies, masked; + uint dim, copies, masked, weight_dtype; float eps; }; // Each copy gets its own RMS dot gate; all copies share the value following the flattened keys. kernel void tf_v41_engram_gate(const device bfloat* hidden [[buffer(0)]], const device bfloat* kv [[buffer(1)]], - const device float* q_weight [[buffer(2)]], - const device float* k_weight [[buffer(3)]], + const device uchar* q_weight [[buffer(2)]], + const device uchar* k_weight [[buffer(3)]], const device uchar* mask [[buffer(4)]], device bfloat* out [[buffer(5)]], constant EngramGateArgs& args [[buffer(6)]], @@ -120,7 +125,7 @@ kernel void tf_v41_engram_gate(const device bfloat* hidden [[buffer(0)]], float3 acc = 0; for (uint d = t; d < args.dim; d += 256) { const float h = float(hidden[hbase + d]), k = float(kv[kbase + d]); - const float weight = q_weight[wbase + d] * k_weight[wbase + d]; + const float weight = rd(q_weight, wbase + d, args.weight_dtype) * rd(k_weight, wbase + d, args.weight_dtype); acc += float3(h * h, k * k, (h * weight) * k); } acc = float3(simd_sum(acc.x), simd_sum(acc.y), simd_sum(acc.z)); diff --git a/zig/src/core/exl3_projection.zig b/zig/src/core/exl3_projection.zig index 0c2074a793..f99e51053a 100644 --- a/zig/src/core/exl3_projection.zig +++ b/zig/src/core/exl3_projection.zig @@ -138,6 +138,7 @@ pub const Pipelines = struct { rotate_in: mtl.Pipeline, matmul: mtl.Pipeline, rotate_out: mtl.Pipeline, + dense: mtl.Pipeline, prefill: ?mtl.Pipeline = null, pub fn load(device: mtl.Device) !Pipelines { @@ -151,6 +152,8 @@ pub const Pipelines = struct { errdefer mm.deinit(); const ro = try mtl.Pipeline.init(device, lib, "tf_exl3_rotate_out", false); errdefer ro.deinit(); + const dn = try mtl.Pipeline.init(device, lib, "tf_exl3_dense", false); + errdefer dn.deinit(); for ([_]mtl.Pipeline{ ri, mm, ro }) |p| if (p.simdWidth() != 32 or p.maxThreads() < 32) return error.UnsupportedExl3Simd; var prefill = mtl.Pipeline.init(device, lib, "tf_exl3_matmul_prefill", false) catch null; @@ -160,12 +163,13 @@ pub const Pipelines = struct { prefill = null; } } - return .{ .rotate_in = ri, .matmul = mm, .rotate_out = ro, .prefill = prefill }; + return .{ .rotate_in = ri, .matmul = mm, .rotate_out = ro, .dense = dn, .prefill = prefill }; } pub fn deinit(p: *Pipelines) void { p.rotate_in.deinit(); p.matmul.deinit(); p.rotate_out.deinit(); + p.dense.deinit(); if (p.prefill) |pf| pf.deinit(); } }; @@ -288,6 +292,64 @@ pub fn encode(p: *const Pipelines, e: mtl.ComputeEncoder, m: Matrix, x: View, y: stages(p, e, a, m.trellis, m.suh, m.svh, m.bias orelse m.svh, scratch, x, y, scratch, r); } +/// A kept-dense EXL3 matrix: a bf16 weight beside the input/output rotation scales, no trellis. +pub const DenseMatrix = struct { + weight: View, + suh: View, + svh: View, + bias: ?View = null, + k: usize, + n: usize, +}; + +/// The kept-dense path: the same rotate stages as the quantized encode, with the trellis +/// product replaced by a plain dense product over the rotated rows, accumulated in f32 in +/// the scratch sums region the rotate_out stage reads. Sequential k; the fp64 reference +/// mirrors this order. +pub fn encodeDense(p: *const Pipelines, e: mtl.ComputeEncoder, m: DenseMatrix, x: View, y: View, scratch: View, r: Rows) !void { + try shape(m.k, m.n); + var a = try rowsArgs(m.k, m.n, r, x, y, scratch); + if (r.grouped) return error.InvalidExl3GroupedRows; + a.su_signs = 0; + a.sv_signs = 0; + a.has_bias = @intFromBool(m.bias != null); + const sums = View{ .buffer = scratch.buffer, .offset = scratch.offset + try rotatedBytes(m.k, r.count) }; + const sums_bytes = try std.math.mul(usize, try std.math.mul(usize, m.n, r.count), 4); + try readonly(m.weight, try std.math.mul(usize, try std.math.mul(usize, m.n, m.k), 2), y, 0, scratch, sums_bytes); + try readonly(m.suh, m.k * 2, y, 0, scratch, sums_bytes); + try readonly(m.svh, m.n * 2, y, 0, scratch, sums_bytes); + if (m.bias) |b| try readonly(b, m.n * 2, y, 0, scratch, sums_bytes); + e.setPipeline(p.rotate_in); + bind(e, x, 0); + bind(e, m.suh, 1); + bind(e, scratch, 2); + e.setValue(a, 3); + bind(e, scratch, 4); + bind(e, scratch, 5); + bind(e, scratch, 6); + e.dispatchGroups(.{ .width = a.k / 128, .height = a.rows }, .{ .width = 32 }); + e.barrier(); + e.setPipeline(p.dense); + bind(e, scratch, 0); + bind(e, m.weight, 1); + bind(e, sums, 2); + e.setValue(a, 3); + bind(e, scratch, 4); + e.dispatchThreads(.{ .width = a.n, .height = a.rows }, .{ .width = 64 }); + e.barrier(); + e.setPipeline(p.rotate_out); + bind(e, sums, 0); + bind(e, m.svh, 1); + bind(e, m.bias orelse m.svh, 2); + bind(e, y, 3); + e.setValue(a, 4); + bind(e, scratch, 5); + bind(e, scratch, 6); + bind(e, scratch, 7); + e.dispatchGroups(.{ .width = a.n / 128, .height = a.rows }, .{ .width = 32 }); + e.barrier(); +} + /// GPU routes choose caller-validated resident descriptors; grouped padding uses maxInt(u32). pub fn encodeIndexed(p: *const Pipelines, e: mtl.ComputeEncoder, experts: Experts, x: View, y: View, scratch: View, r: Rows) !void { if (r.routes == null or experts.count == 0 or experts.count > std.math.maxInt(u32)) return error.InvalidExl3Experts; diff --git a/zig/src/families/deepseek_v41/attention.zig b/zig/src/families/deepseek_v41/attention.zig index 6d7cfe5da7..a1f76f6886 100644 --- a/zig/src/families/deepseek_v41/attention.zig +++ b/zig/src/families/deepseek_v41/attention.zig @@ -120,8 +120,18 @@ fn tensor(t: checkpoint.Tensor) View { fn core(v: View) exl3.View { return .{ .buffer = v.buffer, .offset = v.offset }; } -fn projection(p: *const exl3.Pipelines, e: mtl.ComputeEncoder, m: exl3.Matrix, x: View, y: View, scratch: exl3.View, rows: u32, xs: u32, ys: u32, output: exl3.Output) !void { - try exl3.encode(p, e, m, core(x), core(y), scratch, .{ .count = rows, .x_stride = xs, .y_stride = ys, .output = output }); +fn projection(p: *const exl3.Pipelines, e: mtl.ComputeEncoder, m: weights.AnyMatrix, x: View, y: View, scratch: exl3.View, rows: u32, xs: u32, ys: u32, output: exl3.Output) !void { + switch (m) { + .trellis => |t| try exl3.encode(p, e, t, core(x), core(y), scratch, .{ .count = rows, .x_stride = xs, .y_stride = ys, .output = output }), + // A kept-dense repack: the same rotate stages, the middle a plain dense product. + .dense => |d| try exl3.encodeDense(p, e, .{ + .weight = core(tensor(d.weight)), + .suh = core(tensor(d.suh)), + .svh = core(tensor(d.svh)), + .k = d.k, + .n = d.n, + }, core(x), core(y), scratch, .{ .count = rows, .x_stride = xs, .y_stride = ys, .output = output }), + } } fn norm(p: *const Pipelines, e: mtl.ComputeEncoder, x: View, weight: checkpoint.Tensor, rows: u32, width: u32, eps: f32) void { ops.encodeRms(e, &p.plain, x, tensor(weight), x, .{ .rows = rows, .width = width, .x_stride = width, .y_stride = width, .eps = eps, .weight_f32 = weight.dtype == .f32 }); @@ -161,15 +171,15 @@ pub fn encodeDiagnostic(p: *const Pipelines, e: mtl.ComputeEncoder, w: *const we const ring = try state.ring(layer); const chunk = work.selection.windowChunk(); e.barrier(); - try projection(ep, e, w.wq_a, x, qr, scratch, rows, c.hidden_size, c.q_lora_rank, .bf16); + try projection(ep, e, .{ .trellis = w.wq_a }, x, qr, scratch, rows, c.hidden_size, c.q_lora_rank, .bf16); if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .qr_projection, qr); norm(p, e, qr, w.q_norm, rows, c.q_lora_rank, c.rms_norm_eps); if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .qr_norm, qr); - try projection(ep, e, w.wq_b, qr, q, scratch, rows, c.q_lora_rank, q_width, .bf16); + try projection(ep, e, .{ .trellis = w.wq_b }, qr, q, scratch, rows, c.q_lora_rank, q_width, .bf16); if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .q_projection, q); try rotate(p, e, q, rows, c.num_attention_heads, c.head_dim, position, ratio, 1, false, c); if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .q, q); - try projection(ep, e, w.wkv, x, raw, scratch, rows, c.hidden_size, c.head_dim, .bf16); + try projection(ep, e, .{ .trellis = w.wkv }, x, raw, scratch, rows, c.hidden_size, c.head_dim, .bf16); if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .kv_projection, raw); norm(p, e, raw, w.kv_norm, rows, c.head_dim, c.rms_norm_eps); if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .kv_norm, raw); @@ -217,7 +227,10 @@ pub fn encodeDiagnostic(p: *const Pipelines, e: mtl.ComputeEncoder, w: *const we try projection(ep, e, selector.wq_b, qr, iq, scratch, rows, c.q_lora_rank, iq_width, .bf16); try rotate(p, e, iq, rows, c.index_n_heads, c.index_head_dim, position, ratio, 1, false, c); try p.cache.encodePack(e, iq, work.selection.queryPacked(), .{ .rows = rows * c.index_n_heads, .width = c.index_head_dim, .stride = c.index_head_dim, .packed_stride = kv.cache.index_stride, .block = 32, .kind = .index_fp4, .inplace = 1 }); - ops.encodeDense(e, &p.plain, x, tensor(selector.weights_proj), iw, .{ .rows = rows, .k = c.hidden_size, .n = c.index_n_heads, .x_stride = c.hidden_size, .w_stride = c.hidden_size, .y_stride = c.index_n_heads, .w_dtype = .f16 }); + ops.encodeDense(e, &p.plain, x, tensor(selector.weights_proj), iw, .{ .rows = rows, .k = c.hidden_size, .n = c.index_n_heads, .x_stride = c.hidden_size, .w_stride = c.hidden_size, .y_stride = c.index_n_heads, .w_dtype = switch (selector.weights_proj.dtype) { + .bf16 => .bf16, + else => .f16, + } }); e.barrier(); } const make_candidates = layer == c.candidate_source_layer_id; @@ -250,9 +263,9 @@ pub fn encodeDiagnostic(p: *const Pipelines, e: mtl.ComputeEncoder, w: *const we const group_input = q_width / c.o_groups; const group_output = c.o_groups * c.o_lora_rank; for (w.wo_a, 0..) |matrix, group| { - try projection(ep, e, matrix, q.shifted(group * group_input * 2), grouped.shifted(group * c.o_lora_rank * 2), scratch, rows, q_width, group_output, .bf16); + try projection(ep, e, .{ .trellis = matrix }, q.shifted(group * group_input * 2), grouped.shifted(group * c.o_lora_rank * 2), scratch, rows, q_width, group_output, .bf16); } if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .grouped, grouped); - try projection(ep, e, w.wo_b, grouped, y, scratch, rows, group_output, c.hidden_size, .bf16); + try projection(ep, e, .{ .trellis = w.wo_b }, grouped, y, scratch, rows, group_output, c.hidden_size, .bf16); if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .wo_b, y); } diff --git a/zig/src/families/deepseek_v41/weights.zig b/zig/src/families/deepseek_v41/weights.zig index dc8d508404..77e78caffb 100644 --- a/zig/src/families/deepseek_v41/weights.zig +++ b/zig/src/families/deepseek_v41/weights.zig @@ -8,9 +8,13 @@ const load = @import("load.zig"); pub const Tensor = cp.Tensor; pub const Matrix = exl3.Matrix; + +/// A kept-dense repack stores small matrices as a bf16 weight beside the rotation scales. +pub const Dense = struct { weight: Tensor, suh: Tensor, svh: Tensor, k: usize, n: usize }; +pub const AnyMatrix = union(enum) { trellis: Matrix, dense: Dense }; pub const Hc = struct { projection: Tensor, scale: Tensor, base: Tensor }; -pub const Compressor = struct { wkv: Matrix, wgate: ?Matrix, norm: Tensor }; -pub const Indexer = struct { wq_b: Matrix, wk: ?Matrix, weights_proj: Tensor, k_norm: ?Tensor }; +pub const Compressor = struct { wkv: AnyMatrix, wgate: ?AnyMatrix, norm: Tensor }; +pub const Indexer = struct { wq_b: AnyMatrix, wk: ?AnyMatrix, weights_proj: Tensor, k_norm: ?Tensor }; pub const Attention = struct { wq_a: Matrix, wq_b: Matrix, @@ -170,6 +174,32 @@ fn scale(comptime binding: bool, source: anytype, prefix: []const u8, size: usiz return .{ .name = name, .format = if (have_fp) .f16 else .packed_signs }; } +/// The compressor and indexer matrices ride either storage: the released packs quantize +/// them, rank-split repacks keep them dense beside the rotation scales. +fn matrixAny(comptime binding: bool, source: anytype, prefix: []const u8, k: usize, n: usize, destination: ?*AnyMatrix) !void { + var name_buf: [160]u8 = undefined; + if (has(source, try named(&name_buf, "{s}.weight", .{prefix}))) { + const weight_name = try named(&name_buf, "{s}.weight", .{prefix}); + const w = try source.get(weight_name); + if (w.dtype != .bf16 or w.rank != 2 or w.shape[0] != n or w.shape[1] != k) { + std.log.err("unsupported dense tensor {s}: {s} {any}", .{ weight_name, @tagName(w.dtype), w.shape[0..w.rank] }); + return error.UnsupportedTensor; + } + const su = try source.get(try named(&name_buf, "{s}.suh", .{prefix})); + const sv = try source.get(try named(&name_buf, "{s}.svh", .{prefix})); + if (su.dtype != .f16 or su.rank != 1 or su.shape[0] != k or + sv.dtype != .f16 or sv.rank != 1 or sv.shape[0] != n) return error.UnsupportedExl3Scale; + if (binding) { + if (w.offset % 2 != 0) return error.InvalidTensorView; + destination.?.* = .{ .dense = .{ .weight = w, .suh = su, .svh = sv, .k = k, .n = n } }; + } + return; + } + var m: Matrix = undefined; + try matrix(binding, source, prefix, k, n, if (binding) &m else null); + if (binding) destination.?.* = .{ .trellis = m }; +} + fn matrix(comptime binding: bool, source: anytype, prefix: []const u8, k: usize, n: usize, destination: ?*Matrix) !void { var name: [160]u8 = undefined; const trellis = try source.get(try named(&name, "{s}.trellis", .{prefix})); @@ -184,8 +214,11 @@ fn matrix(comptime binding: bool, source: anytype, prefix: []const u8, k: usize, if ((if (binding) marker.offset else marker.begin) % 4 != 0) return error.InvalidTensorView; if (has(source, try named(&name, "{s}.mcg", .{prefix})) or has(source, try named(&name, "{s}.3inst", .{prefix})) or - has(source, try named(&name, "{s}.bias", .{prefix})) or - has(source, try named(&name, "{s}.weight", .{prefix}))) return error.UnsupportedExl3Parts; + has(source, try named(&name, "{s}.bias", .{prefix}))) + { + std.log.err("unexpected exl3 parts beside {s}", .{prefix}); + return error.UnsupportedExl3Parts; + } var su_name: [160]u8 = undefined; var sv_name: [160]u8 = undefined; const su = try scale(binding, source, prefix, k, "suh", "su", &su_name); @@ -250,21 +283,21 @@ fn populate(comptime binding: bool, source: anytype, model: ?*Model) !void { } if (owner) { const compressor: ?*Compressor = if (binding) &attn.?.compressor.? else null; - try matrix(binding, source, try named(&prefix, "layers.{d}.attn.compressor.wkv", .{layer}), 5120, 512, if (binding) &compressor.?.wkv else null); + try matrixAny(binding, source, try named(&prefix, "layers.{d}.attn.compressor.wkv", .{layer}), 5120, 512, if (binding) &compressor.?.wkv else null); try plain(binding, source, try named(&name, "layers.{d}.attn.compressor.norm.weight", .{layer}), .bf16, &.{512}, if (binding) &compressor.?.norm else null); - if (binding) compressor.?.wgate = if (layer < 20) @as(Matrix, undefined) else null; - if (layer < 20) try matrix(binding, source, try named(&prefix, "layers.{d}.attn.compressor.wgate", .{layer}), 5120, 512, if (binding) &compressor.?.wgate.? else null); + if (binding) compressor.?.wgate = if (layer < 20) @as(AnyMatrix, undefined) else null; + if (layer < 20) try matrixAny(binding, source, try named(&prefix, "layers.{d}.attn.compressor.wgate", .{layer}), 5120, 512, if (binding) &compressor.?.wgate.? else null); } if (owner or layer == 24 or layer == 28 or layer == 32 or layer == 36) { const indexer: ?*Indexer = if (binding) &attn.?.indexer.? else null; - try matrix(binding, source, try named(&prefix, "layers.{d}.attn.indexer.wq_b", .{layer}), 1280, 4096, if (binding) &indexer.?.wq_b else null); - try plain(binding, source, try named(&name, "layers.{d}.attn.indexer.weights_proj.weight", .{layer}), .f16, &.{ 32, 5120 }, if (binding) &indexer.?.weights_proj else null); + try matrixAny(binding, source, try named(&prefix, "layers.{d}.attn.indexer.wq_b", .{layer}), 1280, 4096, if (binding) &indexer.?.wq_b else null); + try plainEither(binding, source, try named(&name, "layers.{d}.attn.indexer.weights_proj.weight", .{layer}), &.{ 32, 5120 }, if (binding) &indexer.?.weights_proj else null); if (binding) { - indexer.?.wk = if (owner) @as(Matrix, undefined) else null; + indexer.?.wk = if (owner) @as(AnyMatrix, undefined) else null; indexer.?.k_norm = if (owner) @as(Tensor, undefined) else null; } if (owner) { - try matrix(binding, source, try named(&prefix, "layers.{d}.attn.indexer.wk", .{layer}), 512, 128, if (binding) &indexer.?.wk.? else null); + try matrixAny(binding, source, try named(&prefix, "layers.{d}.attn.indexer.wk", .{layer}), 512, 128, if (binding) &indexer.?.wk.? else null); try plain(binding, source, try named(&name, "layers.{d}.attn.indexer.k_norm.weight", .{layer}), .bf16, &.{128}, if (binding) &indexer.?.k_norm.? else null); } } diff --git a/zig/tests/deepseek_v41_reference/attention_math.zig b/zig/tests/deepseek_v41_reference/attention_math.zig index d3df4d0ac7..1adec0172d 100644 --- a/zig/tests/deepseek_v41_reference/attention_math.zig +++ b/zig/tests/deepseek_v41_reference/attention_math.zig @@ -168,15 +168,15 @@ pub const State = struct { } pub fn apply(self: *State, a: Attention, layer: usize, x: []const u16, position: usize, c: Config, out: []u16, scratch: *projection.Scratch) !void { const compressed = c.compress_ratios[layer] != 0; - try projection.apply(a.wq_a, x, &self.qr, scratch); + try projection.apply(.{ .trellis = a.wq_a }, x, &self.qr, scratch); self.observe(.qr_projection, &self.qr); m.rms(&self.qr, a.q_norm, c.rms_norm_eps, &self.qr); self.observe(.qr_norm, &self.qr); - try projection.apply(a.wq_b, &self.qr, &self.q, scratch); + try projection.apply(.{ .trellis = a.wq_b }, &self.qr, &self.q, scratch); self.observe(.q_projection, &self.q); for (0..64) |head| m.rope(self.q[head * 512 ..][0..512], position, compressed, false, c); self.observe(.q, &self.q); - try projection.apply(a.wkv, x, &self.kv, scratch); + try projection.apply(.{ .trellis = a.wkv }, x, &self.kv, scratch); self.observe(.kv_projection, &self.kv); m.rms(&self.kv, a.kv_norm, c.rms_norm_eps, &self.kv); self.observe(.kv_norm, &self.kv); @@ -216,9 +216,9 @@ pub const State = struct { } if (self.sparse_snapshot) |snapshot| self.observe(.sparse_o, snapshot); self.observe(.o, &self.o); - for (0..8) |group| try projection.apply(a.wo_a[group], self.o[group * 4096 ..][0..4096], self.grouped[group * 1024 ..][0..1024], scratch); + for (0..8) |group| try projection.apply(.{ .trellis = a.wo_a[group] }, self.o[group * 4096 ..][0..4096], self.grouped[group * 1024 ..][0..1024], scratch); self.observe(.grouped, &self.grouped); - try projection.apply(a.wo_b, &self.grouped, out, scratch); + try projection.apply(.{ .trellis = a.wo_b }, &self.grouped, out, scratch); self.observe(.wo_b, out); } fn observeHead(self: *State, head: usize) void { diff --git a/zig/tests/deepseek_v41_reference/engram_math.zig b/zig/tests/deepseek_v41_reference/engram_math.zig index d57567957f..88772898de 100644 --- a/zig/tests/deepseek_v41_reference/engram_math.zig +++ b/zig/tests/deepseek_v41_reference/engram_math.zig @@ -74,7 +74,7 @@ pub const State = struct { offset += primes[layer][column]; } } - try projection.apply(e.wkv, &self.gathered, &self.kv, scratch); + try projection.apply(.{ .trellis = e.wkv }, &self.gathered, &self.kv, scratch); for (0..4) |copy| { var hh: f64 = 0; var kk: f64 = 0; diff --git a/zig/tests/deepseek_v41_reference/operator_diagnostic.zig b/zig/tests/deepseek_v41_reference/operator_diagnostic.zig index c37682bf8e..66dc039bcf 100644 --- a/zig/tests/deepseek_v41_reference/operator_diagnostic.zig +++ b/zig/tests/deepseek_v41_reference/operator_diagnostic.zig @@ -97,7 +97,7 @@ pub fn attentionMetric(comptime T: type, gpa: std.mem.Allocator, report: *std.Ar pub fn attentionProjection(gpa: std.mem.Allocator, model: *Model, matrix: ex.Matrix, input: attention.View, name: []const u8, scratch: *projection.Scratch, expected: []f32, report: *std.ArrayList(u8), count: *usize) !void { const input_row = @as([*]const u16, @ptrCast(@alignCast(input.buffer.contents() + input.offset)))[0..matrix.k]; - try projection.applyF32(matrix, input_row, expected[0..matrix.n], scratch); + try projection.applyF32(.{ .trellis = matrix }, input_row, expected[0..matrix.n], scratch); const output = model.workspace.view(.logits); const cb = model.queue.commandBufferUnretained(); const enc = cb.compute(.serial); diff --git a/zig/tests/deepseek_v41_reference/operator_moe.zig b/zig/tests/deepseek_v41_reference/operator_moe.zig index eb9780db96..558234569f 100644 --- a/zig/tests/deepseek_v41_reference/operator_moe.zig +++ b/zig/tests/deepseek_v41_reference/operator_moe.zig @@ -134,15 +134,15 @@ fn moeReferenceRows(model: *Model, layer: usize, input: []const u16, ids: *const rows.gate[slot] = previous.?.gate[prior]; rows.up[slot] = previous.?.up[prior]; } else { - try projection.apply(matrices[0], input, &rows.gate[slot], scratch); - try projection.apply(matrices[1], input, &rows.up[slot], scratch); + try projection.apply(.{ .trellis = matrices[0] }, input, &rows.gate[slot], scratch); + try projection.apply(.{ .trellis = matrices[1] }, input, &rows.up[slot], scratch); } moe_reference.moeActivation(&rows.gate[slot], &rows.up[slot], if (slot == 6) 1 else routing[slot], model.cfg.text_config, &rows.activation[slot]); // A shared or identical-weight expert result can be reused only when the BF16 input is identical. if (reuse != null and std.mem.eql(u16, &rows.activation[slot], &previous.?.activation[reuse.?])) { rows.down[slot] = previous.?.down[reuse.?]; } else { - try projection.apply(matrices[2], &rows.activation[slot], &rows.down[slot], scratch); + try projection.apply(.{ .trellis = matrices[2] }, &rows.activation[slot], &rows.down[slot], scratch); } } moeCombine(ids, rows); @@ -306,7 +306,7 @@ pub fn runMoe(gpa: std.mem.Allocator, io: std.Io, model: *Model, snapshot: []con native_inputs_reference.down[slot] = native_weights_reference.down[slot]; } else { const matrices = try moeMatrices(model, layer, if (slot == 6) null else native_ids[slot]); - try projection.apply(matrices[2], &native.activation[slot], &native_inputs_reference.down[slot], &scratch); + try projection.apply(.{ .trellis = matrices[2] }, &native.activation[slot], &native_inputs_reference.down[slot], &scratch); } } moeCombine(&native_ids, native_inputs_reference); diff --git a/zig/tests/deepseek_v41_reference/operators.zig b/zig/tests/deepseek_v41_reference/operators.zig index 8d050300e2..d7c8cb65de 100644 --- a/zig/tests/deepseek_v41_reference/operators.zig +++ b/zig/tests/deepseek_v41_reference/operators.zig @@ -61,7 +61,7 @@ pub fn run(gpa: std.mem.Allocator, io: std.Io, model: *Model, dir: []const u8) ! cb.wait(); if (cb.failure() != null) return error.OperatorGpuFailure; const actual = y.slice(f32, 129280); - try projection.applyF32(matrix, x.slice(u16, matrix.k), expected[0..matrix.n], &reference); + try projection.applyF32(.{ .trellis = matrix }, x.slice(u16, matrix.k), expected[0..matrix.n], &reference); var max_abs: f64 = 0; var worst: usize = 0; var squared: f64 = 0; diff --git a/zig/tests/deepseek_v41_reference/projection.zig b/zig/tests/deepseek_v41_reference/projection.zig index ca0f932c22..5f8f2fe62c 100644 --- a/zig/tests/deepseek_v41_reference/projection.zig +++ b/zig/tests/deepseek_v41_reference/projection.zig @@ -3,7 +3,9 @@ // EXL3 attribution: docs/recipes/exl3.md; pinned checkpoint provenance in THIRD_PARTY_NOTICES.md. const std = @import("std"); -const exl3 = @import("tensorfold").exl3_projection; +const tf = @import("tensorfold"); +const exl3 = tf.exl3_projection; +const weights = tf.deepseek_v41_weights; const format = exl3.format; const block_size = format.hadamard_dim; const inverse: f32 = 0.08838834764831845; @@ -41,13 +43,27 @@ pub const Scratch = struct { }; /// One BF16 input row and BF16 output row; caller must finish GPU writes before borrowing shared CPU bytes. -pub fn apply(matrix: exl3.Matrix, x: []const u16, y: []u16, scratch: *Scratch) !void { +pub fn apply(matrix: weights.AnyMatrix, x: []const u16, y: []u16, scratch: *Scratch) !void { + switch (matrix) { + .trellis => |m| return applyTrellis(m, x, y, scratch), + .dense => return error.DenseReferenceUnsupported, + } +} + +pub fn applyF32(matrix: weights.AnyMatrix, x: []const u16, y: []f32, scratch: *Scratch) !void { + switch (matrix) { + .trellis => |m| return applyF32Trellis(m, x, y, scratch), + .dense => return error.DenseReferenceUnsupported, + } +} + +fn applyTrellis(matrix: exl3.Matrix, x: []const u16, y: []u16, scratch: *Scratch) !void { try project(matrix, x, y.len, scratch); try finish(u16, matrix, y, scratch.sums[0..matrix.n]); } /// The same arithmetic with an FP32 output boundary, for compressor projections and full-vocabulary logits. -pub fn applyF32(matrix: exl3.Matrix, x: []const u16, y: []f32, scratch: *Scratch) !void { +pub fn applyF32Trellis(matrix: exl3.Matrix, x: []const u16, y: []f32, scratch: *Scratch) !void { try project(matrix, x, y.len, scratch); try finish(f32, matrix, y, scratch.sums[0..matrix.n]); } diff --git a/zig/tests/deepseek_v41_reference/reference.zig b/zig/tests/deepseek_v41_reference/reference.zig index 23d811a532..98069068f7 100644 --- a/zig/tests/deepseek_v41_reference/reference.zig +++ b/zig/tests/deepseek_v41_reference/reference.zig @@ -134,7 +134,7 @@ pub const Reference = struct { } math.collapse(&self.h, pre, &self.input); math.rms(&self.input, self.w.norm, self.c.rms_norm_eps, &self.input); - try projection.applyF32(self.w.head, &self.input, self.logits, &self.projection_scratch); + try projection.applyF32(.{ .trellis = self.w.head }, &self.input, self.logits, &self.projection_scratch); for (self.logits) |v| if (!std.math.isFinite(v)) return error.NonFiniteReferenceLogits; self.engram_state.advance(compressed); self.position += 1; @@ -168,9 +168,9 @@ pub const Reference = struct { } fn applyExpert(self: *Reference, matrices: [3]exl3.Matrix, route: f32) !void { @setFloatMode(.strict); - try projection.apply(matrices[0], &self.input, &self.gate, &self.projection_scratch); - try projection.apply(matrices[1], &self.input, &self.up, &self.projection_scratch); + try projection.apply(.{ .trellis = matrices[0] }, &self.input, &self.gate, &self.projection_scratch); + try projection.apply(.{ .trellis = matrices[1] }, &self.input, &self.up, &self.projection_scratch); moeActivation(&self.gate, &self.up, route, self.c, &self.activation); - try projection.apply(matrices[2], &self.activation, &self.expert_output, &self.projection_scratch); + try projection.apply(.{ .trellis = matrices[2] }, &self.activation, &self.expert_output, &self.projection_scratch); } }; From d20aea6b144fd45e3e6dbbe31b4c1667322b1636 Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Tue, 13 Oct 2026 08:20:09 -0700 Subject: [PATCH 07/33] DeepSeek V4.1 reference: the kept-dense apply The fp64 oracle's dense path mirrors the quantized one's boundaries: the input rotation keeps its f16 rounding, the product is an FP64 dot over the f16-rounded rotated row and the bf16 weight, and the output stages are the rotate_out contract. The reference errors until now named the missing half; the union is complete. --- .../deepseek_v41_reference/projection.zig | 39 ++++++++++++++++++- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/zig/tests/deepseek_v41_reference/projection.zig b/zig/tests/deepseek_v41_reference/projection.zig index 5f8f2fe62c..0c2880fbe9 100644 --- a/zig/tests/deepseek_v41_reference/projection.zig +++ b/zig/tests/deepseek_v41_reference/projection.zig @@ -46,14 +46,49 @@ pub const Scratch = struct { pub fn apply(matrix: weights.AnyMatrix, x: []const u16, y: []u16, scratch: *Scratch) !void { switch (matrix) { .trellis => |m| return applyTrellis(m, x, y, scratch), - .dense => return error.DenseReferenceUnsupported, + .dense => |d| return applyDense(u16, d, x, y, scratch), } } pub fn applyF32(matrix: weights.AnyMatrix, x: []const u16, y: []f32, scratch: *Scratch) !void { switch (matrix) { .trellis => |m| return applyF32Trellis(m, x, y, scratch), - .dense => return error.DenseReferenceUnsupported, + .dense => |d| return applyDense(f32, d, x, y, scratch), + } +} + +/// The kept-dense form: the same input rotation boundary and output stages as the quantized +/// path, with the tile decode replaced by an FP64 dense dot over the f16-rounded rotated row. +fn applyDense(comptime T: type, d: weights.Dense, x: []const u16, y: []T, scratch: *Scratch) !void { + @setFloatMode(.strict); + if (x.len != d.k or y.len != d.n) return error.InvalidReferenceProjectionLength; + if (d.k > scratch.rotated.len or d.n > scratch.sums.len) + return error.ReferenceProjectionScratchTooSmall; + const su = try borrowed(.{ .buffer = d.suh.buffer, .offset = d.suh.offset }, d.k * 2); + const sv = try borrowed(.{ .buffer = d.svh.buffer, .offset = d.svh.offset }, d.n * 2); + const w = try borrowed(.{ .buffer = d.weight.buffer, .offset = d.weight.offset }, d.n * d.k * 2); + const rotated = scratch.rotated[0..d.k]; + const sums = scratch.sums[0..d.n]; + prepare(x, su, .f16, rotated); + for (sums) |*slot| slot.* = 0; + for (0..d.n) |j| { + var acc: f64 = 0; + const wrow = 2 * j * d.k; + for (0..d.k) |k| { + const word = std.mem.readInt(u16, w[wrow + 2 * k ..][0..2], .little); + acc += @as(f64, rotated[k]) * @as(f64, @as(f32, @bitCast(@as(u32, word) << 16))); + } + sums[j] = acc; + } + var base: usize = 0; + while (base < d.n) : (base += block_size) { + var values: [block_size]f32 = undefined; + for (&values, 0..) |*value, i| value.* = @floatCast(sums[base + i]); + hadamard(&values); + for (values, 0..) |value, i| { + const out: f32 = (value * inverse) * half(sv, base + i); + y[base + i] = if (T == u16) bf16(out) else out; + } } } From c349413aaf59d606b0b1760aa63300f45d61a2d8 Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Tue, 13 Oct 2026 23:33:24 -0700 Subject: [PATCH 08/33] =?UTF-8?q?DeepSeek=20V4.1:=20the=20rebase=20residue?= =?UTF-8?q?=20=E2=80=94=20one=20counters=20binding,=20the=20state-window?= =?UTF-8?q?=20kernel's=20brace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- zig/kernels/metal/core/exl3_projection.metal | 1 + zig/src/metal/metal.zig | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/zig/kernels/metal/core/exl3_projection.metal b/zig/kernels/metal/core/exl3_projection.metal index 51777a82a8..5bdd86bda6 100644 --- a/zig/kernels/metal/core/exl3_projection.metal +++ b/zig/kernels/metal/core/exl3_projection.metal @@ -239,6 +239,7 @@ kernel void tf_exl3_state_window_check(const device uint4* inputs [[buffer(0)]], const device ushort* tile = reinterpret_cast(inputs + tid); const uint off = inputs[tid].z; outputs[tid] = tf_exl3_mapped_state(tile, (off << 6) | (1u << 11)); +} // A kept-dense matrix's plain product: the rotated rows times the bf16 weight, accumulated // in f32 in the sums region the rotate_out stage reads. Sequential k, one thread an output diff --git a/zig/src/metal/metal.zig b/zig/src/metal/metal.zig index 71bed247d2..6404c4227d 100644 --- a/zig/src/metal/metal.zig +++ b/zig/src/metal/metal.zig @@ -29,7 +29,6 @@ pub const keepalive = @import("keepalive.zig"); pub const clock = @import("clock.zig"); pub const counters = @import("counters.zig"); pub const ane = @import("ane.zig"); -pub const counters = @import("counters.zig"); test { std.testing.refAllDecls(@This()); From 6ad76e17eaf721b56f540e89ca37267045a561c5 Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:42:24 -0700 Subject: [PATCH 09/33] mHC fp64 ideal reference: the mix at rounding level, 1-ULP noise attenuated Ported from the mhc worktree: the hyperconnection mix's per-stage fp64 harness (operator_hc_fp64.zig) and the comptime-Real reference parameterization in math.zig. Verdicts and the 1-ULP attenuation receipts live in the fork. --- .gitignore | 2 + docs/deepseek-v41-mhc-fp64.json | 103 ++ zig/deepseek_v41_reference_checks.zig | 1 + .../deepseek_v41/fixtures/hc_fp64_rows.json | 1 + zig/tests/deepseek_v41_reference/math.zig | 130 +- .../operator_hc_fp64.zig | 1095 +++++++++++++++++ 6 files changed, 1310 insertions(+), 22 deletions(-) create mode 100644 docs/deepseek-v41-mhc-fp64.json create mode 100644 zig/src/families/deepseek_v41/fixtures/hc_fp64_rows.json create mode 100644 zig/tests/deepseek_v41_reference/operator_hc_fp64.zig diff --git a/.gitignore b/.gitignore index 75ba703b9a..9aa8c78c21 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,5 @@ zig-out/ # Versioned GUI build resources, generated app engines stay ignored in app/.gitignore. !app/build/ +# mHC fp64 qualification receipts. +/receipts/ diff --git a/docs/deepseek-v41-mhc-fp64.json b/docs/deepseek-v41-mhc-fp64.json new file mode 100644 index 0000000000..833a67ae98 --- /dev/null +++ b/docs/deepseek-v41-mhc-fp64.json @@ -0,0 +1,103 @@ +{ + "scope": "mHC (hyperconnection mix) stage fp64 qualification: rounding level, Metal deltas and 1-ULP input amplification. Independent CPU reference parameterized comptime (math.zig MixOf/denseOf/mixesOf/splitOf/inverseOf/collapseOf/expandOf, f32 source-contract path bit-identical, f64 ideal path); harness zig/tests/deepseek_v41_reference/operator_hc_fp64.zig; fixtures hc_attention_rms.json and hc_fp64_rows.json (retained inputs only).", + "nrms_definition": "sqrt(sum(delta^2)/sum(ref^2)) in f64; ULP = |delta| / fp32 spacing at the fp64 reference value.", + "inputs": [ + "token671_attention (retained token-671 embedding, four copies)", + "token1_ffn (retained token-1 FFN stream and branch, real hc_ffn scale/base)", + "eight_teacher_rows (the eight retained teacher embeddings)", + "wide_dynamic_range (copy magnitudes 2^24..2^-12, rotating signs)", + "split_args (16 retained unary arguments incl. 1.4393715858459473, controlling post argument, integer exp sweep -32..32, plus sigmoid/softmax spans)", + "cancellation fixtures (half-ULP terms after 1.0)" + ], + "rounding_level_fp32_reference_vs_fp64": { + "inverse_statistic_nrms": [ + 2.7e-09, + 2.8e-08 + ], + "projection_dense_nrms": [ + 2.2e-08, + 2.9e-08 + ], + "pre_sigmoid_nrms": [ + 1.1e-08, + 4.2e-08 + ], + "post_nrms": [ + 3.9e-08, + 7.2e-08 + ], + "comb_sinkhorn_nrms": [ + 1.4e-08, + 6.8e-08 + ], + "collapse_bf16_unequal_cells": 0, + "post_expand_bf16_unequal_cells": [ + 0, + 5 + ] + }, + "metal_vs_fp64": { + "inverse_max_ulp": 0.6, + "projection_max_ulp": 0.5, + "pre_max_ulp": 2.75, + "post_max_ulp": 2.29, + "comb_max_ulp_near_cancelled_cells_only": 34.2, + "comb_max_abs": 1.5e-07, + "post_controlling_exponent_row_max_ulp": 1.61, + "exponent_sweep_minus32_to_32_worst_stage_nrms": 9e-08, + "bf16_unequal_cells": { + "token671": 0, + "token1_ffn": "3 of 20480", + "eight_teacher_rows": "8 of 163840", + "wide_dynamic_range": 0 + }, + "verdict": "every sub-stage sits at the fp32 source-contract rounding level; Metal exceeds the fp32 reference by at most ~1 ULP at the documented Metal exp boundary" + }, + "cancellation_boundaries": [ + { + "fixture": "collapse_half_ulps", + "metal": "0x3500", + "f32_ref": "0x3500", + "f64_ideal": "0x34c0" + }, + { + "fixture": "post_half_ulps", + "metal": "0x0", + "f32_ref": "0x0", + "f64_ideal": "0x3400" + }, + { + "fixture": "branch_product", + "metal": "0x3500", + "f32_ref": "0x3500", + "f64_ideal": "0x34c0" + } + ], + "amplification_1ulp_input": { + "method": "perturb one bf16 stream cell (or branch cell) by 1 ULP, propagate the ideal fp64 stage, output NRMS growth = sqrt(sum(dy^2)/sum(y^2)) / (|dx|/sqrt(sum(x^2))); 256 sampled cells per row; 8 teacher positions plus the token-1 FFN row", + "analog_fp64_gain_mean_per_row": [ + 0.182, + 0.76 + ], + "analog_fp64_gain_max_overall": 0.762, + "collapse_gain_mean": 0.5003, + "collapse_gain_max": 1.944, + "bf16_visible_gain_mean_per_row": [ + 0.74, + 2.382 + ], + "bf16_visible_gain_max": 34.5, + "bf16_rows_changed_fraction": "224-250 of 256 perturbations change at least one of 20480 output cells", + "mean_changed_cells": "8.1-13.1 of 20480", + "statistic_relative_shift_mean": 3.6e-07, + "comb_nrms_shift_mean": 3.6e-08, + "metal_confirmation_row0": { + "amp_mean": 1.886, + "fp64_bf16_amp_mean": 1.945, + "mean_changed_cells": 12.06 + } + }, + "gate_reading": "A single-ULP input difference is attenuated by the mix in exact arithmetic (gain < 1) and appears as 1-bf16-ULP flips on about 0.06 percent of output cells; the stage's own rounding-level output NRMS is 0 to 2.5e-5. Hyperconnection mixing therefore cannot turn fp32 ULP noise into the observed 4.096 percent whole-model NRMS; a specific operator-level difference elsewhere (sparse/compressed attention candidates) remains the plausible cause.", + "kernel_changes": "none required; no tolerance relaxed, no source coefficient injected", + "receipts": "gitignored receipts/ (mhc-fp64-run.log, mhc-fp64-parsed.json, build-summary.log); commit contains code, fixture and this summary" +} \ No newline at end of file diff --git a/zig/deepseek_v41_reference_checks.zig b/zig/deepseek_v41_reference_checks.zig index 61aa30f788..d1f17838e1 100644 --- a/zig/deepseek_v41_reference_checks.zig +++ b/zig/deepseek_v41_reference_checks.zig @@ -4,4 +4,5 @@ test { _ = @import("tests/deepseek_v41_reference/math.zig"); _ = @import("tests/deepseek_v41_reference/attention_math.zig"); _ = @import("tests/deepseek_v41_reference/qualify.zig"); + _ = @import("tests/deepseek_v41_reference/operator_hc_fp64.zig"); } diff --git a/zig/src/families/deepseek_v41/fixtures/hc_fp64_rows.json b/zig/src/families/deepseek_v41/fixtures/hc_fp64_rows.json new file mode 100644 index 0000000000..5894e27712 --- /dev/null +++ b/zig/src/families/deepseek_v41/fixtures/hc_fp64_rows.json @@ -0,0 +1 @@ +{"eight_rows_bf16": "LD40Puy9RL50vqe+8705vaM9QL3Fvn299zzSPZg+aD5MvQY9gzyXPRa+yD2UPbW95j0XPkW+KbtsvnK9lz2HvhG+AT3IPck9I7wLvPo9sz00vjK+BL5Ivi2+YL5Mvp8+Lj2Mvb09c729PTG9+7udvNi9RL6CPaA+2j0WPr8+jT0Jvvw9NT6RPp09+z3uvR89Cr7cPRU+lL7cvTu8xj5WPrE9izw/vQE+ej0jvgG+PL0Bvoq7Jj4HPpu9KT5nPvS9GD6hvlS+Rr2uvW8+ozlivu49VD6+PI48gj1MPqO7m71cvQu7FT3vO9679j0SviO8KT3HPSe+Br5APjk+jb0PvZ69G73vPJW8Fj2evu29I74ivmM+Br4Evpu+Ub2MPLW8Ej6xPQg+9z1ZvqQ9Kb7iPjO+Yr2KvFC+JL6PPpy9DL3kPWw8bb6NPvU8Ib4tPYW8FT5IPjI+lj6VvjI+zj2IPGG9zL7+vfK9RD4yPq89F76lPmy+5D2evFi94b07vT2+AD5FPTc+8z0APhy8hb04vpi+hru3vcQ8PD66vAs+Tj0OPVC+mD1JvvW9Rb6FPAA9lr76vC6+drxpPRQ+Rz74Pd692TzEvWq+Ar7rvAA+Bj3OPWw+9j18PgE+ib0VPo8+cL0GPh+9UD6Hvtm8h71LvjQ+u7tYPjI+xz63Peu8Yj6rPYq9UL2+vSC+FT6mvpS+8bx9PH29Yb4MPHu+MD6cvZq8pz3nPaE+HjvGvd89or1Qvli+AL5gPpS+jz54PdE7oT5Ovnu6Lr0Wvoe+jTxUvhC+/T2VvR6+zr0SvUY+0z3YvoU+gj1FPmQ8/731vSC+/Tx/vs68Cz4SvoG+0r2jvn6+nr7PvZQ+3r2VPSc9TT2jPuW9s73RvBa+eLuoPgA+lT0evhm92zt1PpO9GT2JvuC9ZT4fvgm+LL5GPIW9/b2hPXc9C77pPPw9uL0hPbo9RD7oPWI9azzPvXg+9T1XuwG+azwWPgk9Kr4EvT2+Vr34vSQ+tj6vvSI+o70CPjK9xTzQvTU+BD25PlY9Kr6rvUk9i77gOwa+PD0JPm++QLvJvTM9JL7ivWc9TL6yO6c8pz1GvmM+YL0Nvh28Lb0qPoi9Qb3Yu/k9tL3YPYi9dD0Tvjc+8DywPNk9Kj4fPaI9wD4mPj8++L3YvSg+UD4NPXQ+grybvRI947z3vQ6+gz6ZPYq9ij2nPA88Cz3CPYE9ijyNvZm8t71vvD89Oz0OvtU9bj0bvYK+jj1SvTq9uT06Ps49XD2HvYa9ub1GPcQ9XT7rPe68n7yBPTu+Dz45vXk+E74Kvua9ZbyVvVo9Ar5pPIi9xT2IPrI90r2rvoo+hb1pvSg9Fb3+PVc+4j0avhg+nz2KPXM+Zz41vf68LT4YPUA+OL50vWY+Dj54vm+9cL0XPnO+Eb0Hvhi+Cz4RvlA8VD6bvge+6zwFPsu96rwVvB6+uz1bPbi9rj5mPn6+Fb0hPb09Vz5HPgc+MD3HPGi+Fz4gPZ89Db+ZPZy+qLs0vte9dT6VPc0+Er7cujG+dj4Pv50+ib12vRk8dL7UPRM+mD7OPe08nb0oPgo+Xj6OPi49lL0/vn6+1zwZvm29nb6zvT++Jr5NPpm90TtaPtC9hr3JOyU+hT3FPRA9T741Pr0+CL6HPne9SL5AvLI9iTy1vaE9Tz6evNM7Ab5mPuI9Ez5UPUm8rz3AvZU7bT5nPUg9pTxyvRc+hD0XvMU9SDy0Pa+8KT4TPTm+7j0kvqk8kL3cvRo+8L1jOrO+AT6XPi4+gLxyvhi9N7wgOm8+wz0+vva8OD6uvVA+Fb7YvUw9D74HPli9d70tPr89fbv1PT0+2r3KPS09Vz61PW08Wb4KvpE9WLy8u6q7iby1vRw9fL59vtE+9j3Fvcs8a76PPdS95b1fPlG99T08viI+jj4ev5a9er2ZOyO9Vj54PZc+VL0Jvok9TT4nvpE8Pr7ivJQ9A757Pn8+J75Lu9a7+T2SvTy+KDyMPOy9Sj7Yvaq+i774PcQ76bxcvlY9Rb6XPSK+FT47PS0+5D2zPO69O773vka9bb6Mvo69zj0Xvlm+Ar6Wvi++iD7dPRa+mr6EPjK8bz49vi0+NT6ivKQ8rrwzvf09nz05vkg80j3TvZs8gD7jvRm+sz07vj83hz36vSk+kr4IvLa9Br63PYc+ib6mPvk8cj6xPb898z2APt69wj0CvfA9Ej3/vVQ+2zlePTs9wbujvdq9M75qvkA9rr1nvaM+DT4HvXy+kr1NPea8570YPQI93jyXvdk+HT0avQk+i76tPDK+AT1QPgU+hr1jvho+jb7IvRO+IL2HPhI+Rr39vWW+CT0qPmG9V72qveI8C74Muuk82L2KvYc+Zr6mvfM9VT2rPoO+HT7tPDq+HTykvCk94zxtvgE+Xb4Dvnu9G7thvVY+dzyFvYE+0L0xPjS90r3SO+w+3b2EPem9jz0EvCE9ZrycPUc+ujyjOs89GT6ZvTE+lL02vUa+G7wBPno+pb3XPWO+RD2UPVm9gz3SPWk5uT4oPEm9wL48PAK9pT0BvuE9Nz6FPaS9XT7LOyI+hr5qPbS9Zj32PnK+nD36uwy+9LyZvTk9xz07vA09Ar7Wvqk9zL3qvd+9Cj7qvYu8nj3EPr+9zTtQvSY+0b6PPXc+z71cvvO8Cr1OPr+8Ir6FvdQ+nz4zvlK+IL2NPd495z3AveC+sL0ivo+9jL4jPpC+w72IPVi9YL2/POU8/L3fvFE9ir0fPV0+sb2Zvia9TT5TvVk94r3zPSK+QzwEPmK+u76BPiY+Sz5JvYG9m7xMvhK92D0RP6m+ST4hPnA+eD4qPbm+Rr03vnM9ij7wPJy9bzxYvpa9vbw+vo0+uL0nvsg8ET4CvRc9Xz60vB2+IL4Gvu49jj60vaS90b1UPuS7WbvovTO+NL6XPSK8z726Phu+YT1MvWi+Cz4Kvby8YT48vfC8/jwrvgI+azxXPcE9m76yvWQ94LwrPja9uz4WPVu+p7ltvnu9CD5Xvju+3jz0Paw9er2CvsG9kj2svRA9pT0nvvi9K77bvuI9o7uEPZO8sLxTPqA8Ub6sPNk+5LvuPaw9vLvMvcy+K73HvZS+Ob6JPoI+nb09PYk9AD0IvY08hD5APgM8xr1OvKM9Lz4RPQ4+NrvdvaE9K74DPYK7Bj55vYE96L0MPqc+B766PZQ+LT6NvUs7Oj5EOwC+5zx6via90j1Zvty9jTzKPow857uPPaO8uD3kPQS+QT03PhQ+k74xPri8Ir3fvC4+0DzIvZ29sD17PMK7E7/EvdK94D2UPkQ+FD5MPRI6gj2UPTa6Kz7/vea9GD7kPSG+Zb0TPfi9db4NPn4+Vb1VPoI+IL4PvOM9oL1EPn4+Mr0Wveu9/77HPUk+sT2jPAu+hr2pvLm9JD0uvgu9rD2/vU6+Bb4ovvo93TwQPdy8rT3GvB6+RD5Hvv49DT5QPqo+wL1qPq++kbw8PUa9oL3RvRk+obuIPUy+mbwFPWm+/T3GPeC9IT2NvsK9XT4avu497rwDOwS+C7vsPSA9Ir4XPlA+TD6iPfU9SL6CvQw+4r0HPXc8mb6/Pn09zT3yu0Y+fr4yvuW9jj0+PgE+HjycvsO8o709vsU+qLsRPvY987yCPjI+zbwevLu9bz5jvqK9rT4rPgy9ij2pvGy+lD50vkE+zL1NvZE9Bj7pPcE9nD7zvR++arwzvs+93b40Plc+fz1UvnA+KL6yvii+Aj0hvWK9zz29vb09TjxXvby8dr0RPXA9PLzCPSU+3T1SvMO8jT1bvdi9Bb2hvTU8lD0FPhK+ebwfPmU+iT1bPbA9J74KPbM9oj71PLa9Kz6ivQQ9FT6Hvgc+/r1KvbE+tL1UPZW9Nj5Mvgc7uD1Dvl49jb56vvu9Qr4TPS8+hD4tPUQ+Sz7Uvau9VL4EPWq+Hr4fvP07ur6Lvqq9hj6OPQk8D71SPh6+erymPVa+tT0KvRc+Oj7APbS9wz0fvR0+mLzvO/o9Yj7EO6u8HbzOvBy+z77MPi8+UDzBPE09GT6WPR2+cb7LPsC9A71hvbU9AL3PPQi9vTxfPlg9e765vaE+FD4wvmm+fD26Pp6+hz1Nvmu+7r2oPkW+5D1OPWc+9z6IvGO+oj3kvL2+qz1Oum2+Kz6hvjY+jz5cvgW+zD2zvTI+Nr5jvn0+6T0wPQi+kz7fves9RL6uvfW93TxvPn27aj0Zvjc9lzxlvh69MT66vdA9mb2nPIe+2r2yPQe9/7zNPJi+qr20vrE9sj3bvcE9GT4kvpM9BD72PQG+DD40vCa+2T25vR++Tb2PPJc9E72uPhq+pb2CvZc9hr5Hvac9Qj6rvdy9xb0nPQ87R75ePpi+Db4dPha98j2OPQQ9GT4Mvhq+P77mPQw+oL5BvJm8pj1SPhA+Kr5cvt28yb08vjS9QT6RPqm9GL7TPsA9Pb6/PBq+AL5DveC7xj3pPMg9hr0YPqw9gj1lvre9JL7svU28cL3wvc+9tD0pPqI+tj25vZS97T1vPjs+w72FPp89H7yvvEe+tz0/vve7ZT62vWU9FT4MvfA+Yj4Lvni9CL6DvQw+iL40voU++70TPgU+jD4TPqa95j39PDY9FT4aPi8+1D2tvhS+1z51PUe+gb6GO5Y7Mz5FPtq7hL32vVG+LL6OPn++Cz4hvDI+Gb08Pfu9Ij4NvWi9nj5SvoQ+tb1Xvp4+tL6lO/W9EL4wvpA8TTvXPR89pL7NPDA+ab4/vuk9bL6nvVu+M70bvbo8cb5tvI8+PTzBve49jD6HPhO+vL4ZPi+8Qr1vPrI9aD6mvTK8wr2+PUq+fT3+vCW+kD2YNx4+173rveK9iD43PUK+LD6+PWO9G77+vr29RL5vPpg9k73KPdQ8ojwYPni9U74yPGQ9ij6oPcM8HD4LvAu+7b2KPRy+ZT4LvRm+JL5Dvg46oL3wvXM9gL2aPiC+Wz7gvVO9Cr5Cvj6+/z0kPne+lT6rve895j6RPaY9XL6qvrc9qr5gPCQ+770DPgU9Cz7fPDo7Uz5+vWM+Qz0LPW69h74vPuI6Nj52vBG+wD2HPGC+sb3YPWU9br0DPdM8Fb6KPW8+/zqtPWG9ND77veq9p71avh29qj2eO0o9jT6PvbK8nD1KPni9UL5cPde9Ob3ePbi98j0kvkG9Kj5CPbK+5T0pvV2+Jr3PPve7lz0jvvG8pD4YvjE9LrxLPri90D1QPZk9h74fPgm+GD6uPNG7mz21PUI9h70aPYy+Uj4JPo+9g76lvds80T1FOwM9Gz4HPm++JT4Bv+Y81bs4PQQ94r2NPq29jT4rPsW8t72Ivm28G76pPcS8Gb7CvUg+Xz6JvrG+ET13uwS+oL1jvIe+s71mvao9cT28PZM+wj2fPRO8BT/lvdw8Zb60vRI+kT4dviI+Kr5QPik+qT3iPdo9Oj1XvYC+/70QPgK+iz0UvqI+1D0Pvl++l731vbA+IT57vh4+bD6kvnI7d7wKPnU+rD4Lvj4+EryWPlw+N74pPny9NLs5vs+9CL52PG69ozwRvuM7pbx1vnO+t71tPt68Ez1+Pg29vj1IPjI+TT2kuxQ9xr0evoS9qD1Hvge+srwavKI9hDqlPTO+gb15PZw+SL2LvIW+ND0vPSu91bo6vhG+9LxNvWk8az6Jvq89472BPry9fb6Kvcq9xL42PmA+Abz+PN49P727PVS9B70ePg6+n72Ivb89oL59PjW+4DyMveo8h70Pvb8+hj6Hvkk9Z77mvF++qj3PvdG9Jz5zvUA+EL7IPTo+zz2JvuM9tL68vUy9R74FPgy9KjuIPWa+pTz/PCw+1b23vRg+Mj2cPb89hrybu127Uz2MvrI6ar1ovqw8Kz6PvCU9+j3JvTQ+Hz50PbW8mbyEvbI9Ab6wPai+iLzlvRO9O74rvkg+SD70vcm8rzyTPYw+XT5SPiu9sz2uPeW9OD7Qvss+ST3/PFC9rj24OsY9zL1Hvso9tb18vjC+27x3PFC+8722Phw9tj5NPrq9cb4bvvG9Vz08PA2+DL4TvZS84D1qPi8+Qj6RvkE8Kz5zvpg9K728vT2+LT2/PdW9vD1EPkq9Bz48PqM9nD0nvt29tL2/vZi9jb1AO3++Kj65PGs+Br6kvq89rL2Pvlc+ijwEvhI+Vj4MPKo9/L0dvaE7Cb7PvX8+ab6Cucu9p72CPSo+Wj7kvVy+Fj6DPvY9Ob3MvaI+PT6vvoI+Erxpvpc8Lj3fvYe99L07Phe+kL7GPZw9e7vkvQ8+6r1SPcI9MT6APR0+trviPYM+P759PTG+hL7+u+S8mb2NvcW91j1hvUC9AD08vpQ++TxuPu49xL1mvSK+4zzRPRU+k7xyPie+lz6avZK+krzKvYO+uTyqvQu+X77/vIo8zjygvIO+Hz5jPj2+Fj56vU693LxkPNA9iz2KPea9pb5APXS+/r2IvoK+Or4WPmy8MD2EvZQ9Ij6lvZC+sDzwPRK+yrxIvl+7oz6Hve4+Mz24PQq+NT7TPWG++j0RvrK9zTvjPSU9qD6svZS9ez7qvbu9UL26vq+9LT6LPqO89L08vgg9Sb02veE9RL6FPt89yb2pPdk9lLtTPT+9MT7/vcq8b71/vk898j43vTy9U74KPvM8L74/Pgk+zLzDvUM++TzBO9Q9N76HvhE+TD6HPqK9fz5bPaW+DT4UvWS7qb0fvIM+kD4zvii9Fr4lPhU+cj0ivsm93D0Zvuu9mD1xvUE+2bwNvjY+AT2tvWa+Kj7WvLk+oD4YvRa+db62PlS+Bb1zvRo9cbxTPWy+FL0XPiK+yD3DvRG+Wz4/PiE9tbsPvoS+brxCPtQ8vrzbOo09/rz2vJw9GD52vVu9Tb35PC+8q7zgvXy+Zz40vY89FL7WvGQ9Ij7mPRY8tD1ovRM8oTyQPSw+Br1qvZa+ZT4nPpQ+IL7MvCs+qDwGPtk7ED4JPhe+mz0Hvp88tj5TO4q8F77FPUw+vL1fvty+or2DPlU+Mj6tvS09ZT4uvgi9VDwuPW49cD2mO3a+dD2MPa+9pL0XvkM90rybvni+qD2gPfE9VL6HPcm85j3+PQ++5b1kPi27yb0tPkc+ALytvae9kj47vjM+rT6nPdi9770/uaS9LbxlPeo9EbyJvuS9Ab2CvYe9DD4sPhs+Lr5pvoU9r70qvgE+k73gvRC8ibwaPqW9/z2jvhy+A70HPDw+Iz02PTO9ZT6/PKg97r3FvjY+x734PbO91z2IvlK9RD7GvkY+Qz6Ivm8+Ej4QO1s+gD4JPtK8i7tcPrq98TxBvYo8Dz6nvQi+Ub2kvXu9o72dvYO+t70ZvVy+7DxAvmM9pz4vPpy+Wr3/vTo+hj2ovMQ8l7w+PJs7Lz3cPHw+Ib4Tvvm9JD2bvNK8g706PhI+LL6Uvi2+rz05vgw+E71fPZQ9Or6KvXs9zz1oPky9uTy/vec9yD0evAO9ar26vU+9mr7dvM69s7yjvQC+5b30PY4+lz1yvF0+0L0IvnU+Qz0avnw+1ryrPhE+pD6jPqY92L22vaG8tLwavSO9Hb2sveA9bDxrvnm7XL5WPGy79r2pPAC+db4DPs49OLwGPu29ZD6Ovbg8fL2vvvA9BD7lvvw6TD2ZPZq9fj0ZPVM+CD5dvi+9cb4WPvW8y7wOvTo+K769PoM8t72BvaS+ST5DvQK+Qb7HPXC+DL5JviW+Bb4xvhC+qL03vYu99r3sPRe+2ryQvY+96r5FPsm9172ivpk9Dj0svTk+cj0ZPtE7mz3KPNi8qz1iPQi+0707vV++hj5GvRw+5D2LPqI9873rPlq+CL6Zvsc9Ur1SPl++hj0EvT2+ab7TPXS92r6QPo0+fj2BORg+Er5Tvms9Lr6JPSa+5b3BvRK+lL0mPkS9gb3ivTi+gz0gvMm8jb31PZ48Sz5Fvqu8nb1LvVO+772yvd88Ab5zvrC9Fj5VPSy+1TxFPvS8BL4Yvim9pD2VPci9vL6tvYq9VT2rPUE+Bb4RPiQ+C72SPZq+ML7FPY29FT6KvZk8I744Pp++XT7+vTY+O759PfE9Gb4Cvs+8MD0yu5c9Tjy9PE++s74SPgk7Bj6wvdA8M74TvRC+KrzdvV89tL4oviA977yFu0I95b05vkm96L0avng847ytvSM+FD7tu5m9IL7Fvck7Tj4Cvgy+qr05vs8+hz3EPSG88D04PbQ8nL0jPhc+eL1tO469rDwhPTE8hTyNvdY7Kr6NPQS+1LxLPaa8Yb4HPe68Zb2MPYo+MT0qPn4+kT2IPL08lj0jvS6+mT3rPUw9Cz6+vCi9gr6rPpy9W7x3vmS8ej5APry92rxWvVQ+Lj4yPq+92Tw9Pge/WLufvos+Az79PC0+Cj5hPv09fb5BO5Y8DT4hPqe8Tzv4ve49n76LPWW+vboDPnS+Fr4bPru+2b0ovbO9Oz7DPfs9hT4vOzu9J73avUk9E70uvh++iL5jPEg+Jz46Ppo9tL2uPKI+AT2qvNe7kbwsPRM9Bb6WuzW+Jb1EvZ29Vb5mviu+L72pPRI9a77pvdo9Az9XvWW+hL1nPso8nToUvrs9A74QviO+Lj0QvoK9Nr6QPqc9M7tWPeA9Nj2sPWk9b7yPPYC+lzzbPf+9XT08vfu93T0vPsO+jz7Bvbe9/j3dPcw+eL0MvaS9I77KPiG7IrwOviQ+CT5HvS++lb6OveG9Lz7PvVs+7j0nO1W9Qj0aPuu+Tr7kvZa9nz2RPna+ib3ku0C+H759vBe+PL5yPr89kr21PTs+g72bvRQ9Mj0vPi++hL6FPYq93zuYvhu9BT07vg49CL0+vuE8qb6WvTO+47wtvNe8KrzKPAG+Ur1Svia+Sj5LvjS96jzZPW67p70rvgK//LwdviQ+dT5PvsK8Ir7yvHg+Yr63O+k9lL6CPcQ8t7w9vo2+/r2jvpa8Nb5zPj2+2z2SvYc9s70cPKY9Cr7lvds8oj6lvHM+vD04PhA+Wz6hPti9Wr7cus+8Nz0nvQg9g70qPSi+lD2Tvia+Kj74Pfs9Cr+3vTu+m70ZPQK+1L0Tu4E9IjwbvRm+ED4KPki+uj6bPm69s72zPKy9C74sPdk9Dr4guwG+ZT6BPIA9DL4wPtK8AD74PS29gzzPvZw90j2Xu1m9CD5rPlC+Xr67O0k+RTwPvna+aT0gPLa+jj1nvLy9q73lPgY74D38vV8++LtgvnS9870NO628V735vIa+ub59vVg+4D3evKS9Fb6PPhW8yD3kPV49SzxFPRg+db4SvoO9Xr6rvFK+Hb5dvYo9Lb3qvZ49yL3ZPS8+/L0EvQC+r7y0vjS9sb3nPYq9FryyPQY+677XPZW+ID4Ovti+Bb0VvRy98L1IvQE+jL02PRA+370lvUS+zL0bvcm9T74Hvvg9bbvmvew8Wzy2PXm9gb0sPMU9y70sPhA+Gj7JPTE+hD1gPpc+Qb4LvkY7RT6SPhw+Nj4JvRS9vj3FPT29Qz60Pqc8SDyovlS+vD1fPg0+Yz0yPLG+ULxJvts9Yr5UPcY8I75uvda9gr7gvLo9KL7Xu6u7Kj5ovjg+mb3GPgc+EL6fvhk+kD5APmQ+Y76NPZO9Mr0BvnI9Or1DPCs+670vvrs9tb0qPG4+nr6BPhS+qT0uvT8+HL06Pg+9VD6kub69HL4BPge+Jj44vsk86b0uPg0+1zyyvkI9Qz1kvho+UD5cvco9CD4wvhA9rr5Gvuq9BzyLvqO8mj62vUy+6bzvPRw+5zwBvo+9zz0bPDi+sD5Jvii+iL5SvFM+vD02PEo+4r2XPSa+V770PUc+Eb5tve+65T0kvkk+Yj6rvls7jz1dPYK7lb1mPu88aL0kvvo9/ju3PqK+Hz2gPKm97b2Tvju+h77evBw+Yj3/vG08FD6Yvhy+gr3wPKE9sLwfPDI9jT2FvTA+ML3IvQG8qD18PFE9cD34vXu+yDyBvbG7Or3HPZW+DL6Eu7C95TxuvUW9sb0MPMI8DL0mvOQ8c74Lvss91D19vRi/8LqDvg+9Xj5lvWM+HT5KvTk+aD6Cvb+9Rz6TPjg9az46PgQ9rLxmPZG+Ob5vPic+Wb6pvXI+LL0JPtg9rzzGPSG+cT5mvsy+xj1hPlY+nb06Puw9dL1qPpu9Vz0WPdm9Lb2pOxY+571vvHA+oz2OPZg+QL44vuc+Db0kvMI9Xz3DvdO9S71qvlY9CD4TvuW8Q70Ivui93LxTPqg+Kj08vkC+ej2XvpC+Ij36PDW+kb60vRG+or0tPkq9fj1QPQO+Vz1Ovpg9qb4Xvaq8yLw0vM09rT5BvFM+rz0hvpC9Eb24vOi9qT3xPa49Ij6bvaM9Cb5OPPM98j0nPRq9cT73PFo90b0julW+GzwYPVi9ET6avX0+JL7UPUc+iD2Evti+ST3uPXo+0T0gvY4+1b3DvhQ+7T11vqS9tj1Zvku9gT0JvRC+Hz1fvZQ8Ab4CPo29hT7YPY0+dTyjPei97z0+PoS+gT2Bva69lbusPYY9mz5uvmw+9r2cPsu8tb0XPna9sj5yvQ89RT6fPR8+572Uvcg9KL6nvce9d74BPoK+Cj4dPp09mr6RPI49zz1YPQq+Fb7GPfe9ub0mPqA8g7zkPd49Fz5LvpG81j3yPIa+Zr58PeC9sL3VPWY+474vPR8+sjwSPuk9Rr6YPh++eL6ZPew9iD3pPZ89VL5Ivu89Bb4hPiq+qT21PZa9Qj4BvsE95z2XPlI7ib1Svru9/b3YPik8YL4Fvo09cb15PrQ9Xz1PvlM+Jj7zPTQ+E76SO0m+i706PtC9L74BPsm90T0vPXy+DT09vjU+1bwcvla9hb1TvaI9arzJu8o98jtvvhY9lz3qPbA9lz0Mvcc+yT2DPfS9Xj5Avsk9mr04vrg9s76PPcu9Yr3LvYg+Tj2Mvgu9sr1QvQI+Sr4Evlg+Qb6vPUi9WTyiPog9p73lvIY+Bj5muhA+EL6EPDa+Bb6Rvnk+gj6Vvb49171IPRM+r73SvDY9hL18vIM9+z2Dvha+VL5cvgw+zb2TPNq9b77PPYk+M76KvZK7mT0pviY+IT0MPgW+/L0pPqM9JT4kvds8AD1kPvc9CT4RPic+vj18vbW+N77evUE+u70SPo49/D0cPaq+MD50PpQ+hD36PZW+Hr5Evs69Jz5aPtc9Iz6UvlM+Nz0wvWu9ljo2vfg9zj2RPUO9rLwxPYY8yz6tvoe7kj2SPv+7ML6GvuK8672+vHm8oL07vui9J74NPu68gj0oPpg9zL1APqo+Jj4Hvk2+Vb5FPCk+Nj7UPda9k726vt08rz23vea9rj6BvCC+NDyMvOq+hT34vbY8pb2wvbs8+7sJvYq9Pj6UvTc4Lz0YPgC+jjzRvam+dj5Ivms9X77cvoo+8LxmPpE+BD6MPEE+hj28vUa8FD5xvBE+oz5SvsI8ID3JPQ4+Nb4VPmA+YTvyvDu+rz4dvoa8HT4kPiu+gD6gu3O+xj1SPsQ8WD0tPoc99z3tOYs+h70WvRO+JD68OFg+rzxjPgk9dT4avkw+7z3KPf89iL1SPeW9qDzyPXa7/T2RPWa9Wz2DvKu+7DzivfK8ar7RPfc9qr0CPSq9b75LvgU+uD3Fu7u9VT4rvsa+/rt1vo0+Ob5iPik+Hr6APYA+Hr0uPoE+vb5EPvq6PL6JvoU+cj5HPjG+nb4+vh8+JDzePH2+Xr2IPXk+cD7lPbo9vb2Bvkg+271ePoa9jb5mPiU8hb3BPZO+Tr5pvs+8iDtsvge+PT7vvI89ib4qvOG+Mj79vSg9ZT4GPoC+L71Kvos90jyWvoi9Mb75uxk+DT63Phs9ID6IPai+oLySPfc6HT0RPVS+K77vvVc+xb27vSW+FD1mvoM77z0Jvn+9fL2HPRE9kz46vGu+nr00vjM7rD2PvWy+qL1BvYs9Y7nTPjG9A75+PiW9Bb4VPko+K7zNuys+8D3JvqA+Vz1mPTe+bT4vPga+tzsqPYy9Vz4wPrs7zL0mvh28Vb5BvQg82r4CPoO+Ez5ZPlc+Tr4zPAC+Qz4jPqy8KD51O4U+1DopvrE9KrygPS69F70Tvmq+UL7DvY29G73JPao+v73JvUQ+jL7OvdO8Fb0LvdM9Aj7gPEE+672kO/K9sj1GPmy+Nb6nPSS9xjqpPRE+h70wvDS+2z2BPWG9iL6Qvnm8Rb7pPLu9k74TPQ++jL2HvoO9ijy2vYK+uz0PvmA8ej1SPrk9cL3NPT09Lr4gPt89br7nPaw+ZD5avTW+SL3hPIc+j75ZPZs+DD7iPlE9lD5IPRa7hT4GPrI9zj0DPqo91j2hvbg9sz06vQc+t72JPt69LjxdvQe+Cj4VPks9LL1PvIC9+7wPvrc9gz0JvXc+PL5jPNu8dD4yPni9Hj2TvWy+rT2xPdG8Pr2WPTY+Mr4Hvgw+gT2yvlU9pzzcvW6+qr4zPae9Dr5EPpa9Sr4sPgW9g70Qvkq8Jb0CPW49ob1jvsW9C7yEPFQ9BD4rvSC+ib3yvEG9kr2NPh4+Q77wPKa9Wr1pPr0+0Do2u0g+TTtrvKS+LT6JPRY+tTzsvX89CT4bviu9Pr6zvCM+v76wvRW6t71CvYw9Yj3NvQg+MD6mvl2+gb6XvLA9M729vY6+8z2gvOq95z0pPeQ+Pb1AvcO6rT4zPYs+Hz1xPg2+FD4Ovos9Pr7jvP+9Nr0cPhy9R702PlM9hL0RvgO+FL2oPis6KT46viM+zj0Zvgo+Hr6RvhS+rD05Pqi8ab4ZPjM9Dj6LPJY+p72KPoW+kT4ZvRQ+r71mPYW9LD4bPZ0+Gj4nvjy94L6Bvuc7aT4pvvA8oz0SPuS9kT7NvaY9Kb3Hva49SL5vvYK9Nj4kvoW+wT2GPnO9GL5tvQi+j74JPrY+mr3vPS2+/zyYO2Y+DL7xvbI9BL7PPSm6AT6Gu6m7IL5Evfu9uT5nPUi+wL1Wvr+9DD6KPog8sr0cvrS87z1Uufy88z2HPbQ8fTyLPkA+Wr6NPVI96b2XvYy9873yvFc9aD2xPek9Jr0gPsk9Vz4Zvqi9tjvIPfo8ZL6LPRo+kD3GPTO9Iz5qOvm7t7wqvje9ij7dvRC+Hz55vZc+Kz4yPau6Wr6SPde8qz76vBe+Lr22PUY9u73qPMC9Rb6EPvW9oD0MPs+9Mz5Wvpg9Kr5APeI7gD6dPCQ+Xr68vLM9Dz2iuxY+8r1FPsc6vDxQvaS9i7xIu5k+M77QPXe8xr2Hvak8BD5IvS2+bj5RPfi9A758vdc90z1kvqg9wr1nPRi+u711PnY+WT4Yvo69szolvvg9jbu/vjw+nj31PAI+Dj7ePHg9/zqrvn89uL3+u948WL6IvYe+gr5tOsw+i7xtPPI8Jr76PYM9Fj5Gvt8+873mPR8+Oz4qPda7Z76uvfy9Iz33vSe96D3/PRO8TL3LvTC+Yj4kvG2+W77LvI28Rb1Tvji9zz2JvQA+Vz0BPg67Z75/vdQ8fr3+u8s9oj2BvXe+Ib4Nvjo8o75cPqo9PbylvY49hT5AvjK9JD0bPhg9kj5APiu+lj2svY09br1zvR69GL0IPQW9KD4fvUA+Qz5ZPYu94jwEPQ+8g7skvgS+Pb71vJU9v76DPve9ZTnsPQU+cb6PPZq+9T3GPsc9Aj7TvVu9KjxAvgI+Tb0QPhS+FDxlPdy9gbyMPm4+lz27PMG8bT52Pvs8lT0RP3g+b74dPiy9H771PVk9WzwDvOe7Wb00vQq9h7w2PTE9p71CPOu77zyBvV698DzLPBi8C7whvdW9Qj3/O5k87rwuva08zrzNvMe8Pj0gulS9GDzhvBW9U70RPOM8DbsOPKU8K7xfPTk8RL3lPEg8mzwQPWM9FzylPDY99zySvI47U7zVvAu99Tw5PEc9ErwCvZe8ZzwevaI7zbsTPcu8jLwwvew8dzwPPSG9kztJPI+6K7w3O8W8MLwjPZS8nbw0vTg9ST0AvSS9srwbvHm8gr1BPUM9/rw5vAW9FzxhvIK8lLgIvGU9b7zfvMG8kDyOO6K8/7whPd481Tx1PdA8IL1XvKy8z7w3vV69bT2SvKI7Qr3BvOG8iTs6vUA9gDxhPV87tzsGPdC85DzWPYi7GT2qPJM9iL27POS8Sr2bvQI7oj0DvQS+kbwJPYS9+7vtPII6GrwcPe68Qz16uZM97zuXu2q6d71/vYY9nTupvAe7ATx0vYw8sDxiPdO82LwMvD+8vTw8PLu86ztTvK48y7w9vGM98rubvJS89LwmvZI8bjxPvRA9V7yVu0W8Hzw3vTM9cr0cvQa9AjzXPDi9szyyPLY8HT0MPBW9N7yDPJE867z1PA08IL3TPGG8CTymPO+71Ty6PGO8ejwMPSG8jjxLvfk8bj36PBO81TzuPMm8zTznvBG7Zb2IvCm91bwiPFs9q72/PFs9R72HvaC8xTzGPKQ8gj1zvJO8kD0KvSK8W70vPSY8Cr0KOyW9EL1WvaA8VjyzPK64DLyOOpo8Fj3YvJ88Hr1BvQ893rpsvCu9Gz2UvSc9v7yDvca8S72YvVu9xjrGPZi9jrwnvAS9Q7zhPA+9Lb21PZs8T709PNs8J7wsvQK9PjtbPak8G7yGPUm9qr02PSs777wgPd08QD3hvN07WzzdO7O717tjPEY88rwHvcG9X71JPWK8MLxPvJ28sD1RvFs7Az2SPVw9kbxtPfg8mzwKPNc8fD3SPJK9CbySO149Rj00vYq8nDxUvIE83LxEvcC8Lz2PvQY7tr0bPSA84jxdPRg93bxwOzC9DLxGvZ66Aj26PMq7XL2YO0E91rwuO2i8Zjymu828Fbxquw899DsnPQc9hTwaPTm9eL1VvBC7Az1HPKi9pLwgPA285Ts+PRG8LL0vPdO8EDucOrA83LqNPOY7dz3buh69FDwZvTi81bzqvC09MDvMux89v7xWvay8Hj0gvdS8lz2vO669Mb3ivIw7jLvAueK8ers1Ox09wLyxPCY9+ztOO/889TwfuwG9EDw2vTs9ib2VvC294LsEPZ47Yr3svL08oTrhPAa8eb1wvNI8XTsNPIu9Dr3EvDA9qr27vF28mTy5vZs9Wj1euok8d7zEO6E8sjyMPT27jrszPR484DuHO7m84r0uvTw807y4vB+9dru+u2S9nTxZPYE877yuPAU8Dr2APU27ZTyMvFY9MD0rvA+8dr2UPPK7rzy8vLY8Mj0pPfa8nDsZPdQ84Ly/O6Y8hj1IvFM9YrwUvYW9vDozPYs8Xrxau0W9/bytO3y9kL3FPLg9b7z6uky89jwpPYS84byAu4M8gTtkOuQ8krseveu827wNPSM9nD1XvK68Jr2wvHY8LT1XvD89ar1IvXu9Wb2FPUe6JD0uPY26SL2mPEG8Ij3nvKy8Cr2xOlk9jDxWvGG8Sby6Ox69Ij1TPeA8zLuLPDo8OT0bPTe99DuevBc9u7xSvLS9rrxGvIq67TtCvPU7QD1ZPL24Jr05Pac6g7y7Oyk8bTulvBo6FjwnPQ48QT2YvaK8qDyUvDu9o7wbvRc95TygPG49ZTwEvVA9Ij0OPCm8RTycvSe9Nz24vP68Kj1bOxK9yLt6vag7L70dvFa8cz1RPGO9lDwfvVm7H70vvCi9gTrtPAe9WT1SujW6ND2YPYk9aT2BPEQ9CDwfPTK9zTxBvKM9sLy1PEI82ryyu6i8IL3vPMm8Qj1evKG8W727vH+9cTxmvOy8ET2tPD49b7xvPZs9Vj3ZPBO9dTuQvPc8FzzsvIQ88TwHPRC8KrxiPSO9Eju0OwG9wDs4PGm7X703vM28wLy0vIm8lT07vbU8mDysvOe8Jz2YvSQ8ebs9OxO9DD1avGS8rDsuuwU9ND3WvG08dDzdPNk9J73UvC87crs5veQ8qDtSPby7+TwTvRQ9pD1kPZC7oLkGPAO9yzw9u6a8lLz6vEW9j73+vJI8c7xkvfi8g7zgvEw8kTwBvRw8tLy1vPQ8hj0IO/C6jDs2vVC7fjzjOOM80byFPcU8iz0HPM28bb2mPWy7qDysO/c7zDu1PIe9qTwGvTC9Ab1CvdO8Q72QvRW8dz3wPFQ8bL1vPAc9E7wyvdQ8KzyJPWm9rrtGvIk7Qb3JPLI8+zwkPQ+9Cb0HveA6xzoRu4S877yfPeA8MTtqu5m9Yzw2PKk8z7xDPTe82TxxPSG6Dz1DPOM8WztMvau8pToevCQ9/7uXO7o8CjyLu8k8y7zjvBI9GD3RPa+9uj0dvAu99jwVvMM7a72WPHi8lTy1PE+8YT3mvYu6rzxEu6m7LzyNPI49rjzjPE+9W7zgvIo8Kz2NvBu9Oz3jvCQ9AbwvObG8DzwOPdW8zTuWOie8Gz1/PJ+9YTwPvcG8szwYu5I9iTzPu4+7EToBvGY9pzyfPLs8BryRPY+9Dz2su8i8/rwAvUo9fb2HvIc9zrwVvQ67ML2DPBG9nbxsvf07ij3iOyU8CL1JPZi7i7yOvK68dr1IPYs8Xz39PHs997wfvC+9Jb3ZPEm9Jb2GPD68tDzpO8g8vzwgPHA8DT0GPHU9g72LuhU9nT1svAu8HD1MPZ88YbygPbe7Fj3IvE69uby+O1W9NrwUPY+9b72vvLc8jTw3vZa5tzwCvNo6tTtjO1m7l7wsPZK7Jr2oO7Q6CD1zPOs8yTwFPWo9m70RvLg90jqmu168TbstPYA7xzxfvdw8MTwLvRI9RT1OvYU7sjzrvC+9Gr0PPc06s71JPbY77zwcO7u8xDu5PBM9ITtFOzS9NT0DPN88yTt8vKA8/DztO/e8jjyBvZy8fTy7PLG8dj2Wu5E9jTwnvJO71DyVOhm9Hzy+PHA8wzs0PES8Sj1PPVq9+jyWPKw8vrsmuxO8DD2ruk68iDujvZA8sr3POxK94bwIPAS81jxIvMs8ajuSPby8Fr0bPbK8lDwmvLa8OjwNPRA7jb2mu0A8V70qvFO9X7x0PTO9UL1PPEy7mL1fPca8RT10vP48QLthPXC8mTy9vfg7xzzqvEC9F7wovU09CT1/vKa8Tz0nvcK+xbxtPTg9pjyTPIO9jby0vNI8rTy6PBe93Ds/vQg9iztUPTK8dbwPvVk9lDuCvTE9bjyJPei9ULvYvHC9aD1Ovb+63jzYOys9njywOw49KDxFPTI9/7w2PYS9kjuPPCe7BryBPCs9V7sbPSg8Aj6WvDu9ubxgPEq8QL0iPAO9x7xeO5+8ezxAPXo8lj1MPDU9Ez16PQ09njxrvco817uCPbU7AD3nve67BL0jvRg8PzxKveg7zj0EvGM9Xb0sPXM9rzsau9W6Gb1TPAA9b72EOVE9PDzpvM47hL2cvCU9D7sOOui8nz34PO86hbzMOlI86rpZun89prsDPbe9qT3PO/i8yD3IvMS88ztkveS7aD17vWw8dLwRvP08zzxwve881jw3vZO8FzyUO/K8Gr0FvYI9jThcPP68YL19vII8Eb3ovBe8sLw1vc+7ND1BPdK7Hj1dvfO8mTvKPEY9kzvHu5k96DzOO8w7N7wUvdI8pDwWvc08jbzjPJE8E70/O1o89jy4vKw7PLoEvQk9dbt3vFq9K7zbOyE897wNPU+927yLvKo8iL2AvaM9tbyAvWe7gDxLPRq9DL2HvOM7Vz3qvOk8pLwFvRQ9LryCPBC8QLyuu0o9xjyHvR68XDwFvco8XT0nPUS95LwuPIG8rDwYPQS9Er1BPN48wjzCOwq8sruOPPq69DxDPFo95rwGPJi8hrssPDI6eL3EvQI8JT18PEc9QT6vPR09srvYvGW8+DorPUu9NTspvRa8CL2iPZc9ZzzDvJ29DrrQO609iDxWveq8q70xvXu98bykvBE87LxrPec7szwtvHc9nTyHPfw8T7xhvLQ8PjxqPIe9Bzwqvd08KjyRunq85zvAvKm8XzwSvIg9BL1sOvG8KTtmvZW70jxxO9w6BDxqPWA97zs2PUO6LjwqPSO8Iz3ruuU8rLwQO4A9lTuUvQQ9O726PIS9kDxtvAA9L71hvSS9xDxcPau9oLsOvKs8ET0cvaw8mzzxvAS97LxivZ49ez1ouxE8BDvrOws8/LvxPNA8j7xQPXy8hzxCPb09aLyXuxm9Gj0UPG69CD1XvFq9OLvHOwU8PL0/vSm92TyVvfg8UDxQvFK8wTw3PSW9g726vEs9szyGvWk9ezzjvJW8xDy/PAK+ujuFPGY9F7uVvFc9DT0kPba9nbyDPDy9FbzLPJ+8Ab16vbE7gj27vXs8Ab0DvVG8OD0gPSE8q7sJu9G8Ej1APIQ8D72fPRe97bzmuyI9zrmVPAE9bL29vD+8SDs7vI89aD0KPRw8Hj3BvKM8vr0pvb48iTxFvXs8jjztvLU8y7vBPBK9Zr3SvEw9ubwjvX89VTybvQW8cbxnPK28Cb2DvG08yjyKPUW9MjyaPNi8wLvSPAc9V7xiPUa8DL0hPSI7wLwNPBc6FL20PDy8W735vGc9M7yCu807/zxfPDM9T7yWvSE9yTylvEm9EL0FvHI8ozy7O9+8+TuPvNO8NL0LPWQ9C71JPRo9tDz3vCO7LrzpvJe8Vb3GvCq7c70PPUM8hz1lPHo81DwPvQI9mjwWPXm8Jj18u249Zj0tPWe7Cz2fvCi9vDwkvc+8UL1uO0E8N73bu/w74bz6vF+8kr06uju9pz1GPNC8Cb0ePek8VzyFOy68sD36vO88+bsFvfw7BL1CPEK9sDzxO6c9Ur20vF48eb03vEI9Cz3DPSY8Mz2WvB09M7wcPSo8iD2UPVi8EzyIPHE7rbx5vPi8Hr3qPJO9orzjPEs9e72DvVQ9Gb3pPMI8wL3YPHK9CDxaPZi8oD2XvRk8/jmHPZg9TbwbPe08h706vem8KL1xPM07SDytPKs9lzwsPDE9vztJvXs8BDxNO5U9bj1NvbO8xzwDvQg90btoPCu9cj1APJw9yrtCPGE9Hj1OPE282LtbPLA8LT2NvQQ9FDzyO+I6BT1LPa69Ljw/PAm9wrzmvK48db1XvMW85TukvMk8gTyau2C6Try3vCG8szw1vQu75DvLPIs8Lb0EPYG8G7yFvB0+Tz16PCA9KDwLvHW7Lb0au6c8ZLzDPO48pj3BOgQ8jr0iPcO7Yj11vL28CbwIvKa8ibx2uqM9wDz0vEA98LyBPeE8vLzXPEM9kTxrPSw92DuPva+8obxDvJ87rbw3vPG8VTx3PLU8Lr1evAG9DTucPYo8+TulvEu9VD0BvO08iLwAPSi8lrsrPf27lTxxvHS96DzIPKE8u7zTvOg5S70LPeI8FDxyvK07E70OPUs8Mr16PF0827rdvGk8ELzkuSm85LybOz25HL3ZvD+8fb3pPFe9Cj2+PPc8fD1aPIM95TwBPUQ9OL0rOk68ervTPLg8Djy6uzk9K70pvCc9rLyNPLe8ojuQPV+9wLtYvc48bzxEvUI987xsPE29Pr0sPVW9aTsbvcy9NL2LvUC9Hz3LPHY8tb0vPLw9Tz3CPDc8hTyCve88V7zhPIc94joOPRs917wQPbK9G7yfPOI85LzcvJG8szvvvFO7P716PIY9nLw0PUe9gz2CPT+6f7tpvc+8NrwyvBg9Nr23Pe28Kjz9vGG9Mb1mPEY7EL1+O3g8krwFPZy8FT1RvR890jyGvH+9tzxKPBm9vDyFuza9XDuTPfq8bDwwPTg8+byBPAM90DxbvY08Oz0LPWI9iLwZPZC9hrwEvcy8ejyPvKC8zLsLvSY96bxrvFa9UrzsO1e8iTy3PBu9qLxgPHU9SbyrvCk9gjx1PRE9mTyjPOK5+boOPZg9LD36PFM9crvLvE+5D717Pag8qz3tuy29njyqPP67MLz0u8G9qby3O5+9lbxXvAc8qzygvAg9qLxwPKA9IrxdO/q8i70wPJk8bTwUvcU71zvyvEG9nzw7vCA9rDyzu0Y9Cr3uvIm867rOvI68TDsnvcm8hTxfPEw9ir31POE8tbyjPOi8OLxZvYk83Dwqvfe8CDwaPUw9zD1IPFu8Wz1QPd44sLxgPSe9Lj27vEc9ITzPPNy8TT0ePCW9BzwrvbA8jbsLvBA90Tz9vIK9HT1KPBI9KbzNvbU93by8vOa8Or2XPZO8pzzPu0O83rtLvUa97TyyvBc9hbxrO5G8Vrzkuyw9hbwwvdI8I7yBPCm9prx+PDw85bzQuRu8+zwVvSm9bzxuPKE8JDunvR09ujs1vEC9ADxIPbe7kbwOvJe98zmhvOy8ML1cu2S8vrzYOx09LDsjPLw8nr3IO8a8dLzRPcK8jb2MOzc9MLyMvVu9NbwovKg9gz22vN88ez0lPfo7uTw+vS49gbwlPYY5ST0DvbI8HT0EuzG9kTs5vTo8GL1kPVK9dztCPYE93TylPMY7lTyJOYI8pLtfve68HLyKvXU7Cr1UPPI7yzwfPeE79LwLvMi9V7qgvYm8Ir0mPEQ8Nb2MPKE8BL2SPAu9cbxDPR87WD12PRS8HbuRvHQ8Wrw7vIq92bz6PLW8lr1bvCq9Kb2QPGU9xLsXvXu8qLkNvGY8B71ZPQ69TjxEPP68nzw/vZG9ETxUvdY7q7y9PKQ7BT0Uvee8Cz0mvUC8PT1bvCe9/rhSvPQ8V73uvJg9lzuEPQW9RbyIPFM9aT1oPam7bLyOPeE8D7syPYS8rLuJPUA7yrzzPG27Bj3muyW99LwbPMs8Wr0WvUk8lrznPEo9irx8vWU8KjuMvJi8Ib1lvBk8WTs6PcM8fr1SvEq8eryYvBC9p7woPQI8cL0jPGQ9LrivPMe87rw2vVC9UryHvAg9Ib2dPPC8pruCPNQ9kzyIujg92jxpPOe8jj2gPMk83byRvLk7Jr3LPIo8/DzRPCe9FrxuvDi95bvavJe99r2OPQG9bDxtvG28Or0Zvc28RL1APfi8O7x9O2K9ELwKPKK8Zrx4uhS8mrwrvYA8jbwcvM48QL0GPWk9/LsePW08p7wGPaI8nL0NPNM8vboYvWc92zsQvY097btJvIi7kb2GPLO80LuYPY0957zOuyw9Jz3BvH08JLsXOra9dbrZPJe7rzw6vQ69pD1VvS49oTwjvUM9U72OvHC9tTw+vZi8PzuqvOE9iT2DvXM9pj2QPL+8Xrl8vIc9eLydPJY8cj2WPVM9ej2quuU5irxnOzi66LvdvCG9Zj3AvLm9LT0oOy089zulvPy8iD1DO068ZL28PQ693LzjvAu9Wj3kvJa8OLw3vHW9Oz16uQi8wzzCPeg8sLwAPSq86zyWPA89ozxxvJO8QbxovTk8sTzIvEG8iDybvJO7fzznOq261jwovTU8Tj2ePVU9FD0svUo9WDwjvSc7VD0OvII8MLy4vIw7kTxMPVO9Mb3Fu927XD2xPf48Pz31vDQ9Cr2svFu9sDxVOyM9mz3huzk8gj2qPJ88FD0yPHy73Tvxura7db38vPe7QbyGvCc8Fj0uPbs8rTyjPJO9VTzfvCk9W73TPM68Wjw9O1U9pDwYPUy8jzyWvcS7Iz0nuxk9YbzvPEI8Ur3bPG89pjxsPcS8fD1vPaW9CrsMPY88wD1vuxO8MbywvFg8ZzwKvLC8zbtAvSg94zzcvP476jyqvSO9Fjw7PFA9uzzRPL48gL1jPV68dLyDPGY8N7wIvQY+V709PY+8b7w5vUa9eTyUvPw7lzylPb68ND2JvAs7mDznvC09M70BvWi9+rsSPEY8Sruuu1I9HzusPC29vbqjPIa85jzFuF69jjwZvf88jT2fuy28OD1YvI+9v7uKu7Y7SbwzvRA9EDx5PcM8mzyiPIu9uzyHvEY8DT1aPQA8ILtVPMm8ST3svFc7UzsHPSg95TtPvbG8Kb0zPIC8EL0kPEi9mTzQvEk9qTzQvGk9Aj13vXS7gjxsPWA9XzwGPNU8SDsnvMK8HD0XvUo8Hj3cPAU9dr0HvFW9Cb2EPes7kz0tPT683TyRPKw85rynu/w8/bvyvGY9iD2dvBM9UrwnPE+8BL2+u0e95jvzvLE71zt1vBS9QL37O8U8Gr1WO+e8fz3Fuxi99LwXPda89LomPYu9ZLyBvFs80LzHPUK8xzw7vVC78DxkPV09urz8uqu7hr0EPQ09OzwpPZm8hztEu3g9AT0hPQI92zrEPBa9Xb2zvCE9ED7BvNS9lDzdvCU9Bj2BvJc8LD1su5C8sj3hPFq7tDwavOm7WLv+vIm8bDxpPCY9d7zHvIy8ajxoPdG7RjyDvIE9uzySvT+9+7xdvLa6Or2TvCa9tDtwPes8N72XvHk9cj2MPLw7Kj0ZO6G8Zz3fO4+95TzbPB86NjxlvNo8Tr2KPL28Tz1lPBw9Uj2ZvAa9xryhvGU8pTxvPZA98bx+vSK9fjs5veQ6BDyDvD49QT0KPYW9OTx1veo8Xr2gvE09gbqDvNO8PLzavBO9ST0MvIM9A7ykvJU8kL0wvb48lL3Bu0K9FryvvAi9uz3CvB49uTxKPSg9irwaPcm7+butvGm6LrwFvem8KT1RvLc6Djvmu1q8Sr3BPMS7UL1rPeW6gz2vPPW7Hj2mPEk5Mj2yOxQ9hzulvLu8ib0/PRY8Dj3BvOE8ob3svGq8xjrXPNa8srzbPH09Gr2Kuys9ujzSvUs5Ob3yPD09Gb0HPLw9SL2ivAY9pjwHveG74TxAPU88hD2RPcw7Cr0iPbO9QD10u908gTykPB69AD0LPG89M7tXvag8eb2avX88mz12vaq9jryoPUk89byLPXC87zwUPek7Jr2pvf67DL1KPee6Oj2wPLW7JTt1u2s9DD2sPAq9YDzGvGm9ZLvNvBm9tzsCPW+8mLwkPWk9Fr2xvOO8HrxQOwY8Gz0XPIS8ebuIvIA8uDwgPXQ9jLxQvIg99DxXPUg8jTx4u8a8Mr11vZE9izxcvfk7fjysPKi8FD3vueQ7c70aPRw9DbqRva47CDwnO6Y6QzsJOza9lLzDvOs8h70yvB+9Jb2rPTE8Lz2RvIS+VD14Oxc8a7wrvXe9S70uvb28Cr1Rvbc8tr2XuxA9ZLxcvJw74DyKPQS9gz0PPSy92rxAPU49Uz2GPfa9GT0du8O7a7u0PDu7AT3fvNu8Kj2DPFg8ZD2jvD48dT3Nupq81zslvZo9E73NvbI81rs2PQK8eL1DPNU8HbobPYY7Ar0vPag7nTwzOpY9zbsVuxm9RD3CvMY88DwIvV89EjsNPeK5Fz3avGm9Pz3yO227+7odvBi9WD38PCu8ST0EvZm8W7zGvMY7dDwSvWu9Kj0kvRw8Bz0zvck8TTzLPA28ZDwlvZS9/DtNvZ89ULy9PIW9sLyUvGY8erzAvCU7PD0NvGg8FLtQPAW9BjvHPA49TTxYvBu9hD2ePA88az2yO5Y8GLvXuTY9E72guuc8iz1PPOo9Dr2aPdO7tbwqPSW9GrtRO7M8HT0nPfC7ND3XPAY89TyyvBU9Cb5pvJe8sztRPFI9QT37uzA9pbxyvIu8Fr07vE885byJvfM72TwYuYG92TuMPcc8+zvwvOE79bwhPDy9Rr0UvGO8/jzLPJ28dzsRvV68LL0QPdu7kLyfvVO9c7w6PaE8lT00vME8Eb2xvBQ9wj2Mvb29kD1AvVw9IjwtOjy9lby2vAu9lTs/PWS997x4vEq9Cj0CPZ68VD0KvcQ6Tr2ju7I8Vj1HPPo8W73KPIw8AD17PEy8lrz3u7y8+7u8uls9Sbu2PAQ9LrtgvI+827rTOyA8/zpIvWG7KDwgvRe8rbwTvOS7Nr29vFA7ljucPAa9Ez0rvWW8FL33vHY9oL1APaU9pzxQvW88vby2vPk6gjz2PIm8fz3mPJC8Or3yu7888zsPPLu8J73LOps8ML27vDy8dj1QOmY9kr0LvRu9qzyMvd28Yj0APSM9yryUPbe8zz1+PaE8t7w0u1w82TkWPN68ET3jPP871bwvvQi8trwTPUg94LwdO6s84LyEvKs9irzrOoi7SD0vvIU9Xbz/PJU81Dyauli9Sr2OvVg9x7w/vd6827wGPBy8d71yvLo7TDzZOcq8arykvJy7Xr2DvaK95buQve68DjpmvJU9PT3IvIk9irySPB69Cr3MPCc9Drz2PJE99zupvZs7m7zqvCQ9lzo3vTI9RT2XPKW72Lwivd07MbvVPBC6Jj2aPMG8hzy/O/i8hzyfunO8dr2uvIu8Kr02vJo9c72kvJY9tzwwvbK8oT0ovbE8vbugvLe5LbwiPVU+vju6PLw73zxhvKM8iTx+vLQ7kLxOvQM8K7zwPGA9zL3iO+G8AzwTvWA9EjtXvRY9KjtlvCG7xDwEPVo8Ar2/vLO9rjkKOpy9mDyqvGW9szyZPAq9FjxovSY9rbsVvYs8xj02PNa8BLzNvKW54z20vIA8iD0EPem88DxUvUq6ljyXvdM7oDvquxu9eDpCPLw8njs1vWi8wj3EvHI9TT3IvP68nDy3PEG9iDz4Ooq8NT0IvYq85DyJPW687rwKPXA9qbwPO748nrpyvJC9Ub1GPHm6xDv7vNw8C7zDPI89jj1SPBU93rwdPT88IL1LPSU96LxvvMk7w7hqvLq8X7yYPbk9Lj2OvWM8R7yKPAC9HD0sPLa8/TzBvcS9ED3aPAi9WzzZvVI8szuCvdO8yrucPCA9MjwaPLO8ML2LvI68p7t9vE87crwEPWM8oLwKPYQ8TDfWPA09sj3NPN+8lboVPJG8Nz1dO4e8DzsOvVw9qzw5O2s8Vz0UPR89C71ZvM88ZL1/vRm9RryRO6C9P7sEvFg8QL2DvSQ98bvPOxY9y7xKPeM8JT0ou+K8J7v2PGQ9vzwMvf88pjoUPbg7ibxJvEk8C72UvI49cT0ivaA8oD05PFU92zxau6w7FjyUPIk8BT0GvXs87bwhPWy7Nbv7PG89H70HuEO7H73mvA29HT3GvDA9mz2DPCK9Dj2mOoY8JD0kOnw7AbtQvc88TT3euSy7kL1ZPYm7TLw0vNY8Zz1ivUo807yovEo8d70oPSk9DT1evCa9Eb2rvAy9CLyJPNw8Xj2CvIs9kTvbvLW9/7usvC09tzxmPce8K7uRPaI85TydvIS8gzsdvCS88LwrPc088zzTuoE9cTuJvJ69LrtyPBU9bzxbvA68vrwRvbW8WL1rvAM9bTyivB09wjyOPac8VD0DvLE8Zr1yvUk8QDy0PW08tjzAPLY9hryrvHi8Fr0svSo8+bs0PBe99zyyvKu87jtDvaw8eT2AvQS9JL0bvNu8/LuUvYs8+DuQPOg79zwRPBs9HbynO6M8VrxxvRC9Q72QvSk8DT3IvN46nr2BPJg88Tzauw6927pwPU69Nb2XO1A9gDoVPXU6dDyBOui8VDxVPBi7+LwPPWI8Y72kPMO8vjwGvHg8NTtOPQI8PbzEvUe96LwePNc7k7z0OWs9ZT3APLq81bt3Pec60rwOO3o8yLyDvVk7kru2PCQ9bDs2vRS9hDzDvJ898rsrPGM9hDkBvX+8xzxDvSw90DywPF69TD1PPPm8WTyiu3U8DT30u5E8mLtWPEm98zxAPDS9FD3XPJ86gbydvIM8Ez2mPMu8/LwTPPq8njoiPbw8Mz33O827FbzbvEI8ZT38OmS8MLydvBW7Ar1lvS09q73hPB09hT1IPL+8SjztvVS9Xb3tPEK8t7tRPCG8Dj2sPOo8rjuYPJ895rzjvI88u7wDvQU9Qj0KvB+9obzTu4o8L7s/vKS9kL1YPXe7Ar7SvFO9Fz2tOyC9a73mvDy7oD0UvYQ8arz/vBu8sLx2Ozs7SL3XOtu7CbyoPNA8RTxEPXY9GbsKPbs8djxvO5A9Wj2nO6I9cbxxO3Q8nzzFvBI7nLuXu1G91DsHPKg8DD17urs91bxfvI+8xDyJOrS8lD1mOy28qbz1vDW8Rrrsuqg9U71qu9O8Fr3zvEO9cj00PBI9bD1Pu8M6nbvXOF+8Sr0PPEq807sSvao7Wb0Fvb47ibzEPCK9YL0aPaI8Xz0oPco8hTyAPAO9RDu2vHm81LyIvBQ8FD0Xvcq9cT3iuia7Dr3gvIc85jt/vCA8kj0WvSc9TDyhPA89kr30PMY8lrzAvOS8mrwdvJQ8Fb0Au047hrwCves8sTyBvFQ85Ls6umc6rDx2OwM9hL01PQe8w7zQvJs87DyRPCo9ibwivJQ8/zy3PJe8SbxcPQc9bjyAPLA817xwPWc9Qr2pvdM8o7wDPag5hrx6O5w7ED3dvGy82byoPQq9G70xPai3QD3POh69qDyrvHa8Db1ZvB699ryrvHG9Ir2VOvO9BzyZPYW8bD2hPAg9gLxdPbC8jjwxvbc8+byvvZ26QT0XPY09QjzVu4u8Wj2ovCO9nLwdO4+8zjtJvae8bz2Au6S7tzxMvd68Mz0NPRA8xDzzPF28CzuMPSQ9R7tjvag5tTsSvck8fLu8vKq8mbzQPBU9CruZvU09C7zjvLG8jD02vVU8tryouws9kTySvCa7K70OvYi9zDy+vBe9Jj2ovTU9/TsiPMW8gT1aPMi7gb3Zu4E8UjvRPCa8ITwRPUS9n7zXux88PjycvcE6szxquxE9Jr2GPDS9k7x1PRQ9ljzvvPA8P71lPVa8Gr0aPYe9ET20u289R70Hu9u8Oz0Huye9GTtBvSm+ITyuvNU7MDwzvKM8gTwePUI997z+Oz69lr3FvCO9FD38PFc9Kj0Vu768BTs7vWC5ljxBvfG8FD0uvTq9XLz3PJU8+Dqxu0G9iDyJPZo8FTy5PM68hryCPFO83TyFvIy8MbxzPAW8jrzwOg69zjkvvNa8DbxdOrc80bzNPFY90zx1O447Dz1wu+a8JT0BvPI82jyUuyI94rysPao8Uz3mPA09Dj05PS49QbxKvUU9UzyMvJa6Ej3aO3s9QL2MvMc7Gz2HPTQ9ubyUPas95rwQPfS7YrzBvFi8njy3PMQ8q7zwvAI82DyOPVe8rjyaPZ09Jr1KvD+9Kb0ovYW8BLzUuzM9KT0Lu5m8VjyJvLu8a71yuy88ZDwxvSC9Gj0uvYO9ozw9PRC8CD1sPRE8rbxAPDU8mDzNPMY8Dz2HvIY8ozvBvG68hjocPDE8iLzwO0w9Or2cvAa98zzOPA89Ur2zPO29MLsAvYa8b702PVa6GT1KvFi8gbscPbC8lj2IvBG9Ur0XPbs8VrutvAS9Rrx8vaQ7jLwMvQO8T7yFPHW8VbsDPV+8Hr0wvIy9srytvTm8mD2zPNq8qzy0Osq8FbtOvMg9nrwLvQc9Kr2Ju1I8DT00Pe07obxRvaS7B75fumI9ND0BvNe8LL3+PI48Ij2oux+9ibwzO9w8qLwyPKO8lLxuvai8tD24PFa8zjwLPas8Ob7jO/I8ET0wPSS7WTwJvLC9Dz0YPR486DtoPQu8Cb3mvFS9yLzGvDs9FL28vPS9vDzmvEM9Sr2avM27jzw5Pe45NrzPPP88Jz0bvZS8Qj03vbM8rjy0Ops9b7xhvZm9ujyOvIe9Lj7hvRk+6jzmvNy9qz3MvIg8JD2JvZM9OT3UvJ28Vr2lvNq9Zz3QvI67eTymvQw8A74rPdS9GD1xvee9Dj7ZvRO9z72lvAI96D2/Paq9lD1JPfk88T1Bvqs9WzwaveC9Db6evTg+Y7xQPVe8Cb4lPDI95T2rPNG9Q73QvE8+3r1xvDI9yT0ZuCa+3D0ivjy9yT2mO5G9ib2bPQQ9ezyKO4c8p72fvD6+ej25PKW9Er0bPh29tbysPPM8rbxevcG9cT0DvNg8D72oOhE9ar10PVo9HD5lPmQ5Q70gPcO8+z1GPj69rj0IvVO9iL16PfY7DL2hvEo9mL17upS9Rz36u0K9qjyVPk89Yz2pPEg+vTxtPes8WT1ivrQ9vLttvbo85b0APdC8Zb6VPWY8vr2YvSI+PzxpPak9DT0GvNk7kr2PPLe9RD4uPnE9Urz1PS+8dDyaPXc7GT04vVY97DtDPZ491z2/Pds8i72BPUa8kTyRvZU6sbyCPAe6Pb4vPYU9sryAO4y8NbycvWk7Y72GvRU4QT0EPlA8/L1ePYi94r3lO7A9270MvYS9hzzrPaO93L23PQA+fL2MvTm9Fj7HvXW9Jr2nPHy7gD2qPf89YT2wPHq9Pj2evQM9IT33vHe9yryLvUK9hj0UvGk8oD2RPZq7nr2xPZC9Wr0gPDU8BTw0PpW9wz27PTe9pD1PvSs9Br2uvZs9nz0Wumu+bj0sPBw8yby9vV88dzwIvm280L0cvEk8ojzWPbY9Uj2VPYC9rbwyPQu9hL2jOvA8xb0tvQU+XTwzvmY+Uz2DPe+8EbzOu0i+Wr3LvWM937vyvZ+9Ir5rvJY8AT2qvGK9Hb6mvAy9tzw8vme9PTsXvem86T2SPP+8LL7wPIk84T3dPao9Mr12PNM9gz0EPho9Bz4iPhK+T71pvAy9qj0mvSu+wj0Jvlu8OruTvfY8EL0ZvZU9nT3CPFi+abwkPbE8Kz0aPSA9MD0zveQ9wb2bOgS9Kr6IPKc9Ar3vPbe9G73WvRC+sryrvXI9Yr6jvYc8yb3evPa8hr3dO/a96TsJPjc8i7z+PTM+Yb0jvA++2r0hvU89Hb1EvQa+4D2jvak7lLqlvG69870uvVI9Cbx6PfU7pr0qvcY9SD0NvME8x73lPYa9lD2sPRG+sb3fvak94zt7u6+9oL2gvP28IL0fvXO9Cr4HvZa8CL2tPdi8OD0APV89nL3pvZu8qT1Lvd28Zr0GPqu9ZjwEPoi9hj00PP49tLyuPQ2+pb0JPuk87j01Pai8Lz2LvBW7/z0GPjW9/j2kPNy93L22PXI8wbyavVQ8nj0xPr08Hj0DPne9DrycPS898z1xPVK98z2tvX09drykPAW+Gr3tPZ+8YT1GvUs8Kz4ePjG9mL2gvce7Cj0bPgG+LL7QPbY9Fz2QPDM+iTy1vWs9Lz0nvua8kb1PPAu9lztBPIA9AD3VvA4+Tb1oPQY+ML62vfm8cjzNvKA9x70zvls9hz1yvZ49sT2lPce9HDyEu9C9CjsUPmi9HztRPSg9nr2MPII8br2PvfA8cTynPaE8JD4zPe49pjw5vH++HD7FvEu+GjzTvTy8AD0bvRM9z73OPDa+eL0hvQO7FT1MPBm8Iz3BPWi9sb1pvTs9NT6CO++8b7vbOzE9t70Jvj0+hT08vG88Lz3AO909QDyLvY69nD1JPRs9Dr55Pd+9Nj49uzc9ar65vcG6UTziPfO9Mz2+PQc9pL2vPQE9w72mO2u8UTyhvNy9C76HvRc+Qz4ePtM8sz3Su8G8jT28PYq7nzxYPcO9pz3FPA06iL0yPcS8BT0cuok9lr2sPa49/T1bvau9oLxQPau9nzztPNy8jLytvWi96b0ZPUY9WL5Cvei9QzzkPAm9pr0APRC8cr32vcY9Bj6APR8+EjwIPBQ+2D3kvYw8P7wzOwc9jb28OpS9rzxBvou8oT0EvpY8F76tvNU9IL6JvbQ9B76pvfO9Sz5TPc+927z6vQM93r0MPtQ9ELymO+Y8Hz1QPbM8LDtoPSS+rD2EvTy9jb3HvdA9BD3fvZs95T1svCW8uzsQPsw9ZD1JvYk8Fb1mPQq9FL6FvA6+0z0/vcE6mr1KPQm+YT0IPhE+pz3xPGg9+z2mvfk9I729PSk9Gj6pvbK9hL2CvWC95LxCvsQ8QT4SPCm8B7v3vai9LTtEPR49WD1+vBa8f71uPcY9/DwOPlC8O70MvWc+4Tw7PPe9IT2avV26SD7dvBE+NL5EvYk9T7zrvLu7+T0LPr29wTwZvX89Bb6SvVM9FT12vcg82D3vPTe9Ab0UPSI9uL0GPmC7sruuue29kj3vuxk+PL3UvT29TrzhvZy9D70MPaC8Lb4aPo29pDydPZ090zoavV69BTzmPYG9lTw4vvA9sz3xPJO98DyZOxu+WL3CPTC9kT0qvVW9Cb7DOqk9gr3lPIQ7kT0Xva89i70LPYm9yj3Svag9grzKPNG8OLuWPeW9Nb3YPAu+drzRu5C9fT26PT88ZD3BvTi9Fj6svKG9cT2KvLk8xby8PUm7gz1oPMm9wzzZPII9vD0oPjO96j2MvS89ub3ivPq8vL0TvTw++7vsvc09C70RvWs99j3ZvTu9Q75SPdE99zw8vn+8BzqPvVW8JD5Bvdo9Oj2VO0S7cD06PSA+aD6KuuA8Lr1ePYa9rz0vPBQ+cb4GPli8yj2FvZ49LL5APYo9LT6HveE9BD2RPRc+Fz3mvTS8bj2bO5e8zLwovXe9RDwCvg89Yz2uvR++9borvUG8kT2cvVm9Cb5MvQe+tL1PPvE9xz1dvUu8mr0bPNc9gD55PNC9LDtzPgk9F7uMPEW8lzt6PZE9Ij7BvOC9RjtAvh++FD6vPYU9Ub0FPV09bToRPQC+Tj0YvCO+7z2IPX69pr1hPRo+0TyrvQO9j7wHPga+IrwwvoA9Tr2TvZI9+jyxO768DbwgvUk9qTz+vFc88L1DPHy8xz06Pie9ZDy1ve09rb3EvKg9ezt0vZ89TD2xPXG9W77lvRq+Ij4ovjQ9Gz0+PZ09lr3oO1A+pb0LPKK9tz0/PZ69lD0TPgA+DT55Pei80r2svYq93jwwvkM+tD3+OwG9Oj1wvYG8Sz3TvfK8Cr4+vay9Zz6bvCQ9Qz0Fvu08Iz5jvWu9jD0FPuy8OD0TPt27LDv1vMg9HD6qvdy8i72NvaC7Q71WvZA9jD38uyc+Cby8ves9Wr02vUE9Az7EvWc9DL6YPYi5Yz2mPV69RbygPNY8g72KvQM+jzymO+k8UT2cvdG8bT0avtu8UDzSO7y9NL1Nvns8xD3GvT8+rL1Pvio9KT6TvXC9ab31PcC9Sb03PLM9L71rPak9TL2gvay8tz0cvUK9Xr3xvCu9YD0GPti9BDtNPfO8/rz1vQA9hD3iPYk9E75+PW899r3SPSG9UL48vDw91Ty5va492LyXvfe9EL1MPRi90DzfPfQ70j20PQ29Hz7CvLy9yrwlPTc86r0tPmu9Ab2EvCE9I7wdPXy9H74Pvou9kz0YOxu8HL5svZE9hb30PG49LD3NOyg9nT3mPN09Dj5XvOm8Nj01vH68YbzcPTG9WT2NPT4+Sb4SPjw8Kr6EPQy+k70EPZI9+rw3vTG9BzzDvVg9Jz04vSo+er1AvBG92byJvRY9v71mvB68NL1UPtc9VD4mvaw9rj0vPhe9nzxMPke9Qb7fO7K9Bz7Gupw9vTxKvD494j0SPAG+M74bPbm73LzOvX88JbzTPcA9Zj6CvZK9uTzCvGs9BL3bPTw+gbzXvBo+fbzrO7E7mz3bPQs9Tr49vfu7F72hPWO6+73TOzC+6bx5vU8+Tb2pPPY9l705vmA9ETyHPRu8573pvYc8xj3ZPTK9nTy0Pe28fLxqvY48Ib5KvPy6lrxqvTs9jT2Bveq8Fz2+PGk98j2hvfW7UL2nvfy95bxVveQ8xb1JPQS9ur0xvQW8vD0gvV49MT0rvbE9ST4zO2a+pb19PQo9fT7IvEq91b2NvZk96L0aPB09Tj0fPKe9TbvePDE98Ty2vUa9Wr6APRA9LL7gO+o9wL2IvrA8wD2LO3E8Dj2GvU490z0bvQc+JT2nvQS+sjxHvCe9Vb1DvQm8Ez7xPLy9c735vYQ+tTw/PG09Br67PS49zb0YPc+9Kz0JPXO9jLwPPuI95z1gPMm9kz0JvgG9wbv7OxC90Tw7PR08hD1HPas8kj1APaI82juoPZM9Db2SvRQ9IDxzPdY8pT0ZOxs+Xr2RvVk+tb3QPcG8nLyfvRa++zzBvSW+5bw/PU89orzmvK49n7zmvIo9qz0BvLA8sj0jvaU9JL7ePRw+aD2tPQc+T71yvNU9dLvVPam9nD0UvgO9ajrEvNg7i72LPdy8bD0LvUI93D05vXs9nD1bvQ89o73KvcU9iT0VPgq9FT2bPXO7xD3fPX09kD0dvdy7EL1VvaO9wzwXvce8pDwCvk8+ZT2yPQe8Hz6xvT69mrzVPRm8ib07vE+9oztAvR+9/jssOlC9n7vdPCI9RbwFvrc9hD7ku+W77b3OOwg+lb0bPps9zrxROwE9XrzZPWE9mL3Qvdu9/T1Dvdy8Eb7nPQ68db2QPOS9br17vRU9I70BPak7iz0rPEa9TT0Tvd27j73FvGo9TL1jvDO+yr06vW+7B7yVveK9nbzCveU9qz3APVG8Q70Bvcm9Gb2nvDG7SD23vCy+HD04PoC9xL0euwU94D2rvQA+uz16vQy+GL3pvF2947wJPkY93b0YPpM9hb0xvhe7BzyzPMW8ajuVPUW9Jr5zvfi9GLtavVu9HD4Svga9oz26vYa7Ib29vdw8bD0IvRU97r0rPiM9Oj1rPLY9xDxmvM+9iD17PVa8qLzEvQE+cL3XOoE9DD4BPls+lD2xvHq9YD1GvY09I71xvNy7ET4Fvmm9KzhRPcs9+Ly8vPa98r1mve49Jj2wvdg9AD7CPNK9rTyKPca9jzw1vXG81zxqPYW9oDsEPrW97TslvZC8l70zvaC9r71nPSW+Hj71vVK9eb1NvQE+872FvQ+907tNvRg9ED1mPvE9jr3ZvZE91T0Sva69Xb2xPRW7IL2FPL69SL3Bu3m8Ez3mPWY9N7xQPI69671qvC+9Czu5PdM9pzxYvds9pbzDuvQ9kr2lPbk9yL0jPrm9Jb2+vYo9tzziPKK9W70Gvi+8mD3BvXM81L04vOI7OD18PfS9bb0PPsY9ir2QPSk9tj1ZPKW9HD1TPWu9I72FvVO9sj3iPRY+HjzMvVe9D7y0vW89Kb0OPca84r0zO4k9Wb06uym+kD1qPVm9Jr1jPMY9zbzMvZA9qbuqPTG53DyQvQ++nb23vQq+67s9vdE9JT2QPIw9rLzVvY69m72TvRU+h7wIvuW81TzqvR29x70IPSK+Jb23PQ+9Jz2TPUY9ib1ZvSG+T73xvOi9hTxSvZq9fzwBveI96D1Ovpg8yL2ZPXE9o7ytvQO+PLwqvFW8Jb0OPrq9nDuevXK8fj3HPZ09hLyRvRc94b24PZI8mj3KPX49Gr6UPZS81L0yvgq+mT2fPDo8Xj1jPb+8E72lvQs9qjw8vW69Qj1lvc69arzlvNU8CD1xvQk7KD4fvRY9vL2VPRW+zr3uPek9pT00vgc9lb2Iu9M9ZD0WPVO9Cz5/vc48Jj22vXI9QD2zvI69Rj1aPo+9pzyEPYk9Or6JPTe92D0TvoW8NT02vg+8PDxWPaW9Mr1BPTi9ib0Jvjk+3TyvPXu9jL0Gvh0+Sr0evpi8nLoGPoW8b73LO4+92z09PUw9ujuZPXS8Pb2DPYg90D00PQO+nb36vXY8tr3rvUS+vT1Nvbe96DwMPhc60j3lvWO9hL2IPTo83bytvGS9DL6wPG69Kr6RvVo6nDyMPRE9gbzsPZK75zqFPfU9Eb6BvT07iD1Vvfy9Gb2DPe49kD0bPp29nj1KvAC9nr0rPoS9Dz0VPv48Rz0lPT2+TTtZPI29BD6wPTY92jxBPOw9VL0DvgI+uj0CPf07rDz0PU697z0cvjQ9+buJPYI9iT39PZK9Ir2HvUQ9yb3gveq8/b0UPDq7hD0vPZq9Rr0EvlU9DT4JvlE7Mj3DvWS9Pj08vU29LL7iPVy9JbxUva299DsiPv+7Pb0zvRs9aj1WPSa9Vj4GvpM9Lb7nvVe90r03vvW9zrx5Pdm8ETyiPWM9UL0nvgK+Nz4ovag8Bj5UPco9Bb40PaO9Ib3aPZi6jD3/vAU8ZT3RvQm9FD0BPpU7BTx6O8E9jj3FPRW9tjuwO14+8DwzPYi91z0ePb89/D0RPrQ9gT0avQQ+AL6mvMA9sj3yPIe8/rsoPSO94rsUPBo9gLzBvYS9uj3TPfm9LT3FO6g9yr2uPG6+lr2BvA++qT1FvZe9fr4HPhK9WLzXPTK9vT0Uvbs8Fr1kPQ89671ZOei8fL0ivoa9a7wovdm9Dr4avsy92jqqPJK90z2NvbA9yTzEvMW8fbwtvfO9PD0VvhQ8wT2vvN48ND5Wvqk7f70+PbC9kb3mPSe+Ir4YvWA6Er2FvYM8Wj23PLu9G7zGPbi9Cb1avcI9dbywO+m9T75aPQk+Zz0/vcw8BT37vCI7wL1ivHM9sT1vvaA95bxPPAc+Ij1KPiS+gr2RvY29BbwWPeQ9EbzhO6A9Gj4ovXY9bDtSvgY6A74tvv+9wT30O0s2lz0OPg89Yb3lPZQ9x7zfPAm+rTwFvZI8m7z9vfA9oT37PKs88L1mvf88iT3GPb48SbzXvUK9bj0GvgM+nb0oPh8+FLtRvW+9Hb2MvLE8YD2DvIA8cb33vPE8ub21ve077L0QvQ28F71NPe091z2YvWG9vLxCvGg8/jysPBy7qrw1vuM5ajsrPUo+y728vbw9QrznPQG+mj3hvYq9mDuoveg92z2SvKm9Kz2IvQI+cL0PPgi+qryPPOE9az17PSq68D1Yvaa8qr3iPFy+jD2Bvba8xz21PRO9H76KPYS9pL0Avfa81L07PQa+mDzLPQc+9DygPVQ8nr2dPRu+9ru9PK09iL4CvaK9vDxUvfi8ELw2Pck8dj2YvXY9RT6IvGq9QjqyvYQ9iL0KPZm9MD2uPMa8uL0Bvja947t2vRS+FT6DPTs80L34vZw9aTyNPCC8fzz1vXM8SL4tvKQ9Fb2YvA29+r0lPZO8Dj3SvX0+fL0kPCu+Jz3NvR2+az0Cvqa8Ur0mvYu6Hz1luQo+er0WPQg8Xz3KPJ085T3WvFC84L00Pm49GDrpPJi9Cr1dvas9j73qvRA+ebyvvZs8Fr0TPRU8SDx3PYG89zxNuy89gT1PPW49or3OvEc8ubyePWC8Hj0gPag9DL7cPDc81j17Peq9cLsIPlC8lD3+PFk8Yr0XvIa9uj0IPrg9ST1zPeE9Mj2DvYA9o72Evfk9Rr16Pbu7mb1QvSE7Gj6DPfE98r0Yvmm7kr1kvhy+fTvjvUu9kr2XPUM9UT0evnC8z72ePS09kb3fPbI9RD1LvcI8Kz4tvTm9x70LPdo6OL6hvcy83jx8vX48wbvvPfs9er3mPI49Sj3NO6W9kz37vd89ur1lPDI+p7wgvko9KL1YPaM7dT06vvS9RT7cPSa7qzwtvS2+iL0uvQI91b0UvQO9PD2LPAw8t72FPGS9570mvge+Hb01PRo+jD1AuxA+qb1qvOC8Vz2SvME8Vz0yPTS8Wj3LvI49Gj5iPRu9zDy0PXI9DD1OOQw+FLvaPRk9Ub0DPYE8AD4tvLa9JL29PDM9FT68uxq+mLvUOu89Aj2jPTM8qz0Rvg+9EToQPTK8Mj27Pcq8pDfjPQe+gL0xPd+817wNPgC+bz1KvDi9eL6BPYw7SD2DvB09YryivQg8lj2WvSO+SLvuPAK9mjxBPtw7gj0DvkW9wrlSPls9mT0Hvua79z1avfu9Y7yTvLk9x7yOPD49s71ZPQM8rrzrPBY8Rj0qPcg8n73bPRS95j2nPTY9y7x1vXU8Kb6JPS49uL0YPsu9orwhPu69cb2MPdy9nL1zPdg9oD3PvSq9Ab7puwu+GT1mPa69bT3zu/28JDwfPaA8C7uwvQk9M73nvQW+WD0+Pdm9n7uqvXA95L1sPPk9Nb1ovc+7Cb73PZc9MD2mPaW9Yjw7PWm9DD0jvfS9YD1AvVY9rjurvLc8BL62PHi9J72fu9E8ED07PcO96D0vvXm94D1fPYC9XbwGPqE9bj0JPY48STySvYm9frw9PDK9KT0yPS08+7z3PIM95bu8vb85zL1sPVC9XD2RPTq+Ez1ZvSk91zzhvJa9E77jvN88Or14Pac96DrevJW9Gz3iPFa9D77xPY29YL2wvSC+hL3DPQo7Gr2OvYm9Db1QPEc8oT2/vEW95b2tPd89Uz3KvYC8wj0zvsA9ir1pvdw8UzwOPlo6uz2RvLu8bTxaPWG9Bb6JvV09Iz0yvTG9Ob26vfU8jLwYPIA9yr2qPSK96rwyPha8uL0KvlI9yL2wvfa9hT0fvmA97DwEPZe8hz1LPZ69JT2UvZK9jL0GPfg9V7wyvQo+AL34vN68Zb2SPTe+hT0NvXo8ID6AvJY9PTuDPDI9Hzu/PfE8oL1mPEK9I74bPVC7jj0UPQ8+ib0jPTU6hb07veE9Sr1WPTg+oLy3PU09Fb00vIK9Kr4JPTM9D76qPYS8Fr3qvQ6+L73xvQI9hT0uPb09WjxnvdW9ib0LvYy9Kz4aPjM8hDsBvse7XDwWvTU7gzu7Pb29lTyDPRG9rz15Paw94z06vpU94b3wvCW90T0ZPXk9kz3LPZ29wD0FPl69mj2UPXY8Dz7bOoE8HrsCPQe8BD5sPKE9sDxTvTS91TyaPUW9bD3ivSq+nj2QPf68DT2bPZ69rL33vNk9Bz0bu4W9G77xuyW9uLuJvXQ+IT5/vbi7YL7GvQg8uj3lvfG9072iPKa9BL6ZPbu93j2EvQ69wj0nPrM9mL1IPbg9uL0NPpo9Tb0Dvoy9F72oPQ4+FL7Zvfs95T1hvRE+br3su7K9tjyUvGM8Fr6OOvQ8wz0uPq29AzsSvT+9Cj4KPu28tLzuPAO+Cb3TPfM9sbz2PcM8mLyOPaM8kL23PTM9PT7ouxq9iDy6vQY80r2huZc8vrwAvXY+9bsOPfm9IT64PSy8+TvcPc09qL2CPYw9Nb7OPfq8D71xPZK9Bz45PEe9CT6ZvZa7mr29uya8ir0ZPSG+ir0SvQO+wj1KO0M977zYPXu96zzyPMu8sr2WvQG97zyfvUy9A703PWk9N72BPcw8iT2wvB0+QL1+PSs8kzwcvo29+LwFPQa+jb31PQW9UTxPu7A8972yu5290j0Dvbk8x7yrvPC9oD3evUA+1z0GPQE97b3IvJ890j3NvYK9AzwHPpc9KrwGvho+OLzCvAg+tr00PM88Gr4yvjo9Nj4Avlm80b0xveI9g70AvmI86j2kvcU8nbpZvpC7pLsHPbW9Xz2MPMo9vbuZvfm9HD4kPoa+hz2avBe8Dj7WPXK9Hz4cvp29bL0cPqw9AT0zPUc9hz0cPaM9kr2DPaS9Ub22vUu95jwQPWO8Mb2FO6G9cr2xPXY9Cj0OPLK8Sj7RvRM9Er5vvTO9BL4PvZY9Wr3hPOu7Nj34PW+9j7wZPEc9SD0pPhU9XT3CvFu8Ub1KPGq9fTz6vfg8Pr1GPiI8YL2nPeO9jr3RPa68Xz0Avl69cD0JvjI9DjwjPJa8MD2JPS49gD1/O8y92ry0PZM9P718PXa97T0nPjQ9ubxdPBe+kLskvYS96zlfuyY9Lb2xPMi9hT3IPGO8A75XvbI93zxvPUc9cz66u7e99TywvYg9QL6IPJK8e73SvB89nTyQPo29BL7Bvdc9nLwcvUq8PL2bvVy9AD32Pfg9jL1rvXu8d71NPVK9zDwuPoI+t7wlPRw+Fj0pPYG8cT1zPTQ9DD4kuxM9kb1IvpQ9Rb3GPUE9EL26PaK8qr0RPaW8LD4JvgE9k7w7Pu89ED6Wves9Ej7MPCo94z1TPfW9Rz4FvYa9wb3cPYQ7OT3cPb488DuBPUI+3j2UvQY8ajuqPOC9hDsvvUQ8CjwKPQC9KT3uPC09/jwsPY4+Hz0UvjS8xb0VvhA+Kj6AvIk7/jy8PfQ9AT6nvLS81Tv7uj4+Vz1Bvb49EL0KPjQ9cz2Ru0M+5TzPPGW9jDufvRo9GzwwPD8+Cr5TvEW9oTyRvao8qb36vFS9s73BvJO9Z70ovDQ9Db6oPZM84byCPBO9Iz68vco84D3tPe09tryMvYw9ub2CvCe947wcPS8+KT7WvZY9m73MPXe9KrxJPI89lz1rPfo92D1NvXi9ozxQves9AT2VvQO+uT1kPb09IryhvTi9mL24vZK8WD0+PQu+rr2TvMQ9KbyWvfA7MT04Phm7Nb50vMa9YTyKvLO9M7s4Pb89q70sPc08eDzUvFs9Kj0EPbA9Yz3yPZC9rb30PHi8h72aPE49hjv6PRi+cTw6PoA9ADy6PWA8SD0WPV+9Ir3GPQo9tr2hvae9XD0AvqM7s70HPaI9HL6IvbW9MLw5vbm9sL6HvYa8AD6avaq9OD01Pkm9e73evbY8Cjy5PTI9DT5pvRm9pj0tvsK8Ab6Qu3U8qr0avua96Lypva478b1iPaO9JLqJvUc+Fze6PAo+pz27vWy9ZD0CPVY9vb3XPIc95b0LPRA+5jwWPeY9Cj2nvZ+9SL2DvSa9EjwtPOG9qT20PY69mDyhuqW9ij3Jvao9Cz59vJY9QTw4vr28PbwFvb29/L1+vSI8z7z5vRK+w73hvCc9pz0ivYE7tD1MPTg9bz0KPhi9sT3vPZu9TL6XvN89t7z7PWI8YTufPSC+orwLPmi9Bj2Mvcw77jxGvKe9NT3Huwe+gr34PDg9+jyOPQy8KTwavms96L0KPbc8e729POy9fDybvA07CTzHO9K8GT0zu4W97ryQPbe9+D0cvZ89uj3pvDO+z71nPeu8ID7/vMc9n70WvHQ9Dj67PY49nzwrvfo9kzxzPSo8/b1sPX49ZT3TPKG8nb0OPa87Tb65Owu+lD2ePeC9jDwYPcK9L74OvTa9O7tuvcK8P71ovBs+FrtKOk471L2bvR8+mb2KPZG9sj2Mvey99Dx8PUm7Mr3nOwk8frwSvt09aTqnvXu8ojyIvSU9xT3LPI08mTwAPiG9K70ZvRm+3j3KPC29JD2qPRc+Nz2LPdi9nz2uPVY9qjsvvQE+KzwsPtS8GL7vvI+9Sj33PE88D747vTg+oz2mPbA74zswPqG96z3CPQs95T3UvGy80jsNPcG9DL0LPbU8er3EPQW+Cr3WvV49rL2OvVi9gz2fvfo8S75TvaW9fr3RPUo9er0Wu2i9kT1+vce8i739PWy9jjvRvZs9g7spPra9iD2zPGy9lDzJvZe82D2tvIi8rjuBPX89oL2VvSo+272evdC87rwQvcA8LT0zvQW+QD0NPb+7l73wPAe9/b2gvQ++3L23vR++xDyXPLS9nLt+PSQ+ZLzVPbK9tr0kvVa8ob1gvbA9Yry4Pay9Qb0XvTk9h70IPKO9NL3+vdY82jmfPZI8G7xhvfU9BLwvPI698L3UvSC93j2IPRY9vb1yPdy9dD0AvoE9XL01uUG8PD2LPXM9Gj6gPeO91zwlvvm9QrzrO6i9Tr7VPCM9hT1Lvf+96zs2PCq8ZT6xuki95Tx/vda98b36PQ+9zbuHO6E80zstvUk9oT0kPqC8DD1BvMe81j0rvLa9mT3YPeQ9Hb0CPu697zz3PNM9kr0jPRI8Kz2yvBG9ST3LPBS9/bwwPi87jjwuvVI9DT0pvXu9zDzWvUM8Cz23vE2+7TzuvEm9jz1KvSk9oD21Pc28l70JPQ2+CzxOPSI9Ij5dvYo8hD3kPa69TLzQO0W8z7uEvPw9ST2tuzm9Fb5kPVK9bL33vc28TT1RvuS8pz21PdU9r72jvVy9jrhXve28+7sHvs+9B77XPdO8ij0LPIc8y71oPS28fT0nvRg9Sj2bvAu9Pb1dPL+8jj0/PSu+Tz66PVE9KT0zvAG+vryCvea7fb3wPfw9wrwqvdq9lry5PbA92zsKvle9Mj0ZPYG7h7woPbi92L2iveA7Db1pPNI8hb16PBu9ujyBPZK9kb22vaG7pT1Fu0s9/7zgvcc9v7w+Pas9OT0+vdw9xLx3veY857xyPXq9Z7y5PGw8pL28PJ69ubsEPQy+O72mPd29mb13vLI9cz2dvLO97D36vGM9370TPRC+LL5kvSW+kb2XPbW9ujw6PYK8G75vvSS9cb6ZOwY8iDy5PZ48QL2Svb+9ED22vIO9Wb4EvKw9bz2EvSG9L76pvAc9MD3zvFW9Czwqvba9fj3zPAo9Gz6FPWY9B76SvH89/rzEPcy92z0dPge9mb29PHg9Jrz/veW9/juBPcK9DT4jPSe9Hb0Mve495jyovTw9bD2sOwi9CbyGPdY6Rz2CPNG8tb1IvKq9cj2SPYC9zLxKvaC9+r23PLq92r2YPZU9gL3qPae8Gb1EvTu9aL0Ovl6+Zj2uvfC957obvfK9Vz0yvMk98Lz2POi8S70GPte9Mr5HvXs9B76Dveo8fL1lvPM9Rj0oPdM9B772vQ29hT1DPsw9yLxKO129gz3RPYA9R72IvdI9+zoHvXe9Kz0BPnE8yTzWvRY9Azr9vGO9bjxGvkE+Lj6IPTC9772xPS+9aT6cvRG92zzqPGq9F70dPLe89D0jPBc9Oz1QvXy9cD2uvAW+WT6XPT4+Er16vMI9przNPWI9sjxGvg68fjsUvr094j0lvEq+oz2tPPi9ML6Zu2G9Jryovfo8kD2uPYc9f70SPbi8pL3juYe96z1evNi9Ob0BPia9oLzvPVk9ED2cPBY9IT1+O6A9qDxoPSq+rr0MPni9FTyPPB49mL1/PYU9Sjy9vQI9I776O8k8vDzePYM9QD3TvFI9Ub7eO6U9WL26PY690r2+PeI9qj2OvZw88L0QPq+92b3Wvb89Hr4dPXs9qD2MvYK8YbyOPrG9EL5zPTk+0zxkPUY9M725va+8jLxIPQM+17vFPUM+Oz4APkW7cT3xvHY8fz0yPts9cj7YvBS9yD1mu8Y9qjy3POM8YDxxPQa9rT0ZvH+7Kb0Zvbg89T31va89trwJvNa9WL0hPak9FL0+vZ28JT55vSS+W70pPgK+SzcJPgu92r1QvjI+Eb0Kvq69TD3IOoW9gj2dPX89wb3zPDk9dr6fvYE9mT1xPY08Kz7nPIW9/T2mPSa9GjucPSQ9+7yDvdY9wzyRvNI94r0zPd296z0TvSs94TwPvRO8dj0fPqE9rj0+vYK8QztGvCs91rw+vA4+pL26vTA+0D2Wve29Jzr/vSi9b72RvMG8gD39vAu+pTyCPVK9hb0Ivae99TxGPFm9+71MPEY8C71cvYG9JLwJPvm7+DysvUi8fj1XPfi8Hz1Wve09H777vD+8uLzYvds9fb2+PYw9jb3tvY+8073evd+8AT68veW8yr2wPYo8MbuIu9C86bxuvTe9kL27vVS8CL61PMM9FD2MvLY9Mr6vvCW+lb2rPVa8Fz1lvce9nr2RPZQ8iz1ovaG9MT2OPfc9Hz7vPW69oD2FvQY8kL2WvgW9oL1oPXO8+zxmPU49r71pvck9q735uz69Cj2lvH49FL6tPZS80r0wvY+82j0yPLQ9Jz29u5C7gTwIvSU9FT0Zvdi8orp4vaK8Bb05PYS83zxDPRC8nTxlvJG9Ur2RPSO9krzGvHW81zwEPaM7sTyZve+8nL0Wu+E8ozxqPJi8Az36PH+777xnvYY8sbzgu5+4yDxdPHG8HD04uNo8AD1DvMg7OzxGPT49Zbwyvcs8CbwAPDY9cz0ZPS087jyKvCQ9ib3ZvBq9oLsRPZq8Trwruz+9JjuNPTA9ujzMvLk8hrwvO9C84zt0PL88YTxmO0E9ujsyvO+8aTuKO0I9H7xjO1M6AD1GPV09ij1Xuh69qLrlvM+88zssOwU92DwzuwE9nzstvek8PryiPJG90Ly7O3S8k7wDvaO8szzuO+w7kzysPNm8o7w6PNI7rbyeOx49WbwWPT68B701uxo9Pr06vYa9L71LPW28Dr7FvFI9ALxxO9g8xjzEPD28izuXPJ68fLrZPE69yLu3vG08Wj2hPAW9DT02OhQ8CT1VPRy9mDxlu927XrtQula8Cj0DvRi9CL1cvdE8wbyLup+9CTymvRG95DwjvZi8CTy1u6e5dbyCPPa8jbyAvE29MLywvPy8Aj3BOli9N71OPey8eDsRPQi9Gr0fvec8RbwuvBW8oryJvUo9Orw9PRg9lLwzObs8GjzevBu8gLyjOx09ozuNvPM78Ly8PDU8AD0FvQG9Lb3XPJI7FTvNOye9mz0qvfq8AbzgOlK8nTyou5q8NDyQvKO8AbwrvWg91jmfO6M81rt5vAK9PD19PL08jzxNvSu8gD2QPfC6tTz0u029JzyTvOw8CT1fPaK9NT0Zu4i8z7zvu9a9bTxDvOU8nDzquwE9PzwRPMs8YLyRvP88ID2FvVK8lbwdvA+99LvhPK27MD3ePKq8ib2WvbA89rrrPIK8ljsuvC48jD29u6w8xrz/POQ72TwqvdQ8vLxwvE28ArzrvQM7BLyBPNK8WT0XPe88hj3cPKs8Rz0VPdS7Br1LO3K7B72yvNU8pT2zPBU83Ty3vC49QDwRvIQ5FL17PR69iD1QvEM9Cb0ZO7E8ozzgPJU7Jb1XvSI8pLy4O708wjwSvRg9kTy/vMq8RLvWuoO8YT2/u9u8qDwAPZ+8kDxcPLs8y7wdvUW80rxbPMM8hbz9vKA7hbsNvWE9hDoJPWE94LyOPDq8hbyavJi85TsSO3m8j7vyOxs9GL3kPPg8Xj0qPRe9fjynPBW9b7wCPYQ8IDwWvOa7VL0mvRy9ujoZPSU92bzxvFq86zxdvbc8pLsoPQW8dzu9Ozq8gL27PE68JjsDPem8MTwQvIE8K71Jvao8yDyUO8g8bzwSvdK8M7yfPGw8urs5POC8nDzQvDG7uTzTvJe9Wz2mu8w8sLz0Ogq8ybxcvO+7fz2ivA285rs3PKY9lryxvaW8SDutPCa8EL0RPdy8wzwGPQ49zjvAvGS99Tw8PJm8rruHu7w8LzzJPde8lDx/PO08BryUPde8Wz2XPO08Ar1rPJO8Vz2Pu4G8pzyCPJe8fjudPEa7yry0vHI8sTufvUY6xjxlvNQ8QbojPEC9hLsGvTm9aLxwPLw7tjxQPAe9njyFO6W8Sz2ZPBC8Ur2nvFu7njwmPG89Cbwgvcs8CT34vFw9mjvNvEi9Dr0ZvFA9OTyevKi8d7yKvM08bDyFPbw84jyCvPG8Xz2hPCY9LruavFA6qbyCOo08GT24vGM89rwIPag8wbz9PBM91zziPAU8a73gvLc8mDyevAa9Kr3MPJ48BTymvJw8rTwrvOk7FTw1PTu9FLoAu4c6bDyOPZA8JTwPPfm70zqbO3a7vbwAvdw747sYPLQ7Qz0cvFu9Ab2EPDe87TybPPC8sbwMPcm83jtpvS29lrzrOSi9CL1fPUM7oLuSvKe6y7xJvVG7STwFvXc9iroVPfM8TrzWPKY8BT0fPTa4Tz08PV48f7zXPNO7DD2QPDG7TLoWvGI8R7u1u5q8XzxivUS8ar0YPZY94L04udy82jzQvIY9FT38us08Jj1SPa08q7yGPOS8hz0MvIy51jztu2A8rTwvvUk8MTxLvbc8tryqPfW74jyOurg857zUPD+9hDxrO/c8/rw5vC09wzzZumW9OTv6PGw8vDw3O2y8kbtLPDK94jvGOpI7DDw9vMu6CjxLvTA9vLtZvY08Fj1cPIg8bLvkvEy7CDwEPS89Az2AvJg8LbzqOyY8zTwSu6I6DL2kvf+8TDy+PGy8QLsCPZq9IT0HPYA8AL07vem6mTr5u/k7sLw8OkK9mTuFvKy8KDxyuzk9xbwGPaM6Cb0HPGU8rb32PAc9CzwpPCs8WD3/PFi88LwVO8M7PryUvKi9HD3MOzM8Ab2TO5s8PLwVvRu61Tu7PFk9Br21vBW93rx2vKq82LwdvSU9Qr1TvQM7E7wDPf+70rytvKo8wjtouhy9Fj2QO329yLvCvDQ9B73CvIQ7WzxYPBg9LjxnPF69czwQvRu9PDyVu968fD0bvDA9MT2gPKs82jxtvSg9Fr2wvCU8G7tlPNQ81zsFvaS7O70MPRU7hb2/vAS9Nr0gPY48Hz0+vI08uzyoOZw8+zs5PKs5Vz2cPDU90Ty3vB86STzkusu7qDupvJS7N72kvO48Qz1QvD86vDwfvBI8iDzQvBs8Jz2uO6288bz9PPM7lz3EOze8GD0NPHo9Pr0vPTm94rx8vAI8tjxIvYW81DwzPfK8vzt4veo8ebxHPIW9hbvKO+a8Mb3/ugs8Sz2yvF09Yb3JvLk8mTx6PT29KT0qvFs8B73fOw89K72GvS49STxyPd87grs1vBA8sTyqu5u8Rz37vJ27hzxCPPa8gjwuO8a8gbsWvNs8oLzTO6i85LygPMo8iTwnO7g8LL2VO7S8FD0OvX07n7wdOzW8xrwmPRm9wLz+u3O7Jb2nOxs9Dj2NPE27GrxRPSM8rzwSvWu8CTxhvCa9gLw9u6o8VbwHvQG9Gj0gPca8lL2DPcE7HjsbPWq8eruFPMY8wDxpvZU9vrvQvJM8c7yuvD09cDxcvI48/7sjPdk8DT3Duxq89jw1PUQ8P7yfPIi9KL0zuy88hjw5va47Ijz8u607Lr1hPRI9gT1NuzE7FjtwvEI8nzyvPPA7ybxnO4a7hLxFvVm9Rb2wu0G9S70fPPI7iLx8vQM8C71xPJq8GT24vHA9oj0HvI49kr27vG49Zr0FvRA9CjwgPQi81LzgOya9Tb0DPTE8hDzDu2c8xDsIPAm9DL0xvMk8vTsBPek7ujreuz895jvVPK88RzzBPDw9WrwBvSC9AD0iPXi8+jwNO1q73L65PAQ9ljoAu9E897puvGw8sTwxvDo7NjxjPIm7Gz1sPDA8urwxPEI9Pj0qPSq9W71yPH882znDPK88Fb3vOWG7gDxqvGA9VD02PVc8KjyWPCI9jbwTvBE9pjvLO1e9wjz8PBc9Kz2YvOs8Ib0mPWe7Kr0NPcW7Gj0mPCu9K70LvBG9njyIvdy8Yz3YuwA9Ij2xu5K7iD1TvRA6aj32PJU7D7wfvbW9gLynuyE91rynPN67vLiRPb88A73uPOA8PDxTPce8CzyNvM68KD1Iu4A9BDyDPRQ9ATywvQg9ST32PDy8F71ZPRE97jsTvWG8Kz3xPO28XD1nvBI8kTznPME6IbudPfc8Ury8Oyy9cjyavCm9gjwjveg7gb06OwS9ED1OPQm9nLyMPAC9Dj28vEy8rTxxO/W8rrz4Ovm8c7sYPaS8xjwQveS8d7y8PEs8/bz6O0O9e72ru3G9Gjueu1U62jxWu3+9bLnFvLw8Cj1OvAy9Gj1aPNg8TT2EvTs9rzlbPZU71byrvOY8fryCurM8cj2gvOK8OL01Owy95bxGvJU7Qb2guzm9oz2LPTq9vLrHvMQ8KL0+PeW8Kz2RPK48fTwKOW48CT0xvK+7Rj2nPE+8Yz1KPbq8xbsSPcy9UTtEPTk8pLwnvd68fjzQOxO9cL2uPEE8ij1lPf278bscvaa8Przlu8E8kTyMu488hDuZPd88WTx1PJm9TDyKvBY6yTwvPng8Bz2LPX+90zygPGA8OL3Uu9g7Gz0XPCg8rLzaPBk9xL2guxI9fj2TPIK82ryyPCi93bq1Om08vTucvbI7Vbwdu488Rjy3Ox89pbwnOw65IT3FPAk9Ib1lvJG8lTunuqE8irteu5Y89LzJu0o8Yjw4PBw8Kb1yvGO80byPPaA8QT3MPAK9cTvCvNc8ZDwiPAi8Eb30O087ujwBPK886Ds1uyC9tDqyvIg9Cj14vQc9ebvRPAm9F720PAY9srzavKO8wDxpPBS9B7yiPMW8z7zkO0u9ljxTvKY8pryWvBC8C7smPWs9rjywPYY8Z7yyPEc9Qz0Ouwo9JL2jOi49iL3TPBk99LwMPUG9ibzYvG28hrxJvZq9UTznvAu8rDylPL88vbwDvYC9HD07vfU8ozyavBq8xbrdvD89Ar7AvJK82TzpvMI6ADzRPIC897t3vJg7KLwYO8W8ujzBvMm7Ur02PAu9dr2rvPy7wDunO289Mzy6OxG6CD2jPGQ8Qbq3vBc90jyFOne9Hz2/vM48ljuYvO27oDwVPaC8az2kPJE6AzyBPHC6Bz05vJg8zjwAPPK8PjzPPau84Tw6PYk66jy3u2s7QLyDPKy6OT0Cu2e9MTyDO1I8yLx2PIs8i7zrvBM9kr2YvBI8N7t3PMa8nD0avTc9QL1ivOg8K70OPC09rL2bvcU8QLyPOze98jzlOqc8oLy2PeY8t7yLux69JbzUPIy8sDw+PJi6Ej1FPcS8TjkEvDk8pbohvSk8hTuRvP08Rj0zvC68CL3avKy63bwKu4m8oTsovS08t7wRO6i7/DtNPd+8OT0sPI48Ir0yPT691TzVu+48RLyyvI29XbxGPTa8aDyLvde7ATudvBO9qrxRu6K8JT0pPOs8bzt1PUY7hjuKvXQ8jTw5u8a7R70qPae7HT3jPOq8/TmrOn48hbxHuu08RT1RvK88Mb2XvLc7ybwiPcc9+7uuua88UrwFvZS7F7yxPOM7FjyUvH09sjz5POm7mrzGvH27oL3pvGO80LtHvRa9WT2DPHw9AzxFvci8NLzMOx89YrzjPCW9AT1pvJS7WTxKPa68xrvNvRe9obuAvPu7j7qXPJ+8AT1Ou6G7bTyUvN07/TvqvJI8xjz5PA88BT1AvA+8oDztuk+7X710vMa7NLwaPTo9oDvBPB48Bz2evVk8Pjx2Ow89GL1dPFK98zwsvWo9GT39vB49erwEvRI9nzxJvck8zro6PSq92TyduwW8QLz5O9g587uYPRC9ozy1uBQ8NDxavQw8hbrnvG48Dz3yPK67fjs7PUw9s7xsPPW8AT2ivK689jxXPAw8OD0AOxM9dT1mPSu9ZbyJPNe8Wr34u+A7o7tcPCY9Tj3HPBe8i7zQPCg8xbwVPQe8kj3PPM+9s7yTvJY6Kb0MvNC7lLzgPDW7aj39PL68YL3QOlg8Kb3fvNs8eDzcvIM9Pz2AvWY827wCvRQ9DjwEOgK7Z72ju0g9F72JPNo7E7w3vOk8uLwZPby8abwsvYG8IT2+PPO8Uj1MPRo88jzjPB09MbwLPK88h71XvZ68Wrqouka8jLwIPFo9p7zLPCo4ID0lPEw9gzwpvJo9tDwqu1g9ary+u5E7ErxdPDE9zjuQPAw9o7tHvQc9RL1rvDq9zbxCPD48pD1DvW08Pz0CvN67bLyZPIO9Dr3UPJ+8j7yTvPO7ibyDvXy9RbyzvBg9OLuYvKi5STzevBe9ZrsFvX89lTqiuhS9Jr0LPdM8kbyYujW9YT1TvRk9yzxcvMM8PD0ePYY9Nb3vPMm8ljwivH082ryovB69tbxpvZM9PL3yPPC87jspvb88JbmivAC9pTvzPAI7mDqCPX691jzXPHg8j70YPY28dbtpPMo7hTpuPQ496bzGvJk8mToiPKs8Hb2HvCy9z7vzvKW8K70bvf+7krvjvCC8tDxFPT07nbrdvMk84jxvvTw8yTu7PAQ72TwdPAY9hTwtPRO8Iz2bPA+9kj1GPeo9FbwXOmk8E70YvMS87Ts4Peo84zz1PJG6sbx/PBo8lzuAPFe8nbrlPIa8YbzCPBo9xr07vDK9ob3Vu5M9NzxnvC29ez0Ova28/zxUvLs8Nbw5vae8xjxKuvc8F7zwuws9nL0avEC8JbzzOz69HztJvBi897yRPeW8Ib2+ubQ8h72WPCE93Tv4vC49Bjx0u8M8XLz7PP+8m7xQPPc737wDPKY8+TzMPP08aTyZPP087Lyeu/s80byTPSu8pDxdPJw7l7ydPDc8XjzdOuE8nzy6uxA8xTsgPb28Rr0vPSQ8jbtFvJ+9aD0mvAI9LD04vY08sjyTPPS7zzxsvNk7Nr1fOwS9ijzJu4i81LwSvfg8brznvE08tDwuvMe8Or2JuwO8FL0XOa28lDzzvDw84rwMPYU9OT0hO0u97zvjvP67kL3NvKW8Brz5u1a8ODxVu/W7HD2avH09hbwJu329+bs4PYw9RD3+vOY7n7tCvTw93jzLvDu7iboZPbi95TyiPCy9uD0UPeW81jymPAa9Kz0ePBs9xzsoPDi9nTz/Obm9nrwRPY88lzy+vMw8Ljz4u+s8WL1CPVC7e7sHPPw81by9PMW7hz2eu+M7ebyPO8u8ujxCvZW8wreNPHa7uTwOvI07q70HvRy9jLycvI49JzyJPIK7XT3QvLS8CD1VPM088rsOvIM9fj1APS697ryKOvu7JbvCPMQ7QrwEvDu8Gr0Cvac7Dbw+PUu8ZL18Pem7Kj0SPQm9IL3OvPE8Hb0DPVw8vLsuPXg7Gb0ovFe9/zw4PZQ667xkPUK8dTt/PRO9T72EvM08irv+vJu8zDwCPZc9eby8vEg9sDw7PWY9Kb2CvP48oT30vG09Ar3Gu6k8Djsyvcw8DD2MPK087TzQvE68RLwQvRu8RrrIu9w8WDwgu/i8grsPPWQ7GL35vFo8tLt5vTU8NjuBvZi8BL3EOsk8erwNPYI9/Dy1vF681rtZvF89Or0Lvfa8n7wxvCy9xjzjPIa8RT2kvOE5wT24vPC7Jz0wvdI8Hr1QvVW8Rb2EvIO7XT2vvKM8EL3kPBQ8Mbw8PWS9FLzmPBO9KbvUvNs9sDpivM86nLsrPPW8Qb0vvZY87LxKvPm8lT0FPLk897yEvIO9Gb3RPBc9nLz8OmU6Wzw0vQA9aLouvW28kjwJvD496brMvJU9QjyqvCC9aT1bPE+9WT2APdI8w7wOvRO9DL0vPDw9gDwAPN473zyePbe8yDuDPE+8Nr1bvQM9lbs2vEG9DT3qu5m8GDyMvTy9TDw8vTk8OTwsOty8uzz/PHq8cz37PLC6szz5PNe7dbzGvNI8Jz2RPWU9Cz3xPLs9UDw5PVW8Aj3jvKO7djsNu+q8Mj01u9O8gb2cPGA8Gr0UPRS9XjsXPRw9hTyavJw9g715vSG9JbvkPHe79TxTPSe6mrwDPWO9Cr1/PBw9EDugOwA9xLuDPQ+8Kry7uwA9Bb24O369x7x9PZ+8Pz2EPcQ8ULzJvFM8mzzXPN68YDv8PBY8oj2kOzO9DbxmPJW7pbw7PZg8sjxAPSk96TzuvC49Gb3CvOM8pzyjPX89Gj0VPYG8mT2RvEe7Ur3GvFG88jwhPQ489TuRvAu89Ty1PIe8vzwPPSS9Bj3iPNi8ID30O8U8O712PBs917sTvRO7d73Bu3G8KLx7vZQ7qbqfvA88/jyGPC09UTvxPAw96zzFPcQ8Nzz9vJI8AbyEPOE8Zj2Lvcg9JL2vvLk6fj25PJU81zyqPco8VryuvE28sbyXPZY8krwnPAo8vT3Fuq87KbxLPE29mrwSPPo70TwQPey8ybz+PJE81jy+PGE9Hr0Quhy8gz2EvTY8kTsFPJe9mb2evDk9kT3tPCY9MjzrPFU9K70SPYg8Kz2Suvg81jthPP68kD0QPDk9gDr6Ole9Rr31vKy72LxBPCO90jwgPQa6gDxHPKI84zygPB28grtkPEs9mjy9vOq8DrwLvQo9nDyAuv+8Bb0rvJ+7/zxHPfk8sbqRvFk80joCPem8lzwqvS88AbyOPEK7X7sRve28Mj1yvLs8Xb3yPD69kzwaPbE8ez2RPQG9pjyTPIo9MT25PIk8Cj2GPB29fDwcPCU96jtbPQM8hrvHu1+8lr0BPV67cb2YPYA9UbxJPRu9mb38PJy8/bzLvI283zwbPcK8JLxnPTy9KLwTvCE9L72sPG287LxuPFS8RLuwPB+9Ar3lPNQ8Ij2FPM286ryhvPM857wZPGY9obyhvKO8pb1bvWo9GDzmvPC8C71mvFk9czz5vBo6bD1XO408fr0TvYc98rxZuhI9GTzlO2Y9oLr4PME9ML00PUa9HLtDPv+8/L0iu666gLy+vMq8kLy1vBg5P70fO5g9F71GPVU717x9vAk8u7uJvLa6iD2NvEy9qrwDug09Pb1vvJS7cjxnPLe8ML00vba8BL0WvPA8Vr0CvYc9/LyOvBi9qjw8PSS9sTyKPO48VrzvPDU8hb1sPYA8mDrMvIc8k7yzPMI6HjwLPSs9zjwsPcu5TLoIvVU7/DtvPTw9Mj0aPGA91jqhvJa8Kj27OhK9AT0+O/46tTuOvIW9YjtnPDO7oDtPPR+9m7zmvEW7DT2MPOO8MzwDPC09wzwKvV27b7tsvLq7Ez3Suqi8gbx5PE+84DyavDA96jwHPAU9tLy3PAe9Gz2SPBC9RjwLPTu9tztmPdi8AT1hPBU9yTwSvZA8RjwJvMW7yzzju+o8nbwuPSI86jwrvSE9Trssu648ajyvPC28RTsTvXI89rwFPU08N70DvSU9Gbz/Oyq9vLw/Pbu9KTtgOBc9ZT1WPb88UjyIvfu8qzzGuqg6Tj2DPYK8mTyGPKE8ibqdvJI8tLwNPYE8rLstPaY89bvuvCS9LLwEPQK7jrxgvTU9hTwoPB69Qr2vO3o89rt2vWc9CDyIPII7t7z9uTy9KrukvDQ92Lw6Oyu8Nb2RvCq9RTtOO7W8hLysPBq9qL2cuyS9VjzOPNe7M7wbu4893zu4PBc9yDwgPbW8qzz1u6+8nbxpvaw8f7tJPYc8Sz0TvdA76DxRPLG8DDxkPN682LvPOgk9JT0XPJm8PL2UvVg9Qz0RvL48sTowvZK8DLwyOkK9kLwSuxg8wzyavBM9cTwmvIY92TyHvCi9kDzzvLw8Qz2gPLy8QL7JPKe8mjwCPB06Pj2buw690LyFu4K9OTwgPUo8Er3bOw29jjxYvUW8ZLyLPW29L71fvFy8FjwmvVk9t72LPKw817wRvUg8CLzROrq8Cz1tvAE8yTwnvdy8XrxaPZa8KL3fPO88xDuuPP67VzwfvUC97rwUPfO7mjyzPDu9Bz0evGU8LzwiPKm7kbyKPLo8vryDvC69Sz0CPac8hDtXvZU8hjyjPPi8ojvyPEe8Zzx3PIs8t7uEPZi8Fr2YPU27Nr3UvM08CT3OOw29/LxjPE28Az3uPOw7O71jvOc7b7yoPBS9TLyIPIq8RT0VvMG8Nb3hu5i8zTokvBW9eb0APd88cz0KO+s86Ty5vO489zilvLQ7ML2Hus48qjw/PRQ86jyKPJk8cT3XvJ+8iDwSPYw88rzZvGw8Eb29PNg8SjtTvQo9vjtUPSW9l7vkPC49QjwFvWG7Vj19vW886Lztupc8ETxVPME8GzzEvIe7/DqCuwO8e7vkvLm95LuRvCc8EL0ZvUg9CzqVvQG99rwxvYq8uLybvLU80byCOwA9Xr2vvYi9VLwhvVE9Nj0dvJO89jzCOwO8jjyYPX+8qbyju+Q8OTt3PbO8M71ZPIa9FjzyPCS8DTwfPSQ8CT0NvdQ8e7w5vdu8l7w6PUk8GDw+Pag8zbz9PA+9jTvNO2k7zjtPvZc8PD1UOjA8a707Pb28KjwRvSA9WzsqPXO7VTxLu2+8lDwgPRS8Dz0ivU+7TD0tPa67CryXvIy6Wjz6PEc8cz0HPRU9mLy5O3w9DzuuvC+8LTttvEe8NjuuO7W82rw2vW+9y7zUvPE8u7yzPQE8KL0LvWy93TzEuwG9tLpBvEo9zDuuvb08Aj0CPo08Rz0vvRA9NbwRvAq9wDyxPGc9ubyRu0+8F7ygPQa9HLtAu/Q7fztBPKE7HTwDPbk8p7yVPB08pjzNO0y9D72PPDm8bD0QvD68kz0/PH07jrrxOga9Cz3jPFY8tjxIvFU9e729vOE80DulPBe91TxmvHE8RjtVPGQ7njy8OzI7U7uduxi9N72WvBm9Pb3LOrO9OT0yPXo9Nbx8PPe8HTxRuxY8przdvEC9Fbueu9M7C7rouxW9NLyPvNu6bz1IPQK9O7uoPKY8UbzsO4u9cLyhPBo9Bj1dPAA8Ajz3PLu8ir16uke9hzzVO7u8ZzvRPFy8kz2XvOC8i7xVu6C8IL0gPRm9qjyAuwO9hDrFvAq9VT4vvX67BTyIPZ88YzwDvWA7qrw8vZe91jw2vQA8OLyavMu8zrwpPFg9pDzovMW8CbwoPNw6Kj0JvHe8Eb2PPAe9Pr0WPR69zrxRPWG7E70GPUk9gj0VvVc8Rz1HPBe8Ar3hPYu7PrwlvfK8KD2oPSG9tLzhO7G8JrysunO9Kr0wPTC8CLwivMe8Ljz1PP06orwivP284rwzPRm9Ij0cOzS9jbzwvCG80TwwvZC8yzuXPRu9t7ylO3U9+7xoO5k8CD0zvZs7y7w/vLo8D739uyG9gjvHvBa8sTyTuwI9Hz1vu5g8Br2Hu688Vz0svVa9Qz0lvQA9VT35PAE9CDw2PYo8mTxvut28I7tUu9A8eLx/PYa9z7wUPea8r71gPHo707rzPEm6kr28PMW7s7zJPMo857zcPIa8gjxPPaU8AzydOyY9bLvYPE09cjtuvAY9vzx0PKI8EL0nPZU8Pb22vI89Ej3sOgy8ZL0/OxQ9HrqaPAM9ED2bO5c9jTymvKk8Bj0zvCQ88Lxku8m8LL0LO/c7srufvWW9VTyjOt07jrwKvJ07nTyBvRQ8f7zivCU87rozPUa9CLxCOwA9/LumPZM71bzovMm6mDxePIu9Ir1xPSi9jLwwPUo8/7tqPfU8KbwaPYK9Pj3huu67nbwQvFI9iDxWu4w9ybumvJM8b7wyPb670TqGPLU8pztUPBw9CrwDPI48nDw7PNm8z7uZPGa9CD0IveA8rzlmPDa7Dj2BPPO8BD00Oz49uLgivdg8JT3lPDy8O73lPK28L72OvRW94zvlPHs7TzwZPDE827v9OzI9bz0YPJW807w6PV89EL38OxO8X73ZvKk8mrw3Oxk9irxYvRM9f7slvQE8Dr3IO/881Lx+PbM8BzzyO4w7Bj1ivGI7vTyBvIc8ULrwPAY9Jj0zO9s8JD3tvNC86jxrvVU98LowPca8gD26vHu9/DxuOyY9xjzlPHI7crqnvBa9sbykPMO7hjzcPKM8wbxTvWy8KrwCvEK9Yb0wvOY8abyJPAC9e7oUO6A7PLzavKK8rb2UO548mrpwPcE3IDwbvTi89jxPPcC8JTz7vBQ9vLwQvZO6CTyXvEi9W72fPPg70zwPvLk8LrxIvGA9rLxnvY68ubsdPJG8X73pvLw9EjzWPEu9FTukvCs7nTymvGQ9MT1NvPO7KLwZvYk6lDzBOwg8AT2IPEy8Ij1ePQE9eDwrPWU8ybxtvE688DyLN9u8ab3xvL681Lz0PJM4ET0ovHI7TzxRPAU9nTz8O4I6/rwou249+rymvNO85LzxO9q7Xr2HPSW9VryMuzS9g72BO847+bwMPaM9qL04PT08eLuhPCO9HLwyO8+8rjwcPAU96zt7O+o7FDz8u3S9NT3DvPo8PzuevE48b7xqPFa9Ur0Svfg8MbuLPNk88LstPae7Sz3FvZ08Ljy9vMa8hbwPvR69rj2ivHQ84rqFvOY8hjyZPJg8E73dvIg9qjxTvA29wL1ZvAW93rtPvJw8wToePQy95DqGPVg8CDuCu9Y8Aj2bvI07szxyvS+83bwVO4U77zxaPCY7ljxPPdY8Oz2YPJK7vD24PLQ8P7wfPdO7gD3lu6w8Mr04PJi9qzwhPWG9SD2CvfU8zruKvLY7zLy6PFe9gL1VvTi8fb2MPBi9iDx1OeY7mLxYvZg817yAPTc8gT38umS9Kz0EvfA8TjmWvAq9nzylPOI7I7y3vCu8Nj0UPTk95juTvZi8ETwSuxI9Fb2qPGI8rDz6PD69Bjx9OoU7NLwRPQA8r7xjvSQ60zsmO4K8zzw3vbA7gTwlvSu8Kjx/vI68sT0mvYC8EDyGvbY8Fr0BPbA7CjyVOxy8wLu5PBu9PbyNO4y8Wj3+PAo9GT2tO708JT2zu6s837skvRO9LL0MPJa8SD30O6E6qTxxPOW8MzwTvU08rDstO/a87bzFPA09iLzyvJ68sDxqvWE83jw7PAs9gbyQPb286DzVvKk8pbzuuzI8BT0UPX09CT2VPI88gbuWvJq8BjtVvS47TjuKvRw9NL2bu6M8Cr2DPCc73riIPIM8u7wUvXG8drzBPNc8L70BvaA8Zz0ovTg8nrywPWq8vrzku+27DryJPGc9SL2husc82bszO9q87jvYPLo9Bb29vFo8Jb3mvEg87Dw9O106X7z8O5a8NDvQPF694TsBvNw9ITz1u+Y7+DpZO6+6oDw2PTO92rwHvJ88G72sPB28AL2kvBy91bx+uqU827xOPa67ozwuvXM7vTuePY+5XzyIOso7Kr2ZPFA8e7whPUI9g72TPJo8pbz4urM6IbzxPFm9ZbtSOo88Bjxuurk8kz3wvPE8qT0ZPXO86jySvGs9ALwvvBe8UjxJvb08czybPeK71bzIPB48Ir6fOwm8tDwFvTe7sjzAvLo72jzKvLg89rz+vDS9nr3vvMI8ID0+u1c8+zxBPDI9hryDvDa9RLssPbs6jrxoPVg877xSvc+6r7wPveE8xTx1OrK8wLySPao7uLzcu8q7Eb3mumK8Dj3svJY7VTpxvJy68Ty/PN68Qz0nvAs8ojznuwi9IbzhvOm8GDwuPfu8gbx3uzk9Nz2dPNQ8U7wCPZI8rDwNPQU9rbzOPLa8DD2bPUA9ljyMPDq7jTyMO/w8kD2rPRE8jzygPNw8nTw3PBE9DjwhPCC8ML2GPAk8zbxrvPG87jyOO3M9hLxaO7Y8uDx8vdM6frqvvcK8vztRvC+8Hz2AO/O7Ab1Bve48yTvyPDg9lLyyuz67qjumPFQ8v7vtOww8Nj0ovfE7qjxHPXK8WjpKPci8Bz1BuoA8aT2iPPK8WLxTuzQ86zskPdO7lD0MPWc7QbzZPVq8pjwBvKq7Sr1dPDO8B70BPMU80LxoPYm8SL3nPA0817tvvQK8vDyXvGk8gjy8vJE8i72LPKq8AL3du6s8lLxMPNi8kLylPNi8o7zpvBm6UrwavRQ7p7oaPZi9DjxNvfU7BL1IPfy7mjwivU698zwMvE+9ozsiPWw9OryWvCM8obz5vaU8lrw8PBk9bD23u6c7ez32PLE8e7wCPbM8IzwKvdW8jjzFPNa8sLwSvYw8QL1IPKW7XL0JvtA8VLwquig9oLy+vBi9hLy3PKa8e7ysvCk8FbyvO+K8bL1tOVk9Cj0QvUu9yb0mO1C91j3QvLs8OjzAOmA9Ij0YvT+9g70CuxK8zrtqPfq8LbsWPTi9SD0OvTC9hr2IPKC9FT10PJO9xD3vvYq9Eb0yvXG9ir0yPqq+4b1LPNs9wz0qO6O90L2kvVq9ajwKPpu9Gz1HvTq9nb3FPdG8hL1HvKs8tjyZvAO9qj2Fvew9pz39PaK8vb2tvTU9KD1zPQu9hD0CPea8Fz7ovBQ8UrqMPaO8YL0DPmQ8FL66vNA9Hj02Pbe9eD0Futy9SL0mvcS9Br7APUK8Ob2mPa086r2OvRw9tL2rPZi9/7z2PZC4073tvHc9iz2rPZ+9Uz28vc27rL3aPay9tD2PPcY91D2WPMQ6jD1FPQa+grxsO9U7RLyuPG09yztMPSU9xD0qPVO9kb0KPiA9a72nPAk9rD0ZvMy9tDyMPa08/z1SPQo+kL0Qvhy+XToDvVA9vD3yvFQ957zKvV29Nr0fvdO90bw5vv289TtFPYo8iD35Pc49Br5HPU+9gr0Xvp68JT4fPpQ9HT36PWU94Dzou3Q92DyHvdO9lz1oPaC7aLySvXS96b3TPQo+nLwFvRk9z7w4vcg8H72vvey9Jj2ZvSC9YrziPOQ99LyDPMy9B73rvA2+8D2KPQo+E723Pdq8CD7PPay9/r2WvSA90D1ZPXM9Nz1wvcO7b729vUY9iD2Rvaw8GL1PPm49AL61Pb+9HjwgvTS8xLywPAm9oD0evae9or3PPBW9/b3+vcY8jL2NPQw+xLsEPv89jr2dvVG8GL0RPUA94L2tvYm92bvhPf682D0CPBg9Ij6wvRE9UTyvvUM+k7swvg28Rr65u9m9gL1qvc89BL3qvTU9wTvNPfO98rz6PO+8kz2VPNi9CD4IPUQ9PzyyPOi9ur3QPe49/L23vfS9oL09vu26xLv5u8g9pDwjvD29Hj38O0C9zL0Hvqg95r2sPJY81D2gvea9Rb5LvDM9XD2qPQS96LydPUo9Zbu2vRQ+wb3LvJy9kr2+vA29/r2vvPq90TwKPGa9kLzZvTw9mj3UPA09ET1zvHY7gzy2PbW9zr2PvcO9QT3iPa0947zfOdi8TT10vZy9zb1GvdS9B76XPcQ8hLyePIA9Oz3avAM+7z3PO2e9Gb6UPTO8QD0hPdM9Kz34vHq94rxSvN27gz33O6c9zz2ju0M9Nj7/Pf+7q71LvaY9HD2+Pem9Xr73vNu8DbtkvAS+Fr0TPkQ8RD0hvGU9obsmvvm9ID2evNS9rTyiPJo81L2pPSM+az12vdI89rx6vAA+3j2Zuom9AL6JPSS9xL0DPg4+HL7WPZW9oj1evfI9yr0/vYE9ZDyJvts8ijqEvR6+mDxavN26ID2GvVW9pbytPaq9cD0vPpI9VL30PPg8Bb5dPeE8B77lPNQ9JD7QO0+8TL7TvKk8uTz9vC4+2D1+PPG8prsAvi29pjyUPD4+rj0PPQg9FD1nvZ08nL0MvV09XT3pPAK+Fj0ZvCe9RD0PPVM8Sj1uPQi+Sb0IPRc9WL2+vYm8pLx3PAW9Ur3Evdk9iT34vXE9YjxdvFA91LpJPYM9XbwzPom9Ej7bOhk7hby4PUS9BjyHPDY9rr15O988hT2aPT6+fj3gvZS9171+vaw9S72yPaI96TyaPaq9mj2TvY490L3svbA8sTwRPjQ+8zrfPJK8kDwGvb25mr15PPc88D3avai98ryivUc9Ez1tvFk9TTzdPT2+lr1CvL48QztBvsY8iD3xvMm80LxhvD28uz12PCy+HL5cvai7xz00PaM9dLwYPh4+lr38PPm8aDyhPUM+ib2OPQA9ZL0+vgk8WDzMPRQ+AT0MPJ69fb0APTK9Kj3Su8+9WT1PvBm+yT0tvQe+ozyLPcO9Jz2TPGK7hDyzPKK8T71pvci9ALvVvRq9Ir16PdO9W7wjPZe9DD3lPY88CbycvTs9i70Cvse87z2DPLC9azwzPU29trsfvoi9Fz6sPIQ9tL0WPdS9ND52Pdu8az3kvMS9hT2Ou9M9DD2QPea9QD5SPKW9Xr1TveG9BD4vvdS8dr3XvWm92TxXvQI+vL2OPV8+wb2jPem6Aj6uPe099zozPQO9ED2EPRg+uD2KvdS9tjtoPS+9gD2kvXq9kT3+vSG+OL7NPQo9FD2cPEO9zzryO/69Cb4cvhU9Iz2OPZU8VT3kOym8xb0wve+7MLkzPae7kjqBPcS8D76avcG76jx/PCY9A77+Oyq8rjyoPVI9zbxWPQW+Qby0OxE+yb3ZvT6+1bw7PS877j1IPYs9GjyjPT694jvnvYs9BL4lvfA93b3wvQo+E70oPpc88z3XPfc6CD06Pf+9Wz2JvC6+gb0DPpa9FT2gvLq8lj2kPfE8GT5APNG9tz15vEI9m7t6vbI8PL0VPks9FDxUvRC9iz2DPDC86TyXvBM+Dj5du7E8Sr2IvDe997yWvvM7Rj6DvGi9sLxivYy89TwqPMu8Ib2NPJk8n7wFPoG8cr1aPXs9aD3yPNA9ob0CvLO9yLyQvbG97j3hvRI7K76VPZ69Kz3oPbM90724PP+9CjwzvY09gL3LPbc9DbyZPYg83LwVvcc90jzFPSE+B73KPAo+yT2VPYw8n71SPYK8Cj0evWm9njxoPik+nzzgPJk85zwQvra9Jb5vvdG6rjwhPo28rj3APZq98LzNuzS9Z7xeOnq9C70WPc+8izsKvcY8Gj1IPSw9/r2TPSo96zwbvTC+Xr3fOuQ8Wr3BPYu9h7oNPhY9TTwuvac9hjmKu8o9kzzjvDg9DL04vIm9Rz0AvaI8oj2avZe9ATy2PJW96T2tPJY8Vz2HPeg9jj0dPG+8dL3IvQe8yzuLPV+9pb3ju6a8rD2QPeA9Jjyxve67rr35vZw6+L0avki9tT15vTo8br6rPX68NLr4PEy9iD2Cvd695LwUvZk97j3nPCg9lz2rvYy9izsDPX89TT1RPRY9kL2GvO08rD2EPQy++zz2uwu9oDyTvUG+mD0SvsE8sr1cvXo+GL2aPHA8LL2lPd893z3FvXM9x71uvOc8Aj6oPJe8yzxjvIu9Lb5KPKs9aryGvPy9Qj4TPT09lr2tvTi9JD3CPYM9H77BPZO9P72+vQS9/7yMPZi9rb0lPsC9Ib2GOhu+oj2yPLe9z70OPme8uz3VPLs9qzy4vSq9fLz5vCg9jLw7vO+9FL0AvV+9rruEvAw+jrwqunq8/7tgPiK7er0FPtW9sL26PAa++70EPRA+lr3zvCo+l71CPn+9HD4rvSU9ub3ZvI67tj0qvCa9x7w/vcI9hzyzPCm9ujwpvHs8Z729PXc9UzyMPRQ9DL7Nuua73j1FPb69DD0hPam9dz2ZPTQ+Xb0pPb49qD39PA++zrwTPYe96r2bvP89Er6DPVM9ibxHve260j2YPZY9nLsMvps9vr1XPRa8FD4JPhY+370MPgq9Bb4qvXy9oj2OPcs99r1WPj89vb0QvHA+wD0YvaG8r72MPGe7KD6NPQi95L28va+7Sb46vVM9Gr2KPS+9LbztvQI9xb3YvRg+Nb3gvQQ+cr0APr66Kb2ZvT49RL5uPR+9Rb6uPSs+GD6bPfU8Cb1GvTO95D3WPHk9dr3WvUK92T2iOpg99r3hvRI+4jt3vKs8bz28vZU7Aj6bvDs8uT2IvEg9lj0QvoE99T0/vdw9AD3VPcs9NzwqvXc9dz10PZ49Qr3Bu+W8xLxPPSi9Wz4ovrY8dD2Ivbq9qT1JPNI9v72JvUG9rjwEvaq9CL6qPPE9Er3/PRW90L3cPUE+Aj6aPeE8W70xveE9gbxYPWE8hb7qPAg9aj2XvFO9l7ydPRS9ZD5CvC49nLoJvv86rD15PFA9GT5/Ooy9Nj2PO5K9BDwovLu91b2zPZ09Ub3uvZa9Cj1TvQW+9z0ePI88sr0vvYc9k70jvi099z1tvIi9jrzqvOI9izwbvrg9fr1mu/Q8+L1iPbK9j71pvYQ8RT6Kvds8b72DvS89Dr1EvKi9tz3Ru5c9ED3mPXY7ez2qvMC9Ab4bvoE8Rb3GPGC9Vb2/vDW9nj3iPfI83r0NPgY+tr0vPV6+Y75dvNu74DwavsK9OL0bvY+7KT2LPQ49Aj6DPce9yrz0vfU9vr2aPKQ7pjx1vcw9ob2cPZg9Lj6gvQO+vztDPbC84b2CPRs+zrwoPGY9jz0hvU29lz2gPQA8o71nPSM92z3zvSE+3L1xPfy9Ob0Fvsk8OL2MO6o9cbzevVC9vL2bvCI+or22vQU+Y73LvQ89pTxsvIk8PrxMvXy8gz0LvYc9Zj3eu5S84zw9PTq+/TyOPR09NL02vc89L70CvIG92LzVPNy9T717vbC9sDxAvYa93b2YPc49cjxYvQe9dT1Cvlq9lj0hPXC9QT23On49vjxtun09Ab5vvWK9F70evuC8Nj3GPYU8db2mvJs9bT2kvC49Vz2WvYK92j0Ovjo9lT2DPHu9Gr5+PTO+5708vRq9uTyuvIo8V7xmvby9oz2dPFc9pj2ZPDQ9/jzIPcK8TL3hvUu+ub2cvBA+7Tzou4O+F73gPTq+5b0cPMC9JbxavXS8Er7RPOU92jpyvVi9Dz22Pb+74rzMvJg9K70xvRC9or1cvgs+zbvYOw++rLzaPQO90TwKPQc+ST1QPHE9SrsBvbk9ED07uq29obwmPnq9QL2fvaQ96D0lvQq8B705vsC90DvNvQO96juBvP+8vzxmPaS8TT0vu+K9wLwGvhy9OT1vvSk9Hr2mPZK9/r1qvQG9nzxPPnQ9TD2WPaw9rjtqvU49Kb25Pbm9Gj6OvCY+3z2Fu8O8CzzCPLA9rLvRvGM8IT0FO+a9oz2ZvUg9wb3dPSQ7iTyqPd298T3CvXk9L7zVPLS9Xj1cvTA9ET1dvRC+lL0NvoI8j7y8PLK9yr3nPK49xD1rPSc8+LzKuZA97z3tPbE91jwXvms9wb0XPpO9lL11PQI9FTwsPQI9ezw6PDC8hLwlPdC6yz0HPao9Fr0LPLW9oL0Hvsi8OL2rPXY8ijqdvWa9Ob1dvRq9170RPa48DL6MPMS8IL4gPoI9IL7Uvc+8Yj3kPc89gT2zPQ4+5D0KvSs8q7ylvUa8aL0pvdg7j720vW89gr1fPOy9oT0evWE9D72kPQC9sr1IOwU+fr0vvco9Ob2JvVa9pj0IPhC9Iz2HvG89yj2ovO48Qj0HO7E8z7wAPq29gL2/vdu8s73YvVW9Mj1Cveq8vbxqvbA9kD3APW+9vD1rvWq9jz3YO/K8vb0TPfW8YTxqPYU9dbskvAU+Srw5vqO9dbzUPcm9Fr6jOo090z21vRQ+c7z0PDS9Dz47vdE92j07vQO+rr0HPsE8yD0gPYa8KT0kPte9GD1hvhM7P70ivja9lzzlvYo7GD30vdm7gby6vXW7sb3AvS69cT3zvLE8ST7cvA6+9b3fvPY9vr3dvIc8pbxgPoU83jyZPfy9tD00PBe9NT2Pvfc8Hz0DvEM9GT5Yvic+qTyNvZE9irwSvhY9ZT2OurI9JT0Hvi28G75rPUa96r3OOyI+yD3Mvem7zr2QPSg9Bb2uvcq9Ub1avZe9LT7vvQ48zj2TvC29dbzWvQq+8z3gOow95L2nvXS9DL7pvG89vj10Pba9jr19PVq7jb6mPbc9Nb2CvCc9B76Su709gD22vcW7kL2+OxM9Nr2JvaM9Dj2NveA9Db7+PZ07cLyLPKA9mLx2vd89gD2xvUk9NTuEPQo+xj2avJk98L1YPtM9nD2bPMC8zzyYPTM9Tb1zvQs+Rb35PYi8FL2CPTi9eL1Dvl+9wz30Pb6977wTvBM+Db5APUo94D3bvZ69Kj6dvR097jyEvJM9iL3UPIy9tr0PvBa+Yj0gPg+9xbxEvSM8Pr1Muxw+gLyjPRC9qb2KvVi9uTxoveK9EL26PaO9IT2GvBK+Gr7nvGS9vL03vba8mr2UPEU9wTz9Pae9SLz0vB69aD3SvZq9cj2QPW09F70kvm498D3nvHq9jD1Yvf48l72IvVy9j73JPBQ+Wb2aPeY9E7wpPT29or1TvP09kL0DPIK8AD7TPdW9ar4wPFm9N72/vUk91D0yO3y9oD3xPQK91Lx+u+A9Kj4lPQ29Er2hvPo8fjxxPRS9SL5KvNY9hr3RPRM8ET7kvUo9m7t6vC88hTzkvPc8J71gPQU9mTuPvP28JDwQvk29VD1QPcA92D3ZvQg937xbPUy9x7wsvba9VLwHPsC8Dr3mPbk90L0vPcQ8xju3vQG877s2PL896jzEvfs9wb2MvQc77L0Tvny9PT38PSO+MD02u/68Pz2kvVi9s70PvnW9Rj6OOxa9oD3lvVS9sT1RPOU9Y70lPSG99ToAvW08hz1Zvbc92T0oPVw9vL0ZPca8lL3yPLC8MTvdvYC9sT29PQa9Wr18PbQ9Tb0xugc+5r1hPQk9u71LPaY9jTyfvTe9Vb7tPRa+r73vPOu9wz0uPZO9A74RPto9/7wuvt29EL0xvuc9ebwTu6o9VT4EuYk9xr1uPhc+Nb4dvq49zT2WvYA9BrwrvQ09PD1MvGO9nLzHPQ0+sbz5PRy+jb2HvUk9trxAPfI7try9PKc8Wz22Pey7ur3zvLg9HL5evXc9bj0pPiq+Fr4lvlm9ar1FPb092j2pvc68wTwVvQu+aDy4PVU9lb2DvYS9sDwYvZK917wuPSG96TxmvXy9+D2zPek8rz0RPZG9Iz0qvhy8K745vQC9p73TvTG9CLscPcS9+z2IvVM+sTzDvIq+KT7zPZ29M74tPk09Ij0MO1O+ij2nvaO9FD2FPTG+Cr4jPZ+9rLyUPUk9ZrupPTu8jLyTvIs99T1PPAy9AT7PPTW9CT4cPge9ITw1vOs8BDsBPZi8lr2jvVM7c73PPBq+yr0NPuw9P72hvNE9or1GvPK8h70Evq09wT2Qu+K9pj1SPKa8bjwpu1W8FTugPFk8/j30vRi94j0pPDw9TjwaPcC9mb1PPq27ML2FvE29yD2WPKw9Oz5HPd67lz2fuRU+0b1Cvgg+djzAvCc+fb2nvbK8Q725vZ29yzq4vaE9SjzQPdm7mL1GvQc+mz3kPV89nDw+Ot08w71+vTY+Ej3Avf89lj2zvCM9wL2bvYC9xL0Rvha9Fz0YPbs9Gj0nvRo8uD3lPXK9Uz3cvZ08uz0Mvb+8EL7uPT496DwaPMI9C77cvZA8Xj2DPcQ9ZrygvYY9pDuLPaI9xT1lvYQ96byNPaO95bxePb+9UT2dvSY8mr03On49ojwsPO28WT2IPJm9gr2CvQ68PTvovJw9or2JPf+9M72GPES+kDzvPEi8iD0avr69Wb20vRG+Y709vR0+B7tPvac7g7zxPLU9r71dPXa9m7zNufa7ozwTPMC9nD1ZvYA7LD2WPDA+vD3WO0K+D71fPTa6TT35PAa8PD2GvZc9Cjx8PXq9cz5DvSA9Sr2WvaO9CL4QPqa8c7wTvZS8mz0jPi29zr2xPCG7bjzcvcs8vLwFvhy+vL0dPWu82T03vdQ8W72aPSU9aj6Fuqu9nb3qvaa9sD2VvZC8yztGvRc9pj2CvZ88xD2hPLs9Ar2Wvc68ybwevsE9c72hvPu7mD0Kvum9yD29vCm93z1hPb29Cb7fvYM90TzsPcU9Dr0RPgo8iL18PSa8Qz6mvQm9UT78PSe98b1LvWQ9mz3/vDM9cL2gvR+9FT2pPHO+Hb4vPtC8gL2NPCC8kj1hvNM92bynvNq7Bj2fPbS97b1Lvte9273wu8G9XDz3vIy8x72sPYA8zb0XPvi9171BvXQ7eD2KPGA9xzyOvY48qrwlPaY9br0zvV68EbwUvVW7jb3WvYu8yj0OPl892j2ePIM9I70EvZq927wQPnq9ar2hPWK9uL2cvNa9oj0vvUe+qzxSvW+9Lr0BvtQ9Yz2OvGq8Db2GPcC9rbw0vps9/j13Owe+ZL3YvPi8C71TvZu9RL0AvsY9Wb18PKo9BL49vfm9/7qWPWy9Jb0EPo28ZTxxPZO9Kz6DvDO7gLwSvYe7LrySPLs8OjxyO8u94b1qPsw8/j3SvN07DD3hvfo8tr05PZu9Krzlusk8zT2wPJE9gj21vIU8fL1UPT+95T3MvRW9dDtWPrS9DT3dvc29Fj6TPZu9KrwtPTe9H766vIO80L35PU89Wr2/vUK8YDwxPuw9/73yPfy9kT1Wvba85T26PCw+WD3fvXQ99z17vAw+L7zCvQe+D72bPR89obyZvSc9ajycPM68sj1uPZ886L3QvYw9ELxePZ29KT1JviK9HLyoPRE+Fz2qOwQ9nzxoPsG97LyPPWm9ML0vPRa+fzkOPL89K74rvaQ8Nj7+PPi94bzvPLq8k705PLs94rzYupQ8lLwHPtM8AbwkPVG8Vb1bPPe9Cz55vZk9x70zPgy9uz2bPAc9rb3XvVw9Rr0cvTE9qrtlPVu8WL2yPcs8+DyOPUE+4D0Avga+zT39vQq+NTxrvM+9cT1tvcy987ytPa69Nj2sO509jj2EPdS9jT3vO2G+Bz0yvoq95b0HPbm9qz3gPcK9Mz2JvSS7BL75vTS+rj3KPQm+kjz9PTS77b3UPCi99T3CPem9jbxKPag9T76pPM295z1Pvck9mD2PvYi8fzxmPV09v7quvTW9oLpDPKm9sj34vBU9/LyLvZa9tL1+vXY+Kj36vDa7rb1Wvqc9jLrAvH+9bj4XPSu9KzwnPd49aD22PMu8Qr08vtw8rjtWvmA9LjssvW69P70APAE+iTzIvVc9zr39vSW+pDqdOEA7Gz2tPae9WDy6Pd+8jzuwPeK8fT0xvZi9C7wQvUM8ij39PP09kr0iO3O8RrsrPpu9hz1DvYK94j0tvsI9YD2UvMK9iLyHvdM7FLuSvQs9Gz4PvSg95juwPCA8/zyEPeO9ebuGvNi9QTymPT49er3XPOy8xzz1vaw9Rr2huz88wzzgvWc9Ab4QO1K9NT48PWY9dj2fu/w9Nj2MPcW88DziPbI5Dr51PQM9jTv6PTS80TxsvcC7/jxMvSy+iLwUu728Nz5GvlY8FLoBvtU49D2IObm9ljw7vD+9UzyEvZY9b71KPOA8hDyGOyk9wr3lvZw9uz3APeI9Fb6mvME7OD4GvSc92b3XPDu9nb2bve29sTtfPey8Djy+vf88iDyUvds96D3CvRa+hT3rPb490zy+vQ69oL28vaI8vzzDPIs8kr2UvY09/z0SPd+7kb2kPeA9uzwjPaU9mr2UvHU7g73EPWA9yDyyPYK8WT2xPXa9SD2IPZq8Dj3hOiS9/7wHPg47zbsOvT28xzwBPds9Iz1Rvcy8LLwHvWU9z73ZvQK8x73gvKG9Qry5Pdo8mz0SvSk9uL2Gvam9RT5KO3A9Ab0/PHC+hDyVPAy+hb3mvV89wb3jvSW9m7xSvqI8P75JvSi8tL0QPUQ+Fr6MvJe8Vr0NPKg96z2jPZq9ez10O6s81b0dPbi9Qz3hPTc8sT32Pbg8W70Gvoq9JL2PvbG9yTzvvI68pb2BPfW9QrtKvaO9Sz14vaq8Pj3vvCM94z0lPY27y70MvYM95DpjPaQ8lT1YvJY9kT0ku8w92jpUvbO8vj3IPMo8qzqFO8O7sLpvPRu6kr18vRk8Xj0hPsU8/j0hPQo8iL1xPRc92L1GvWq9rz23Peo9wz1UPfC9zzyXvaq90T27vZS9yz0QOgW+6D1KPYE+sT3wPQS9Ar0Rvng8nz0ePtO9Nb2WPbG91TxlO6K9uTwJPYk7ir0rvcc8Rj0TPfa80b1dvF48GT3pPO28Cr4UPBC+IzykvYy8Yb3Zvci76D0+vBa+wD3+vGo9Jz1cPcO83L3HPb+8Czzvvbw8jb1ePHW9Mz6PPee8jTtyvSm9T725vce8PL1/vS0+17sTvRO+Fr0KvZW93z2uvR++9DzvPIQ9XT6YvV49VD2RPWA8GL2QvS+8BT30vQm9Nz22PRm+rr3dvd29XDy6u7m9QD0BPgO+zr22PTc9rryhPKu8Zz06vUU8cLs5vku9B7yaPZk9vL2oPU29qbxtvLe9Iz2SvRc+K7yvvZs9cjsZPre9qL2IPUw+zrzxO0E9yT1dPXy7572kPNW8rr2IvSI9LL2DPdO9xjydvRy+Hb4TPqw9gT5quzK8cD2NvEA9T70fPae9qD2ZPdG9eD2pvW29gz3VvQo+OD0DvTo9lT2qPfm7aDzzPZM9zLwuvKg8Y72OvbI9Iz4dPIS9ET2TvbM8wj2rPDK957yDvAa7NjxBuki9yzxmu6i95bx2vIE9rDwrPSC9ST3UPbM9oLzIu5695DtJPfM9z7z1PT+9Eb7mvcu9ozulPaw9+T2UOhy+C751vVg9fD1QPLi8BT64vSK+BzwSvn49eLw0vWW98L2FvBy9U70evEC8LT2bPG+9AT6qPT6+j7v3PRk+17ysvOA9q7yyPYo8HTyVvPS96DskPQe+fz0sPUg9bbyaPP68mr3MPb29Fb4LPZe8ar7CvKg83L2QvSw+FT0aPfa9dz0Lvrk9NbxPPrK9A72/vTg8QjxTvow9P7xKPcI9tbk/vmy+Dz2KPXW9wLrBPaA8gD1mPR49UL1dPeq9jLwTPUy8mr1avKC9O73FvcS96T0lPp282DzTOpm7CL7aPCy8wD0fva89+TyOvG09F723vUc+Ez26Pau9Aj4jO5I7LT0LvSo+Hj7WPNA9nby3vf68xT27PYW9Lz13vc692Lwevd09i70pPdQ97bx/O4E9oT2bvQs9p719PVs8gL25vMm9aLwsPZC9IL3Tuzo9iT17uoI997pKPZq9wjzmvbq91b0cPeY9AL0FPYw93D0bvss9wDxyvHA9JrzUvYI8A73HPbs8Vr04vn69PD3IPAE98zxMvps9WTxVOSA8FT3wPEq8KD4JPJY9Yr05vLU92L2SvTW+Fb2fPJ2+Oz7xvYU9ojtMu3U9sb39vFa+uDwYvBK9Y70WOtU9SL3ZvYi9K75rPYg9hTwTu5I8Gz39vVM6HT0HPqq8Tr3evdq9ST7NvIQ9urwQvT2+uL2tvAI+Ar6TvSK9ET7HPVu9XD27vfG8Aj2vvA2+Sb3ePcu9Cz0dvR29Ar76PFG93L2lPUM9vryUvc89CL7KPMu9aj0RPpO8xT37u6I8tD0svVO+Bb16vSQ9rr3YvQK9sLyovUC6KD0RPW28Yr2uvDA8Er2MvS89lT1mPXy9vL0Jvqm99bw5vri9HjylPdg8TD5DPoc95bzXPMi9oL0tPYe8Uj2rvaK9l72GvYM9cj0QvjO+a7xxPVq9Wz08vn89Yr0ouio9xD2sPRI+mr1CPXe9sD36PO09UT2CPAI99zy7Pby91LyYPX69nT3wvcq8uz1iPVw+0L0vPJG9Pz31PKi9Vj3Ru327vT0LPnW8Bz5FvEe90b0JPkk9Ej0wPbm9770bvEQ90j08viS9ET0oPJg9rz3KOg49TD0pPQy+Qr0jvH48jLwRPfi8CL6/Pdo94DwNu9W9Kz5TPeq9kT0Cvi08kD3PPYi8uz1ivBy9Jj1FvSA6oTxXOyA85rwGOw48AL2tPS+9tT1KPek8a72+vQO+uTubvb49Pb2/vDg9e71hulU9yL3QvAe9lLyyvYY9E77JvMG8ej0hvEy9jb0zvDU+nb2dvVW9GL2Dvc+8XD1UvTo9h7yVPVK9kDwnPlC+azyvvY69Dr7CPBa9TjioPAO+8zw/PaO9sT1sO2g9YzyTvY89Gj1dPIc99TwdvDs9IL5uPfo8kDvhPMQ9Ub14vSe9mL2/PSU+xb0uPBo9jL1UvJi9Nr7tPQM+5z1cvVc96zzgvRS9Sj2QvVY94TvhPek9xb3IvC++Hjy2u/I9UTxUPVO+Fz1hvb09Rr06vYM9sb1kPTm98b08vhK+gTujPRo9bL2hu2s9jDu6vIY9BDwNvUa9PzvIvTK+iL0Xvli9x72HvQk9Njx8PAq9A70MPNE8mr0bPe49NL2MvI0+qb3PvQW+Dz3lPQA9MjzKvds9bD2XPUO82T0CPbI9PL12O8A9uT23PQS+mb0+PqW9obxyuwk+9b0evn89vr2/PNC8wTwQPoI9VD0ePZi9IL2cPRU+AT7BvQO+cT0bvqY8VjwUPQ4+Cz6QvYa9HL3/u/+8XT0wvdQ9UT4OPgG8hb3XvGa9sb3QPVK9S7zZvPg8+73/PVe9oD2DPQc9GzyfvBA+xTw4vac9Zr30ve+8kz0xvZG9Fr7ZPRc9gj04PcK9ujwhPve9Rz1aPVE97z2zPPi9E73wPC28Dj32POC92TuGPQC9DDzJvYo79Tynuqk80TopuhO9Ib1Ou6O9vr3GvG49ujsJPR89Cb4EvS69srxzvZG8mDxjPQS+PD0GPXA99TvdPS87HL2fvbu93r3svdY9njwNvbe8y7x8vOI9lTwDPZO9E72xvee9Lj4SPkU8OzsUvs69Gr1TPhE6eL0Pvky+fDwBPB+9UruhPZi93b2YPXu9Rb1YvZE9Bz2LvJ89OzzLPQI+eLyuPcy9STsrPRK9Ez6Cvdu7/r1PPkw9kr0Dvtc9Zb1oPp08IT6XvJ09Bj64PBQ8Zr0KPU49yrxcveO8272SvYc9cryUvf88mb3KPBK+zT3jO4u8M74fvqi92j2ZvZ+8Hb1Iu0a9FD75vR49E72FOzw9Jr1CvgA9cT1jvOk7kzz6PLi9wr12Pd06wT2VvXi9+zznvUa6CL77vCW9MT28PJ+8BD6OPb07Tj2yvKo9hD2Cvc+9br1UPSk+CD0ZvDA9br0cPUy9Az07Pac9iz3svSu8gz0GPDo9NT4KvgY9iL0BPW085Tzfu/08VD7avbm8FD3Ruw89SL51vYu6YD2TPge9ib2Ivm49Ij0qPYM80r1Lvos6Zz3BvIo8Oz35vLu9nT06Pa29ET5SPaC70r3hvZy9kL2rvYe8bD0cvgM+oL38PbQ8Ujxevto9G768Pfi9qTwkPXQ9E74KvCc+Nz0nvLg8srzAvfy9kL1/PQc9ArtWvGo+d72NPRU8cDr2O/29ozwavru9Rr3KvUY7Xj2bPYq9JD6VvQs+sbywPVu9WLwLPiy9Ar7fuwI9O70SPcu9JT7SvBe9072BO0I9bT6Cvd89hD28vcY8Kzz7O4Y9Wb3AvdK9Iz5fPLS82D0wvWM9sT3ou/o9/7yPvAc9BzwqPTw+fj0vPdI9dT1rvVg9sb0xvLI9vLpKPgS+Ub14PV69Yr0Jvn68ST5WPTG9Ub22PY+9pL39O8G8ijvVPTG8xb0Fvuy9iTxzPGa9rj2KPJQ94brGvBA+Vr4TvYU98j0avUA9RD1NvRa8y73UPEW9TL2tPVw+pz0mvdo80j1mPYK9Eb7tPZc8sb0qvGg+h70SvKA92zovPc49nr3pvKI9eDy8vSY+2zthvEg97T1ivfO8/7xkPq89v70JvtE9ibwEPr89Oz20vbE5Ab2HPRc+wD0wu9C9WD6zvGi9VD5TvdQ9Tr0QPnM9bz1EPKq9VD6xvBs+wLw8vpk8nztnvZ+9b738vWa9YDsGPC69v7tEvVc7rT0zvdO8Ir4zPSo+T76PPQm+tj1DPOA8szwAPlQ90j3LvII9gz1SPSg9HL4rPJy8Ib1QPa28ur3UvRC4xzsrvRM+ljzXO/Y9lL1zvQC+yDxqvYO9JT1HPSK+2TwGvTs9kr4ePiw8OL5lPTo9nr3uvZq8Mj0Nvfm8mLxmvAM+8D0ZPOM9aT3PPK09Mry2vY6+iD2BvG+9ar1Xvb09djyivMC8TT6gPD27Ab4UvRa7WTzdveA9hLurPPu8eb2FPYs9yj1NPD69OT0fPQC9Lj1IvES927zDvFa9x7yfu847YL0KPA09Dj1Svc28gr1GvZs9FzwUva68IT1tPfg7V7vsPCe9/ryHvUM82zpAvNk8Y7x9vC68zDuovFO9mzzjvFI7Dz20PCO5vrwrPAE86jziPI48yLxgPE684ztLvUO9Gj1DPaE8zj10PQ09lTwMOj+8RT3LvPi8Jb0IPSQ9MLzguRq6cb2sPDo7Ej0ePIW8TD24PE88e70WPKU9ij3gPGM77Tp6OhQ8KL2UOz29GLxZvfq8bjy5PEY9kDwoPak8Dj01vd27Eb1TPG08dLwxvYw9CD1EvYs83DwLvYW8CbvLupI8kTs6O4q9vLyEPG+75zyKu4O8Bbx4vJY82LsNPDy8jDzcu189w7tLvW48Lj2NPHm9ar0lvVA9yrzkvb86hD14Ou68WzyvvJU7/TxnvA89+zyePIo9Mzz0OwO9HL0PPao7hbwQPTE83bwUPHw9GLsXvJS8/rxMvVY8Dr3tPBC9g71xuqO8ezqgPJ09mb3NvIe947uTPK+9fb1PPIg8ub3JOkA9Ej1+vKu7jrzju/E8CT0aPD88Cr29PHk9ljyLPOg6mrtjPNO74TzgPBc90DwdvPa81Dw1ves7gj3uPLQ83jsxPc+8nr1VPDy5vzw1OTU73zshvYA8AbxdvBa9S7xJvbY8gbsgPcw7FDynPcq9Cb2iOU+9Lz07Pak8aDyFvei8ZzvQPAe8hz33u6m81zwJvbW8iLw9O7Y7SD36PC+9rTuOPM08F73oOuc8Ir1vvGS7+DzVPJU8qb15PA+9sbuxvFe9kL1mvJQ7nTwBPV29ojwrPZi7+jw5vYe8xTzyPAG9crr2vBs8M72UPNu8LjtdPcA7+TzvvKK93TzuPD89bL2jvJM9/7wePR293DwovOW82zoWPVS8Rj2QvF296Twlvb+9yryWOuk8wDtSPQs9Rz0cPQK9mT2pPFo8QbzhPBM8ybxivA87Az2fPbg7jzyEvGO8tTwTPcm8Jr0nvYI97Lwjuwy7LD2PvA08gzvNO9C8jjulvCO9a7ojPJC7zDtVPV+9PjzKvHq8Bb0HPcq8JTwcPYc8urzCPAo8yzwou4q73zxfvYy8ert9vNu8hD2vvFm91jzZvKG7pjrrOzA8pD2EvBy8VTybPEs8ZTyLO+U8Eb2nO5s8WTz1vPQ6Dj1ePde7fjsbPcU8Gr2qvLu8rjwivIA9GbwavY+9Er3xOyG8uD0+vRe9fzpDvby8Ij01OwM9ZTwhuwI9srxyvHa75LwuPXI9rjzOPFW9wTstvW69Wz2MPGu8Wby2PPe8BL2ru1G8gDzIPAi9ETziPBS8hrwXPRS8cL12PcW8nzzTvCC8/DvFvCY8ibzHPMg8Jzz9Omq62T2HvHq9Q7wVvQK9VrxqPIs8yLzJvMA8dz0/vc+83bwyvZU8nTwSvYs8rjyIO089NL13vG69OTwzvXQ9grvEu2A8Xj1kvOu8jL0RPUI8jLxoPBy55ryqPM08gz2ovLe8Djx6PaW8w7t8PBq9MzzYPL28M70TvLI8E72YvYK77rxQvWg7r7y8O7o8BT2dPV68ALwPvZK8EruFPBo9kjzIvLe8Vj2dvDs8k7wpPVe9jrx1vGO9ez0fvQK9w7ykO3K4ND28u2A93bocPbq8Rb1ePbc8wjwlu3e9s7yruVQ82zw8PKA7HD0+PVk9NLxfvFo9HzsIva08k71ZvSu9Q7ySPLA8hrwPvB49OruqvEe8Bb3YPLo84zyIPDo90LykPDg8nTsYOE08lDyrvOo8Vr2qvNC8bLtUvN68Fj1TPKQ6MD2TPNu8gb36urk8kjwsPIo77Ly8vKs8RDpyvR48CL1HPP67xbxqvPY8CT2ZPGs867tuvQK9yjwBPSe7jDy4vH88AT1jPK475T0QPcc9Aj2xPbw8h7wFPdo80zzDPFk8Fbu5PLW63jxyvOS8aTxnvQW87bySvek6sDyFvQ07BD0KPaK99zw3PRS9dzyVPSk9RbxQvd68fL1VPeY88Tz3PCy8hTs6Oyy9aj1uvYq7Bzy0vIg9CbwuPXq8Pjxsu0I9jr1EvTg9GzwoOui8Mr3LO3g8PL3EO0I7Ej0OO488qLuku4688rxFvJM7lLygPFe8bjxKPbW7lLtxvK28ULwGPTY93TxPPMG5fbpvvDg9qD0fvRq8LT1UPHe73rs/vO88Lr2EvSK9srv8PF88CL3sPEo9lbuHu9w8erwovY29HL27PDw8GT0RPMS8NbxOPJU7F70ZPK47gDyjvEc9Fzrpuwi9zzymvUM9Hj1FvQ28b7z/O4C7LL3EvB68+Tu0vHE8cL1YPZC7gTx0Om08gD2qPGO8sjs1PDa8rDuZvRy8yrwLPSy8oboNPVK7Bj22vZ29gLv0vF09lLyrO0E73Dw7vY68Er2LPLs7Hb2nO7K8Fz0pvYm9gbxKPSc9v7x5uws9kL39Oxw8vLzIPGO9j70lPWg9wDyFPQO9Yj2NPQa9NT18vae7Fr0DvfO6S72CPI08KTxjvQ89Fb1Ivaq8Sb2CvF89xLvCPP07i7urvJ08tjw7vUI8vTz9vOa8cz2sPGM80rx8PCE8vbxFvJq8Jr2cvPi8k7woPSK9JT1kvJg7hbz+u8q8EL3yPG88Hj3PvCE7jTsHPLI7lLz8PJW8Vz0mvTY9frwovfc8wTuCPBO9rTtCPSI9G70fPdW8zTssvBi9l70JvF49+zugvC69ID2EO8s6tz01vJm8lTxvPEI91bxTPHK8djtjPPu8Sz1CvRy8hj0ePYs8GbwRvQi9bLzEOzY9nbw/PMG8NDySO6A7xjyavO88Hz2fvN68IT2EPd68njsEuzY8BTtnvLU85jwtvd281bwou1O9wbwdu2G8hrxfvR683bxCPA49lLscvSS8BT0PPcI8L7yuvIY7M7ySPTq97byPPbK8q7x4vY299DwrO2C9Zr0JPdC8Cr1cvBM9+7wjPRo9IL2hvXU8Lr2RPBm9jj0ovYA9oLuMPGm9cD02vI08STyUPHo82bvuuz48/Tt1vJ48rDwxvJ07Y72cvQS7DDwHPRS9MjxBu0q9Ar04vT09+DxqO0M7rzyfvDa96bwhvb48rbogvAs82TzOvME8B730O9a8Eb0Ivc48hjylvKi8njz8vB49s7sGPCK8Frx7Pfc7FD0AvZC8xDoEvQ48Dzz9O7S7NzrQvOw7lL0kvZc8kDwkvfa7OjzavKE8Gj2WPC69FD28PE49ZLwzvfq86Dy7uwm97TySO0E9mbvTvKK9V70nPJo8qbxjPbs7wDzFvhU88zyMPL88IDyCuwO6ATwfPBs9oryIu0O8pbxpO1m9nDx3Oz28Jj2OPL67Ir1ivW26LTwNvZ+8rbzhvJA8Ab0VPRi9oDyBPJA7vLgpPU49jz1cPfM8Wz2PvIw6KTypPIw9GTtmPHy807xoPJQ9BrwSPcM7sLyau3m9gb0BvWW8lDzcOz29VjyKPX48aLxHPQy8Rzy1PCu8rrziPKy8BD1HPT49lr1EvTQ9+TvWu4I9kDx6vIc9SzwJvYs8tDwBu9E8sDzvvK+8BT2rPMI8ITyaPRE9/jyXPAG+JD0ePfE8Hz2fPFA9Iz2zPMK8WLxWvCk89zwRPVM9kbwBPFK6ebzZvJM9Zrv7PHO84byyvAc8JLzqPGi9RLrHvNw85TtEPQI8kbsgvA49EzsbvTW9q7yMumE9Mb0HPX69Ir0KvTE6R72WPDa9D72IvI89Zrz2vKM7Yr27vJU7lL2CPDo87rw6PZS8kr0AvV48y7xgPJM8gr2qO948s7xjPRG9SbwPvUQ83DyEvRW9EDx/u068mTtrPIQ8YjwsvSs8Pr2evHc8FDwBvXw7nL2xPTo89LzUvLU84zyzvOw7k7wBvR871LqKPeq5ATtBPaa8oTyxOxE9yrqrPZQ8RLyqO748Cr1ePOA88zsavcK8xLzWOu482zyAvUC83Tx3Pas8arxPOZG9Nb3xvJK8jDxhPSM9MTs1PJw9Hz2GvKw5yL1Eu028aLyCPB8+ajzIPLS8OrwzPNA8grxnvR29czurvDe8/jxavAo9QbuovUQ8PD13PbG86LwYvem8p7zLvBy8rTzePCe9Pbx+vHu8HT0JvQA8Kb0avNU8MLx9PAk8X7w4vbY7m7xivaK8SzwrvSs8Wj0rvPO7ortjPYU7CLxZvIm8Urw+PPc64jzruws8n7taPD+9NzsoPWI9vjofvFy9ezwOPcO83Dz6PLy8mL1RPJC9Zjw/PGW9WD3bO4q7q7wxvMo9+ropvC29KL3VPB09SL22PDY9Qrw/vPu6Ir3SPPw8zjdDu1i9Fb29u1U9UD3JuzY6kTqUvNU7RT0yPLC8yjxYvWA8Bb0EPQk90Tw2vZk8uLyFPXu9WLyjvDe8KL2yOza9SL0APcs8sjzBvK+8R73mPGe93jyXPOe8Rr1NvRq8RD0Mvgy6B72cPJm8+bpDuwq8Jz0LvJc8b7zZvDG9jTxXPHu8Wb3BvI05yL2FOki90zsJPV08Bz16PZ28AL1evRA9YT14O9U7wj3pvLW7Cb1JPJm7Sz3lu0O9Gzx7PUC8Gr14Paa7Pry6u/I8Tz0HPb68Rj0ePRG88TtqPe08RL0JPJy8WLyTvL68xDxpuzG8Cj0yPXG7L70WPX87DT0bvQY8kLysvQs7Uz0nvRq8ML3fOye9Fb1ZPEs8bj0NvU69Dj2tvd+7xDzkvE+99jzJPJC7SL2vPHS9mrxJvYE9nzwjvA68J73ePOC6Ib03PGe9e70hPJc9Mz2ju/g707rXO9c7ET3tPM68PT3wPCe9HD1tvEM9DjxEvMc82bwIPXS9i7vhPLU8rzmvPMs9U72bPUI9Bz18vMw8qDz2PBq81jwTuvi7YLwJPRY9HbzvvOS8gTwnPRG947xAvSW9HL22PIm8zDyBvK89hrzMO629wjypPKA85rs+vZc9Kb0jPfk8j7ytOyk9RD3EPK67bzxfPZi9Wry4vMG6yDwDPAU9fj2euqu8jrw7uy29hTywPDc907sVPdm8ED3uvJK8Pb1VvU+9vLxOvV26mDydvDW9FbzGPcw7gz0SvfG8Kz3EO9Q8Ab3LukA9kL0gPcu8BTxzPBA8q7wFPTW9ub0EvTK9jLxDPBQ9DL18PY+8IbwvvNO8VDuKPeo8d7pOPZM9uTyzvHw8jzw1u/C6Nzk+vT68FLw9PQY9NrzDOqK8TTsdvS69LLwLO9O8RbtmvVO8U714utW8xTySPG69ez0evAQ9Vz24Owy9Pz1uvLi8d7xSOla79rtSukM8JDrSu5E9LL2VO+O6Wzz+PNC80TyGPEa8wbyBPLE92Twlu2u81jwlvQY9E70rPS87EjwePae8grtzu4y8G72GPUg9Rr1HvMa8PTvZvG48BD3sPIY8kjydPYc7tjxTvRU8Gj1ZvPc8QrwmPZY8tL2API88kjx5vMk48LygvAg9NrybPOM7ujs9vQ46Lj2EvdS8Or0SvF68hDxoPak7Fz37vIi7bDxvPV28ITn9vN287TsmvUk8sby9OZy9Bb3hu6w9Ez0vvUu9Fj3GPLW8Vb3uPK49+zyoPNQ7TD36uiS7VL1pPCG98zwhvTi9l7ySvHm6jrwuPc68VDsmPdy8HD3HvCC9ij1oPN88oDwsvQG8D7wXvVU81DzGPHI9lj2MOwQ8EjwevZ28TrwnPZG9Szx4PVA99TzuPHe7Kjs/vdc84byEvUc92bxPPAQ9nzsSvey8TrwjPQi9xjwdvAy8AT2KPB69DD20PPQ6Az31Ooa9wDzlu1q8sjvqu468VrxKPcm9RD0jPaC8zzzuPDs9NT2FvIg9QTwjPfu7tztePGY98zwKvDq9iD1hPAM9WL0+Oxq9hzyCOxK9WD1IPDs8BbxtPFU8rb21vMw8qjzIvcA8QjuDvBU9Ej2sPOM8xTxcuzu8CD28vBm9cjxqOyC907wBPKm73bl4O8i9Rzxqu2S8JbtVPGs99TyGPDI8STxKPB69Vr01vG490jxLPDa8sTzwPGo69DvOPEc8N73sPOK8hD37vOI77rvQOy29QbyMOw09pjykPEC7s7vQPAk8DD0LPXY9OzydO2+8AryWvKw78zy8vWy8R71fvR88mzzAO7k8u70MPbC8Z71UPMi8ojyVO++8ujzKPCY9Dz0CveM8qjxIvXa9ejyHvL+80bx5vCO9ObuZvBU87LyqPHo5ijuQvdw7fju0uxW9QTwBPV08RD0cvYA8kL1QPJa8djyBvRU9Rr13PSg9ZDwpu4Y94TxEPYU8BT2JPD894bytuzo9GTzhvBw8L7sMPZ26Fj1cuZS9prx0vCk98LyUvSI89Tz6POm7ab0LPZy8+Tw2PR29hz3oPNA8kjwsPZK7Bz0zveE8kL0pPJy8mzwuvYe9hjzQPAy84TyqPJ67zLv1vAm8I7wGvUO8GrxMO7K8TTxIvaM8bj19Opi8Ar2dPB67D72qvXS9pDzDPAe9CT2NvO+8trvuO9U80T3QO6Y7k71DPBe8Mj2mPcS8DrwGPVS9lT1EPKq8mrwrO8Q8er0XPFw927ygPaQ8UTsIvdA8jLx8PTQ9gT24Oma8Pzz0vJq6uL0WO6E9ebySPAw9hr22PBQ84rzSvI89Dz1MO9E8HT1hvS67srwTPfM8rjy4vDO8Qb3WPO28HD0dPBC7Wj0EPQg7HDyHvX67Nb2PvN+8Cj2OPAM96jwdO4s8xLo7vI68WjysvKC8iT0EPe66QjwfPLq8iTwIPOo8qbyTOuW8dTuCvZq8HTvCvPW8Vjy1vM48w7xcPI07PL1SvMu7DD3OvBI9Bzwdusw8eLxkvYO8ar1Vu/C8WrsTvbM8tLqmPF89O71cvTe8fj0DPe86gryAug89pD0QvVS8Kz1FvWE9xjw+PNS8UD01PSi98TyEvN87FjxbPRo68zzvvNy8RDzovC47+ruZO868Cj0Huss72jyKPJs7i7xMveE6jzyHPGK8mrzQPDw8dzw1u4q8sLyzvK884Tz7PMU7FT2bOcq8KL0LPYo88jyRvcW8dTvovGC8Q7qEPB49GLsnuwq9PDyaPZy8ozsxPek8wDynvJi88Dwcvci8zrwUPdY6Pz29u4k9irxtvMo8E72Du0g9Yr0kuz296z3xuog8JD0rvUO9Wr2FvAu8gDwQvP+7Lr14POO8oT2tuoK6obwDvR09lz3BPGe8wryrvKy8nrsNvUW9wrsNvc+8o73gOxA7ZD3QOj48Xr2dPSQ9Mb1BPMs8KL1FvI48ITmNPFM91TwNPIK6lb3YPJc8q7xjvV48+bs6veG8zLsivTw7EL0jPbw8ubzPPIG9hL2GPGC8yLsfPJW9Tr1/PIe9c71wPaQ8jz2APVY98jqFu5u8CDw9PfY8+bpCPWA94j37PBU9bjxrPZQ8vjyfOzA9Pb2vvPE8k726vLS8MT0cvRE92ryJOzM9iT0+vGq79T0hvTK9cr04vLg8TLyhvBa9A729OzU8Kzwivd68/zzjPHo6kzzWvE49ILz1PD68Ez3Du088CL1Xu4Y9WbwIPCM9UT0/vLu6+DtHvHE8lz3xOjQ99LuYPVq8gL26PBm8nbylvIM8Cr0FvAa7Oj00Pb47kT0BvZ+8hDyBvcE9WT0DPO47tLw6PeY8AryBvWs8pLu8vPs8CLpdPFk9JDxoOvi8lDzgvIs9J7zOvAe8fLwmPJI7hjvWu609Yz2DPOq8VzygvdE8UTyju2O9bDzUPDE9J73lPNA81zxZPI487zyVPNw9Kj0vvDa9PTytO4c8MDqaPHq93DsYvS87GzviOya6zjwvPcY92DyIvCQ8iztGPCc957wAu4i8HL2zPS47TTz0vAc95bzYt0Q7krx7PQM8zztBO5Y7AbxgPdm8Bz1guym9M71JPZO9Yz0BvMg7nb2cvbC8iryjPWk8kT1YvVc9ST09O9M73zvgO0Y9wLtSvQ89Mr2KvDU80TyjPKS9Bz1QvZM7FbynvNC6AjwqvaO7Crz5u148Sb2NPWi8pDzSvbu8RD2iPGq7cb1XO2s8r7s8PQw9Nr2AvTK7T7z9Ohs807uwPFq9GD1zvf08sTswvN67tT07PfW8F7eqPGe86bsqPdC8Cjw5vW49jb1kPIo8QrwtPTI9H73SO9A8HD0bPYS8QLn2PKw8bryYume8hDqcvGY8ZjyKvJe7WLt2vVg9qTxDvUM9xj1Mu3w9RLszvQO9C7wYPHG9bL2+OhI9hryIPEU9MDqbPPc8TT2kvdG7R7wHPRw82LwXPd67Q73xPOU7OD0+u7M7SL1AvYu7jzwbPAW9rzzdvOS8AL2JvQ69ujivvVa90by8vKw8Az0IPZK98LqFPKU78zlyvQc74jzDvBU7Fj3ZPIO8l7wxvIo8RL2Wu/C7pr1kOzI+a710vWc8pLyAO/48jzziPAM8QL2Eu5E7kzxJvS47yLonvCU8Ub2xPda8lbwcPaI9Jr0iPAi8FbuJvAa87LyFPdY8XL3wvLq8Mj0JvF886TwdvTm6bT2evFa93TwPPUo9L70FPDM9iTyTPOa7Ez0UvIW8sztgPAm90LpMvAG94DsFPDK7CryVvBw9vrovvSu9gTzPvIM7uj32PK67xzwMvRu90byovM+8+TuVuXC8YTzWO9c8wrxavBo8trycPEI9qryzvLm8prmEPN27/LleOni9z7w5u4a9VLxwPN877ry5u8g75rosu3E8drx/PI28DDw9PAK7Ej0hvdm8Lb3sPHI7JL0vvRO8sryPPJo8ATxcuo08hjyWvF68Dzz8PKc7Kr1VOxU8xrsKPHc97LxuuyG9MDwQPdI8eT00vFo8j7wavGw8DL1CvEc8bLymvP+8VT0kOjI80js/vMo9BL28PJ08dD0avYU8Bz37PKS7/rw3PFA7OTv2u6Y8m7y8uyu8mDxvPa08vTwwOgA9MTtcvWU9ozzCOt48mrsPPfk8qTvAPJW9WbykPK68Pb2JvP06+zwLvNW8GT0EPQw9iLzmvFq8E70GPKo8Cz3kvBg9K72fPEo8OL0cO0A8Gb0OvVA9i71NvK+8bTz4u8G8t7xHvDu8sT0nvAO8QzutPA49xrxXPIm8fzwVvfu8Qzw+vJU8iTw7Pe08EruWPEA9+jsOPJm7Gr1OPAW9hDwjPLq7ybrVvVO9tTyyPKA8Dr0MPZq9rjwTvX68gr0GveO8CLwRvb47+bshPA89pjyQO8y8Jb3tvJc53DsRPbG8MrxivgM9UjzBPGY8Wb0jPTU8Br11u0o9PTx8PGa8+DuWu++6Gr1yvJm9LrrBvIc9+btJvYS8Fz3GO2e8Lz27vT49WDw3vRu9lLy+vN26rDvTPJC85DtROqQ8CrvFPHw9KD0DvaI9kLrbPNa6XTzdPDq9E71KPP08Bb3pOuI6pLzvuz697TzAPB89kLriOww9zjqKvP87DbxtPb48Gj3APKO8hLsBPDE8qDyQvB89Ib3nvCs9ZjzOO8A8PT2bOwo91by/vDG8wDuhOxk9p7zCOwY8vDwBO5o9sDy+vPm8VD1BPMC8mrqrPFs9JzwjPRa9V7t5vQ497rwNvVy9Fb0mvaq8Wz0hPSI9sD1qPR69lrtbvHS96DwnvTQ9cDxFu1U8uTwNPa88zDxGPaM8HzyiPDM9ZzwPvTi9wT0avTO8g7tzvIK87LzOvKo89zypvDc9GT1APRq9GjwDPWo817shvA48mjxePQ49ArxfPYa8V7y/vP+8u7sUvAw7VL1ZvLK7HbtZvbC8tToqvBU8PL3FvG29oTxoPPW7zzvivOy8Dr08vey8urxwPMi8Zj0cPVM9Qr36PJe74jo2PFk9QzygvE68jrxZPSk9rbzvu2Y7X70kPRk9+bvQvKS88DnDPG29izyMvD29Rr2sO3E92DyKPEE9MLysvK08yrw0PRY9AT1Svb67jLpRPIe86jq9PIg96LtSOte8ADu2PKI9LTySPOE7M7z+O1U9lTsDPXo8VbwnPR899Dx8vWw8kTwYuys9rDqGOu88KD12vbI8Mz1Tvfm7iju4PM28EjwOu1s9mLxzPKW9lbx7Oyy9kLsVPK49+TsoPf68Dr3QPEC7gLyxvA29zTw0O8y9AbsWvOw9KT3BPTK98TzFulG9xrx1PdU8Ez0fPAi9vzx6PLc9OrxRvK08OjsDOzI8gT05ve48KrwPvBO9yTwwPYG8I70avRY9QzyHPfK8G724PbK8mjzcvNg8S7wIPR687DwvPH48hT2pvSG9KT3fPKw5qL0jPEa9FDzdOqS8HL3rObq8Kb3FvMg8KrzfvHs7yTt6OyE8h73hOzA8Gz3XvIo8vLsUPXM7Fz1AvUC8BL2+PEW8SruNuye9gr13PGg8Czv8PGs9U73vO488ujyjPD49OL3tO728XT1oPNk8S7wDPQI6vLuAvac8p72ju5O8dLsVPdc56DwHPQi9xLw/PQq9g72SPPA89LwuPeg7GzwAvdO5qDtrPqo8TTzNPNM9IzzqPAq9Kjs2u6C8hL1QPam8az3jvLK9krwFu2S7Dz19vSC8Dr3XvLa8n7sFPaY89TsIvJ282LxQvV091LvhvGM8hLxqve471DsPPTu89LwePQw9jT0FvdA9oLzNPD68XT1PPDw9nryoOwg9izwTvaw7Bb33vKU8Tb1VvNq8rrxuPVA9z7yMPKo8hbwuvQw9ubyOPSy9pr3Xuhq88zxgvUO9qbwIvKI9pL0FvYw8pz3oO3K8Qj3LPMA8N707vEO8qjzavIM7Tby+vDK7wDyiu948Kz2fu6+8m7upu3C9nbpwPZ+8cjz0ura8kLwxPAo9Hz3Iulg9tDyFPJy8xrzFPB07CD0YPUE9UL30PBo7gr27vWs8EL2avVc87LtDvMs8Pb1dPD884zxkvLo8Sj2KvFe9ejxCvXs87ry0vGW7ijwDvSI8Cz14PAI8GDs1PF49ITw0vTi9Wj0+uko9CTwgPLA8Tj3vO/u6VjyyvBQ9gj38PNC8/7srPDG9Kb0LvIe8qDsgvee8LzwhvZW977xXPDI9PjxLuwy8oTv0PIC98buvvKK8CLwNvSo9D729Oo88IjwSvUU9/rxrunu8orwhPIk8q70kvTI9bT3ePIw7Obx3PPM89bytO+w8Ob2UvIy8nbs+vQy9xju0PEo9Jz1FvQ87Q7yVO308wLz3PK09IDuNvHM8Db34vHo8J7w/vB073ry0PBo9Cr0MPPG8jT1rvRE8gjzqPP88gbuhvAK8bzuqPIa9dztzPV09ST2XvN+8Hb1YvWW8sLwJPKs8vTswPIu7hDwcvfO7trwmPRY9hLyIvDg9+TyhPBe8V7uZvRE8Fz0kvKO7tzsIvI+9M7wEvRg8YTy9veY8aDuDvA49LT24vEq8Ir0dvAu9Fzt+vPU79DsSPC49Bj1sPVU9y7uoOw+8BLzJPOu8MT3jPCI9a7xJPL48rLydPHk9hbo/vHA9qzshvIS857w3vB48SbzRvIk877slPUc8iLwQvEy9Dr2qvEE8hzzmPGQ9UTz/u728Z7n4vP+7QL18vSW7lzw0Orc6ujzGPIi8Kz19PRI9jjuUvew8Nz2EPBW92zoKPWO8pLwlvZq6xDw7PJe8iTwJuzU8FD1YO6i9SDxhvQk9pbwbvT29nj2pvM47Kr3iOsW8a7r5Oz29Mj2EPD49dT2QPRK83jtPPOa6kbxAPIE9T7w5vOW5Uj1hvO48Q7y/vMI8gb0tPQS9rTs4vRy97rwGvQw9ILwwPb+7YDz/uz+8Fz0VvYS9bTzau009BD3yOyO9NL21uyi9c73/vAQ9WL2NOww8C71wuyI8H7wVvPK67z0SPLw8dD2evME7sDyFPLo84bw2POA8UD2ivBO9ljuRvLK9bL2+PGm967uLPNK8lDzNOgE91bw1vcu9SbwZPK48G70EvUM9QTxJPXO9yLyTvCC8p7x+PDo8/rwePcm8BT3NvEG9EDxIPNY63ju1vGC9jj0iPXO91byGveU83LxLO427Iz3MPPQ8Or2kPK08Mj17vJG81zrTO4O9ubwPvJC9jLtAOYM6n7w8vam3MD05u/48tz2mPFQ9xLuTPcE8hTyUu8M7Fb3tPGq8Y7ycvXA8iL03uyg9TL04PRG99LuYO1+8Pb3dPIg8Fb3VvBe9Mr2uvL28M7s5PPG8zTq3PIG8lTztvF86zDyNPBM9b728PPi7ujtMvIa8w7xfPHC8ED1Qvcu8WD0bPR89hj3HvI+9Dj3BPBs9SD1yvIo9Fb2hPO26c72zPKy8n7wLO9Q8Krysu3y6HzzxPBU8ir3SvAm9F71DO5y8wzzZOsa8GTwPPec8FTzRPBK8jzyFOg49s7xSPBu92bsyvT47tDsPvZ48nLx3PX47ATwxvIg8Sry8ud87ZTwDPJC9UL36vLI7Or0gPc09gTx2PQK8ojweOyq98Dw1O0A8TLoavQU9cz38u+m8kb1FvFW9wDwtPEQ7kLu6vAk9qryJOxi9ij2NvV68tbzQO+C88zxKvHU8zLyBPA+9qjwZvWW9wbrdPLO9lDxrvdi8lTzLvJw86DtRvLC8+jxYvfo8jDwrvS08WbzJvIW7lD2VPBO99zz8PIo9hbtTvYC8w7wePco8ZDslvU48Nb2JvYs9N7xVu1I8Pz3qO+68DbyUvMk8Dz0cPRG9W7yHvW88jLxhPXE9Cb3dPNS8lT07vXC8MD3AvPa7iLwZvGs9s7wrvQO9qzuCOq28wDz2vNm7DL07vfw77jwkvA29MLz/PGi99junu8M9nLySvL67Cb10vZe7Yz1GPLi7kz1ivXm82ryGvBu9yjwzvRI9Hr0Su3o92rySOxG9xD01PSO8tLyvPbg8uTqyPO+7DryyO+G7lLwqvYe8Bz3dPI095Dq+u3o8ZzwLvuG8Fj1mPf68uzyNPX28YjyiO847G7yLvFq9Yb37vKG8JT33uhC9tDv6PNQ8ETqSPHY8vrwePdO6gryRPOC7Dr1wu9g8CD2BvZO8qzyhPM68WTy/PFo9qrynPDw9TLySvL08JLwFu1m7E70Evas8RLyrPNe7jrwxPfg8H7ymPJC8+jwMPVg9Pb1BvYs9hzrpPMe8Dr2NPTo8JD3XvI09gjtXPRq8Aj0CvPw8Jr2UPcU8LT2Yvd08PT0NPTK94zxzPbw8zzuXPcy8VT1zPSy8urz/PEY8hboIvaU8Fz28u5a8Mr2/vL+86zv2vNC8ID1ZPNa8m7xpPDe9BTwzPb88Hb3wPKw8o7zGuqS8lbzFvJ26UrsKvR68wLwKvUo9vDzWPPO8sDqDPbc7mT1TPIY8Db0EuUU9oDy/PHm9vrtAPQ09Bb1HvC09YTwOPTY87zyKPQa8irxvPJ09JLyEPVm9SjwuvdQ77ztVPCo7Aj0nvLs8nDoYvQc9HrwhOxy8Rz3OvEY7AL0SvXs8/ruEvag86ToZvQe9xbupvD89zDx/vOI8MLwavQy9Hr10vC+9BjwTPLw98bt0Pba87TwKvU096jugu429AryDPFY7Br1+vDQ9tTyFvDu74TxhvQy+Pj0eOqE8yrvSPIW89TqKPSM9DD0UvAe8Lj0LvVE8g7yRvIw8sDyLvCm9WzymvCa8Kj3gvDC+rjwGvDU8MT3TvIi9V72fO4o7Ub2fPKy8Hz1DvOQ8SDxxvUy8H7rkPGA7JL2zvbs8CL1/PZQ7lTwbPcK6aD1ePc28QL2xu3q7JL12u6g8Tr00vEE7P70+PZW9A72NvRw82b0bPeY9eD1oPjU8773Pu9+7rry0vUK9SL6APQm+Rj64PVs94boVvli9gL3NPTs+47tRPaa9qLxlPUQ+5r0Hvto9ir3cPJC9Kz0tunG9gDzYvFQ+77sbvkg9lLwGPVE7Oj0+PVE9gb0dPpY82z25vYK9xL2Qu4s8IrsmvuQ9sz09PoW9xb1xPfI98b2wPIU9ML0Gvv48AL5sPR+4/LxCvQu+Dj6tvS09AzwUvr49vTw6PWA8vT1jPCc7pLxTvMu97z3MvMY9db4JPnI8nj2HvLq9FL0jPsW87L3yu4K9IrxZvVs9TT6QPcw8+7xWPZY9FT3bvRw+WLuZvTe9STyYPfw9Ib0pPQO+kz1+PXU+/z1IPCu+WL4+vl49Fb1DPpi7+D3DPDC9Zr29u4W9BLwwPU2+Ib22PUo+9L2gPek9njx3O0G9Bb3vvdw9lT1UPhI+nL06PbA9Vz1IvOM9uj05veU9zL3IPU++XT1APIO8+71JvTA+BT6YPDW+XT3DPbm84Tw5vKW9hT35Pdu82b1Gvfw9E7zxvVm9cj2hPYg9Br7DPQo+jz3dveg9pLwgvT49LDtNvTa9Pj3fPRa+Cb0oPBw+lDsvvYK9Qz1CPbC9yz0Kvi8+J73zvb27G74uvp68bD2RvYy7ML76Owe9Lb4hvgQ+QL3OvFS+Bz4Rvq+8yTztPaa7bj3OPfi98D2NvX+9jj0EvKe90L2PvZQ9uzzEPQa+2rtgPsq9670wvEa+1z2lO6O9oL0Qvjq9HT46vg++8j1TPYa+1D1vuwQ9UT1/PH46jD5wPds6Kr2VPBK9Zz1zvQQ+GTwLOoa9Ab7YPQG+Kb7DvCi+jL2PPQQ+UD4RPtw80T3jvFc+DL1UvSS8x71ivbI8nj3KPF2+Cb7jvd+96z3BvXK9DT2Jvco9vT0SPVG+oD04vSc+YT0svhm+Xr3eu8S9Gr5yPQ28DTwoOyg8m70zPvo9ar1CvPM7Gz5KPL08cb20Pbk84bsrvm88FT3FPPu9mz1MPXW7hL2mvQ686DkfPZ882Ls4vUQ+oj2Lu608LT68vJw8mT3qvfE9tT1RvRi98j0XPey93r1pvCC9Lj3EPRS+g70BPmQ9Pb2CPTG9FL6kve29sT1cvA+8+b0fvpe8gTwgPVg9Ob3/vbg8Tz17PfI8CDulPVC+kL56PI09j72QvJm8Br68PJs87T0TvYO9QT2yvRI+1D2zPUa9q71nvni+2bsKvLq9Kj38vT4+W71Muza9ijuMvXc7wD3JPJW+5z2kPOm7kr3OPXq8B76Uvbw9hL0Ou2Q6HT3qvHW9eb3BveY9hTzJPAg9uTwjvuk9+z0iPky9Ij5lvck9Dj26PaQ9iT4uvTU+tLzlvAK+B71RvIW75jtoOcC9Mz2KvJs9Wz0JvV48QD3uu4+9tr0JvQo9P77YPTQ5IT0FPXO99b3PvZI9k730vQ67w7wfu8g9CD63PCe+A75IPp69Hz4SvQk9Vb2zPJ88pDzOvcs9aL21PS2+br2uPCs+vjxGu8g9zTsOPqw93TwdPtc8hL1jO8W9F74OPY+9rjyauzo+nryPPGs9RzvVPdS9Pj35vdG9pjz9vbU9MT2GPQS9Sr23PdQ8lbzrvbu7O75yPdO8BL2ovVg91j23PdY8Lz5TvuQ9zb38vFs9HL7IvAe+C72ivfY9tb0Wvgg+Ar5PPZm7+b3XvGu+Iz7LPRO9jD7nvEU+uj0bvVO9uLzPPeC7CD7QvZg99z0+vcy9cLwOvQ69Wb3DPa08Iz4svpU9rr1APc68qDzVPYq8Rr6EPWO93r3ePR0+Wb4Jvne9/bzcvaW7A74cO5+9lr0rvHa9tD0pvgq7PL4qvMO8kL2UPRY9Yrxevda9Szx8vIM9o72SuXY9rr29Pcg9Ur0APD69jb38PBq8d7zKvS48pb2YPma9sT0mPCK+JL7uPec9+Tw1vG492b0iPti81b2PvVu7Ur2sPXW+Ar7HPRu+E70qPha+Nj6BvSM9kD17PZk93T0mPhE9j71BPNQ9IT1GPrq6lz0FPgC9zb0IPug9hr1dvNK9BrxxPZC9Yr2YvTs9TD6CPZk9OLwFPhM9/ryMvDq9XT1KvJ89Ab3DO4U9wb13Pb27Sr0vPTw96rsYPkI97r35vME9o7xUPMS7mb3lvQ69Lj22vRc9Ez3DO5a8/L2IPco9xzpuPHu8k72OPZY89r3oPJM9d726Pda9Er73vFy8+LmHvQ49OL7NPTy+2jvxPLy9V72LvFs9Pj4tPcK9AL4DPTW+ID0GvrY93r2RPZa71D0gvO48NDwvPk49Er7ovco9ADyevYO7pL3xPQY+Ij75PRM8iL2Jvrg9mbyXvJa9DT11Pck9r70VPtS9iTs0Pci9Qr0mvY08o7yXvYI9jbxoPTC76bsDPSs9+zz7vSE9B72EPbs8qDyYPYK8+bwcPcq8M74TvW+88ryFPbS99L3ZvWU90L0mPnM9qTw3vlo9Mb2BPQE8Nz0qPUE+5T0CPSu8JL2gPQC8iDxEPve9KD4VPsm9kL3lPe09Jb4TvrY9rz18vE+8ub3QvcE9vzwZOww+Ib2iPek7gjwovsC9dLxBPTW8Cr3/vbM9hr2/PBK+wD2ju8u9J7wQPg+8pLzlvb294T0ZPlK7l7wgvYS8BD4RvR09NDyfvKA9Pz00vrw9nL3oPSs+kTyIvai8pD0SPAc+nT2JPbO9CrxPvS69y73cvIG9Kr0GvhO8G76QPC68H77fPIO6Lj4kPla8MD56Pdo8172DO6+9dD3APas8i7yvPHk7TL24vQQ+Jj5aO869tDqovdW9Fj0QvmS+f71AveS8sD1GvTw+ib1CvG+7Gz60veW9O72NvYe9vzwfPsU9vzsZvZ67cb6CPLY80j0PPpQ9iLxyvRA+AD7YPIQ8fj0vPs29iT0ruyi9s70MPvW9oT0gvk09lD6hvSM+nL0jvg08zLwCPqq9+D0GPma+lb1EPUw94D1IPGW9SL1uPHa9u72cvSc6nzwrPm28iD1uvWq99LpRPqo8TjxgvRo7N75xvB49jTydu4q93r3wvUq9j73BvQk+ib3Lvcm9G74KvhU9uz2kPUM8sjxIPJ+9Tb3IO4+9iDyPvbw9d71rvVU7UT1jvXg9dT2WPZk8S72cvYc+ij1MPcu9/r1kPeW8ur0tvkW6C75IvgU9Mr72O9g9Fr1MPsm9tT3mPak6Yr21vZM92r1Yvay8sz1CPhE9Yr4JvaI9q7wQvjC9LbyzPX29kb07vf28jb2qPQq9wjwNPjg8Bjs5veo9gT0ivcG5JDyIPRs+t71uvdi8lj3evbm8CD41PJ490r2qPaU7Tr0oPqS9gj0lvrs9xbxUvp+9ULzEPQ+9Jj6ZvJ49ljzFvO89DT2KPdK9kT3KvUI+vD3fvf083j2gve69Ab65vak9Jz0NPpe9hj3+PQg9qb0wvj29lT1kPdY9VbsFvt69Ar6AOJg9lj1Bvsu9Iz6zvFI+37xuPHG91j03vi092b0RvQ49Pz5pPME9BD6uvXq9Fz4hvKW8M73AvYu6n722PCA+LT0rvfC8GD79vSG9Db3yPSO++jwgvaO9JD2EvSc+AT2uPGC9rD26PVM9GD33PR89Nb2nPdi9Z72CvfE7HT3kO749lr1OvCK9Pb1vPgO+KD2EvJ29Cb3YPd677j0JPb69mr3xvLE84z3gPPU8fD74PDc9n72SvIo9jz5dPZq94T1IvH48zrw2PoU9l72VvdA7lbwBPqQ9N7w4PrW8gD0OPQ09BT2iPVu+zDqZPf28XT0BPBo+lj13Pt89rD2bvHG9sL1nvoI9qj39vIu8Dr4vvYu81b3WPPK8c72OvZ+9zbzFO869Dj4aPiq9v7wWPmQ9hryzPTS+tryYvUO9pL3SvZc9L74hvZg76bxnvbC9Xz3/PLu8X70vvqK9RL3RPJ89pT1AvDC+Tr0HPcq9973GvV6+GD4cPSK8Fr14vOq9BL6ePeU9Kr1xPHC9hbxmvco8Cb68vbw9LD1WvJk9kD1kPGS9Mz5rPv29ND0dPo89ib0CPbS86D3cvHK9WrsWvSm99z2cvKo9AT5wPgq8W73OO+094b2LvT09rTy/vCo8BzxNvTU7L76/PTE8qz0SvoK9Bj2UPcm9gTyhPAo9vL3ivEy9Rz2/vZ49PTyxvBC9C75fvga+YrzJPae9CT0Dvn6+OjyEvOC8Uj2iuwo9sLv8PTY7Oz2HPbC9mT25vE+8L764veE70jxUvT09UD1ivBO+TDqMvYG8nb0jvTC9mrzWPZS9v73JvRK9xD14Pe29Qb69vbe9pj2dPjS9nLxrPSy9UD25PI88Zb3svMi9gTwuvba6fz2YPd89r7xWPbu8qT3ju9Y9sT1Yvbq9gj3fPWY8Az19PAU+nz0IvhW+NL4cvaY9jrzGO3q8nrzAPaO9Br55PhI++j2CPRm+7TtHPf27pT0zvWs9j72zvPy9Fz7UPRo92T0bvXI9D75WPcE9aTy3uy2++T3tvb+7XL1OPZC+jTy/vaI9ND24vee9UT4oOw262b3Yvd49rztQPEs9ZL7Tvam9hT0lPt89HT7IPdg8wb2IO2u9cb3CPVU75b2TvJk9jT1nPUO9Mj6HPZm8ELyJvS29sL0hu6m9Az25vbQ9D74tvco8Rbw6viM9yby7vRs97b1BvD89Jr0wvFC9lDstvbW8VbyUPH0+kD5JPMe8Y729PUa84D3LvWc9y73avXm9OT5jPpy8jzy9veY88b2bvZc8lb2IPeC9Tj3ivOo9srvnvQM92D2qPZI7oL0qPYu9Wj6lPF29Pz2SPQi+mTsAvCa9Qb06PXq9Q75kPeY8770IvVo8bT2JvBk+ej0Ouvm9+T3/Pfg9A71JPu68uj2GvQU+RL4cvYI9KT0IPQQ++b0nO7U9NT0Mvse9vr09vFG9YTz/vQ88ez08vQe+4z1Fvlo9tD3+vL+96jwWvje93b2qvWg8tTyOPJa8+jo4vl8+8D06vs69Kb1YPqY8lz20vJM9hD3FPSM9KT3APQU+0j3RvDG8Qj6fPZI7Sz2XPQE+HL4JPlo7JTuvvYk90z3bvay9VT4JPew9TT6DPdC9sj3YPUo+Fb05vRi+nz0OPoI9tbziPT680j2svYo93zxHvSu+KbzgO8q8KL6RvHi95zyrPRg9mDz1vAu7UL3JPaK9Rz0hPjC9kb3bvbu9P73tPZY8sD2kPQO+dL06voi+D74zPd487D0uvvM9PjxiPgm+nj0yPnA6Z7u/PaI8Cj3gPag9Eb6wPP490LzMO6M9nLwTPPM9Or0WPPq9Mr56vSS+/rwIPiS+mDwGPqg9pzvNvaM9tT1jO5o9drvwPDu+2zzUPfy9qb3RvUE8mzzovbi9m71EPQw9pz1YPl49oL03PiA+zLsLPjK+5r2SPSa7cT0cPCO+zD2Fu1i8QL5mvYm98D3vu329qD2pPUe9B72MvEg9XL0BPcU8ET0Cuha+/zzTve49sT0Jvf+8+D04vlq9ar0MPl+9uj3GvOi9Ub1BPXG9jT0TPlY5Dz6ZPAu+WL00vRO8nT2FPZA99T2RvTC+EL6Kvrg8P72evS09VD1Fvkw8MjyHPSe9JL2lvZw9kL1ovdE9CD72PaK9/T2HvSU+cDw+vFq9lLxZvWq9hz0ZOyC+LTy3vQ+8qj0jPpw9Tz4IvqU9Dj1dPEq9iL2avTQ9zj17POO91z3EvFi9HT3pvWA90Ly+PeY99L1tPZM9Kb4kPPw7AD7XPBu9Z71kvvY9STx6PsI6QrxNvgi9ej2fvRk+Db2FvY+9Rj3GvYk+KD2lvYM9/LxQPLI76D3IvAo9Hr3RPQ29wDxsO5E9Tb0bPUW9Cr6BvYq9qb1zO7O8DD4kvk89PL7XPKA9lL3dPWw9gz3gveC9uT0jPVm+cb2aPZw80L2mvWK9Nz5fPfi8Ar7rPYa8hT0Cvqk8BTzNPcU8pD2fvHU8/buBPRo91L0APde9Dj5YvcG9mb22PNY9Kb5GPWI9fL2SvWu9pD1LPis9Yb7Bvfc92jzQvSk98j3BPWS9fT1UvO08Cb4vPUG8ML0JvaO7eD0aPUg9hjybPaS9EbzqvYY98b29PGc8Bz7MPf+8pr0Mvg89jzzsPca9pb0kvHM93j3ivHa+gD17vQk9WbycvOC8Gj2vPSs+sD3UvAE+Hz7mvTg+S7ykvBa9+rvqvbI9vTzFvTo91TyfvQO+/DwBvbe9SzyivA0+Gb4FPTE+er5lvNe9Gz2gvZM9dL3yPQk9BrzovaG8br0TPuE9lz2cvAK+1rwDPpc9EL3cPZq94z00PtS7qrzGvBc+HroHvpO7Eb56u3S9Sb4HPhc+PLyjPf89YzySu/a8uz0Svvy8gz3xvR485zsGvL68Jbv3vbE9SbwMPa09AL24PUk8GD6dPAc9Cb6WPWi9Nb1aPZK+Bz7UPBw81z04Ps87OT0kPf89qzwcvsi90j2mvES9iTykPLi9uzwIvXA8ET7YvBm9fb2dPho+n70YvJm9XzsJvTq9yL1RvYc9dL1TPvw9xL1dvau9HL0svp+6AD51vJ+9fb64vWO9WT3rvZc8MD6HPcu9Rz1mvd69zz20Pfc9kT0PvTU93b3CPAc9372hvQ28W71ZPBK+Fz1oPOA8Lj0+PHG9wr1kPey9mz0wvjY82T3rPCk8h7smPZs6dbzbPbW9wj20PZU7Xr6mPXI9hz29vds9xDwkPbm87r0BPZm9zL2ePSU+Ib4YviM9Ej3ePYI9d71uPts8drwcPrI8VD1OPue9nb1MPcQ9xL0wvbI9qb36vQG+jT0CPNK9FD6BPeC7r7wkvgw9MDwlPY09GL3OvTS+lbyHPNy9CzzPvM08sz0oPVi9qL1gvpC8ET0cviY+i73OPdQ9E7xnvau9pD0VPgq+yj19PSa8QrvbvQS+gDvovcm9nzomPQo+yT2hPY68072fPJg9vj3QvZw8173aPc490zzAPYs8JT1ZvI09Gr25vYa99z30PBi9nj2VPSi97z1vvJo9lbyfvdq9Nz3dvZA9MT7bPS6+kD1DPU88zL0GvQu7GD1Mve69sb37vdC6Cj0gPai9XD3WvSI+MrzjPX+9HD0DvqK8hb7gvBG8Vz15PQc+/z0IvQ6+orwQPQ2+Eb3IvWU9FT26PUi9ErwYvYq+FT6xPag9g7xnPds9PD2Pu4y93b20POY8vTvmvVE9yTzWvU87/z2vvXK7dzs6PfM8Ir78O+w7aL79uyY+TL2nPXw9JL0SPX6+fryCPSg9jLwqvTW9AD4tPBa9XjyJPYq9/z2Uuyo9OL40PQc+FT6rvdu9sL2ZvV69Mz4JPsY9/D2pvLg9/L2dvbU9wr2fvMo9zjxwPdu99D2wvZ89TL2XvDm9zr1QPKM9f73MvUc9RTs6vV+9Kr3rvBw+0z2tvLw9gT0nPQS+6b19vS2+iL7vuiQ9oD25PIc85r3VPYU9Fj3wPdE8Ib5OvZQ9Cj2XvEu9a70KvuC9bL3qPQg+6bymPUY9bD3tvY69wb2YvbU8ELv/vX29Gj5hvYY8eb4rPhC+3bw1PF0+E74lvYG9iD1XPKM9KT1KPYc97LxevVg+fz1wPfs8X76/PdE9i7zGvYA8Oj23PYE9yj28vO09KL4DPe48YL74vZQ9UL26PRK88LyOvSI96rxcPSQ9Yj2nvDe9SL6SPUo9x72FPXC9Sr3DvOy8QbyaPd493j2ZO/C8hr1/vcq9CTwQPT49xT1mvYG9hT3GPPg8CT2UPMO8v71lvYK7ET4GvZO94j1APvs9+j0APqK9DD4vvrq99TvdPIm9Aj6Dvec9yDwHvoO9qb3DvIw94L2qvUm87r2qPUO+AT7JPZE9oz0Qvhw+wbwQPky9bjzVPci9Ab2QvBw8HL7VvYM9MLyhvIE8Wj51veu8GT6gvSq+gzwFvmy9m72UvYI8br0Uvg0+lL1Nvac9aL0FPtA9xz1TvAe9NzsFPuo9NbxHvjs+y73hPUg9grzYPd68lD0tvaS6EL0ePZW9vDxhPSc+3z02vs492D0ovCc+9L02PuW8o72qPX49djycPP+8zryPPQs9pr10PVU9Gr7kPOG957wxvfs9hj01vha+0b0Mvgc+nT1DvRo+A745PR69gzxhPom9tT38PcG9L73HPaQ9iL2+vBM7Rb2zvUO8wbrYPYk9uD12PUc9ij2BvOC72D1OO7M80z0cPde6oL2ovL481r0xPKS7nT28u9o9fzwgPtA9l7rRO4637L18vSO9Er49vRE9Gz0wvgK9qDyCPSc9TL6TO5c+Yj2yO5S9Tz2VvWm9BT3lvIE9Jz7tvUy9GD1gPcm9xL3OvS49Gj1xvbU+Aj64vA++mD3Cug2997uKvuc8KT2tPAw8Kj4uPT+9vz1ZPYw8GL1qvfy9Nb6cPdG9LLqBPWg8tr3IPeq7eb5yvMc9NLtlvYQ9Tj7jPAg+gb0VPAy+ib2QPT2+0L0CvYI8qLvYPe89CbxIPTA+BT3SvQa+HL6BvTc8G75BvbA9Mr3QvQy8H72OPYI90j0VPbO7xT0fvRE+ODxgPfM8DD6WvYu9cr0EPf097T0yPQW8Nj5NuzO9Zb3Vux880D2+vAo9MTzaPAe95j0ZPMG9sTzovDK+yT0FPj497zs/Ph8+Y73ZOwK9hj0XPrM9Ib1QvAC+OD0EvfW9Y70UPpw9kL3evSI+GD7Zvc69GTwMvha9Z74xPRG+jz0ePmQ9Fb6EO2g8hD2jvc290by5Paq98bzqPM+9GrwHPgu9Lz6gOx6+jTwSvf89QTzcPW69Bb7IPYm+qz0avuu7pr3EPRU+2L2LPOU9krupPSE8pTysPTI9LL0iPBO9g71NPf28qj3SPUK8yb0dvhw+EjwXvYG8q712vo+9Bzy2Pde9lj1BvqE8GL3UPNs9rTzGPC2757yDPQY+hj3AvDK9FzwsvdU98r0XvDI+Fb4HvP68JDxFPSu9Gb7SPSy8RT2+Pd69zrwovDy+bT0EPqM9gb23vEO93LuNva29zjwEPki9H76tPRC9Ej01PNq9ED4UvvQ9gD0UvgI9wbxHPpK99z2FvjM7H73OPV+8ITuEvae89DzEOzK+MrzwvEk82D3APDC+/70Lvpo9+z0QvSC8Z7w5PEg9pLwNPDi9UD1TPXO9GL1+Pjg8G72dPSo9YD6XvcA8gTzRvT2+BD0jvjq62zwhvVM8y7xqPQM+hDsiPoY8AzyKPG893ru0PQy9NT3fvUG+T73YOw0+jrujPa48xb0hvdU8pz21PVy8AD5ovdw9T77VPYM9Hj6jPQc+573ju7w7VD0cvVK8gz1NPI49WL6WvZg9H70zPXC+9z2ovY68v7xQPSS+VL3zvQy+9T2avQm8nz63u6u9tT1bOkY8Tb30PXy97TxnujM9hr2Gvbi9tL04PmI9Gr0VvYI9gz0rvbq8Lb6QvQC9vT3UPYc9UL6Jvim9PT0avim9mj0xPT88FrySvfS8NL0hPgo+3zw0vjy9Yb1oPUU95T1Gu5e9hz7pPUm8Kz21vXM8kD0wPi69ZT1DvQm+FD48vaU9cr1zvde9+L25uyQ9Fr5NvBO8y70cvpq9cT3JvXC92zxDvRW7jz3RPJE9ETtAves7/738vfi9A764PRK9IL7uPS29rD2JPpk8UT1iPQ27xjwCPsI947x3PWc9Ej2DvQ+9C72TvQY9JbwrvfE9wD2oPXO9Jr0xvhq8Db2CvJc8SDxevSe8l73CPUG96byiPQC+zL3Vu8U84L2EPZu99D0rPYM6kbyPvdM96T0nvS2+WD7AvTW907xXPQa9tbx5PQI+ib2CvTy8cL2rvbW9uDy1PAS9bb62vVA9wTxxvQC+Ob4SPgK+tT3KvOe8mj28PeU9Ij4qPQg9mj3bvXC9nbzsPTM+nL0aPgO+rL1evbK9Mr6JPtA9Fb4wvYg9Pz0Dvp49cT1jPnm8hjyFPRG+Sr3QPS0+JT4ZPW09Q76BPM695T0MPpu+2j2wPea8/TsWPp08y700vNo9Fj4dPto9Qb6gPcs9lbvOvSU9SD1jPe89grxdvf+9jjypPGE9A76yvZw9PbxYPtG8WTr7PYe9+zvxPfK8LjwWvTs9QTyLPca9gj3RvHQ8iD0rul47l709Pae7WbvnPUc9tD3lvJu9wbyJOoE7Cz11Pai8Tr0NPZs9Jb6cPgc9Yb2hPTU84TyfPe09Kz2kPXC7V75EvQY9jjz/PVM9aT6hPlU+rz3CPa090DzHvdW9CDx9PDS9Ij2JPQW+mryLvS+9dj3tvas9pL1mvqI7OL2svR68qL2DvNG95L2bPfW8pztWPVO8Gb7rvA+9Zj0avtc8xL09PSq+srwKPhc+Kb0uvYA89T17vV28rD3nvb89pbzAvBy+Frxpvf88c70PPcY9FL2FPc48Q74evnk9JL7XvS89f72Rvok8+7swvhe+Ir0OvW8+Gb6PPQm8QT4fvbk9ub3IvJE9Fjw6Pc+9Ab1Zvay9OT75PPi9gb37OzG5pj1Zvbc8gT00vSk+3zyVvU69Ab6lvUu9xj3qvfA8lr03vom9v72cPdM9TT2QvY69tTxpvjY9gz1yPaO9Ez6lPYW9Kz1BPZy9iD6nvSC9aTzTPXS7+DxMvX479jv8vFY9Rz6bO4K9870KPhS9Or7jPXu9wr2PPOi83D0FPYA9kD2pvDY8VLzZvDu+eT1LPSo95bvpuqs9Ir1Tuhi9PL2Qvdi8Er6CvBg+Dj7mvXO8sb0Vvku+e7yxvfM8Uz4FPZs9t7xnPTw9pT2DPS6+Bj5xvbi9zj2Zu1+8lzzDvLE5xzzAvXa7c720vAW+gb3WPWC8UbyKPJc9cbskPn09xrz/vBo+0j09vhG9nryIOQm9EL4oPQS+BbyLvSc9jr0cvsq9vr1bPYg9uL2svKe8Jzs9PaG9g705vUg9JT25vSI9hb2RPQa+nT1fPUi9Ir40u2q9Ar2sPVM9kD3dPMW9bL0BPeG8Fz5nvQe9t70qPgY+WzyLPAq+BL19vaW9OL05vBu8j7xyPBg+MD2vPDQ+iL2pPBy8Jj60PVc9Bb6/vdy8R7ygPXU9SD36PBe+Yz1ZvaA8eL2JvZO9tTwCPW49gb37PM29A7sDPlO9uDxNPP+9yz2vPSq9bzuHPY29jLxQvRW+Vr5rPZa8Ab1RvbY9xT1wvc+8lTy0PQ89BL0BvZq8hLtgPtG8sb0qvnc9jj2iPUi9370GPYk9Hb4hu+E81T3Kvb49RL5FPm49lj7Avbu9UrxHPZK9xD3avQg9Nz2fPb49jL0TvRY+J74YPpY8d7yHPR09HD5Qvpg8kL1Cvus9172rPB68b71EvYM+qLv6u4e9+jwOvi09cT2/u489SL59vYO9z7xEPRU9lr3JPbe8T73fPTs80Lo8vK49ZL0VPie8YT0LPky9iz0ZvYQ9bD0XPvI8yzxSva89F75VPV29BL4LPsI9oTwkPh4+VTzLvBy+Qb5+Owk9OzziPNY95ryMPXo9yr0cPdI9VD2LPck8Ar6Pu0u+W73VPJC9dr2HvT8+AT7jvf+8qz1HvaK9wbs/vue95T3gPfG9h70XPYU71D0avus9z70Qvjy9Fj0jvMa8Oz3nvMM9/bsNPOs8sb0TPrC9FD2fvZ09kz2XuYg99Lt5PS89Cb7SvPK9nDyou589Tzw7PSS+8Ty3PZs9Jb0dvpy9Zz3PPQi8qzhmPYU96D2mvYQ8/7tBvhy9Rz2+vfa6Cr7tvR8++j3SvDS9cDzQPZW8vr0OPr88CD4Cvoo9mTyxu448y73rvV+90T0PPQO9ML4mvcy8frsqPE69hj2rPBQ9ML3Tvc+9kryjvIg9+T2AvLU9Cz1fPi09kz2jPE09HjpHvXC+F751vn6+NbzevKq9N75ePUQ9LT5LPsu9wTwHvha8BT0Uvo+9ST5bvJi9hL7+Pb288j2PPHY7Kz3SPZ48kLuHPcg9mzxuvaq8yLyjPCY+rL3QvaY9z70FPQS9Yzssvm49tD3svd285TwHPUw8kz0vvXc9uL0Hvke+jz2YPMa8vb2APKW9vL3VO7c9qT6FPYU9ubw7PfQ9IrxhPRY6CD4lPoY+Cb60uhU9Yr2wPRE9cDvxOyQ9OzwEvcU51rzYPZI9mb22uuw9nz3XvHe9zb0FPpe8Kr28PRK98j2qvRk+KT0BPj49pzv/vLc8z73Du049Dj5APmY9Ob1IPcU9Cj17vTU9t71zOto8wzyGuz+9tj0BPhc9Dr6BvdM8Rz5LPRY9mL3+vay68j2Cvbk9JzzYveK91LxevUY9qbxwPO89wDuUPQc9EL5cuyo+KT2bO/S8/ryFvSK+Fz1hvS297b01vuq9S70Evky81DxKvoG9Ybz6Pac9G70OvD+9ULvMPRs9mD1hvQ89673nur888T0GPZe5sb2SvTO9MTxMPdi97z2KuoK9Pz11vSQ9dD1APbk9yb0oPEw8Or4tPiI8jD6PvFU9/LtmOyG+oj1JPac+BL55Pew8nLnOPZe99TwDviq9Ur0kPaw8az0dvvO7Oj2PvFi9kj23vZc9Crq/PaG9k70bvt+9mr2zPRs+cb2SvQE+Gjz8PCM9Kz7GPXo9Zb19PQO+lL10PKK+Cr6cPT67GD1MPGi9t72kPTS+zLyQPCy+3701PQ296L1pvYS9Nr6Gvbg9p712vB6+bT11PQ6+2rz8vK08tD1HPdC8TT7UPcc8wr1zPMc9Bj1MPdC9O74tPjI9Az2cPXW+jD00vZM9ybyLPGQ9ND1zPmS8A7pzPSw+EL4Rvhc9mb1Qvcc9RD2cvcS9db0mvQK9ED2evKI9qz1jvbu8Ib4MPcO9/b3nOzY+972VvHi8QbxDvda92j0BvbQ7K77oN8m97r1rOzs+wz1aPQ2+TT4kvsw9ADzFvQq8zj2/vaY8Qz4APlS9+zwkvcO9lzoVvXg9vD3qvCy9OL1xvOM88b2SvDk92L1sPRu+7rwNvUi+Fb4RPhA9FL2UPQC9RTywvQq9Nj0DPmc89T3NvM497TwrvEs9Y73RPEe9jr0nvYM9lT1WPga+KDzZOmY92bwcPY09+zyWuyC+vzy1Pam93T06vbw81bzEPfS9Bz3pvEg9q7yrPZk9Cj6BvDO8gjvVvBC+TTxwPQU+6jylPYA8Fb4pPhW9nD2tvdi9/rsGPug8JTwuvvQ90jl9vLY9+LwWPu49Rj3NvXC9BDsQO7Y82L1dPC28GL3UvOM8Fj4gvv+8EjwKPoI9zr04PZ09FL7VvYM9ZTzzvfu9Gj1KPta9kb0ivQw+Gb4xvXu81zwzvbM9kz6evAo+3z2OOfQ7Mj7qvSu+vT3QPYy9eDw6vQe9xb0ZvuM9Az6svRw+OD3uvMO9Tj2hvAu9ED4TvYc8mzx8PW0+ED4cPkI9Ir0NPhW9/zy1PXm9FD4Evky9zLsTPWo9cr1BPsK8mD2gPA2+w70XvR09w7ypuzU9l71JPAm+b72oPXy9Wr2fPQ29lb19POI7+rwovuW7hL7XPNw8Yj3tO4C6uTyxPf+7OT36Pd28Ijw2vQW9mLyJPfW83bxZvjC+GD1gOsi7/z26PQw9VT2NPBQ+Xr5Jvd69vjsUuwI+0r2xvaW99j2Avl08zT2nO389rz3OvY6+pjzCPRS+RLwXvYC9UD4pPr48GD4uPZ86+z1zvVS+ir47PBg9dr0bPBk9Lj50PfM86Tv7PUq8XDvmvVM9nD3VvDS+hj1sPQK9Ej3JPYW9VT2IPas7KTzAO488V70zPOo7Mrx4vME7t7xUO/e7zDy2vIU6gLueO6K8d7yuvCq9bz3RvI48yLzWu9M8E7sWvCY8srwoPfa8LTySu6w8YjwfvBM7vTwcvN68VrzGPCW9TLwfO9Q8kbyWO5k8n7yxvI68DzyGPH88nzrZuya8C73kPIM8c7vauxQ9Cz2sPE08LLx9PIK9BbyyvHM7Aj09vf+7srvOvA88Ez1HPPq6AzyvO+k7xjzcvEU8Jjz/PHY8YrwfvN07nLtFvAW8tzlWPJe8Ub2uvGq8hLyDvDk9+TsHO0+9yzt7vCe85DyLPMQ7VzwxPNk7A739u3O7LTsCvfo76TuQvP46OL1VvK88xbynupU8+zm1PI68XDtEOkw8MDrmPMq8ojyfur08YzxuPOC8djwRvSQ8FD0KvCS+XbxHPIa8R7rgPLm89zyvuqg8prswPAE8xTyGvH68sryqvAk9szwzvOY88zzBvKM8izx2u169ITujvBK96zzMu5i8C71wvMi8w7snPSC6lDyXvMi8Hr28Oxm8gb0svSw8tjtZvLc80TuBvCg8i7zFu7M6EL0vPNO8Cj2NPI28lzxjO8O7TzwWvIc8OruYvJu8oTz2O7i8D7yDvIC7DD0hPeY7sLwSPeA7J70NvYq7Lj2XPLe8uTytvEs8Dj2CPES8WbzLPBY7d7wLPOI8C71Du1c9+7yUPAe9JbxAvYM8TzzdO/G8+7ysPH48iryPPTK7+rx8vB08Cb2DvYE7Dz2GvIs8oLwGvQI9TTx5PKg8Sjwxvf671zqwPLG7DrydvMk8lbwlOaQ8HzxxvAu9JT0FPQO9vrwIvfS8nrnBu4e8B72LvAI9fbxhu068aDwuPJC7LryGu7Q8frwPvDq8Nb32O2C9Rjy+O9o8RDwBvNc8gLvbO507D7sMu5Y8lbwau4y8k7xjPDm8Ur2cvI68mjwEPRM9fjyCPYM9zjytOyA94zyxO4s6gjzFPE+9KLz9PLU9DzoHvEg8Xzz2uze8xLx4uw29Lj1au5+7y7xAPYe8ojxMPMo8PTwGPbG85ruqvBC9HbwEO+s6Db32u0O827xiO8+7Wzydu688Nr0Nu5I8J7pEPc479rqPOyK8gbuBPPY5WjwYPTc7fbuxPDM7pzvlPMa8NzraPJq79byHvC08mbwevIU87zyQvNi7bLukvPq8dzqDO1s8HT32vGY8gjw1vXQ7P73GvI48rzzHPCa9n7yPOp68+zyxPHq9XjqnvDe8qTvduxC8gzuLOo67djv4O4g8pjwUvM88ODwDvcK6WLxPPQM6y7kQvRQ8urw6uw09xbwAvaC5yrwmPMe8grunvI68CDwJPVC8HDycvTA8lTwBPSQ8wrt2vJk8ezzTPE262ry8PLq8h7vNvMK8Sb3XvD87Z7xHvEc7kDzgu9k8RzyhPIG8VjxSu2i7nLw7uoY8Rbz6uLs7GbtwO2u7tbtEuy08pDwxvNs8DD21PMe8r7uNvKE8+bqIvN08izwyvak88DwJuy69Abt7PDg9CT1KPEM8hbqNPJ865rzjPFk8hrvYO2o7xDzbPMM8pLu9OyO7u7yMvBc9rjyWvMo4cbwFvBA97zw5PJG8XLziOx49qzyDvMq7SL1FPNK8P71/PZY7DryIu/+8gDwAPfy8Ej0JPDO8Br0DvfE7ybzhOwA9ajyYPGO8ljyEvDW7UryTOqg8LTzjPKa6Qz27u8M7LjyOOzK8zrv2O7S8zjtRu+Y69bxZPJ08Gbw4Okc8i7x/POk8yDvIvBE8Wbs+PKs8gru+vCu98jyhu5K8dDySvNQ7wboJPQI98Ls6uxA9ODwhvTW7Q7wpu6g8ZbtDvQK9OTybPJQ8VrwxvHc8dLxUOg69Zjy6vO+8CD2evCC7u7z4uhI7hrxcPKq8JT3puvG70DzmPAw8/jzEPCA9gz3TvJ28tjyJPDI9wjyePCk8lDw3vCc8n7zsPIc65bm0vEq9WroKvDK9irwBu6c6xLxMPI27X7wwPD09PjzuvAS9hzzOvMW89DoYPbe7S7uSPME8srwtvEO9o7x1vKy8+T1IvAc9+rugvOW8VDhhPPe81DshO+07jryrPAo7/7vRvPe8ETrHPOA7gbtZPI88sLypPOk7Ljrqutg8kDuzO+g8oryuPA08uLwQPKK7DjzEPBI7i7ZAu4U7Iz3qPNC7NzyKOwQ8d7wOPJG8XDpXvFQ78rwfvRo8tDxAvCM6Cj3CvP082bvKPJS7Ybx9PO48rzxxPAG9DDsNvfO7iryGvHU7ET0kOm88rLwNPQk8F72BPUq9GTyoPFi8GD3yuyO9kDwPvU69PryYu3K7Vrs1vS87lDvlPFU8k7yYvCk8ozuMu608tbzuPOe8Bb0qvSO7Q7xxPMy8HbutuiW9lbiyPEa8DD0jvKW6JzmXOw04PDxgPCs8ojyROzM9S7clPTM7O73wOiE8qzyXPK08pDzyvKM8HT0hvfA8O7xnOyw8ljzrPCU9xbyKOk08rbwGPNW8vzsKvA+847zZPKg7b7odvJm7WjtNvIS8BT1wvDW8JDzdPAs9u7w9PDy6mDxjvDO9MbtUO5a7qbv6PCm6MbwYvBu80bzFvNq8sTnIu966frqoPP48ubzovCG9R7yJvPk7x7tIvMI5dDy1PJa8b7ybPAY95TuTOvU7iLwMPRi9nTylvC+81Ds0PCW7Dr2Gu6a7pTwrPI48xryHvKa8N7wpvV+7XbxkvFc8Lr2xPGC72bzcPGq767wUPKM7Cz1JPAm6pDxIOku7xLwUPV28cbxUPWs8CT3wO2k8f7xuvJ88WjzNO/I7ML3wO4g8+jy9vEQ88ztSPMc5IDy8PMI8VTzAOke7STvMO1q8xbybvFe9f7yWvBg8SjoDvUw7GDwoPDS8pDzUu3M7YTrdO+i7i7rEOwc9Sj2mPAK9e7zpO508Jb0kPE88R71BOm88p7uCPCY9nLmxvFU9TrwvvTw7f7v7uz085jwivUm9urySPAk8A71EPSG80jvru5u7iryfvJu7ZzqOPP46TT37PCo8sbz7vEk8kDwdvBq8RzxgOgK9Hr3MvIa8rbyHu0287LwjvCU85TufPG68aTnxPMG8+LwfvD86iTudvOS8DTzLugw9szonvRi8oDy9O6+8ND3kPMk8gbx0Oze8K7sHvYw8nDy5Omk91DsXPRi9GL0OPOa8hTwku5G8hjxYPY+8DTzLvD88u7v+vDA9TzwtPKa6cDw7O0e8FLxKPLo8lTwZvNS8QrzbPHq6sLtSvFC7qTuQObu8Rr3cvAA8ijwUvcm8DD0/PcO+XTwTuoM7FrwyvIM7vTuIOzw8jDwaPM27B73BvKk87zsrPBC82TooOwQ9MzzQvCq82DuiO0+9hTylOmC96TzcOx88HD2uugs8QTyHvBg9ED01PKe7g7w4vPO8pDsjPIu8AzwVPKc8czuoPFK8Zj0LvAw9YDzUOp08krywvGy8p7yjvEI8GL2qPIc8RD0oPLE81zyoO+M8jDwbveM7hbysPGI8N7xLveG8fDzivMi5cTyHvG25yDtrPKi88rowvI+8aruLuoi7h7wAPeg7DrysPIi7bzzcPIq8gr3buQw91zsOPKO8HTyMPLK8LL1aO7o5oztfvCk9vTwmPI87gjsSPCm7Mj0kPLk7lTy+vCK8ojv+Ojw7RzymO4s8rLoQudo8M7vrvIu8Pzx/OzU8hTwkvLI8ZruCuxi9XbxRvGi8ELwjvaA7djylvNO8vTzbvLa8JT0HvRg8Ezl9vCs95rsrO8U8cbvevMW8G7q7Owo8kbzNvJo6ETx5vLA86bwEPBY7UbyYO+a7iTtKOxE89zuHusI7A7zXvMu7qzzSvBe8OzzkvA08aLtuvOI8wrt/vHA3GLzbPB+8GTw7PIa78zzcO0E8yruIvLK6A7sku0u8ZbvFu6M8VD20vNG7vrt9vaQ8az1yPMa82rxWu0g8FzwGPRO8nbwEvBM9Mz2zvPS8Ar2SvJ28urxLPOQ7qDw6vea78DwTPVo8wLyKvU+8LjvZvEo8Fz67PJU8RTevu9S7ND3gO2O8+zwLvLK6jLxePKM8nTz4uye9iTthPFa85bzavHE7Mr3SvCC9ODzIvI08mLuKO927qzsHvXK8qTzQPNe8VbxxvMQ867qdPPq7+zrTvPm6e7sgOoS8gzzlOc68Xbu+PEc8k7vNO9u80buPvFg5HjynO+U8/7qcvFe8F7yQvBw9gjyru5W8ujuhvIE8nbyaPDQ8gDytOuk857yNPL28hbueOho9XbyuPHa8Hj0yPCk8IDwIPAW89TyNvCQ8RDwNvG07r7zBu548ZjyvO9K64LxUup87Uz04PZK73DymOyQ7TbqwOXw9XDxOPAq8lTyMvAy8GLwdvJA6dLz2PB88qrqpu+k72DuJvQq8zrqqvI878Tw1PQy99LwGvdM8Er2wPIw6zry1vKC8lry4POK9oTtDvXk91rtcPEk9U7vBPAS87jz8Oio8aDw/Og+8p7wGvC68ljwhvd47IjyTPAu9q7ziPIA7YryFPMw8BD26PJ28sjz0PCC9DLwGvdQ8tLzTO5o8Vbx2vEa8sjoGvW099Dvcuci7wbwIPew5ULuGPB49WLw9vak8Pz3yuzC8SzzQO08747xxO408Nzvnu0o8wLlBvUU9WzyyOuO87Ts8vFm93bwXPSm9zDklPNm8ibqWO+88QryvPKG8Ab0zPdq8lzxevMM8yrxpO0I8KjsBvLO7brwCPb28+TwiPMK8gDxUvGC6NrzdvLK8Hb1Ju4c83TwbPJe7urymPOC6Djy2PIQ8bbzAuqo8XjzEu0m8EjyyvCM8c7rJPMI7ML0DOx88xzxFO188ljzmOps8XjxEPKK6Mz0zvNk8K7yjPEO8fzyHvPA7KT0KvD+8gbtBPE88E7y8uyM8a7qLvAo8qbstPD68AD2rvKY8C70DPU+8W7vnOsq8Cj10OzM9KD0bvYk7sLpGPbw897sePC08ULyuPMs8DDzePFc8Mz3jPL+7XrszvBg8sbzCPPw7BD1TvLK8ZTyvPOO6nbxNvNG8sbxovHm9ObxmO4I8R7ulvIE9kzzXPHY8ErzcuxS96DzmuY88TTwTvZa8pby0PBM9PjzjO8i8cL0ivVG7PjvDvJY8mDwTvA68FT1lO9S7sDzkO2Y83DqSPN081jwIOt88QTpAvKA8Mbx/OFy9lbxovCC82bsmPPs8tDyVPHW8g7qBvBw8wbuMPIA8hjyjvAW8Cr39PC69JTukPLy8mzyXPE07JbwGPeI7yDzauug7XjuDvAS9azzTu1K8dz1lvJc8ADw6PKU8hLtUPNQ8ALz/OBE96DwgPDq7wjmzPCe84jvOvJ07GLwOPCs75TvDulu8dbwvPIU8ID2DvLo7k7tjPOk607vcPDc8b7w7vBk7X7sjPeq8q7xzPMy8Czwwulc92DxUvSs8ETwnvQU9JztdvK88nzycPNM8cbx5PAK9Iz1QPVG8HbsRvaK8eLyNvP48FTojvVm6T7wovJ87yLx9O7e8qTtdPeu79LqVPJw8+ryLPNo6Aj2hPOa8CbxfPCk8WTgjvRs8TT2tOoU7cbqjPAi9hjtsO2Y8Lb03PBK9ML0VPBS8ujxuvAc9ID1tO5s7IbzRPO07azyePME7gjytPIS85bx5vAe7oDzRPIk8ibsbPIk8drzcPNW8/zsIvYi8gzynvHQ8Mr2vPPK7SDw7vK+8NzwdvDe9jLzLvJo8FT3Eu4S877xZvNw8o7ydPIo8oLupu2i7iru6uxk847w5vK66SrxcPEq92TyPO+87BrwivAY8TL1XPCY9NLxevKm4LDwlPNe8Pz3duiI8r7x1PLM64TyUO0Y8I7zJOwa9yTwDPJK8SbwxvCC8UzzPO/k8Vjy8PCG9ErorvfE83TvpOlw7AT0evPC8ozyMue28xLyxPMy88Ts8vAq8Mb3YOwm6mbyhOx+8JrwYvCU8mLxlPK27trx7vCE7iTwsPLA8LbyFvKE8D70gO/y8gzx0vA88DLvJOoY8A7yfvKU85bvQvJY8CDyAPeS8lrkiPFO93TwXu4o7lT3/PIe7BbnauhI8SLq6PGA7OTw3PCu8XTp6PLu7DDz8PPe9bLzIvGW9H7wEPQi927sbu0k8gDzuu6Q8CzyIvOC8sLx2OT484rwYvFw8BLzVPPK7c7ybu5E6iLzAO4C7EL0jva+7SrzRvMC78DxXvBS9D7xOOwu9m7y/PHU8vjp2PWK897sjvZ48yrz6O728nTxVPNo857uzu7m7lDz3O+c7jbzLPGs8kDyivEQ8F7snPMG8Fz0VPYO7bDwCPEC8G7olvJ47Dj33vBq8Yzs6PYU8wro+vfQ8fDteuwE87by7PPE7sDzmOxW8p7ztO+u8G7v3vFE8pbxRvAO88Ly+PBM9cru4vEM81DsvO9u8+zoXvNW8LbygvH28Zz0TvN+5Mro7PRe6Mrytu4g8rrxxPDC93btnvBy9qbyivFk7qzvZPDm8CLtXu1M8STwNvco7Vrr8PNg8XbyOPAe9q7s7PSK9T7spvFg9nbwBvS498DyEukA9nDy5vMi8rDyjOw882rvdPJk8XbxJvLm7CrsGvUU7rTyrPG66eDyXvJs8F7w8O5C8Dj06PXs8o7zkuoO8CT2pPFQ8FTyKO7U8I7skvQo8KL25uRC837uRPNQ8bDkkOki9ujvhvBO9Tb3MOso7F72bPFa8ozvgvMG6qbxhOwE9Ejy4POU8g7oGPFw87TttPLe8obsTvKO8GbyrvBK8Xb0yPKq6SbvJvK27lDkDvN087jvfu7q83rzoO0e90zz2ux+8frybu+Y7zbxrvKQ8nrqru9O8bLzCvE48QzygvH+86ryDOwg8xjsPPAK8XTwIPeU8oLsXPaQ80zwgPY05vzstPYI7cruePD699bvAvHo8y7wnvJU8abyoO+m6rbwQvBy78LxCPKA8VbwxPCo9pzyjvJu6h7ywPCq8WbvNPPy7JTyuO6U8RLuyvEK8lTx5Owe9dzxXPSQ9p7xdPAg8ULvDuhI8sbz7vAy8j7sCPVC7oLsKvM08iLuFvB09obxEO+08wbsMOnm8/ruuvHC6rbwUPAQ9sTwqu9A8GD3auVM8GTwEvC88wTwkvZW8Wr0sPZa72DuzvGg8Br01vei7jrx4O8O8wjtxvCw8ADxlPDm8jr1GvS29xTpzPJ87vjsKOww8mLsBuzu8qrv1u5U7Eb0zPe88GzvQu5k8nDwVuyE9ljy3vHA8sryuvFe8try7vD+9L7wiPTw9sDmAvJI8NT0MuSe70DwDvJO9FTuyvMK6BjyavOg8Az0AvLQ8orubuWC7Xr3hO5k8jLsHu7u4GryvvLs8zjyAu5K7xTwMvGu8QL28ugE9hDt6PPA77zrhPIE6vDxCvO87o7xtPDI8XTydvAk8xjwgvWS93TvPPM264zznOvo8Uj21vAU7+TuVPOm8Q70zvZi8B7zhu5O70LsTvIY8CT0Nvf28ebr/PJ28HLwaPFO8ujyEu988sTuUPAO9Tbs0vXO7szwiPBS89DztO6I8cbwhvA49QjzQvLG7abzHPAE8jzx+vdI8ijuPvEo8Fz1qPD09+TwOvDE9Gryiuxa9L702PdU5/zwhPQw9BT2OvB09FT1NOrm8Aby/PAo9vDw8PNG7CT1BPOi7cLthvAi9QDwQPAO9SLtQvGa8nbxpOY+8QDz+PLE8AjxjPE69zrpRvDQ8lDsFvJQ3ZrueOy83JT1+u/27NTzVul28zDxfO109MrwLOle7rbzEvP08Wb1APRS9vDsBPPk8X7xBPMU8jj0TvJq8ET1KPC+8KLzrPPu8MrzTvGM9iTyvvGm8ADwRvfG8hje6u2m8ETybOLa8r7xuuk47q7zrO/C5YLsmOgA9Jb2lPC69yjucvAW9Tzz1uZg8CzwgPUQ8IrsDPBg9ljyQun88rzyWO7u8FTyNO8S8hjtCPTK7CrvsvCu9T7ysvAy80jrhvFS8zLsIPOc8LDwjvIg8+DxJu1a8Cr2OPBE7DL3TvPW3prwJPdM8rDsUvb+86LzTu5U8GLtAPb68kDonvA69qDwYvU69rjwbPbU8oLuLPJk8wryOvCA8DrxQPIW8Kj18vRe8lzwfOjY9Cj07vZU8sLz4PAq9vLwFvXI8fDwlvYu8/TooOyy8YDwNvJe8DryHuym9lzyKPOe7sDypPGC8CbtZPEO9ojnJPI88NTyvvBQ9vzzWvAQ9gTwOuGs83LwMPEO9EjzqvI66Y7vOvM08azzdOQI9I7zXOdy86jurvB+9Ib3+vOU7Pbx4PHO7bzzePCW99rzUPCq81Dvpu4K7ajyuPAM9urwpPAY9djtMu488Pbt3PGS8WDsqPbi6Dr0MPXk7cDyiPIO8rroLu/m7Hj7BO2a9FL10vBi8IDz4u5K8zLwHu/u7AT3aPCu8NDysO4u8PTzcO588ZzyiPG070jzdvOM8F7wlPRS8irvavB49SrwFvc68xLwIPcG8mTwjOwK9gTvkPMS8JLxou2K8PDu2vAq40zzhOz68yTu3u2e8CrwYPJc8JbxBvDG7KTzXPGI70Dy0O3w8jT2xvKK86DvQvBu8Aj3GPLw8ALzPOvq8M7xxvI26rrvTvBM9yDxYOlO8ybz7uz68dzopvEg9hDwavNI7MrwOPIU7GDwvvRg9p7zfu5E7H725OwY9dTtzO+A7sLwpOj677Tt/vBc71bzqPAc7cbzyO3I8trirPI88rruxvHk6QTxhu5A797tfvCC8iLtMPIG89bsuPDs8IbuAvLy6pLyXvAw8+DwEPFc6zrpCvHu7NzwTPW66PjyivL68uzvOO6y7vTydOWA80TwXPCK747zqvCk98bwevR+8sLwgPYI83Tyou5E8o7xOvLU8drz2vAA92jp3PLw8rrw9PaE7XbyePPY8f7vYvFk7tjz2Opm8mDy8PHG8aTySvBA9Tb26vOk8gTxivKy8BbzJPKy6djxmvFw9UT3Quie8irwivAg8GL3IO+Y8I7xhPCq9cjzUuzg8trvMPKy8Hj38vGy9nLy2O6+8sjuzu/O7V7wgPBY9jLvxvLq7+zxBvIe8CbuAvJq8e7y8POu8ODzMu1E8ZbvMuzc9/zyzOtW73DuNvIs8xTqkPNg8gjyPPIm81rrfOzY8RTxTPNc8Jr1bO5c8qrs0vEe8FTw8O7081zxEPIY8PryUPNI8Mr3tvBm8ubwQPRU7wzyWPG6+KT3BOs87CT33vJk8OTzbu+Q70bwHvVq83rzTvNc8E7svvQ09hbwlPVq8fD0yvcs8wrw2vJ27vLx0PKS9gjyuPDC8WruiPBw88zu+Ogk9ijx4vDs85LzKO7W8yTuKvPu8oLzcPOo89zq3u5S8SrzcPNS7Dr0yvK+8cD1iO5w8ML2pO/c7bjwmvaI7ZbwIPMW8Mrs3vGU81TwCPJ+8zrwDvJi8qLzhO/Y86TymvBu89bydvPi7NDylOqK8brvQOxa91rwQuya8uDtbvQU7KD3iPNo7Fz2XvJS8JzzsO3K7J7zBuse8KDwbvA49PzxnPEG74bunPAO7XTs5vFS8IDwDPbw8F7ylPI88QjtTuwO7prwevZS77TwyOze7RbuMPDI8Ij2QPEo7vbvoOzQ8rDwZvE091rz9PPc7E7tHPGC9uLvLvKO7/jzfPMS7g7yAPNK4ObwbOx89IL1LPN685LutvDC8PjyoPE+62bxFPFm8wbxAO3q7fjwOvUa8mbzDO3O9lzybPEa8F70FPNa8Q70UPDA8+rwJuqg8DrqoO6G8wjwJvOG8WbvTPDA7eDw1vAo81zs5Oxc6uzxXvIa8Pbw2uzY9TD1Lumq9nTwivLe7Ez2guyO9o7yXvCU8U7vTPIE8rbzYOqO8mjwRPIo8Mz2nPJu7XzytOwi6KbtNPPA7rbyOPEI82ztlvDS8LTxZPCm847wePK87+TuUuCS88zmEOw883jy0u565PL3iuyQ8fjwyuk287rxTu6M8aDydO3K7XzwPPag8iLy4vNk76zktvIQ8i7y/PL48ZrxFPMM7z7w2vVg9GTmQPKG6Rz2ZPMC8uryIvCM9irxUvIc79LzYO8g8u7yzu9K7yD2HORw957yDvFw7Pb0YvG68uDyxPJw7QjwBPOE8cj0YvJU55LvfOwe8ibywPMa6EjyqPA44ADmJuyQ9UzuZvDi9sDzFOiY9ertLO7I9frubPCy9oTyju0S8MjswPQQ9wTzePH69LL3Hu1k6tjwcvR28w7utPAa96jxGPKE8A72avJA7d7xwunq8zLvevFO8cbpavZm7gbw4PI87uTyluXg9rLvzu0i8c7zdPKI8x7xYPGc7M7zju0S6RTykuy49PDzivLC8irwBPKg8fbuFvbA8Mzx7PMW68DtAPAA8krzuu0O9yzwFvZo7RbssPLs7nDxVvFw9hrsEvZA7szxGvMG7oDwlu9Q81LxOvfa85ry0Ozc+ljylO5I8Hz3BvNo8dzq/u5g7cL2ivAK9rTwVPZw8vrvaPIK8uDq7PCe8srwtvJ+8yTtEvJc7nbzGPDM87zsWvPm8KT07PEq8mrzuvJg8qjw6vIo8WTmEuws9pjwUPLm7Cj0YPIU7Mr0mPMM7qDxRvEM8NDx6Oo289TsQvWy8g7wQvTk8dbzZvDs9iTzeOyq7EbmMvFy8Ij0IvT08IbxcvPs7brw1vNW7JbzruwS9KD2mvDA8nzxRPVA7P7zIOyA9XzwzPL68Ir1svPa6RLrbO327dLzQvBY9+zy7PNQ7PDzOPMa8+rxQvHa8ATtMPNo8urzBPD87VDzlPLY6ajuYPIq7HrzuvI27J7y5OxS9QbwqvZY8tDz7uVC96ru9O1S9W7uSOmY8hDzKPEA8TLuuvDA7FLysujI7nrw2PMi7TDyPO9C7JzyjPBS8GL3rPAk9Sjw1vLW8+jw5PCG9g7wTPZ68eTzAu0G8HLxVvAM9oruCPBm8rLqKPS685zsSOxG7DLwCvAq8brybu/O8YjoOOw+8S71iu4A8vbyovI87B7pUPKe777yHu6y8OjywO7+7ljwKvOE8G7mOPBC9QjxvvHo7CL3/PN88RTxgvbC8BT2DO/g7qLtOPJI7xzyRPLM86jyLvRC8v7yQO4Q8NTnqPGI7wDsNPb+8CjzNO0a8Qz0UvL48Aj2SPL87LD1/O4E8Jz14u528aju+vNy83jupvMa6t7xEPdK8NbxDOya70TzIvLm7CroKvGY6bLzkPBI9IzxGPIC8UbstvIG8ITzKO8I8ET3vvC+6Hr2/PEy91zwqPXs8gjxDvN48tjyTPLI8SzyjvBi9/rwYvWM8nTzZPB+9X72CPAw9hzwsvTO9ZDwAO4m8cjwiPd08x7rovNG7v7vXOki80DulvEe8Z7yqPLq7dTyZPOo7VzsivaS8Dr3LPIA7eLz1unM8rjsivbK7UTzMvMO72zyWPME8QLqpvDw9dDs+vKE7bTy0POA8Gjyzube8+LwXvT06jzyCPLW8ArvVPLQ8izvyPIG8JDx/vE68Wrw8PDW7tDsxvMM847yqPLO7tDwJPEm7RLyIPNG6DjukvIy8uDqEOhK8obvqPLC7Oj2rPDa9Iz3svAs8gr2jPAK9hbuXPG072bzoPBo8NL0Uu/m7tLuZvJ87bLwFPbc76zw1Pcq7eLwFvGE9RLsNvEY8rTwivak7grzWPAY9MTynOgS9czocvT898juLPK+8nbwAvGO85Tz/u7A7wLomvJu8Jb1XPNG8U71tvf2307vsPJ+80Lycu1C8n7qBvP26tTxsvaK81jsmPBu97DpVvFU8pLzePJK9DT0tvI27njsIPVk7STwqvGe8QDxpPHU8Z7sePNK7vru5vNK6J7ySPK87VbvgPDi99zw/uwK9Kr0wPDa9Hz24O8S7zTwavEU9o7xBPKg86bwgu2o80zwLvSE9lTs2O/w4KLuWuyy8fzzxuqe6IL0XPZw8WrxnvCu9CD2bOmI8Db33uw895zy5vP87zzyuO728zTpSPCE8OLyiu7+7NL00PPA63jzFPFs8lLsUPX08hTxUPVE8Cj34vAs927tPPRS7vzwhveM8rLxtPAy9mboIvd27dDw7PCs8qTqFuxc8LrzWu428pzwDvay7AL1SvKK8iru9vIA827xRu++7F7z/uqe86zwVPHa8rzzXuwY70bwEPIe8hLwMPJQ80LxZPG+81rz7O2Q8ZbxCPfm8IL0aPaQ7LTrIPKc8Az2gu6A64TsjvSu8pbwrPHw8Ej2evAq8JzyCPJq8i7oOvLW7I71hvAU9vTxWvCC8orzOuUI9Rbx7vNi7s7yzt4u7YbtoPAM9ebyOvIi8XzzSuyS7Jr2FvKg7urs0PFa8aDzCuy88Azy/Opu8mLzAvAK8Cj2UvGA9QjzjPE28ujwkvZE83bwqPJm747yTuiM8SzwKPB28x7sWvYg8ZTimPJ47njv/vBM8MjxPPIU7TbxruQg9xbwmPGO8w7s8PLi8Djx+vJG7IzxMvEm78rs7vDa8vLzkPN+6Ubk4PNK7/jwFPBw8RzwzPGy7CTyivDC82rySO967obzgPFQ8NjwsvM87qDvFvAc7yTzsO5S8jzyrOxO8STuePJc77bsrvEC8STwIPcO8x7wCPZm8s7ywPG88kryXPBS9SjspvAo9+DyuPFK8+7yfPJ28kLt6vEm8GDwUO3K8Mj14vL+8rbw/PDW5C7zEuyy8J70pvCi9XLwMPIm8LDuXvOE7m7xzu4e7Ej2NPCe81DmPvI68iDyavBU9xDpfPK87trwqvJe88LqtPE28i7sUvFI7n7w8vIC8V7zxPBw9hLydOyg9H7yXPAA897w5u5i82zxdvNE8ITzJPOG70rshvB48/7uNvDa+pDtgvAy8s7tbvK88UDxPvLc8wLyOOt68NTznvIO8RDzGPKg8QzuRPMM8E7u3vPG8ATxTuoU8lLziu7s8CD1RvJO8jLwmu688zLw+O1I99Dr3vP07UzxqPE88ELwJvew7+DyEvDq8Ebv2O6E7sTyZvDK8Azz5ul48Bz2XO5g8gTzIvEo6drwOvRe8GT0VPEA8jjyNPNY8nrrKO+87ID3tPAE9CDwKPa08UrxRvdI8crwAPXq8Lj2GPG+8qTreulg88Dy7PPq7wjyGPHs8tzyBvDc8bDzDuuG8Tjvzu6u8aTynu0U8CD1jPLO7/bsePT882zupvMk8I7wVvLk8gTxHvEM9zTwxPEK8SDsyvKi7NzzjutO7xzmzvNY8gTz/Ow88IDwvvBM8MLu/u7m78DsjvM08uLvVvGk8ZTzxu208TrwXPCU8DjyRPLY88jy3PGc9dLw4O446dT2/OzY7XLwQPKy8UbynvCa9TDxCvLa84rsXu868zzuBO5e7JrxJu3e8JzwgPKe8yDyBvWG9bTyNvG+7m7ziu0Q7pjzkO6G8wbxTO9i8AbwKvX+8yrxHvIG87jv5u9O88byLvK48LjygPJk8D737vNI8lbqtvMy8mjy9O0A8F7pivLE88b0fPCs9PD3DPFQ8rrzevMA8x7pZPOq8FTwvPbA8QLwUvH28kTw+vGC7Y7y2O9q82LuXPI+8572qO7S8srsgPUU8Br0SvSI85zujvMy7OryBvLw8jTz9uwu9j7wzPHG87bzWPES9NrwSub09M7x8vOc8m7vxPAU9pjtuvKe8Wbwduxk91zw4uv876jwIvfc7k7wsvYK9FTw=", "ffn_stream_bf16": "WD1XPAG867tYvTS9Cr2FvDU9Mj2mvUQ87rvsPIG9Xb3xPMw8FLwMvCG91b1BPfE7lDzxvC+9qzzSvM+8xrw+PeG5Vb0YPOK8Fb1SvRY84DwQuxA8ozw0vGE9QDxDveQ8RjyVPBI9Yz0aPKQ8Nj38PJK8kTtLvNO8C735PDg8Rj0NvAK9lrxfPB29rzvHuxQ9y7yMvDC96zx4PBA9H72nO048qbomvDU7xbwvvCI9k7yhvDe9OD1IPQC9JL2vvBu8ebyBvUE9Qz3/vDq8Br0YPGG8hrzJOQe8ZT1qvN68w7yWPIk7obz+vCE93jzSPHM9zjwgvVC8qbzRvDe9Xb1tPZK8nTtBvcC83byCOzu9QT2CPGA9cTuxOwY90LzjPNU9k7sYPas8kz2Jvbg85bxLvZu9ETujPQG9Ar6QvAg9g73ru+48gzoVvBs977xFPR+6kj3pO6S7NLp3vX69hT2YO6W8DbsEPHS9ijyvPGA907zXvAy8QLy7PEI8vLzyO1K8qzzKvD+8Yz0AvJq8lbzyvCS9lzxpPFC9Ej1evI27QbwXPDa9Mj1xvR29Br0GPNg8OL2zPLU8tjwbPQs8Fr05vIM8jzznvPY8CDwgvdA8Z7wCPKQ8+bvSPL08Zrx2PAw9HLyLPEq9+DxuPf08DLzUPO08yrzLPOe87bpkvYS8K73WvCg8Wj2rvcI8Wj1FvYa9oLzFPMk8ozyBPXK8lbyQPQu9KbxbvTA9ITwJvR07Jb0RvVW9oTxRPLE8JLgKvJ46mzwWPdi8oDwcvUC9ED35umq8LL0bPZS9Jj3AvIO9yrxKvZi9XL2hOsU9lr2LvCW8BL06vOI8Dr0tvbQ9mTxNvT884DwmvCu9Ar1aO1s9qTwavIU9SL2pvTU9SjvwvCA94Dw+Pdy81DtYPOM7prvWu148QjzvvAe9wb1evUk9W7wtvEm8nrywPVK8UDsDPZE9XD2OvGs99jybPAg82Dx7PdE8kr0EvJI7XD1HPTK9h7ycPFS8gTzbvEK9w7wuPY+9+zq1vRw9JDzfPFw9GD3bvG47L70JvEK9oLoEPbk8yrtcvYY7QT3YvFs7bbxuPJ27y7wUvFm7ED3+OyQ9BT2EPB09N714vVe8CbsCPUo8qL2lvBU8EbzmOz49E7wtvS090rwmO3A6rjzeuow81Tt2PdC6IL0VPBi9OrzVvOq8LD0eO7m7Hz2+vFW9rLwcPR+90byYPaY7rb0vveO8jTuBu4G65LxhuzI7Hj2+vKs8Jj32OzU7/Tz0PBe7+7wPPDS9Nz2JvZS8LL3guwU9oTtiveu8vjwpOuM8Cbx3vXC81DwmOwY8i70MvcO8Lj2pvby8W7yZPLq9mj1ZPYm6ijx2vMk7ojyyPIw9XruLuzE9IDzeO2Y7t7zgvS69PjzNvLi8Ir2Cu7i7Yr2ePFg9dzzvvLE8CTwPvYA9TrtkPIm8Vz0vPTC8Drx2vZU88buwPLi8szwxPSg9+LyoOxg90zzhvMk7qjyGPUW8VD1lvBa9hb3kOjI9ijxjvGO7RL39vJs7e72QvcE8uD1svNO6Srz2PCk9g7zhvG27gjyCO0I64Tycux696rzWvA49Ij2cPV68qLwnvbK8ezwrPU+8Pj1nvUi9er1YvYU9KboiPS89vbpHvaQ8OLwjPeW8rbwKvQw7WT2SPE68abxBvLw7Hr0fPVM95TzIu4o8Ozw3PRs9M73xO5+8Fj2+vFK8s72tvEm8Y7riOz+8/ztCPVY8BLknvTk9EjqBvL07LDxnO6K8SjoNPCY9CjxCPZe9obyqPJO8PL2lvBq9Fz3oPJ88bj1fPAO9UD0iPQg8IrxGPJy9Jr02Pba8+rwpPWM7Eb3Fu3i9tTsuvRm8Vrx0PUo8Y72OPB69W7sevTO8KL2MOu88B71XPcS5A7o2PZc9iD1oPYE8Qz31Ox89ML3MPD+8oz2vvLY8RDzYvKi7q7wgvew8x7xCPVW8nrxavbW8f710PGS87LwPPa48Pz1pvGw9mj1WPdc8Er1NO4689TwXPO68izzwPAg9E7wjvGE9Ir3sOsE7AL2eO0A8drtdvTO8zbzBvLW8ibyVPTu9tzyZPKy85bwmPZi9Ijx/uyU7E70LPVm8YbywOy+7BD00Pde8bjx3PNw82T0mvdS8Kjthuzi93DyqO1E9urv5PBK9Ez2jPWM9jLskugQ8A73OPHK7p7ySvPu8Rb2Pvf28mDxvvGO9+7yGvOG8TDyRPAC9HjywvLW89TyGPRU7B7uTOzW9TruAPC055TzUvIU9xDyKPQU8zbxsvaU9YbuoPL077TvLO7M8h72qPAa9ML0BvUC907xDvY+9D7x4Pe88VjxrvXE8Bj0SvDK90zwsPIk9ab20u0m8nDtAvck8rjz4PCQ9Dr0JvQa9rjq3OhK7hrzyvJ494DwmO1C7mb1rPDc8qjzQvEU9NrzbPHM9I7oPPTs85TxJO0y9rbzLOiK8JD0CvJU7uTwIPHG7xjzKvOS8Ez0XPdA9r725PRi8DL32PBq8zjttvZk8dbyWPLY8TbxhPeW9VLqyPAG7pLsvPIw8jT2uPOM8UL1VvN+8ijwrPY28Gr07PeG8Ij0EvEu5srwNPA891bzRO9E6NLwZPYE8n71cPA69wbyyPPa6kT2GPNm7i7sUOgW8Zj2lPJ88vTwKvJA9jr0PPaq7yLwAvf68ST16vYO8hz3QvBW9AbswvYI8Er2fvGu9AzyKPec7JTwIvUg9pbuHvJK8sLx1vUg9ijxhPfk8fD32vB28Lr0mvdc8Rr0mvYI8Pry2PNQ7yTy8PBc8bDwNPQg8dD2DvZu6Fj2cPWy8DbwcPUo9oTxkvJ89vLsVPce8S725vMI7Vb0+vBQ9j71uvau8ujyNPDa9hrm2PAS8/DquO307eLuZvCw9j7skvbQ7eToIPXg86zzGPAY9aj2avQ+8uD3mOpy7XbxJuyw9gjvIPF293TwqPAq9Ej1EPU69hTu1POq8Lr0XvQ494jqyvUo9vzvvPBs7vLzRO7g8Ej1GOys7NL01Pfg73zzPO3i8nTz8PO47+ryRPIG9m7yCPL08s7x3PZW7kD2MPCy8ibvPPIc6Gb0lPL08bjy2OzY8P7xLPU09Wr39PJQ8qjzCuxe7ErwLPa+6TbyKO6O9kjyxvc07Er3ivAc8A7zYPES8yDx6O5E9v7wVvRk9sbyTPCO8srw6PAw9HjuMvai7QzxavSa8U71hvHM9NL1PvU08PbuYvWA9yrxDPXG8/jw/u2E9cbyaPL29/DvGPOe8P70bvCi9Tj0IPXO8o7xPPSe9wL7DvG49OT2mPJI8gr2LvLm80DyvPLw8F73TO0C9Bj2ZO1Q9Mbx5vA69WD2dO4O9Mj1sPIc96L1eu9O8br1mPU+9wrrhPNw7Kj2ePKs7DT0rPEQ9Mz3/vDg9g72HO408LLsNvII8Kj0suxk9LjwBPpW8PL29vGI8Srw/vS88Ar3IvFw7n7x0PEA9cjyVPVM8Mz0TPXo9DT2cPGu9yTzau4E9tzv/POa95LsDvSO9HTw7PEq97DvNPQO8YT1cvSw9cj2rOxe71LoXvVE8AT1vvQk6Uj07POu8uDuDvZu8Iz3qujM667yfPfY8BzuHvNo6TDzHuoe6fz2guwI9t72oPb47+LzIPcq8xLz2O2S93rtoPXy9aDxtvA28/DzOPG+98TzVPDe9kLwXPJI78rwbvQa9gT05OGE8AL1fvX68hjwSvee8IbyvvDa92bs0PUA9zbsePVy977yLO8g8RT2XO8q7mD3rPNQ72zs2vBO90zygPBa9zTyOvOU8kTwSvUQ7Vzz0PLq8qzs8ugS9CD1mu3e8W70qvOI7ITz3vA49Tb3bvIm8qTyIvYC9oj2zvH+9Z7t8PEw9Gr0MvYW86ztVPeu86zyhvAa9FD0vvH88D7xKvLC7Sj3IPIa9HLxTPAO9yDxdPSY9RL3jvC48gLynPBg9Ar0QvUI83TzCPMc7AbyRu4s8Frv0PEw8Wz3ivAc8k7yMuyY8Kzp5vcO9/jsnPX48RT1APq49HD29u9S8YryhOis9Sr0gOyi9GLwIvaI9lz1jPMK8nL1QutE7rD2HPFe957ypvTC9e731vKO8EzznvGk95jusPCy8dj2aPIY9+TxTvGK8tDw7PGU8hr0GPCu94TwoPFG6d7zqO8W8rrxdPBa8iD0DvdY587wJO2a9nbvVPIc74ToCPGk9YD3mOzk9AropPCs9ILwjPfe66DyrvBA7fj2PO5O9Bj07vbs8g72SPHC8/zwuvWG9I73CPFo9q72Xuwu8rzwSPRq9qTybPPK8Bb3svGG9nT15PWG7DTz2Ov87CTwGvPQ80jyQvE89gLyJPEE9vT1lvJC7Gr0ZPRU8br0IPVy8W706u9U7Bjw9vT+9Kr3ZPJS9+DxWPES8U7zAPDg9Jb2Evbi8Sj2vPIW9aT15POG8lbzDPL88Ab6nO4k8ZT3hupW8Vj0NPSQ9tb2avIA8Pr0VvMo8n7z+vHm9njuBPbu9fDz/vAO9U7w5PSA9JTyru/y60LwRPUA8gzwRvZ49Fr3tvN67Ij1/uZU8AT1tvb28QbxwOz68jz1nPQo9JDwePcG8ojy9vSa9vzyJPEW9gTySPOi8szzJu8M8Eb1lvc+8TD25vCO9gD1RPJu9AbxyvGQ8sLwKvYe8bjzHPIk9R708PJw82LzGu9Y8BT1VvGM9SrwMvR89LDvAvA88MzoUvbA8OLxbvfe8ZT00vHK7yTv6PFw8Mj1KvJa9IT3GPKW8SL0QvQG8eTylPL8727z9O4680bwzvQo9ZT0MvUs9Gj2yPPW8H7ssvOq8lrxVvcS8FLtyvQ09PzyHPWU8eDzTPAy9AT2ZPBU9c7wlPYK7bz1oPS09Z7sLPaK8J72+PCS9zLxQvXg7Pzw2vdC79TvjvP+8XbyRvcO5Ob2nPUE80bwHvR496TxOPI47KLywPfm87Tz+uwi9/zsDvT88QL2xPAQ8pj1RvbW8Xzx7vUO8Qz0KPcE9Ijw0PZq8HD0svBs9LjyJPZM9W7wXPIg8bzupvHe8+Lwgveo8kr2gvOQ8Sz15vYO9VD0Zve08xDzAvdM8cr0DPFk9mbygPZa9HzwdOog9mD1PvBo97jyHvTm957wmvXE8zTtQPK48qj2XPCs8MT3JO0e9cDwFPD07lD1tPUu9tbzHPAO9Bj3Ku2M8K71xPUM8nD3Tu0A8YD0dPUs8RrzJu1s8rjwrPY29BD0WPOk7CDsCPUw9rb0wPD88C72+vOe8rjx0vWC8xLzyO6O8yDyFPIi73LlRvLi8IbytPDO9/7rjO8s8jTwsvQI9gbwlvIC8HT5QPX88Hj0mPAq8f7suvQS7pzxfvME87DymPc06BzyMvSE9y7tgPXa8vLwJvPm7obyJvJS6oj2/PPK8Pz3vvIE94zy6vNY8Qz2TPGo9LD3eO469rrygvD28lTurvDS88rxTPHM8rzwuvWO8AL0aO5s9jDzvO6i8SL1WPQK86TyGvAI9KLyQuys99LuTPHO8dL3nPMY8oDy/vM+8sTlLvQw94TwSPHG8qDsVvQs9TDwxvXk8VTzIuty8bzwSvKq5LLzkvJk73rkdvdm8PLx8veo8Vr0LPb488jx+PV48gz3mPP88Qj02vYs6RLxvu9Q8uDwTPLy7OD0svSW8JT2vvIs8t7yeO5A9Xr3Au1e9yjxuPES9Qj3wvGo8Tb0/vSs9VL1eOxu9zL00vYu9P70dPco8dDy1vTY8vD1NPcU8NzyIPIK97zxSvOQ8hT3hOg09Gz3TvBA9sb0jvJo82zzlvNm8jryyO/C8gbs/vX08hj2cvDQ9Rr2CPYM9RLqAu2m9zrwuvC+8Gj02vbg98LwePPy8Yb0uvWY8WjsNvWc7czyQvAU9nLwXPVC9ID3SPIe8fr2xPEw8GL28PIa7Nr1XO5M9+bxwPDA9ODz1vHU8BD3QPFq9izw8PQo9Yj2GvBY9j72HvAS9z7x8PJa8nbzCuwu9Jj3ovHm8Vb1NvOw7XbyIPLY8G72pvF08dD1KvKm8KT2CPHQ9Ez2aPKI8w7kMuw09mD0sPfo8UT1Yu8+8OLoOvXs9pTyrPfu7LL2aPK0897spvPm7wL2qvLw7nr2TvFu8BTyqPKK8Bj2nvG08oD0kvFY797yKvSw8nDxsPBa90jvbO/K8RL2hPEG8Hz2uPLW7Rj0GvfC8jLwau9C8jLw7Oyi9x7yAPGA8Sj2Ive884Ty3vKA85LwzvFi9iTzdPCq9+bwKPBo9Sz3MPUQ8WrxaPVA9K7izvF49J70tPbe8Rj0jPM882rxMPRg8JL0HPCu9sDx/uwm8ED3TPP68gr0dPUg8Ez0rvMy9tT3dvLu85rw6vZc9kryoPMu7Rbzbu0u9Rb3rPLG8FD2CvGE7kbxYvM+7Kj2EvC+91TwfvII8Kr2mvII8OjzkvHe5Grz8PBe9KL1sPGw8oTwyO6a9HD2wOzS8Qb0DPEc9ybuQvBC8l72SOqG867wwvW67Z7y9vM07Gz0iOxg8uzydvdg7w7x0vNE9vbyNvY47Nz0yvIu9Wr02vCy8pz2DPbi83Tx6PSY97ju4PD+9Lz18vCM9/jlIPQG9qzwfPdq6Mr2NOzi9PzwYvWQ9Ub1eO0E9gj3dPKM8zjuTPAQ6fjyju2C97LwcvIq9gTsKvVo8+DvJPB89yzvzvAe8x71auqC9iLwivSE8Rjw0vYY8ojwEvY88Cr1rvEM9ETtWPXY9ELxLu5K8cjxYvD68ir3WvPQ8sryWvVe8Kr0ovY88Yz3Euxm9f7wTugu8XDwGvVs9Dr1OPEk8+byZPD+9kL0QPFS95zurvLo8oDsFPRW96LwMPSW9Prw8PVm8J71YuVa89zxYveq8lz2BO4M9BL1IvIo8VD1pPWg9lrtyvI494DwJuzM9hryeu4g9PjvFvPQ8ZbsFPee7JL3vvBc8yTxYvRW9RzyXvOY8ST2IvHm9ZDwyO4y8lrwhvWm8JTxmOzo9wzx8vVW8S7x0vJi8D72pvCY9/jtwvR88Yz3Rua08w7zxvDS9UL1NvIW8CD0hvZw88bybu4I81D2TPLO6OD3aPG0857yNPaY8xzzcvJC8wzsmvck8jjz5PM48J70PvGm8OL3pu9m8lr32vY49AL1tPHO8c7w5vRi9zbxFvT899rw2vHU7Yb0OvAU8nbxhvDC6EryZvCm9fDyOvBq80DxBvQY9aj30ux89czyrvAg9oDybvQg81DzAuhi9Zj3fOw69jT3ou0m8fruQvYU8tbzLu5g9jD3qvL+7Kz0oPcC8gjwfu9E6tr0kutY8i7uuPDm9Dr2jPVS9LD2gPCO9RD1TvZC8cb2yPD29l7xMO6m84T2JPYO9cz2mPZM8vLyfOHy8hj14vKI8kzxzPZY9Uj14Pay6JTmLvFU7Abrmu928I71lPcK8ub0qPUU7LDz6O6S8+ryIPTw7S7xkvbs9Dr3evOS8Cr1ZPem8lrw2vDe8dL06PdA5BbzBPMI96zywvAA9LLzoPJU8Dj2jPHK8lrw8vGe9PzywPMm8PryHPJ28kruAPM86wbrVPCm9MjxNPZ09VT0VPSy9SD1dPCO9EjtUPRi8gTwtvLq8jTuQPE09U70xvdG73LtaPbE9/Tw9PfW8MD0Jvaa8Wr2tPFs7JT2aPd+7NTyCPag8oDwWPTE8g7vMO/e6srt1vfy89bs7vIm8IjwUPS49uzyuPJ08kr1QPOC8Kj1dvdE8z7xXPDs7VT2jPBY9S7yRPJe9w7sgPTS7Fj1hvPE8QTxRvds8bj2oPGw9x7x8PW49pb3jug09kDy+PXy7FrwxvKu8WzxiPAy8sLzTuz+9KD3iPNq8BjzoPKq9Ib0ZPDc8Tz25PNM8vTx/vWM9ZrxzvII8Yjw5vAe9Bj5XvT49j7xkvDi9RL13PJC88TuXPKU9wbw0PY28ETuYPOS8LD0zvQC9aL38uxE8PTxRu7K7Uj0yO6w8Lr2uuqE8h7zoPPS5Xb2PPBi9/jyNPaK7Lrw5PWC8kL2+u4e7vjtKvDO9Dz0SPHg9wTyYPKI8i728PIm8SDwPPVw9ATwhu1Y8yrxJPe28YTtXOwU9KD3mO029sLwpvTw8erwOvSA8SL2SPNO8ST2mPM28Zz0CPXa9gruHPGk9YD1dPAc80jxWOym8wbwaPRa9RzwdPdw8Bj12vQ28Vb0JvYQ99juTPSs9OLzaPJE8rDzjvKG7/DwBvPK8ZT2GPZm8Ej1YvB08TrwFvbi7RL3mO/K8rTvjO228FL0/vf47xDwavWA75bx/Pce7F73xvBc91LwJuyQ9i71vvH+8ZTzRvMc9PLzHPDu9M7vxPGQ9Xz28vOK6vbuFvQQ9DD06PCk9mryAO1C7dD0BPSA9Aj0BO8Q8Fb1fvbG8ID0QPsW8072SPNu8Iz0HPX28lzwuPYG7ibyxPeE8TLuxPBe877tRuwG9h7xvPGk8Iz1/vMS8jbxpPGk9zLtFPIW8gD24PJG9QL0BvV+8oro3vY68Jr2yO2497Tw5vZO8eT10PY48xTspPSE7orxlPdw7j73jPNs89jk4PGS82TxOvYw8vrxOPWY8HD1PPZm8BL3DvJ68ZDyoPHA9jz3zvH+9Ib2LOzm9GDsEPHy8Pj1CPQk9hb0zPHS96DxevZ28TT16uoC81rwxvNq8Er1IPQ28gz0AvKW8lDyQvS+9ujyUvb+7P70VvK+8Cb27Pb+8ID23PEg9KD2LvBk9y7vxu668V7o4vAS967woPVG8sjoQO9C7ULxJvb880rtQvWw92rqDPa889rsdPaM8sjguPaE7FD2KO6e8uLyJvT89EzwQPcS85Dyhvey8arwEO9Q80LywvNo8fT0YvYS7KT27PNG9xDc6vfI8Pj0ZvQU8vD1HvaG8BT2mPAi94LviPD09RjyDPZA9yTsKvSM9s70/PWG73zx7PKI8Hb38PAw8bz0nu1e9qDx3vZm9fjybPXW9qr2QvKY9Sjz0vIs9frzwPBM96jskvai997sLvUo95bo5PbI8wrsdO3+7az0MPas8Cb1mPMa8aL1iu8u8GL2wOwA9cbyWvCQ9aT0WvbK84bwkvEU7BzwaPRg8hLxRu4m8gDy4PB89dD2MvE28iD33PFc9QDyOPIK7xLwxvXS9kD2LPFy9ADyAPKw8prwSPey53jtyvRw9HT2HuZC9uDsJPBw7dDpPO9k6M72UvMa86TyHvTS8HL0kvas9ODwuPZC8hL5UPXI7GDxovCq9dr1LvS69vLwIvVC9ujy2vZ27Dz1ivF+8pTvfPIk9Ar2DPRA9Lr3YvD89TT1TPYU99r0aPTO7ybtgu7Q8Orv/PNu83rwqPYE8WDxlPZ28QTx0PbO6nLzbOyS9mj0Rvcu9rjzQuzY9ALx3vT080jwJuhs9fjsBvS89sjucPPA5lj3PuxC7Gb1DPb68xTzxPAi9YD0EOww9UboWPdq8Z71APe47ZrsRuya8GL1YPfs8KLxKPQO9nLxhvMO83jtxPBG9a70pPSW9HTwHPTK9zTxLPMc8C7xiPCW9k70APE+9nj1LvL08hr2tvJG8Zjx9vMC80Do6PQ68aTzZuk48B73+OsI8ED1TPFS8HL2EPaE8DTxrPa87lDwku9+5Nj0SvZ+65zyLPVI86z0NvZk91Lu1vCo9JL34uj07tjwcPSc97rszPdg8DTz0PK+8Fj0Jvmu8lryuO048Uj1CPfS7MD2lvGu8jLwWvTm8TzzkvIm97jvTPMG5gb3DO4w9xjzzO+682Tv0vB88PL1EvRO8aLwBPcw8oLxuOxC9Y7wrvQ894LuRvJ+9Ur1xvDk9nzyUPS68wDwRvbO8ET3BPYy9vb2QPT69Wz0bPIs6PL2WvLe8DL2PO0A9Y73xvHi8S70KPQI9obxTPQu9rzpNvam7rjxVPUE8+jxbvc08izz9PHY8SbyZvPu7vrwHvMG6XD1Qu7Q8Az0ou1S8jbzWutM7IDwgO0a9crsgPCK9HryrvAq86Ls2vby8QDuaO5o8Bb0TPSm9Z7wTvfW8eD2fvUA9pT2qPFC9ajy/vLW8mzp9PPc8jLx8Pek8kbw4veu7ujzkOxM8vrwnvaw6nTwxvbm8PLxxPUA6ZT2RvQu9HL2rPIy927xhPQE9Ij3JvJQ9tbzNPX89ozy5vES7XjyVORY83rwSPd88ADzTvC+9Dry0vBY9SD3gvBg7pzzcvIO8qT2IvAU7jbtHPSi8hT1XvAA9mDzUPKG6V71JvY29WD3HvD693rzZvAg8I7x1vXG8qDtHPJo5ybxqvKG8mbtevYK9ob3qu4697byKOmW8lD09PcW8iT2HvJA8Hr0Kvcw8Jz0LvPU8kT37O6m9oTuavOm8JD3TOje9ND1EPZQ8rLvbvCG94zsau9M8NbolPZY8wLyGPLs78byFPJ66eLx2vbC8jLwpvTC8mD10vaS8lj25PDC9sLygPSa9sTzSu568VLk0vCI9VT7PO7c8ujvgPGW8oTyHPH68rzuPvFC9AzwnvPA8Xz3Mvdc74LwEPBK9YD3+Ole9Fj1RO2S8JLvCPAM9WzwBvcK8sr0ruU45m72bPKS8Zr2wPJk8Cb0aPGe9Jj2puxa9ijzHPTo81bwBvM28ajniPbS8ezyIPQQ957ztPFO9P7qVPJa9zTuoO+G7G70/Ojg8ujybOzO9Y7zBPcK8cj1PPcq8/byaPLk8Qb2KPOY6i7w0PQe9jbzkPIg9b7zwvAk9cD2svCg7vDx9unm8kL1RvUk8sLrFO/283DwNvMQ8jz2NPVo8FD3fvB09OjwfvUw9JT3pvGq8zDstuG68urxgvJg9uT0uPY69YTxBvIk8/bwcPSU8tbz7PMG9w70RPdg8B71SPNi9TjyxO4K91rzNu508Hz02PBI8sLwvvY+8j7yhu4G8WDtovAM9YjydvAs9hDzwudk8DD2yPcs83byAugg8krw2PUo7hrwjOw69XD2rPDw7cDxXPRQ9Hj0JvVe8zjxivX+9GL1HvJU7oL1auwS8VTw/vYO9JT31u9A7Fz3IvEo94jwmPQK74rwIu/c8Yj3APAu9AD2+OhI9sjuDvEm8TTwKvZK8jT1wPSG9ozyfPTg8VT3YPF67tjsdPJU8ijwDPQW9gTzsvCM9Zbs3u/o8bT0gvQO5Ibsevea8DL0bPcW8Lj2bPX48Ib0OPWk6hjwiPUU6aDv/ulG90TxLPcA3JbuPvVY9l7tMvDK80jxmPWK9TDzUvKa8Rjx3vSY9KD0MPV+8KL0Svam8Cr0LvI082DxePYG8iz2YO928tb33u6u8Lj22PGY9xLwxu5E9oTzlPKC8gryAOx68KLzsvCo9yjzyPAG7gD19O4O8nb0mu3c8Fj1vPFO8Eby7vBG9trxavWq8BD1rPKS8HT3EPI49qDxUPQe8sDxnvXK9QTxCPLM9cDy1PL08tT2DvKu8dbwVvSu9KzwEvDA8Fb32PLS8rbzpO0K9rTx5PX69BL0kvRq82Lz0u5O9jDzwO5E84Tv4PBI8Gz0fvJk7oTxVvHC9D71BvZG9LTwMPcu8lzqcvYE8kzzrPN27D72vunA9Tr02vY07Tz1OOhM9hzpwPGE66bxYPFI8G7v3vA89YTxkvaU8w7y/PAe8djxZO049+js5vMO9R73svB882DuTvKU5az1lPb48vLzfu3c95jrTvAg7fTzJvIG9YDt3u7I8JT1xOza9Er2HPMG8nj32uyc8YT23OQC9grzGPEG9Kj3QPK08X71NPU0897xbPJy7eTwNPfa7jjypu088Sb3zPE08Nb0UPdU8mjqAvJ+8gjwUPak8y7z6vA48+rxvOiA9uTwzPQM8ybsNvNu8PTxlPQU7arwlvJ28G7sBvWW9Kz2qvd88HD2GPVE8wLxPPO29VL1cve08Pby5u088HbwMPas86jynO5s8nj3jvOG8jzy9vAS9BD1DPQi8Hr2gvNi7hjxou0C8pL2QvVY9dbsCvtC8U70YPaU7H71rvee8VbuhPRK9hTxxvPy8E7yyvHc7TztHvds61LsLvKo8zzxIPEU9dT06uwk9tzx6PFE7kD1bPao7oz1pvG87dDyePMS88DqYu4u7Ur3QOwc8pzwNPYy6uj3YvGG8kbzFPHg6r7yUPW07LrymvPS8OrwRuum6qD1TvVm70bwVvfO8Rb1xPTU8Ej1rPUO72Tqeuys5YLxIvQk8QrzUuxO9qjtYvQW9tjuKvMU8Ib1fvRo9pTxePSY9yjyDPIM8A71BO7a8drzVvIi8CTwUPRi9yb1yPQW7KLsOveG8gzz7O4G8HDyRPRa9Jz1RPJ48Dj2SvfE8wzyVvL+847yfvBu8kzwUvei6VjuEvAC96zyyPIG8VzzVuxS6iTmwPG87Az2DvTc977vCvMy8mzzrPIs8KT2KvBm8mTwAPbY8mLxJvFs9Bj1vPHw8sDzWvHA9aD1Dvam90zyhvAI9AzqHvI47pjsOPdu8b7zUvKg9Cr0avS89RLlBPbI6IL2oPK68d7wMvV28Hb31vK28cb0hvZI69L0FPJk9fbxpPZ08CD2AvF49rLyLPDG9tDz3vK+9mbo/PRY9jT1IPNW7i7xcPaW8I72avBk7kbzjO0q9qrxwPX+7obu1PEy927wyPQs9EjzBPPM8YbwbO4o9JD0xu2S90zm1OxK9xjxlu7y8qLyavNA8FT0cu5m9Tj0OvOW8r7yLPTS9TDy1vKO7DD2TPJS8KrsqvQ29h73MPMC8GL0lPai9Mz0DPB08wbyBPVg8yruAvdC7gTxKO9E8KrwkPBA9Q72evOK7Gjw7PJy9qzq2PGm7ET0mvYM8M72RvHU9Ej2YPOq87Dw/vWQ9WLwZvRo9h70RPa27bz1IvTG72bw6PdK6Jr0YO0C9KL4dPLG81Ts3PDa8oDx9PB89Qj31vO47Pb2VvcO8Ir0UPfo8Vj0pPRG7wbz5Oj29r7mUPEK97rwUPS29Ob1hvPc8lDz0OrG7P72IPIk9nTwZPLY8zbyEvH08UbzgPIK8jbw0vHA8BryNvA07Db23OTG81bwMvGE6tTzRvMs8VD3PPHk7gTsOPXm757wnPQe87zzZPIa7IT3ivKw9qTxSPeQ8DD0MPTc9Lj1IvEi9RD1QPIq8uboSPeQ7eT1BvY68yDsbPYc9ND24vJM9qj3nvBA96rtivL28W7yhPLg8xzyovPC89DvWPI09UbyuPJk9nT0mvUK8Pb0rvSi9hrwAvNK7MT0oPfW6mbxWPIu8ubxqvYK7LDxuPC69Ib0YPS+9g72nPD09ErwIPWs9EjywvD88MjyaPMw8wjwMPYe8hjyhO768c7x+Ohs8LjyKvOg7Sj07vZS8Br3vPM48DD1SvbI87L1SuwC9h7xwvTU9xrkaPUi8VbyRuxo9rryXPY28Er1SvRk9uzxQu6i8Bb1FvH29sTuMvAy9+btXvIo8fLw/uwE9XbwdvTO8i72wvK29O7yYPbc82ryrPKY6ybwZu068yD2evAu9Bz0qvY27UjwLPTI9AjylvFG9obsGvj26Yj0zPe672bwrvfw8jDwhPZ27IL2MvBE71zyqvDE8oryUvG29pby0Pbc8WLzMPAw9rTw5vuY78jwSPTA9NbtYPAa8r70PPRg9GTzzO2g9B7wJveO8U73KvMS8Oj0Svb288729POK8QT1KvZW81LuPPDc9xDk2vM48/jwoPRm9lLxBPTa9tDypPL86mz1pvGC9mb28PFk9WjwCvOi7Wb00vQq9hrw2PTE9p71DPOy77jyBvV698DzLPBe8C7whvdW9Qj37O5g877wuva08z7zOvMe8Pj0TulS9GDzhvBW9U70SPOI8DrsOPKQ8LrxgPTs8RL3lPEc8mTwRPWM9GDylPDY9+DySvI87UbzUvAu99jw5PEc9EbwCvZe8ZTwevaY7y7sTPcu8jLwwvew8dzwPPSG9mDtKPJa6Krw2O8W8MLwjPZS8nrw1vTg9ST0AvSS9sbwbvHm8gr1BPUM9/rw5vAW9FzxhvIO8ZTgIvGU9brzfvMK8kjyNO6K8/7whPd481Dx0Pc88IL1VvKu80Lw3vV69bT2SvKE7Qr3BvOC8hzs6vUA9gDxhPWQ7tTsGPdC85DzWPYu7GT2qPJM9iL26POS8Sr2bvQY7oj0CvQO+kbwJPYS997vtPII6GbwcPe68RD2yuZM97Tubu1u6d71/vYY9nDuovAm7Ajx0vYw8sDxhPdO82LwMvD+8vTw+PLu87TtTvK08y7w+vGM99rubvJS887wlvZM8bTxPvRE9WbyTu0S8HTw3vTM9cr0cvQa9AzzXPDi9szyzPLY8HD0MPBW9OLyDPJE86rz1PAw8IL3SPGO8BzymPPK71Dy7PGS8eTwMPSC8jTxLvfk8bj37PBG81TzuPMm8zTznvAq7Zb2HvCq91bwkPFs9q73APFs9Rr2HvaC8xTzHPKQ8gj1zvJO8kD0KvSS8W70vPSU8Cr0POyW9EL1WvaA8VTyzPJW4C7ySOpo8Fj3YvJ88Hb1BvQ895bprvCu9Gz2UvSc9v7yDvce8S72YvVu9vDrGPZi9jbwnvAS9QbzhPA+9Lb21PZs8T709PNw8J7wsvQK9RjtbPak8G7yGPUm9qr02PTQ777wgPd48Pz3gvNs7WjzfO6+717tiPEU88bwHvcG9X71JPWC8L7xNvJ28sD1RvFg7Az2SPVw9kLxtPfg8mzwJPNc8fD3SPJK9CLySO109Rj0zvYm8nDxUvIE83LxDvcG8Lz2PvQQ7tr0bPSE84TxdPRg93bxvOzC9C7xFvZ+6Aj26PMq7XL2TO0E91rw6O2m8aDyku8y8Fbxluw899zsmPQY9hTwbPTi9eL1WvA67Az1IPKi9pLwdPA685Ts+PRG8LL0vPdO8FjuSOq883bqNPOE7dz3Yuh+9FDwZvTm81bzqvC09KzvHux89v7xWvay8Hj0gvdO8lz2tO669Mb3ivIw7ibsMuuK8c7s0Ox09wLyvPCY9+jtHO/489TwduwC9EDw1vTo9ib2VvC294LsEPZ87Yr3svL08jDrhPAe8eL1wvNM8TjsLPIu9Db3EvDA9qr27vF28mTy5vZs9Wj1suok8d7zFO6E8sjyMPUa7jbsyPR483zuCO7i84b0uvTw80by4vCC9eru8u2O9nTxZPX8877yvPAY8Dr2APU27ZTyLvFY9MD0svA+8dr2UPPK7rzy7vLU8Mj0pPfa8nzsZPdQ84LzCO6c8hj1HvFM9Y7wVvYW9xzozPYs8X7xdu0W9/byoO3y9kL3EPLg9brzvuky89jwpPYS84bx7u4M8gTtbOuM8lbseveu82rwNPSM9nD1ZvKy8Jr2xvHc8LD1VvD89ab1IvXu9Wb2FPT+6JD0uPZq6SL2mPD+8Ij3mvKy8Cr3NOlk9jjxUvGO8R7y7Ox69IT1TPeE8y7uLPDo8OT0bPTa98zuevBc9vLxSvLS9rrxHvIO66jtBvPg7QD1YPNG4Jr05PY06g7y8Oyo8azukvCc6EzwnPQ08QT2YvaK8qTyUvDu9o7wbvRc95jygPG49YzwEvVA9Ij0MPCe8RTycvSe9Nz23vP28Kj1dOxK9x7t6vaw7L70cvFa8cz1PPGO9kjwfvVq7H70wvCi9hDrtPAe9WT0zuie6ND2YPYk9aT2BPEQ9BDwfPTG9zTxAvKM9sLy1PEM82byvu6m8IL3uPMi8Qj1bvKC8W725vH+9cjxlvOy8ET2tPD49bbxuPZs9Vj3YPBO9ajuPvPY8FzztvIY88TwHPRG8KLxiPSO9Cju4OwG9tzs6PG27X702vM28wLy0vIm8lT07vbY8mDysvOa8Jz2YvSQ8e7s2OxO9DD1avGO8rTsuuwU9ND3WvG08dTzdPNk9J73UvC47bbs5veI8qTtSPbu7+TwTvRQ9pD1kPY+7z7kFPAO9zDxMu6a8k7z6vEW9j73+vJQ8crxkvfm8hLzgvEw8kTwBvRw8s7y1vPQ8hj0MO/i6jjs2vU+7fjwCOeM80ryFPcU8iz0GPM28bb2mPWm7qDyxO/Q7zDu0PIe9qTwGvTC9Ab1CvdO8Q72QvRO8dz3wPFQ8bL1vPAc9E7wyvdQ8KzyJPWm9sLtHvI47Qb3JPLE8+jwkPQ+9Cb0HvdI6wzoRu4W88LyfPeA8Ljtju5m9ZTw2PKk8z7xEPTe82jxyPSK6Dz1BPOQ8VjtMvay8sDofvCQ9ALyWO7o8CTyGu8g8y7zjvBI9GD3RPa+9uj0cvAu99jwWvMY7bL2XPHe8lTy1PE68YT3mvYK6sDwyu6i7LzyNPI49rjzjPE+9WbzgvIo8Kz2NvBu9Oz3jvCM9AryOOLG8DjwOPdW8zjumOiu8Gj2APJ+9YDwPvcG8szwQu5I9iDzSu467EjoCvGY9pjyfPLw8B7yRPY+9Dz2su8i8/7z/vEo9fL2GvIc9z7wVvQq7ML2DPBG9nrxsvf87ij3jOyU8CL1JPZy7iryPvK68dr1IPYs8YD38PHs997wevC+9Jb3ZPEi9Jb2FPD68tTzjO8g8vjwePG88DT0HPHU9g72PuhU9nT1svAy8HD1MPaA8YrygPbi7Fj3IvE29uby/O1W9OLwUPY+9b72uvLg8jTw3vZK5tzwCvOM6sztqO2G7mLwsPZG7Jr2rO6U6CD10POs8yDwFPWo9m70RvLg91zqju168TLstPYE7xzxevdw8LzwLvRI9RT1OvYU7szzrvC+9Gb0PPdM6s71JPbk77zwcO7u8yDu5PBM9Kzs+OzS9NT0BPN88yzt7vJ88/DztO/i8jzyBvZy8fzy7PLK8dj2Wu5E9jTwovJC70zyROhm9ITy+PHA8wDs0PEO8Sj1PPVq9+zyVPKs8v7siuxO8DD2suk68iTujvZA8sr3OOxK94bwIPAS81zxHvMo8bjuSPb28Fr0aPbK8lDwlvLW8OjwNPRQ7jb2nu0E8WL0pvFO9YLx0PTO9UL1OPEi7mL1fPce8RT1zvP48QLthPXC8mTy9vfk7xzzpvEC9GLwovU09CT18vKW8Tz0nvcK+xbxtPTg9pjyTPIO9jLy1vNE8rTy6PBe92js/vQg9jztUPTK8drwPvVk9ljuCvTE9bjyJPei9VLvXvG+9aD1OvcC63zzZOys9njyvOw49KTxFPTI9/7w3PYS9jzuOPCi7CLyBPCs9S7saPSo8Aj6WvDu9urxhPEq8QL0mPAO9x7xdO5+8eTxAPXg8lj1OPDU9Ez16PQ09nTxrvco82LuCPbY7AD3nveu7BL0jvRk8PjxKvek7zj0EvGM9Xb0sPXM9rjsZu9W6Gb1SPAA9b72rOVE9PDzpvMg7hL2cvCU9CLsYOum8nz33PPg6hrzQOlA84Lpnun89pLsDPbe9qT3KO/i8yD3IvMS89DtkveK7aD17vWs8crwQvP08zzxwve881jw3vZK8FzyTO/K8Gr0FvYI9fzhdPP68YL19vIM8Eb3ovBq8sLw1vdK7ND1BPdG7Hj1dvfK8lTvJPEY9lDvIu5k96TzQO9A7N7wUvdI8ozwWvc08jbzkPJE8E71AO1k89jy5vKw7PLoEvQk9cbt3vFq9K7zdOyE897wNPU6927yLvKo8iL2AvaM9tLyAvWe7fzxLPRq9DL2HvOU7Vj3qvOk8o7wFvRQ9LryBPBC8Q7yvu0o9xzyHvR28WjwEvck8XT0nPUS95LwuPIG8qzwYPQO9Eb1BPN48wjzDOwi8qbuNPAS79DxGPFo95bwGPJe8iLsqPDA6eL3EvQE8Jj19PEY9QT6vPR09tbvXvGS84DorPUu9LzspvRe8CL2iPZc9ZjzDvJ29ILrQO609iDxWvem8q70xvXu98rykvBE867xqPec7sTwtvHc9nDyHPfs8ULxhvLQ8PTxpPIe9Bzwqvd48KTyGunm86DvBvKq8XjwTvIg9BL1JOvK8IDtmvZe70zx5O906BDxqPWA97Ts3PTG6LTwqPSK8Iz3uuuY8rLwQO389kzuUvQQ9O726PIS9kTxuvAA9L71hvSS9wzxcPau9nrsNvKw8ET0cvas8mzzxvAS97LxivZ49ez1muxA8ATvwOwo8ALzyPNE8j7xQPX28iDxCPb09Z7yVuxm9Gj0UPG69CD1YvFq9ObvLOwU8PL0/vSm92TyVvfg8UjxNvFK8wTw3PSW9g725vEs9sjyGvWk9ejzivJW8xDy/PAK+tTuGPGY9DLuVvFc9DT0kPba9nLyCPDy9FbzLPJ+8AL16vaw7gj27vXs8Ab0DvVK8OD0gPSI8q7sGu9G8Ej1APIQ8EL2fPRe97bzkuyI9uLmVPAE9bL29vD+8Uzs8vI89aD0KPR48Hj3BvKM8vr0ovb48iTxFvX08jzzsvLU8yrvBPBK9Zr3RvEw9ubwjvX89VDybvQS8cbxmPK68Cb2EvG08yTyKPUa9NTybPNi8wrvTPAY9VrxiPUe8DL0gPSU7wLwNPB86FL2zPDu8W734vGc9M7x/u8w7/jxePDM9TryWvSE9yDylvEm9EL0EvHQ8pDy8O968+juPvNK8NL0LPWQ9C71JPRo9szz3vCK7LbzpvJe8Vb3FvCS7c70OPUI8hz1lPHo81DwOvQI9mjwWPXe8Jj1+u249Zj0tPWe7Cz2gvCi9vTwkvc68UL1xO0A8N73Yu/o74rz7vF68kr0iujq9pz1FPNC8CL0ePek8VTyHOyy8sD36vO88+rsGvf07BL1BPEG9sDz4O6c9Ur20vF48er06vEI9Cz3DPSU8Mz2XvB09MbwcPSs8iD2UPVm8FDyIPHE7rLx4vPi8H73qPJO9orzjPEs9e72DvVQ9Gb3qPMI8wL3XPHK9BzxaPZi8oD2XvRs8BzqHPZg9TrwbPe08h706vem8J71xPM07SjytPKs9lzwsPDE9wjtJvXg8BDxJO5U9bj1MvbO8xzwDvQg9z7tnPCu9cj1BPJw9zbtCPGE9Hj1NPEu81LtbPK88LT2NvQQ9FDzvO+86BD1LPa69Lzw/PAm9wbzmvK48db1ZvMW86TukvMk8gjyVu0G6T7y3vCG8sTw0vQi75DvLPIw8Lb0DPYG8HryEvB0+Tz17PB89KDwLvHi7Lb0Uu6c8Y7zCPO08pj3EOgU8jb0iPcW7YT11vL28CbwFvKW8ibyCuqM9wDz0vEA98LyBPeI8u7zXPEM9kjxrPSw92juPva+8obxBvJw7rLw2vPG8VDx2PLM8Lr1gvAG9ETucPYs89jumvEq9VT0BvOw8iLwAPSi8lLsrPfu7lDxyvHS96DzIPKE8vLzSvNk5S70LPeI8FDxyvKw7FL0NPUs8Mr16PFs81rrdvGs8EbzUuSq85LyaO4K5HL3ZvD68fb3pPFe9Cj2+PPY8fD1bPIM95TwBPUM9OL1IOku8d7vTPLg8Dzy7uzk9K70ovCY9rbyNPLe8oTuQPV+9wLtYvc08bzxEvUI98rxsPE29Pr0sPVW9Zjsbvcy9NL2LvUC9Hz3LPHY8tb0xPLw9Tz3DPDc8hjyCve88VrziPIY94joOPRs91rwQPbK9HbyePOA85LzbvJC8szvvvGC7P717PIY9nLw0PUe9gz2CPUC6f7tpvc+8NLwxvBk9Nr23Pe68Jzz9vGG9ML1mPEw7D714O3c8krwFPZy8FT1RvR890jyGvH+9tTxLPBm9vDyFuza9WzuTPfq8bTwwPTg8+Lx+PAM90DxbvYw8Oz0LPWI9h7wYPZC9hrwEvc28ejyRvJ+8ybsLvSY96bxvvFa9UbzsO1m8iTy3PBu9qLxfPHU9SbyrvCk9gjx1PRE9mTyjPNm5AbsOPZg9LD36PFM9a7vMvLC5D717Pac8qz3xuy29nTyrPPy7Lrz1u8G9qby4O5+9lbxYvAc8qzygvAg9qLxvPKA9I7xbO/m8i70vPJo8bTwVvck72DvyvEK9oDw9vCA9rTyzu0Y9Cb3vvIq8/7rOvI68Rzsnvcm8hDxfPEs9ir3zPOE8tryiPOe8N7xZvYk83Dwqvfi8CDwaPUw9zD1HPFu8Wz1QPYk4sbxfPSe9Lj26vEc9IjzPPNu8TT0cPCW9BzwrvbA8ibsLvBA90Tz9vIK9HT1JPBI9KrzNvbU93by8vOa8Or2XPZO8pzzOu0S83btLvUa97DyyvBY9hLxoO5G8V7zeuys9hbwwvdM8IryBPCm9pryAPDw85by5uRu8+zwVvSm9bjxuPKE8KDunvR09tzs1vEC9ATxIPby7kbwPvJe9KDqhvOy8ML1hu2W8vrzVOx09KTsgPLw8nr3MO8W8dLzRPcG8jb2MOzc9MLyMvVu9NbwpvKg9gz23vN48ez0lPfc7uTw+vS49gLwlPac5ST0CvbA8HT37ujG9kDs5vTs8GL1kPVK9cDtCPYE93TylPMg7lTysOYE8pLtfve28HLyKvXk7Cr1WPPQ7yjwfPds79LwKvMi9WLqgvYm8Ir0lPEU8Nb2KPKE8BL2RPAu9b7xDPRs7WD12PRO8KruRvHQ8Wbw8vIq92Lz4PLS8lr1avCq9Kb2QPGQ9xLsYvXy8y7kMvGM8B71aPQ69TjxFPP28nTw/vZG9ETxUvds7q7y8PKM7BT0Uvee8Cz0mvT+8PT1avCe9GLlTvPU8V73tvJg9kTuEPQW9RryIPFM9aT1oPaS7bryOPeE8DbsyPYW8qLuJPT87ybzzPGu7Bj3muyW987waPMo8Wr0WvUg8lrznPEo9ibx7vWU8LDuMvJi8Ib1mvBw8XDs6PcM8fr1TvEq8eLyYvBC9qLwnPQE8cL0iPGQ9E7muPMa877w2vVC9UbyGvAg9Ib2dPPC8o7uCPNQ9kzyUujg92jxqPOe8jj2iPMk83byRvLw7Jr3KPIs8+zzQPCe9FLxsvDi95rvavJe99r2OPQG9bDxvvG+8Or0Zvc28RL1APfi8Orx7O2K9ELwJPKG8ZbxkuhS8mrwqvX88jbwbvM48QL0GPWk9+rsePW88qLwHPaI8nL0MPNM8vroYvWc93DsPvY097LtJvIW7kb2GPLS8z7uYPY096LzKuyw9Jz3BvH88I7tgOra9X7rYPJS7rzw6vQ69pD1VvS09oTwjvUM9U72OvHC9tDw+vZi8QzuqvOE9iT2DvXM9pj2RPL68C7l8vIc9eLyePJU8cj2WPVM9eT2rur05irxiOym657vdvCK9Zj3BvLm9LD0wOy08+DulvPy8iD1BO028ZL28PQ693bzjvAu9Wj3lvJa8N7w3vHW9Oz2FuAe8wjzCPek8sLwAPSu86jyWPA89ozxxvJS8QLxovTs8sTzIvEC8iDycvJO7fzzgOrO61jwovTQ8Tj2ePVU9FD0svUk9WTwjvSE7VD0RvII8L7y4vIw7kTxMPVO9Mb3Iu927XD2xPf48Pz31vDM9Cr2qvFu9rzxXOyM9mz3guzg8gj2qPJ88FT0yPH+72DvzurW7db38vPa7P7yHvCY8Fj0uPbs8rTyhPJO9VDzfvCk9W73SPM68WTw8O1U9pDwXPUy8kDyWvcS7Ij0ruxg9YbzwPEI8Ur3bPG89pzxsPcW8fD1vPaW9A7sMPY88vz1zuxS8MbyvvFk8ZjwKvLC8z7tAvSg94zzbvAE86TyqvSK9Fzw6PFA9ujzRPL48gL1jPWC8dLyDPGU8OLwIvQY+V709PY+8bLw5vUW9eTyTvPk7lzylPb+8ND2KvA07mDzmvC09M70BvWi9+7sSPEQ8TLuvu1I9JDusPC29ubqiPIa85zxOuV69jjwZvf88jT2guy28OD1avI+9v7uJu7g7SbwzvRA9ETx5PcI8mjyiPIu9uzyIvEc8DT1bPQA8ILtVPMm8ST3svFo7VDsGPSg95TtPvbG8Kb02PH68D70jPEi9lzzRvEk9qDzPvGk9Aj13vXi7gzxrPWA9XzwGPNQ8TDsovMK8Gz0XvUk8Hj3cPAU9dr0JvFW9Cb2EPe47kz0sPTy83DyRPKw85bylu/w8/7vyvGY9hz2cvBM9VLwkPE+8BL28u0a95jvzvLA72jtzvBS9QL38O8U8Gr1ZO+e8fz3Fuxi987wXPda8/bolPYu9Z7yBvF480LzHPUC8xzw7vUi78DxkPV49u7z1urC7hr0EPQ09OzwpPZm8hTtHu3c9AT0hPQI95jrEPBa9Xb2yvCE9ED7CvNS9kzzcvCU9Bj2AvJc8LT1yu468sj3hPFa7szwZvOu7Vrv/vIi8bTxpPCU9ebzGvIy8ajxoPdC7RjyEvIE9ujySvT+9/bxevLC6Ob2SvCa9sztvPes8OL2WvHk9cz2NPL47Kj0bO6G8Zj3eO4+95DzbPBU6NzxlvNo8Tr2LPL28Tz1lPBw9UT2ZvAW9xbygvGU8pjxvPZA98rx+vSK9gjs5vfk6BDyCvD49QT0KPYW9Nzx1vek8Xr2fvE09gLqCvNS8ObzavBO9ST0MvIM9ArykvJU8kL0wvb08lL3Au0G9FryvvAi9uz3BvB49uDxJPSg9irwaPcm797utvGS6MbwFveq8KT1RvLY6Dzvgu1e8Sr3APMi7UL1rPeK6gz2vPPW7Hj2lPCo5MT2tOxQ9iDulvLq8ib0/PRU8Dj3CvOI8ob3svGq82DrWPNS8sbzbPH09Gr2Iuyo9ujzSvRo5Ob3yPD09Gb0GPLw9SL2ivAY9pjwHveG74Tw/PU08hD2RPcs7Cr0iPbO9QD1vu948gDyjPB69/zwLPG89MLtXvag8eL2avX88mz12vaq9jryoPUk89byLPXS87zwUPek7Jb2pvfy7DL1KPea6Oj2wPLi7Izt4u2s9DD2sPAq9YjzGvGm9Y7vMvBm9tTsBPXC8l7wkPWk9Fr2xvOO8ILxNOwY8Gz0XPIS8bruIvIA8uDwgPXQ9jLxPvIg99TxXPUY8jTx7u8a8Mr11vZE9izxcvfs7fzysPKe8Ez3uueI7c70aPRw98bmRvbE7CDwkO5o6RjsBOzW9lLzEvOs8h70yvB69Jb2rPTM8Lz2RvIS+VD12Oxc8arwrvXe9S70uvb28Cr1Rvbg8tr2ZuxA9Y7xdvJ874DyKPQS9gz0PPS292bxAPU49Uz2GPfa9GT0ju8W7aLu0PDu7AT3evNy8Kj2CPFg8ZD2hvD88dT3Gupq82DslvZo9Er3NvbE81Ls2PQK8eL1BPNQ8GLobPYQ7Ar0vPas7nTwjOpY9zrsUuxm9RD3BvMY88DwIvV89DjsNPQu6Fz3avGi9Pz3xO2u7A7sfvBi9WD38PCq8ST0EvZq8XbzFvM07czwSvWu9Kj0kvRw8Bz0zvco8TDzKPA28YzwlvZS9/TtNvZ89T7y9PIW9r7yTvGY8e7zAvBQ7Oz0NvGg8CbtQPAW9BDvGPA89TzxXvBu9hD2fPA48az2xO5U8G7vZuTY9E72guuc8iz1QPOo9Dr2aPdO7tbwqPSW9ErtMO7Q8HT0nPfC7ND3XPAg89TyxvBU9Cb5qvJe8sjtQPFI9QT35uzA9pbxwvIu8Fr06vE885byJvfI71zxYuYG90zuMPcc8+TvvvN879bwgPDy9Rr0UvGS8/zzLPJ68dTsRvV+8LL0QPdy7kLyfvVO9crw6PaA8lT0yvME8Eb2yvBM9wj2Mvb29kD0/vVw9IDxKOjy9lby2vAu9kzs/PWS99bx4vEq9Cj0CPZ+8VD0Kvb46Tr2lu7E8Vj1FPPo8W73LPIw8/zx6PEu8l7z4u7y8ALy9uls9S7u1PAQ9LLtdvI682rrTOyA8CDtHvWa7JjwhvRm8rLwRvOW7Nr29vEw7lzubPAa9Ez0qvWW8FL32vHc9oL1APaU9qDxQvW48vry2vN86gTz2PIq8fj3nPJC8Ob3wu7487zsQPLy8J73COps8ML27vDy8dT1MOmY9kr0LvRu9qzyMvd28Yj0APSM9yryUPba8zz1+PaI8t7w4u108xjkWPN68ET3iPP871LwvvQq8tbwUPUg94LwcO6o837yEvKo9ibz0Oom7SD0tvIU9W7z/PJY81Dyculi9Sr2OvVg9x7w/vd6827wGPB68dr1yvLU7SzzIOcq8aryjvJu7Xr2DvaK95ruPve68MzpmvJU9PT3HvIk9ibySPB69Cr3MPCc9Dbz2PJE9+DupvZ07m7zqvCQ9pzo3vTI9RT2WPKe72bwivd87K7vVPBq6Jj2ZPMG8hzy+O/a8hzyfunS8dr2uvIu8Kr00vJo9c72kvJY9tzwwvbG8oT0nvbE8w7ufvKK5L7wiPVU+wzu5PLs73zxivKI8iDx+vLM7kLxOvQM8KrzwPGA9zL3fO+G8AzwTvWA9DTtXvRY9NTtlvCK7xDwEPVo8Ar3AvLO9TTnkOZy9mTyovGW9sjyZPAq9FzxovSY9rLsVvYs8xj03PNa8A7zNvC+54z20vH88iD0EPei87zxUvUe6ljyXvdE7ojvnuxu9aDo/PLs8nTs0vWe8wj3DvHI9Tj3IvP68nDy4PEG9iTzzOoq8NT0IvYu85DyJPW6877wKPXA9qrwWO748lbp0vJC9Ub1HPIu6xDv8vNw8DLzDPI89jj1UPBU93rwdPT48IL1LPSU96LxuvMo7pbhrvLq8X7yYPbk9Lj2OvWI8RbyKPP+8HD0qPLa8/DzBvcS9ED3ZPAi9WTzZvVE8sjuCvdS8y7ucPCA9MzwYPLK8ML2MvI68pbt+vFI7b7wEPWM8n7wKPYQ897jXPA09sj3MPN68j7oRPJG8Nz1YO4e8FDsOvVw9qzw6O2w8Vz0UPR89Cr1ZvM88Y71/vRm9RrySO6C9RrsEvFc8QL2DvSQ98rvPOxY9yrxKPeM8JT0du+K8Hrv2PGQ9vzwMvf88rToTPbY7h7xJvEo8C72UvI49cT0ivaE8oD05PFU92jxbu687GDyUPIk8BD0GvX087bwiPWq7Nrv7PG49H71yuDq7H73mvA29HD3GvDA9mz2CPCK9Dj2YOoY8JD0tOnY7AbtQvdA8TD2euSq7kL1YPY27TLwzvNU8Zz1ivUs807yovEk8d70oPSk9DT1evCa9Eb2qvAy9CbyKPNs8Xj2CvIs9kzvcvLW9/busvC09tzxmPca8LbuRPaI85TyevIO8gjsdvCW877wrPcw88zzguoE9dDuHvJ69LLtzPBU9bzxZvA+8vbwRvbW8Wb1rvAM9bDyivB09wjyOPac8VD0EvLE8Zr1yvUc8QTy0PW48tjy/PLY9hbyrvHe8Fr0svSo8/bszPBa99zyyvKy87TtDvaw8eT2AvQS9JL0bvNq8+ruUvYs89juQPOY79zwRPBs9HryjO6I8VrxxvRC9Q72QvSo8DT3JvMs6nr2BPJc87zzbuw69z7pwPU69Nb2UO1A9cjoVPXw6czx5Oui8VTxUPBm7+LwPPWI8Y72kPMO8vjwGvHc8PztOPQE8PLzEvUe96bwePNc7k7zeOWs9ZT3APLu82Lt3Pec60rwMO3s8yLyDvVs7jLu1PCQ9bTs2vRO9hTzDvJ8987sqPGI9kjkBvYC8xzxDvSs90DyvPF69TD1OPPi8Wjygu3Y8DT31u5A8nbtUPEm98zxDPDS9FD3WPJ46gbyevIM8Ez2nPMu8+7wSPPq8kzohPbs8Mz37O8y7E7zbvEE8ZT0AO2a8LbydvBe7Ar1lvS09q73gPB09hT1LPL+8SzztvVS9Xb3tPEG8t7tQPCC8DT2sPOo8rDuZPJ895bzivI88vLwDvQU9Qj0KvB+9obzUu4k8P7s/vKS9kL1XPXe7Ar7RvFO9Fz2rOyC9a73mvEO7oD0UvYQ8bLz+vBm8sLx2O0E7SL3YOtm7CbyoPNA8RjxEPXY9IrsKPbo8dzxnO5A9Wj2oO6I9b7xxO3Q8nzzFvAs7m7uUu1G90zsHPKg8DD2Curs91rxfvJC8xDyFOrO8lD1oOy28qLz1vDa8N7rruqg9U71lu9O8Fr3zvEO9cj00PBI9bD1Mu8k6nbv6OF+8Sb0NPEi807sSvao7Wb0Fvbw7ibzEPCK9YL0aPaM8Xz0nPco8hDyBPAO9Qzu2vHi81LyIvBE8FD0Xvcq9cT3tuia7Dr3gvIY87DuAvB88kj0WvSc9TTygPA89kr3zPMU8lrzAvOS8m7wcvJQ8Fb35ulA7hbwCves8sTyBvFU84Lswujo6rTx0OwM9hL02PQO8w7zPvJs87DyPPCo9ibwfvJU8/zy3PJe8SbxcPQc9bjx/PLA817xwPWc9Qr2pvdM8orwDPcI5hryCO587ED3cvG282LyoPQq9G70xPYu4QD3HOh+9qDysvHa8Db1avB699ryrvHG9Ir2UOvO9BjyZPYO8az2gPAg9gLxdPa+8jTwxvbY8+byvvZy6QD0XPY09RDzVu4u8Wz2nvCO9m7wcO4+81DtJvai8bz2Au6O7tjxMvd28Mz0MPRE8wzzzPF68DzuMPSQ9QbtjvbQ5tTsSvcg8dru8vKq8mbzQPBU9D7uZvU09DLzjvLC8jD02vVM8trynuws9kjyTvCe7K70OvYi9zDy/vBe9Jj2ovTU9/zshPMS8gT1ZPMi7gb3Xu4E8UDvRPCe8IjwRPUS9n7zaux48PTycvbs6tDxquxE9Jr2FPDS9krx1PRM9lzzuvO88P71lPVe8Gr0aPYe9ET2yu289R70Tu9q8Oz3+uie9GTtBvSm+IDyvvNU7Mjw0vKI8gDwePUI997z6Oz69lr3EvCO9FD37PFc9Kj0Uu7+8Azs7vYG5ljxBvfC8FD0uvTq9Xbz3PJU89zqxu0G9iDyJPZs8Fjy4PM68hryBPFK83jyEvIy8MrxyPAW8jrz8Og69yDkvvNa8DbxeOrY80bzMPFY90jx2O4o7Dz1yu+a8JT0DvPE82jyQuyI94rysPao8Uz3mPA09Dj05PS49Q7xKvUU9UjyLvKC6Ej3dO3s9QL2MvMc7Gz2HPTQ9ubyUPas95rwQPfG7YrzAvFm8nzy3PMU8qrzwvAA81zyOPVW8rjyaPZ09Jr1IvD+9Kb0ovYW8A7zTuzM9KT0Gu5m8VjyKvLq8a713uy48ZzwwvSC9GT0uvYO9pDw9PRG8CD1sPRE8rrxAPDQ8mTzNPMU8Dj2HvIY8ojvAvG+8hDocPDA8ibzuO0w9Or2avAa98jzOPA49Ur2zPO29ObsAvYa8b702PTa6GT1JvFe8hbsbPa+8lj2JvBG9Ur0XPbs8VLusvAS9Rrx8vac7jLwMvQG8UbyGPHe8T7sDPV68Hr0xvIy9srytvTq8mD20PNq8qzywOsq8FrtOvMg9nrwLvQc9Kr2Ku1I8DD0zPfM7orxRvaO7B75WumI9ND39u9e8LL39PI08Ij2lux+9irwqO9s8qLwyPKO8lLxuvae8tD24PFe8zjwLPaw8Ob7kO/I8ET0wPSm7WTwIvLC9Dz0YPR086ztoPQq8Cb3lvFS9yLzFvDs9E728vPS9vDzlvEI9Sr2ZvM+7jzw4PeI5NrzPPP88Jz0bvZS8Qj03vbM8rTy3Ops9bbxhvZm9uzxZPVs8A7znu1m9NL0KvYe8Nj0xPae9Qjzru+88gb1evfA8yzwYvAu8Ib3VvUI9/zuZPO68Lr2tPM68zbzHvD49ILpUvRg84bwVvVO9ETzjPA27DjylPCu8Xz05PES95TxIPJs8ED1jPRc8pTw2Pfc8kryOO1O81bwLvfU8OTxHPRK8Ar2XvGc8Hr2iO827Ez3LvIy8ML3sPHc8Dz0hvZM7STyPuiu8NzvFvDC8Iz2UvJ28NL04PUk9AL0kvbK8G7x5vIK9QT1DPf68ObwFvRc8YbyCvJS4CLxlPW+837zBvJA8jjuivP+8IT3ePNU8dT3QPCC9V7ysvM+8N71evW09kryiO0K9wbzhvIk7Or1APYA8YT1fO7c7Bj3QvOQ81j2Iuxk9qjyTPYi9uzzkvEq9m70CO6I9A70EvpG8CT2Evfu77TyCOhq8HD3uvEM9ermTPe87l7tqune9f72GPZ07qbwHuwE8dL2MPLA8Yj3TvNi8DLw/vL08PDy7vOs7U7yuPMu8PbxjPfK7m7yUvPS8Jr2SPG48T70QPVe8lbtFvB88N70zPXK9HL0GvQI81zw4vbM8sjy2PB09DDwVvTe8gzyRPOu89TwNPCC90zxhvAk8pjzvu9U8ujxjvHo8DD0hvI48S735PG49+jwTvNU87jzJvM0857wRu2W9iLwpvdW8IjxbPau9vzxbPUe9h72gvMU8xjykPII9c7yTvJA9Cr0ivFu9Lz0mPAq9CjslvRC9Vr2gPFY8szyuuAy8jjqaPBY92LyfPB69Qb0PPd66bLwrvRs9lL0nPb+8g73GvEu9mL1bvcY6xj2YvY68J7wEvUO84TwPvS29tT2bPE+9PTzbPCe8LL0CvT47Wz2pPBu8hj1Jvaq9Nj0rO++8ID3dPEA94bzdO1s83Tuzu9e7YzxGPPK8B73BvV+9ST1ivDC8T7ydvLA9UbxbOwM9kj1cPZG8bT34PJs8CjzXPHw90jySvQm8kjtePUY9NL2KvJw8VLyBPNy8RL3AvC89j70GO7a9Gz0gPOI8XT0YPd28cDswvQy8Rr2eugI9ujzKu1y9mDtBPda8LjtovGY8prvNvBW8arsPPfQ7Jz0HPYU8Gj05vXi9VbwQuwM9RzyovaS8IDwNvOU7Pj0RvCy9Lz3TvBA7nDqwPNy6jTzmO3c927oevRQ8Gb04vNW86rwtPTA7zLsfPb+8Vr2svB49IL3UvJc9rzuuvTG94ryMO4y7wLnivHq7NTsdPcC8sTwmPfs7Tjv/PPU8H7sBvRA8Nr07PYm9lbwtveC7BD2eO2K97Ly9PKE64TwGvHm9cLzSPF07DTyLvQ69xLwwPaq9u7xdvJk8ub2bPVo9XrqJPHe8xDuhPLI8jD09u467Mz0ePOA7hzu5vOK9Lr08PNO8uLwfvXa7vrtkvZ08WT2BPO+8rjwFPA69gD1Nu2U8jLxWPTA9K7wPvHa9lDzyu688vLy2PDI9KT32vJw7GT3UPOC8vzumPIY9SLxTPWK8FL2Fvbw6Mz2LPF68WrtFvf28rTt8vZC9xTy4PW+8+rpMvPY8KT2EvOG8gLuDPIE7ZDrkPJK7Hr3rvNu8DT0jPZw9V7yuvCa9sLx2PC09V7w/PWq9SL17vVm9hT1HuiQ9Lj2Nuki9pjxBvCI957ysvAq9sTpZPYw8VrxhvEm8ujsevSI9Uz3gPMy7izw6PDk9Gz03vfQ7nrwXPbu8Ury0va68RryKuu07Qrz1O0A9WTy9uCa9OT2nOoO8uzspPG07pbwaOhY8Jz0OPEE9mL2ivKg8lLw7vaO8G70XPeU8oDxuPWU8BL1QPSI9DjwpvEU8nL0nvTc9uLz+vCo9WzsSvci7er2oOy+9HbxWvHM9UTxjvZQ8H71Zux+9L7wovYE67TwHvVk9Uro1ujQ9mD2JPWk9gTxEPQg8Hz0yvc08QbyjPbC8tTxCPNq8sruovCC97zzJvEI9XryhvFu9u7x/vXE8ZrzsvBE9rTw+PW+8bz2bPVY92TwTvXU7kLz3PBc87LyEPPE8Bz0QvCq8Yj0jvRI7tDsBvcA7ODxpu1+9N7zNvMC8tLyJvJU9O721PJg8rLznvCc9mL0kPHm7PTsTvQw9WrxkvKw7LrsFPTQ91rxtPHQ83TzZPSe91LwvO3K7Ob3kPKg7Uj28u/k8E70UPaQ9ZD2Qu6C5BjwDvcs8PbumvJS8+rxFvY+9/rySPHO8ZL34vIO84LxMPJE8Ab0cPLS8tbz0PIY9CDvwuow7Nr1Qu3484zjjPNG8hT3FPIs9BzzNvG29pj1su6g8rDv3O8w7tTyHvak8Br0wvQG9Qr3TvEO9kL0VvHc98DxUPGy9bzwHPRO8Mr3UPCs8iT1pva67RryJO0G9yTyyPPs8JD0PvQm9B73gOsc6EbuEvO+8nz3gPDE7aruZvWM8NjypPM+8Qz03vNk8cT0hug89QzzjPFs7TL2rvKU6HrwkPf+7lzu6PAo8i7vJPMu847wSPRg90T2vvbo9HbwLvfY8FbzDO2u9ljx4vJU8tTxPvGE95r2Luq88RLupuy88jTyOPa484zxPvVu84LyKPCs9jbwbvTs947wkPQG8LzmxvA88Dj3VvM07ljonvBs9fzyfvWE8D73BvLM8GLuSPYk8z7uPuxE6AbxmPac8nzy7PAa8kT2PvQ89rLvIvP68AL1KPX29h7yHPc68Fb0OuzC9gzwRvZ28bL39O4o94jslPAi9ST2Yu4u8jryuvHa9SD2LPF89/Tx7Pfe8H7wvvSW92TxJvSW9hjw+vLQ86TvIPL88IDxwPA09Bjx1PYO9i7oVPZ09bLwLvBw9TD2fPGG8oD23uxY9yLxOvbm8vjtVvTa8FD2PvW+9r7y3PI08N72Wubc8ArzaOrU7YztZu5e8LD2Suya9qDu0Ogg9czzrPMk8BT1qPZu9Eby4PdI6prtevE27LT2AO8c8X73cPDE8C70SPUU9Tr2FO7I867wvvRq9Dz3NOrO9ST22O+88HDu7vMQ7uTwTPSE7RTs0vTU9AzzfPMk7fLygPPw87Tv3vI48gb2cvH08uzyxvHY9lruRPY08J7yTu9Q8lToZvR88vjxwPMM7NDxEvEo9Tz1avfo8ljysPL67JrsTvAw9q7pOvIg7o72QPLK9zzsSveG8CDwEvNY8SLzLPGo7kj28vBa9Gz2yvJQ8Jry2vDo8DT0QO429prtAPFe9KrxTvV+8dD0zvVC9TzxMu5i9Xz3GvEU9dLz+PEC7YT1wvJk8vb34O8c86rxAvRe8KL1NPQk9f7ymvE89J73CvsW8bT04PaY8kzyDvY28tLzSPK08ujwXvdw7P70IPYs7VD0yvHW8D71ZPZQ7gr0xPW48iT3ovVC72LxwvWg9Tr2/ut482DsrPZ48sDsOPSg8RT0yPf+8Nj2EvZI7jzwnuwa8gTwrPVe7Gz0oPAI+lrw7vbm8YDxKvEC9IjwDvce8XjufvHs8QD16PJY9TDw1PRM9ej0NPZ48a73KPNe7gj21OwA9573uuwS9I70YPD88Sr3oO849BLxjPV29LD1zPa87GrvVuhm9UzwAPW+9hDlRPTw86bzOO4S9nLwlPQ+7DjrovJ89+DzvOoW8zDpSPOq6Wbp/Paa7Az23vak9zzv4vMg9yLzEvPM7ZL3ku2g9e71sPHS8Ebz9PM88cL3vPNY8N72TvBc8lDvyvBq9Bb2CPY04XDz+vGC9fbyCPBG96LwXvLC8Nb3PuzQ9QT3Sux49Xb3zvJk7yjxGPZM7x7uZPeg8zjvMOze8FL3SPKQ8Fr3NPI284zyRPBO9PztaPPY8uLysOzy6BL0JPXW7d7xavSu82zshPPe8DT1Pvdu8i7yqPIi9gL2jPbW8gL1nu4A8Sz0avQy9h7zjO1c96rzpPKS8Bb0UPS68gjwQvEC8rrtKPcY8h70evFw8Bb3KPF09Jz1EveS8LjyBvKw8GD0EvRK9QTzePMI8wjsKvLK7jjz6uvQ8QzxaPea8BjyYvIa7LDwyOni9xL0CPCU9fDxHPUE+rz0dPbK72LxlvPg6Kz1LvTU7Kb0WvAi9oj2XPWc8w7ydvQ660DutPYg8Vr3qvKu9Mb17vfG8pLwRPOy8az3nO7M8Lbx3PZ08hz38PE+8Yby0PD48ajyHvQc8Kr3dPCo8kbp6vOc7wLypvF88EryIPQS9bDrxvCk7Zr2Vu9I8cTvcOgQ8aj1gPe87Nj1Dui48Kj0jvCM967rlPKy8EDuAPZU7lL0EPTu9ujyEvZA8bbwAPS+9Yb0kvcQ8XD2rvaC7DryrPBE9HL2sPJs88bwEvey8Yr2ePXs9aLsRPAQ76zsLPPy78TzQPI+8UD18vIc8Qj29PWi8l7sZvRo9FDxuvQg9V7xavTi7xzsFPDy9P70pvdk8lb34PFA8ULxSvME8Nz0lvYO9urxLPbM8hr1pPXs847yVvMQ8vzwCvro7hTxmPRe7lbxXPQ09JD22vZ28gzw8vRW8yzyfvAG9er2xO4I9u717PAG9A71RvDg9ID0hPKu7CbvRvBI9QDyEPA+9nz0Xve285rsiPc65lTwBPWy9vbw/vEg7O7yPPWg9Cj0cPB49wbyjPL69Kb2+PIk8Rb17PI487by1PMu7wTwSvWa90rxMPbm8I71/PVU8m70FvHG8ZzytvAm9g7xtPMo8ij1FvTI8mjzYvMC70jwHPVe8Yj1GvAy9IT0iO8C8DTwXOhS9tDw8vFu9+bxnPTO8grvNO/88XzwzPU+8lr0hPck8pbxJvRC9BbxyPKM8uzvfvPk7j7zTvDS9Cz1kPQu9ST0aPbQ897wjuy686byXvFW9xrwqu3O9Dz1DPIc9ZTx6PNQ8D70CPZo8Fj15vCY9fLtuPWY9LT1nuws9n7wovbw8JL3PvFC9bjtBPDe927v8O+G8+rxfvJK9Oro7vac9RjzQvAm9Hj3pPFc8hTsuvLA9+rzvPPm7Bb38OwS9QjxCvbA88TunPVK9tLxePHm9N7xCPQs9wz0mPDM9lrwdPTO8HD0qPIg9lD1YvBM8iDxxO628ebz4vB696jyTvaK84zxLPXu9g71UPRm96TzCPMC92DxyvQg8Wj2YvKA9l70ZPP45hz2YPU28Gz3tPIe9Or3pvCi9cTzNO0g8rTyrPZc8LDwxPb87Sb17PAQ8TTuVPW49Tb2zvMc8A70IPdG7aDwrvXI9QDycPcq7QjxhPR49TjxNvNi7WzywPC09jb0EPRQ88jviOgU9Sz2uvS48PzwJvcK85ryuPHW9V7zFvOU7pLzJPIE8mrtguk68t7whvLM8Nb0Lu+Q7yzyLPC29BD2BvBu8hbwdPk89ejwgPSg8C7x1uy29GrunPGS8wzzuPKY9wToEPI69Ij3Du2I9dby9vAm8CLymvIm8drqjPcA89LxAPfC8gT3hPLy81zxDPZE8az0sPdg7j72vvKG8Q7yfO628N7zxvFU8dzy1PC69XrwBvQ07nD2KPPk7pbxLvVQ9AbztPIi8AD0ovJa7Kz39u5U8cbx0veg8yDyhPLu807zoOUu9Cz3iPBQ8crytOxO9Dj1LPDK9ejxdPNu63bxpPBC85LkpvOS8mzs9uRy92bw/vH296TxXvQo9vjz3PHw9WjyDPeU8AT1EPTi9KzpOvHq70zy4PA48urs5PSu9KbwnPay8jTy3vKI7kD1fvcC7WL3OPG88RL1CPfO8bDxNvT69LD1VvWk7G73MvTS9i71AvR89yzx2PLW9Lzy8PU89wjw3PIU8gr3vPFe84TyHPeI6Dj0bPde8ED2yvRu8nzziPOS83LyRvLM777xTuz+9ejyGPZy8ND1HvYM9gj0/un+7ab3PvDa8MrwYPTa9tz3tvCo8/bxhvTG9ZjxGOxC9fjt4PJK8BT2cvBU9Ub0fPdI8hrx/vbc8SjwZvbw8hbs2vVw7kz36vGw8MD04PPm8gTwDPdA8W72NPDs9Cz1iPYi8GT2QvYa8BL3MvHo8j7ygvMy7C70mPem8a7xWvVK87DtXvIk8tzwbvai8YDx1PUm8q7wpPYI8dT0RPZk8ozziufm6Dj2YPSw9+jxTPXK7y7xPuQ+9ez2oPKs97bstvZ48qjz+uzC89LvBvam8tzufvZW8V7wHPKs8oLwIPai8cDygPSK8XTv6vIu9MDyZPG08FL3FO9c78rxBvZ88O7wgPaw8s7tGPQq97ryJvOu6zryOvEw7J73JvIU8XzxMPYq99TzhPLW8ozzovDi8Wb2JPNw8Kr33vAg8Gj1MPcw9SDxbvFs9UD3eOLC8YD0nvS49u7xHPSE8zzzcvE09HjwlvQc8K72wPI27C7wQPdE8/byCvR09SjwSPSm8zb21Pd28vLzmvDq9lz2TvKc8z7tDvN67S71Gve08srwXPYW8azuRvFa85LssPYW8ML3SPCO8gTwpvaa8fjw8POW80LkbvPs8Fb0pvW88bjyhPCQ7p70dPbo7NbxAvQA8SD23u5G8DryXvfM5obzsvDC9XLtkvL682DsdPSw7Izy8PJ69yDvGvHS80T3CvI29jDs3PTC8jL1bvTW8KLyoPYM9trzfPHs9JT36O7k8Pr0uPYG8JT2GOUk9A72yPB09BLsxvZE7Ob06PBi9ZD1SvXc7Qj2BPd08pTzGO5U8iTmCPKS7X73uvBy8ir11Owq9VDzyO8s8Hz3hO/S8C7zIvVe6oL2JvCK9JjxEPDW9jDyhPAS9kjwLvXG8Qz0fO1g9dj0UvB27kbx0PFq8O7yKvdm8+jy1vJa9W7wqvSm9kDxlPcS7F717vKi5DbxmPAe9WT0OvU48RDz+vJ88P72RvRE8VL3WO6u8vTykOwU9FL3nvAs9Jr1AvD09W7wnvf64Urz0PFe97ryYPZc7hD0FvUW8iDxTPWk9aD2pu2y8jj3hPA+7Mj2EvKy7iT1AO8q88zxtuwY95rslvfS8GzzLPFq9Fr1JPJa85zxKPYq8fL1lPCo7jLyYvCG9ZbwZPFk7Oj3DPH69UrxKvHq8mLwQvae8KD0CPHC9IzxkPS64rzzHvO68Nr1QvVK8h7wIPSG9nTzwvKa7gjzUPZM8iLo4Pdo8aTznvI49oDzJPN28kby5Oya9yzyKPPw80TwnvRa8brw4veW72ryXvfa9jj0BvWw8bbxtvDq9Gb3NvES9QD34vDu8fTtivRC8CjyivGa8eLoUvJq8K72API28HLzOPEC9Bj1pPfy7Hj1tPKe8Bj2iPJy9DTzTPL26GL1nPds7EL2NPe27SbyIu5G9hjyzvNC7mD2NPee8zrssPSc9wbx9PCS7Fzq2vXW62TyXu688Or0OvaQ9Vb0uPaE8I71DPVO9jrxwvbU8Pr2YvD87qrzhPYk9g71zPaY9kDy/vF65fLyHPXi8nTyWPHI9lj1TPXo9qrrlOYq8Zzs4uui73bwhvWY9wLy5vS09KDstPPc7pbz8vIg9QztOvGS9vD0Ovdy847wLvVo95LyWvDi8N7x1vTs9erkIvMM8wj3oPLC8AD0qvOs8ljwPPaM8cbyTvEG8aL05PLE8yLxBvIg8m7yTu3885zqtutY8KL01PE49nj1VPRQ9LL1KPVg8I70nO1Q9DryCPDC8uLyMO5E8TD1TvTG9xbvdu1w9sT3+PD899bw0PQq9rLxbvbA8VTsjPZs94bs5PII9qjyfPBQ9Mjx8u9078bq2u3W9/Lz3u0G8hrwnPBY9Lj27PK08ozyTvVU837wpPVu90zzOvFo8PTtVPaQ8GD1MvI88lr3EuyM9J7sZPWG87zxCPFK92zxvPaY8bD3EvHw9bz2lvQq7DD2PPMA9b7sTvDG8sLxYPGc8CrywvM27QL0oPeM83Lz+O+o8qr0jvRY8OzxQPbs80Ty+PIC9Yz1evHS8gzxmPDe8CL0GPle9PT2PvG+8Ob1GvXk8lLz8O5c8pT2+vDQ9ibwLO5g857wtPTO9Ab1ovfq7EjxGPEq7rrtSPR87rDwtvb26ozyGvOY8xbhevY48Gb3/PI09n7stvDg9WLyPvb+7iru2O0m8M70QPRA8eT3DPJs8ojyLvbs8h7xGPA09Wj0APCC7VTzJvEk97LxXO1M7Bz0oPeU7T72xvCm9MzyAvBC9JDxIvZk80LxJPak80LxpPQI9d710u4I8bD1gPV88BjzVPEg7J7zCvBw9F71KPB493DwFPXa9B7xVvQm9hD3rO5M9LT0+vN08kTysPOa8p7v8PP278rxmPYg9nbwTPVK8JzxPvAS9vrtHveY787yxO9c7dbwUvUC9+zvFPBq9VjvnvH89xbsYvfS8Fz3WvPS6Jj2LvWS8gbxbPNC8xz1CvMc8O71Qu/A8ZD1dPbq8/Lqru4a9BD0NPTs8KT2ZvIc7RLt4PQE9IT0CPds6xDwWvV29s7whPRA+wbzUvZQ83bwlPQY9gbyXPCw9bLuQvLI94Txau7Q8Grzpu1i7/ryJvGw8aTwmPXe8x7yMvGo8aD3Ru0Y8g7yBPbs8kr0/vfu8Xby2ujq9k7wmvbQ7cD3rPDe9l7x5PXI9jDy8Oyo9GTuhvGc93zuPveU82zwfOjY8ZbzaPE69ijy9vE89ZTwcPVI9mbwGvca8obxlPKU8bz2QPfG8fr0ivX47Ob3kOgQ8g7w+PUE9Cj2FvTk8db3qPF69oLxNPYG6g7zTvDy82rwTvUk9DLyDPQO8pLyVPJC9ML2+PJS9wbtCvRa8r7wIvbs9wrwePbk8Sj0oPYq8Gj3Ju/m7rbxpui68Bb3pvCk9Uby3Og475rtavEq9wTzEu1C9az3luoM9rzz1ux49pjxJOTI9sjsUPYc7pby7vIm9Pz0WPA49wbzhPKG97LxqvMY61zzWvLK82zx9PRq9irsrPbo80r1LOTm98jw9PRm9Bzy8PUi9orwGPaY8B73hu+E8QD1PPIQ9kT3MOwq9Ij2zvUA9dLvdPIE8pDwevQA9CzxvPTO7V72oPHm9mr1/PJs9dr2qvY68qD1JPPW8iz1wvO88FD3pOya9qb3+uwy9Sj3nujo9sDy1uyU7dbtrPQw9rDwKvWA8xrxpvWS7zbwZvbc7Aj1vvJi8JD1pPRa9sbzjvB68UDsGPBs9FzyEvHm7iLyAPLg8ID10PYy8ULyIPfQ8Vz1IPI08eLvGvDK9db2RPYs8XL35O348rDyovBQ977nkO3O9Gj0cPQ26kb2uOwg8JzumOkM7CTs2vZS8w7zrPIe9MrwfvSW9qz0xPC89kbyEvlQ9eDsXPGu8K713vUu9Lr29vAq9Ub23PLa9l7sQPWS8XLycO+A8ij0EvYM9Dz0svdq8QD1OPVM9hj32vRk9HbvDu2u7tDw7uwE937zbvCo9gzxYPGQ9o7w+PHU9zbqavNc7Jb2aPRO9zb2yPNa7Nj0CvHi9QzzVPB26Gz2GOwK9Lz2oO508MzqWPc27FbsZvUQ9wrzGPPA8CL1fPRI7DT3iuRc92rxpvT898jttu/u6HbwYvVg9/DwrvEk9BL2ZvFu8xrzGO3Q8Er1rvSo9JL0cPAc9M73JPE08yzwNvGQ8Jb2Uvfw7Tb2fPVC8vTyFvbC8lLxmPHq8wLwlOzw9DbxoPBS7UDwFvQY7xzwOPU08WLwbvYQ9njwPPGs9sjuWPBi717k2PRO9oLrnPIs9TzzqPQ69mj3Tu7W8Kj0lvRq7UTuzPB09Jz3wuzQ91zwGPPU8srwVPQm+abyXvLM7UTxSPUE9+7swPaW8cryLvBa9O7xPPOW8ib3zO9k8GLmBvdk7jD3HPPs78LzhO/W8ITw8vUa9FLxjvP48yzydvHc7Eb1evCy9ED3bu5C8n71TvXO8Oj2hPJU9NLzBPBG9sbwUPcI9jL29vZA9QL1cPSI8LTo8vZW8trwLvZU7Pz1kvfe8eLxKvQo9Aj2evFQ9Cr3EOk69o7uyPFY9Rzz6PFu9yjyMPAA9ezxMvJa897u8vPu7vLpbPUm7tjwEPS67YLyPvNu60zsgPP86SL1huyg8IL0XvK28E7zkuza9vbxQO5Y7nDwGvRM9K71lvBS997x2PaC9QD2lPac8UL1vPL28trz5OoI89jyJvH895jyQvDq98ru/PPM7Dzy7vCe9yzqbPDC9u7w8vHY9UDpmPZK9C70bvas8jL3dvGI9AD0jPcq8lD23vM89fj2hPLe8NLtcPNk5FjzevBE94zz/O9W8L70IvLa8Ez1IPeC8HTurPOC8hLyrPYq86zqIu0g9L7yFPV28/zyVPNQ8mrpYvUq9jr1YPce8P73evNu8BjwcvHe9cry6O0w82TnKvGq8pLycu169g72iveW7kL3uvA46ZryVPT09yLyJPYq8kjwevQq9zDwnPQ689jyRPfc7qb2bO5u86rwkPZc6N70yPUU9lzylu9i8Ir3dOzG71TwQuiY9mjzBvIc8vzv4vIc8n7pzvHa9rryLvCq9NryaPXO9pLyWPbc8ML2yvKE9KL2xPL27oLy3uS28Ij1VPr47ujy8O988YbyjPIk8fry0O5C8Tr0DPCu88DxgPcy94jvhvAM8E71gPRI7V70WPSo7Zbwhu8Q8BD1aPAK9v7yzva45CjqcvZg8qrxlvbM8mTwKvRY8aL0mPa27Fb2LPMY9NjzWvAS8zbylueM9tLyAPIg9BD3pvPA8VL1KupY8l73TO6A76rsbvXg6Qjy8PJ47Nb1ovMI9xLxyPU09yLz+vJw8tzxBvYg8+DqKvDU9CL2KvOQ8iT1uvO68Cj1wPam8Dzu+PJ66cryQvVG9Rjx5usQ7+7zcPAu8wzyPPY49UjwVPd68HT0/PCC9Sz0lPei8b7zJO8O4ary6vF+8mD25PS49jr1jPEe8ijwAvRw9LDy2vP08wb3EvRA92jwIvVs82b1SPLM7gr3TvMq7nDwgPTI8GjyzvDC9i7yOvKe7fbxPO3K8BD1jPKC8Cj2EPEw31jwNPbI9zTzfvJW6FTyRvDc9XTuHvA87Dr1cPas8OTtrPFc9FD0fPQu9WbzPPGS9f70ZvUa8kTugvT+7BLxYPEC9g70kPfG7zzsWPcu8Sj3jPCU9KLvivCe79jxkPb88DL3/PKY6FD24O4m8SbxJPAu9lLyOPXE9Ir2gPKA9OTxVPds8WrusOxY8lDyJPAU9Br17PO28IT1suzW7+zxvPR+9B7hDux+95rwNvR09xrwwPZs9gzwivQ49pjqGPCQ9JDp8OwG7UL3PPE093rksu5C9WT2Ju0y8NLzWPGc9Yr1KPNO8qLxKPHe9KD0pPQ09XrwmvRG9q7wMvQi8iTzcPF49gryLPZE727y1vf+7rLwtPbc8Zj3HvCu7kT2iPOU8nbyEvIM7HbwkvPC8Kz3NPPM807qBPXE7ibyevS67cjwVPW88W7wOvL68Eb21vFi9a7wDPW08orwdPcI8jj2nPFQ9A7yxPGa9cr1JPEA8tD1tPLY8wDy2PYa8q7x4vBa9LL0qPPm7NDwXvfc8sryrvO47Q72sPHk9gL0EvSS9G7zbvPy7lL2LPPg7kDzoO/c8ETwbPR28pzujPFa8cb0QvUO9kL0pPA09yLzeOp69gTyYPPE82rsOvdu6cD1OvTW9lztQPYA6FT11OnQ8gTrovFQ8VTwYu/i8Dz1iPGO9pDzDvL48Brx4PDU7Tj0CPD28xL1Hvei8HjzXO5O89DlrPWU9wDy6vNW7dz3nOtK8Djt6PMi8g71ZO5K7tjwkPWw7Nr0UvYQ8w7yfPfK7KzxjPYQ5Ab1/vMc8Q70sPdA8sDxevUw9Tzz5vFk8ort1PA099LuRPJi7VjxJvfM8QDw0vRQ91zyfOoG8nbyDPBM9pjzLvPy8Ezz6vJ46Ij28PDM99zvNuxW827xCPGU9/DpkvDC8nbwVuwK9Zb0tPau94TwdPYU9SDy/vEo87b1UvV297TxCvLe7UTwhvA49rDzqPK47mDyfPea847yPPLu8A70FPUI9CrwfvaG807uKPC+7P7ykvZC9WD13uwK+0rxTvRc9rTsgvWu95rw8u6A9FL2EPGq8/7wbvLC8djs7O0i91zrbuwm8qDzQPEU8RD12PRm7Cj27PHY8bzuQPVo9pzuiPXG8cTt0PJ88xbwSO5y7l7tRvdQ7BzyoPAw9e7q7PdW8X7yPvMQ8iTq0vJQ9ZjstvKm89bw1vEa67LqoPVO9arvTvBa987xDvXI9NDwSPWw9T7vDOp271zhfvEq9DzxKvNO7Er2qO1m9Bb2+O4m8xDwivWC9Gj2iPF89KD3KPIU8gDwDvUQ7trx5vNS8iLwUPBQ9F73KvXE94romuw694LyHPOY7f7wgPJI9Fr0nPUw8oTwPPZK99DzGPJa8wLzkvJq8HbyUPBW9ALtOO4a8Ar3rPLE8gbxUPOS7OrpnOqw8djsDPYS9NT0HvMO80LybPOw8kTwqPYm8IryUPP88tzyXvEm8XD0HPW48gDywPNe8cD1nPUK9qb3TPKO8Az2oOYa8ejucOxA93bxsvNm8qD0KvRu9MT2ot0A9zzoevag8q7x2vA29Wbwevfa8q7xxvSK9lTrzvQc8mT2FvGw9oTwIPYC8XT2wvI48Mb23PPm8r72dukE9Fz2NPUI81buLvFo9qLwjvZy8HTuPvM47Sb2nvG89gLuku7c8TL3evDM9DT0QPMQ88zxdvAs7jD0kPUe7Y72oObU7Er3JPHy7vLyqvJm80DwVPQq7mb1NPQu847yxvIw9Nr1VPLa8qLsLPZE8krwmuyu9Dr2Ivcw8vrwXvSY9qL01Pf07IjzFvIE9WjzIu4G92buBPFI70TwmvCE8ET1EvZ+817sfPD48nL3BOrM8arsRPSa9hjw0vZO8dT0UPZY877zwPD+9ZT1WvBq9Gj2HvRE9tLtvPUe9B7vbvDs9B7snvRk7Qb0pviE8rrzVOzA8M7yjPIE8Hj1CPfe8/js+vZa9xbwjvRQ9/DxXPSo9Fbu+vAU7O71guZY8Qb3xvBQ9Lr06vVy89zyVPPg6sbtBvYg8iT2aPBU8uTzOvIa8gjxTvN08hbyMvDG8czwFvI688DoOvc45L7zWvA28XTq3PNG8zTxWPdM8dTuOOw89cLvmvCU9AbzyPNo8lLsiPeK8rD2qPFM95jwNPQ49OT0uPUG8Sr1FPVM8jLyWuhI92jt7PUC9jLzHOxs9hz00Pbm8lD2rPea8ED30u2K8wbxYvJ48tzzEPKu88LwCPNg8jj1XvK48mj2dPSa9Srw/vSm9KL2FvAS81LszPSk9C7uZvFY8iby7vGu9crsvPGQ8Mb0gvRo9Lr2DvaM8PT0QvAg9bD0RPK28QDw1PJg8zTzGPA89h7yGPKM7wbxuvIY6HDwxPIi88DtMPTq9nLwGvfM8zjwPPVK9szztvTC7AL2GvG+9Nj1Wuhk9SrxYvIG7HD2wvJY9iLwRvVK9Fz27PFa7rbwEvUa8fL2kO4y8DL0DvE+8hTx1vFW7Az1fvB69MLyMvbK8rb05vJg9szzavKs8tDrKvBW7TrzIPZ68C70HPSq9ibtSPA09ND3tO6G8Ub2kuwe+X7piPTQ9AbzXvCy9/jyOPCI9qLsfvYm8MzvcPKi8MjyjvJS8br2ovLQ9uDxWvM48Cz2rPDm+4zvyPBE9MD0ku1k8CbywvQ89GD0ePOg7aD0LvAm95rxUvci8xrw7PRS9vLz0vbw85rxDPUq9mrzNu488OT3uOTa8zzz/PCc9G72UvEI9N72zPK48tDqbPW+8Yb2Zvbo8jzvcvDI84LzcvAO9d7ykPGQ8oj3NPPU8o7yQvJC93bxIPSg9+DyFvB29Ar6LPG29jr26vX29GLzNvYy9crxbPd08mb3pO1S9y7yavGk9yrwpvMM8mbzDvQI+mT3fvIg817vJveY9JD0LPZ27Pz39Pci8hjyGPXM8y7zWPWg7qLoePSa9c7twvQq7hj2tPIU9srwovMu8YDzrPIc9kTzGPXY9AL0kPSy7rrz5utQ6lDu0vRG+HT2AuzG9trzpPDC8k7wxu049az1vvaC8m72gPHS8r72OPR67gz34PCy8gL0APpe8oLuMvOw8tzwbvaG8NbwZvUM9/jyEvUm9yLp8PaG8m7y9vJ27WT35vIi9kj05PcM8PD3CvNg8p7x4PFQ9ab2vuxU9Wj3vve28PL2ivZy9GD3vPRM9TT6SuzQ8vLyHPU49MzsPPaA7Xb0DPme9X7yvvIq9/Dx0vQa9r7mRvDs9jLwBPUe9g7zjuhO977yGuwK8jbzHu4U9ML0jPcW6/7zwu/W8fT2WvRM8Lr2BPBw95z0IvcK94T2nvQc91jyFvUS8kTwMvYi9yLxDPSs9Db2HPK49/TwnvQs7hr0DvZ88gLxIPSM9Eb0gvQC9jr1XvRy8VL3vvK49I72QvKk8Hz0mvfi8PTyRPaw9bj1pPJg6Pr3Uu/m8dj1YvF099L1ZvY49DTxyvZU9azwCPZo8NLznPJM9e7u8PN67U70SPZK9l70qvZ49Fr1juzs9Wb1hvZW7IT31vBG82TslPKU8GD0QPbu8LT1HPa+7tD0Hvc87jb2yPLa9/bo3vcG9u72Zu529qr0qvfs8cj0KPaE7t7yQPT896ztlvVW7ILxGPOU8+z2curW8tryNPYg9zDxluzc837yWOsU8mj0fvb88tz3+vHI9D70pvBY9aD2Mugm967zUPJ68v73OvHk9RD2iPCw9Ib1WPbi8u7yiPFO8Vj0NPVg6VLvFPCW8Nz27PK46LL0hPSw7Ar2uPRM9Az2RPB28hjwfvPk8pb1rPNO9jrzYu5Y9Lz0IvZc8UT28O6u6JbyAPNs9ZLvBPQA8pLtZvZ+9Gz1ivds9gL2yPRQ9ZDx5uxk9eD1jPXe9Gr3gOwY+6zx9vQu9WTzluyU9pL0avcC9Lr1EPDc9ybyqvbO8sjpdPSa9zLyQu+S7mL36OzM8A76gPIY787zgvAy9H7kfvak9ZD1iuwW9vbyVvHy75jwePg69Er1iPEy9FjxJPcK9c71dPVy7lj0RPLS9VT1evF+9SLybPI082D2puy09sL2Xve+7uLsOvHg9nzx1vXW8Lj20vWo9A70ePDS8hz0Bvlq9ZL0KPd67e7ydPEe9FDulPAm+DjxLPPu88jzau908Fz2rPG09pb0KPDW9xDyDu7a9qjxFPUm94Ty2PWC8C74Xvbs8mDwhPc08tb3XvJk9QT1evbU9w7sSPD491D3APHi9azsvvTE9cboCPWY9KL2jO268e717PUY8QTw7vVI9zD2CPY08uD06vem9O71DPeg7Q7t4vdG8E7zpvJ+9HL0+vUu9qj2EPDQ9kjsFPTg9eTqxvCc9LLwgPJ68B71NveC8i7yGPZs9sLpEPZu9uT2YvYq9hD0pvYI9xTw0PYO9LL0IvWw9iTzivKo9aL2Wuz+8kj28PSQ8M71Fvfc9Ez0BPoc9vr2IPWs89byHvSQ96T1hPDO8njyZvDs93z3GuxK9gzyxvYO8ODy+uhO94DwtvaM8Xz3kPUC8tLt4vSI93r2OPIw8Hj05vBc96Dygvdi74LyRPZ46djtjPcg7k71bve+7njynPYQ7Yz0cvQO7Cz0IPUC9Xj2dPMG9LzwBPHo8Nz1sPLU8bjsnPCC7iD12vL48j7zBPUW9ML3IvaK8ErxxOk290rxgPH49s7wKvIQ96DzRPQU8oDv6PKk8FDztvWY9Az0OPOA7dj0VvEY9AT0IPDE9kr1JvQS9QzwVPZM9Dz0Kvbo9o70gPeU71LwBvTg9mz0dPRi9C72EPXq8bzy4va88YrwuPIi9ET4xPJ49Jr1uPcI7/bqAvYk9drscvq49DL0GPNE80LwmvT+9JbzEPXi9Zj0ePcm86ztXvNq937uMvFa9Fb0QPN66fDzVPKy7KrsoPRu9qjwmPaY6nT2JvPO8BrwXPYy8AL51PPs8cju0PBe86jmFu7A8YDxLvWK8Qb2aPQG+Ir2CPFS9aL1Bvai7Bj60PH28or2WvTy9RTyzPBg7zDxlPeS8Sz2LPQQ9Gr0gPfo7ATsCPQ88aD2fvTE9CDx0PCW8vryJvC48uTzwPKw9I71WO5e8l70kPfm8QL3kvA881Lz+vKQ8UT3jPYc87zyevPM8Pjx2uwy91zt7PKk9g70JvSq9uj3uOgY9Ob0DvWs9uLpNvYe7ar2LvKu7cL2ovZc8ED20vGU9WL2wPaI8QT1UvfQ9P7uNPQw+67qHPIG9jj0bvTa9fb09PUu9ZD3kvJm73DkXvKc9Eb12vBu9fz0jvBg9W71FPSE9qL3GPGC9bj0Tvow9cjwKPRk9GTw9PVS9Gj2tPRw+jjwOPMk7BrvcPAc9p708PbC7sDxSPZy8X7t7PU47Qb0DvWG9Fb0TvKw9sLzGPI49CL4+vQw9pr0UvX880rzdO4Y9SbxEvWC9gzwgO0K9Uz3JvKA8bT0wvaa8nLyzPBA77byKvfI7gjwUPV09mz2KvRi97Dxhvaq7tb2BvVu8Pz2QPfQ89jvfvN88h72IPau9S72tvIA9jToIPhW94j1dvDo8ozt1vYW7Vz2RvVq9/ruGPb69Uj0RvZW9zLzOPM88Dz2evZ28ij0dPZi8+7yxPHG8YT06vSI84rwzPK67dj0Bvbs8Hb25vaE8M70UvW09nj0qPFk6kjtyu8m8Kj3jvIU9mr1dvT09/DuMPII9gr3bPG09xzwDvYI9Tj0OPbE7oD3IPCI9/rrWO3A8XjxTPQk9UD1bvSQ8RT2Wu4y9eTupPWq8+Lp0PZQ71zxJvL89RD36PIc5Kb2HPYq7ijy4PW+9R70kPWi9uDwIPZs8Eb2wPFU8tb2cPeG8U7uePWQ9cL2sPea6Bz2jO2C9Lj2RvXy8K72DPTA89TlYveE8Gj2SPUG8S721Pdy8t7zBvAc9vrlAPMi7v7tRPJi9Qz1euxS7Q71LvTg7hjqQPds837wnPV88j72DvAu9PLuGuo48RD2BPBE8ED3rvHy8Hz0gvvI8F70MvYU8qL0UvNm7Bj21vdE9uL1pvDY8tzyUuoo9urzmPJO9yjxtPPM8BztAvfq8zz0svME9Iz0LPSS9Aj6nO8U9zj1+PKO7rryaPO+9frxNPV89V70OvYe96ryPPX49xrtMvZw8lDw4PdC9qD1HOV293b0SvYc94TxSu5S9mrukPdA8AjxJPKS8wLsrPRE8iT3KvO89Cbw+vbi8bbySvTE9KzzGPSe9hT1NvVE7mL3TvQk9hrzSOwc+zbs6vbu6j7xVvXY9gb11u6g9jbwIPTU93TyNvCi9dDyXvJY8iTyHPJS9JD0UvGa9cT3svCe94zxJPU87H7yQu/I8Iz03vH87O7q9PKW7Xz1xvSs9qz3BO229yL3aPN27grxxPbY8vb2QPby7Gz1QvY88Pr0gPQC9dj3APJ+8w709u5W90LybPV29k7ylPJa9mzxFPem94Lw8Pf48pDyvOea8fT2FOSi9CT3AO727p7xovXC9/Dtju4E9f73NvOy8wz1evZi8z71YO6W9U71EPQM8kDxnPeG8Jj16vSC8pDu6PJO8xDytPQg9mD2Gu2u8Sz1AvRW99zw1vY89nDypumI8l7zaOnO97DoNuqe8sDr8PI68qb0fOyI9MjzgvJA9IT2PvFw8YjuFvZm9sjpPPCC9i7uavJ89Br1SvVw8Pj3BvEq9dD0HPZe9Bj2RvP28ervUvYW8Sj2MPai7+DuKvTQ9sLxOPfe7Gr0ivC88ubqlvdw8Qz1ZPZs87TnbPPM8kj0XPgO9eL0MPcs9pj1RPWs8oD33vCq9T7vBvSm8srzxPRI9Gb3YPYo8irxjvWU9aTzMvSU9FLw+vXe8+bwGvbU9aT2jvGi8hbwhvRI8BTw4u5K9uTwCPau8UL3UvQK8xTxoPdu8bDvwvSO7Az0WvZ4717xbvci8ZjxgvAi9OrzoOqG94j1LvDw9ejyzPOS95r0WvEK98TzxO5i9hL2SvUO9I722PZE97DvLuxU9QD0MvQ0+Fz0FvaY9kDzXPH28vT1Ku+c62rzNvMu7xz17vSA9xLxtPSG9ezskvDi9Y7yNvIG7tr0VPb88uT2CPeg8Mr3bPGm9pr3FvBC9LbuBO0o81LygvM09WLyhvbo9hj0svek8Xb2BPds8mz0XPPQ8k71uO4g8kb2JPG69qL3su5E9hjyvvRW9u724PPE8tjyMPdA9lbzpO449DL33vTk8ujxyvX+8fz3auwA8wLwAO708oryqvbI9rzy0Paa8YzytPEI9x7zvPBC95r2uu487Prw8PRO8rb2/PPW9qzyUPJ28Br2kPew8Sj3Tu8Q8g7tOPOg7NLzqvR49oTsWvQQ9UT24PLE8ojzJvbK83rzGPRC9iz3VPLY8rT0nPdy8lrsbPGM9TT1ZPFW9rD3IPWU94rugO2E9irzkvPw8aD3qvDm91z2tvJi9Aj2gvH68p714vci92Tz6vH68/r3TPWY9AL0MvdU9Ur20O6k9PL3FvBS90jx6vM48jDyyvG+96Tx7vYM8BbyyvBw9JrxpvSi8HzzxPJ698DzAvFe88bxGvQE9pD2HPdI8Jz3gPIo7dzyuvGY8xj1fveE9Uz1wvJm6+jsbPDW9ETwxvZM8RD0AvW290LyOPTY8XTrZO4I95Dsdu9E7GD0PPL+8oD36PRY9VbvyPJW98buEPea8Hz0Mvd888ruLOzw9v7yNvfq9ATyHvFE9KD1VPQa9VL0ZPdM8qTyRvTM9Lj3XPZi8kzr5vA2+pDwHO5W83DwBPfI9tLxrvDW9wzwRvvq9uj1IvJW8yryZPb+9njxYPSc7ND3tPXo8KL1JPZo81bhEPVY7D73pvRQ9PrxmPEI9Mz2fu769NT0yvcc9XT3AvXe9VL0WvfM7Ob26Pdq8hz2UPPQ9Tz3yvOo7Rz2QvU28cDscPWE8mDuyPRM9Gj1qPHA6cj1OPSE8ur2EPA69AjxBOyg9U72wPPy8w7zlPOW8Z73/PCM9iz1HvXC7yTwQPIm8Xz2DPX48lrx9vJm9tjy9PA+9Yz2BvcE9+7sBPTc8x70uPUK98zy3u8K9DDuPPfg7Bjy6PaM9hj04vUi9AbzDvQ49yzxsOwc9Wj26vEW9Q7zSvZg9Kz6xPXg9Nr3Nuz471byQvUQ9sDwWPa28grxEPYI8Ez2WPQ48NL0Dvbq8krvPu8w9kT0JvPS8eLx+O3475DyXvJg9lD2BPHY88zxqPQ89Iz0WPdm8sjvEujI9JL12PKY8V70LvMe8w702vYS9YDwIPXO8eD0XvY+9Zz0APpG8Kr1dPLk9bLzJPPQ8CT2xvBm9Nr1uPAi8Vbu1vVQ9gLwsvZg9vTvcu267oLzhvXS9uzwbO607db2lPHi8lj3ivIc8Eb3KvJy7Gb14vae8XzwEvVM9aLxZPdM8gr32PVk9pj04Pce8y7xMPIA9pj2tPEw90TxVPYW8tjynvb08N72qvYi8yLxBvIM9HbvjuxO7NL0CPAK9aD3/PAu7Br2ZvSC8Zby7vC29pr1rvUe94rzBvK47Fbp0vZ49wz0tvLY9JzylPau9HT0KPb09kr1DOpG8Gj1oPSs9I7yvvYy9yb1fvaM8IT3pOiS96r3uvDI9Vz2UvCE99byHvOA9ebuyu4G9ObuDPU087z0ovTA+pb3KvTO8a72DPV48Tj2GPUi9BL0qPAg9RLzEPfy8vj0APfS8Mb2zvfM8XDzzPCq8EL0OvFA9abtmPRI9TjxKPdm9lD2QPA283rywPd87SD2nPIW9zrwsvUy9ob30PBK+ET0rPRK9PT3euxW+wbwIPdY7lL3Guok7Cr0bvTu82DyfvOs7Jz1dPMw8xz0QPfK7DzwevZG8KT03Pds83rtsPce9nr2rPEc9Fr19PZm9OrxAvZg91DxmPQC9RTsZveI8nDxcPDy9BryWO0295LztO2i8WT3qvGO8pDxGvNe8rD3YO9y9iD3CPBO9EL5pPYi907t8PWS8XT3KPZS9jb20vWi9ajwXvaK9xTunvbM8vbw4PZm9yDx1vQK9LT3/PGC7KTxJPU+9mL3OPKE8SDuPPb+8url1PI49t7yQvW6827xJPGU9Djz6PIo8iTzXPEa9UbsJPP+8vzxvPdI7pTxiPUK9UL3lPJ67qT3qvE69sz2vvCa72bzlvJQ9Prs9PWQ8AL3rOzS9Jbt7vGC7fL0jPaa8Qbz3vLY9IL0ePDy8nT3CPBs9kr3AvJ09ULuOvMY80jpkPcO9p7tavIu6aDwTPYu8Tbskvfi6tL0gPec8uL0kPOS8d73MPb+8D7wIvTq9IL3yuja9l7ykvLq9KDyQvKM9GD2IvKo9hj2bvVE8eT3cvBa8z7yFvD297TuXPYu9brwDPKA9QL2Fu7S9ez0tPYS8ED3lOxY9zL3KPVY9ob1ZvK+8YT0TvUE9j7pevYM80z2ZPCO8LT1IvBg9E72juZq9QTzpu8K9Ez3DvJY9Dz1cvAM9w72SvAA9qbwju6C9ZLvUvBO9/jyautK9OT27vA+9IbsYPVk98rwJvDc98zzgvRK9jrrSOyS9070BPZW9Cz2VvcE8/bxtO827Br2wu+y9Xr0XvTA8qr2lOAs+Q72MPHg9YT2wvem8qbwJuA69rD3zvMy8QLwDPXu9Pr2OPVy8BDyQPA08Gb1XvF69oT2dvUQ9kjzFvTS6S7wRvU09pz06PW09rD2HvYo9ETwzPJI9bb10PSc897qKPUk9gTyauzu8qbxzPdm8P7wLPC68FbwMvYk89zy5PAg9Uzu1PCe8NTxJvVK98D0HPQk9nDz3OjO9p7wuPWa8TTxsvSm9lLyVvdy80Dxgvbm8bj20vXw8KL0OPc08hjw/vZ67Jb05PQg8tz2qPFG92jyUPGM94ry5uv09nLsau8k7XT31vHi8sT3nvA69X71gPRU95bzbvCu73bwavhU9qDuoPI+9l70qPGE8/Lymvfk7xzofPWm87rzpOxW9mD0QPSc9yjtGOz09tLz9vAI8bD2IvQs9vj0BPcE9mD3Svd89Y7y+vC29MT2uuxG9rTu9PDw9Sj1vPCC8Hz21PKS7ar14PHg9YDylvYM9mzyJPXe7mD0cPCA+rL0+PQK9TT0Ou7+8TL3Xu/C8GL2SO0e9lT0UvTu9q73ZvKK8GTsMPQ078T14PZ29Lz3UPYw9ET0zPTa8uTyevPU9HL3HPTw9wDyuvHq7Lb3nvCC9AD25Og69A75sPIa9rr2AvY49Lju5POo7CztJPWi8kjx4vR49xryUvUW9qTyePOq92bzOO2W8CbzXPMc9tTyLvMs9pz3hvO48BL2jvJQ7DTzmPMy8m734PLy8kj2/ux+9hDwkO4S9yLr6PNm8zrw8PKO9fby8Ows9Xz2hPQ+9vLxuPUe9P71UPdC9rrt/PFC9DzzzutI9jb0DvXi9yLqqu5U9YjykvPa8+r03PLI9hLwZvZY82D0yPPk6w7ykPQ+8Hz3aPVU627yRvQ68ezw8vbC8HTswPZq9Eb3ivN085zwWPbe9Iz0EvUG9hz3gvVG8Pb2LvCO7Lj2UOwm9+rt8PRi++LpmvQm9Zr2TvI49ozocvNc82TuJPT89k72hPQU9tr1iPV09Jz2EvQm9G71CvJU9Oz3ovM68grwOvYG8Mz2JO488kT2GvI69WT0iPb+85zyHvHI9fLoLvTc9sb32um+747wMvWe86z2RvZM9Fry7PRy69zx2Ok49NL12PIM9qb1PPba9gjw7PAA9SjwfvVC6M72JvFW7i72avM+8KT1APZE8i72DPKu8/LyQPWm9d7w3PYY7HjzCPaO8nryoPbO9Mb4quyU8Jj2ovNG8WDzmPPk8nbz+vGc8or09PVO9DD3APQE+ljywu5I8ML01PU+94zxFPFK9GD0fPPQ7GLxcvck9Fj0hPeW8Qr3avZa9HD0ivSA9irs6PR69Ib3WPRi9hz0Gu1Y8wbwSPeu8arxEvUu8ZbytOA09dj1yvX+9Gb2nvGI9aj2zPSS9Rj3ivNc8ujzmPMo8vzwJvRC9Bz2cvUc9GLyUvbS9STpnvbI8gj0SPI68O7x+PYA947yHvLo8fLssvd08DDeBPWm8Rbz2PME8EzscvVa9nL3qvYo80j0lvQA+JT3zPGe9gz1kPYI9/z2NveU8tL3dvLM8QLyKuvA8M73zvAS9mr0DPQU8+TxBPdM88LvsvYw8CbwQPsi9Tr29vFE827yzPd88vDzmPV693j2RPLU86TwqvZY8FL1IPLm9uTw9PZI8Yb2yvfM8NL2aO9U9izyGOk+91zsfvV08nr0WvhO9tDxFPZk9Cb3Gu7O8eD3xvUc9Uj0SPnI9Oj2+Oac8HL2kvBm8er1ivAo9p7wJPcy7ADyFvXg9JL2jPMk8Cj0tva+8bz37PAs9PzuKPck9jjyMvbS9LjxyPSu9uj3hO6g99zzQPXC8tL0wvRm9RLxnvTo9Lz10OyA9nb2yPfW83zvRO428nj2kPBG9i7tIvdg7Lb1BvZg7iz1FO/C8Yb1oPes8yz2KvOO8AT0wvVi8brzzPCS9HzzcvUa8j71OPCm8hLvtO8A9rD2qvC28kL1tvbo9PzydPeo8S7zFO/q8hLzJvZy9DD2CPFW9zDyjvRQ9uLzNPZG9nz2yvc68RbyiPfK8uz2iPDs8mD2xPLg8Jr0JPXW91LyovQc9ij1CvSG8fT0xvDS6grw5PIa9qLkrPW+9j72hO+s8Hbz/vI49sL3mPCo9jj1NvZ+8BrxUvao8ez3VPDS92Tw4PO68KDzhPZi86b1DvVq9xTwwvIU9FL4mPY88KTzyPJO82DyfPBc9TzqOPFY9hb2DvN68jj2mPBg6AzuNPa68+rsHO1U8OzvpvCW9DL2ZPLA8RT3dvAe9rzuFvcK8ejw3PH88pby2Pf28Ojx6PJQ8gj23vJM8KD2yPUc9hr3sPPW8xzuMuoW87bzAPHO9Jz0dPWc8qTwuvfk5pLwlvMo9mT0uPTU7VD2KPLa8TL31PIS9Vj2uvJO9Lbt1vc68mj12vNg9lj2lPO45F76IPSq8gjxePJi847yHvXi9Fbz/PDK7mj1UvQK9jTsJPDq9QT0RPGU7DT3mPJw9Br5XPEI8JTxnPU677r13PV69CL29PIs8GjnPvCc9mL0yPdu8FzyhPcw9MT1gPPA8TL3QPLC7lz07Peg8Ur21PEo9+zsNvUK9wbwxPKo85byqOw09WD2uuwq9pT1yvBQ8E73qPGg9yDsgPU29zz3xvJ47Zr2mO6G8xjyyPTu8Tjw+vbi9ybx0PTY8mDyMPWM7jr2Rvew87j0lvIs8Yr0uPKa9njzlPJO8wT0MvDu9tDu2uwi9LbztPOq9Ez0UPZQ83r0nPTw9Pjw9vd+8EL4pvYa8tTy4PTi7t71svH+91z2KPag8bb22PZU9HrxpPQ28xbz0vDW7Xj0vuvm5ojybPS49Mz5vuWS8TLygvCQ98zuKPTG9mT0zvGU9azoCPD49lz0OOyo9dz3tvQO9vLuCvFm8bj29PdI8aT2EvFE9ML36vCo8Wzygux29ibytvQy9b73BvZE9BDzsvHA88bxbvB+8YL1wPCQ7f73PPVQ9qr2FvEW8gb17O4c6+Lwxvau9ILzqO6Q8v7zEvEM9PzzXvH29kr0jPcK9y72OPcQ8CLtQvXw9HL0LvSe9qr2wvK09vru1PYe8jL0iPZk8nr20PGK9vbxEvP28Nr2rPDO90TxHvaw9yzuovPC8Sjyevcm8YL3Evfm71D2ZvMS81Ts0PM49kzx4O/872zuePQk9LL15vd+9l73PPJY92rxTveK6Db23PI28j7tCPTg99bwjvHY8CD6TvEQ9hT2YPXO98byMvUe83b09vTs9n70wvZ49+rzNPNI8l71+vTU9or0QvQ29Tz19vSo8SrzjvQi8ljyIO5+8eL2VPIK9IjvMPIY9Gjz8OM89lTyJvLc9gz1kvSO9+zygvA48EL16PUG9UTwSPGS9i71VPBM+MD0BvQW8a705PaC7gr3jPB892bzRPGc9Gj0dPTE9hT3CPBW87TrovOI6Vz0CvZ+89Ly/O8g8k72LPCO8n70JvZS8NTuTvDI9LDxavca8Eb0FvY09ZLyiPdi6CT1YPQc9pD0wPTm8JL2fvM08Vz2IPGI8hT3EPI29/jyjuyO8rzyRPfW82z3QPCG9FL2jvZS8Fj1UPbm7tryAO4G9JLyGu0u81D1XvKY5hr0/vVK9Nr3pO0M9J73Cvbi8xz1ePUO9gDzzu+I8kzzWvcs8tTyhvWc9Oj6xPR69jbtGPUu9krydvFi8o7xtO9+9pjvvPM888TyqvTu9NbylPEw8Vj0svQ29vzzBPY+7I7ymu1S83TzEO629wbyavU29VDyGPaw9yb0VvXo8ITstPXC7UD1qPI69h7ooPjY9PbqvPJa8qD1hPdy8C708PWQ8tTvHvAC9wju6u7w8q7w0PRA9orwFvam9rLwsvDM9DT0fPbg8Oj0JPlu9qLslvGs9YL1uPZ68N70GvMw7or3mPBc817yavUC7jT2dvXg9tLsSPZ+9G70fvS09eb0kPI29BT3hvEA9az20PLQ98jsxvS09Eb2EPKk9Uj05ve48nTwAPF+9trzYvIs91T1mPZ69gLswPe07Bz3OPHO9azm3u5u9/7yJPWS8aDyRvU29y7xqu6C9tL2tvDY9lrtIPY+9zzzau7e9G73JPHe9yTyYPV486DsGPZI9ATyTvZw9Nrx+PXS84ju7POu9DL1RPCS9QrpFPVC9Gz2nPBg8dj0zPe888LtsPSQ7BTzaPJC9HDu6vLM8LL2IvbW7m7wOvLS9gT3JvDg8jj0QPSg9DTydPbM96LyQPWQ9RLpLPdo7bT3uPEe9p7zFPTK8Sz03vHU8fTzuPCw6qD1aO8w7XT3KvEO8Vj2bPQM91zxbvYq7kj0Bu+49PjwKvEs8ULytvWm8mz22Op+8SLr1vHm6hbwjPXG9nbuWPGe9EDy0vKU8Mr39Ori9fj0gvYs9XzygvFy9j701vDE8Sr2/PB+9/DwmvVI88bxqvZC8NrzWO8u8yr2dvZI8/jwWvbc9Ir2GPSk8xD0ZPYe9vb3fPIm6rz09PBs9AT2OvLc9urrjPJa9ojwavL28TL0dPVe8srwaPGO927wAPbQ9rLx+PIE9lj1jPHU9B70bPcG8E70Pvh+7gD2Bu0y9Yj1VPcg9Hj2GPUK9ODycvVG9hr0OPZI8Hz2sug29yjxFPdu8GTzcu0M8kjycvfe8Az2gPF69eL2AvFC8Aj0xPVe77bzzvD271Dz+PNW7Iz3kvCk90rxGPao8Yz33vG69qbzyN828ezxxPN+9Uz1zvLS9pb0+PZ08m72WvaK8bL1NPYY9Lb2QvTK9mjzovNy8dTzPvJS8RL08PZK8CLxPvBo99DujvRM9krwzPYa8J7uzPQU9G73jPI+8Rr3MvZw8IzxIvLm8mT0jPRO8fL1evT89kjo1vTS8IT1PvV09nTzPPXa9uT2DPBG9TD2MPQU8YTzovL6817x3PLY6a70ounE8Kr2bPBC9lr3XPQK8BTz5PMs8Zz2HPJK8HL2vvVS9Sb29PAU+wr1fPVu8mrvHO3u9KjuPPZQ9orxOPBS9EL00vRy9Fr1OPac9STyNPd+8Cb1TPZM8jL2+PWi8fbxBvDO9sbziO+W7CrzoPcg9Jb1oPba9Dr2kvO48BD1cvJ+7+zwnvYk6+jzTvI89uzwPPS48xjx4vbG9kbyQPcQ7c7yeO/u8bL0LvsS80b0OvcS8f7mTvUs8Fb2YPQe9bjs/vUm9eb0KPrA8ET2qvfg8hj1avbE7Sz3CvNc70jy4vFQ9NDwbPdI9wzyhvcG7Qr1VPYi9JD2gPaU88D1oPWW5RzyQOmC8cb2APE09oL1UvC88ZbuKPZS8sDydvem8X70PPWm8mT0dPZw8srz8PMe7d734PLc6Wz1qvRk9XjtmvKe86L3aPII8Pj2OO9I82jwmvBo8s7wZPUK9dD1ZvHe95zuKvDG9CL0ZvUI9dbxPvD49nz0DPO68Cj33vIU9K70+u8W8UzwbvdK8x73SPJu95LzWPUq92btOvUm9UL3TPT69/LztPD29aT1/PQO9CjyWvb68+byFOjW837vUvQs81js5PNo8qzyqPOE8Gz0kPR28KT2BPa48wL22PUG8Qz0YvPw9Cj4Xu0Q9ZTy3O7K9CDswvZw98z02PYI66bxsvHc8mTrZPJ680jx6vIk93j2RvXe9Az3XPFO84jwxvaw9Sz27vEI8Pr1vPfA99bx/vJW80rzOPQW9Ar5pPJm9o7wtvGG9YjxavFi9fb16vCu7Ur5PvOE90D1+vVG9BT1/vJY9aj0YvQi9Eb3ytou9aDsTvQE8nz2TPQG8cLwPPtc8aL13PMe7T73VPZq9lr3MPbK6ETydvBC9+jzLPDy99zy+vAI9VL0fPYS94zxKPZu9VDyfO0S9yLxIPau8IDwfvag8Lz00vUK9tD0bvW29nTztPKo8jb2Uu4k8iT1ZPW69R7zbu4Q8GTzFPH29gb1APOu9nLxBPR+9Oj0JPem7YLxKOww9TDyIvPM8RL0XPWi88DkPvHG9D72CvMq9w7ySPbG5CD0QvSi94bvQPJQ9Kb1qPXg9Ib0BvQU9FL38u708tL33PNI8hj2+vc29XTxUPIo9PLxyOYM7PL3nvI69BDyjPSe9+LzkvJ89DD3quom93rztvLA8IrwqPXG7QLkVPN47mL2VvOC9mbxVvLK9Aj05PaK8grt+vbY8Q7osu5i7MjzMPL09lj05PSi9zLtmPFG9vzubPR49GL0xvT+8XrwcO1I97DrPu9q837tMu007jLz0vEi8Ebw0vUU8cL2Ousq8Ur3ZPXO9/bwTPHE9GrwLvXU9lrm3OJ27c7y1vEK8Nz2VvSw8CTybvL08Lb0rPWE9MDurvUa9Qzy7PF49ND2uOy68Oz1Nvdc8KT2RvC49M72SPSo9kz0qPdm8hr0JvJ47Jz2LPB67Jz3QvIA9ojzOvSG9Hr0APU87xLyYOxY9R7wMPGK9ZTz2uz29arzUPZE9kL35vHu9u73BPSw9+7yjPGo8uDyRvQi7pLyEPQo8Sb2TvXe86zu7u+I8aL3iuwS7VrxUvfS8NLygvf89+rwkvQg9cL2PveU6nr2nvfe85Ly1vSu8iD19PbU7jzyhvTi9xzz0Pdm9ir1XvcA9kjw8PKI9sr2wu9S9gj1avAG9Uz2rvdE9nb1IPd+8pDvouw69ujwnPOS9Db3EPbg9tLw+PH28hrsyvCO8kz3UvDW93TxJva+8QTxWvTm97T2zvRq9AzwQvZs8kT14PK09db0UO5670LyRPEQ9iL2HvZ+9i71Nvao7RTs2vIi83zycPVM7Dr2Iu6c9hz0Cvrk8ET12PSg9Mb0eO4k8A70BPfI8G71xPUs93jwuvdk8vbxfvZM83DxHPTm9gL0+PUw9rrz7vIw9J72iPEO9QrxgvLM73zuZPck8wrxgO728ST2fvWU8hj0bPdm7Pr2CPQ==", "ffn_branch_bf16": "Y761PKu+yL1vPD68BD/6vTM+jD5PvZI+fz1rvji+eT7IvRa//T6XPci9ZL7yvpY+hD72vSC+uz69PaS9DT0rvn2+uDmuvrE9Db8Yv/Y9BT4Lvcg+qb0lPsu97b2Svbc9ib2GvlW+nz1uvna+V70cPny+Bz6XPGy9eDxHvgy9wTttvim+9L4ZPp++zT5dPjE+gz4EP6S9uD2EPre+Xj53PuY8Sj5Dvum9vT2Kvg8+wrvhvVs+kb1qPrQ9Hr9vPiI+MD45vl28Sj03vj0+0z0SPss93LxqvgK+p72WOxA/qb5Kvg0+tb5DvBA+H77tvsC+jLtYvtY9Lj46vbQ9ET6rPoO9ZD2cPlK+E74lvTe+tL0ZPoO+8D7dPDo7Wz0BP0y+nr72vuC9cr2uveo8mT2BPui+Bz3FvYy8arxyvT8+Yz6lPpM9QT0SPhK+U74svgC+2Tw3PRA+Ub7yvoG+Ab87vve9OD6+PnS+z73MvbW9hr6qPnO8Tb7Svqu8IT0FPcI++T29vae+SDymvLA+970XPsU9Q71kO/A+HD4bvgO+CT09vTC+m71VPqI+1jrNvu++xTy+Pio+xb2wPdS9sT1tvr08Ib5Rvjs+s75Jvn0+ST6GPou+K77jva49mz6ivom+l7wePkW+/zyQviy8oL4avY886L2evH8+tb6lPQo9Ob5dPgI9LT2KPgK9jD5yvNW97r1DPh6+q72UPgk9j77pPfo96j3rPDO+iz4lvTo+er5VPkc+Uz32vjM+jr6CPga8p7ysPei88T4Ev1k+Uz3NPqg9Mz5iPIM9aL6NPiC+Qb6dPX0+mD7qvhS96b3qPby9Wz6gPkY+hz4mvrE9CT78vZE+dL6Xvjw9B73wPQA+T70BPjG+hLuVvbQ9WT5tPZo9uT2Evf490D3QPc08Rz1lPUm+xTxfvhC+gj7svh0/Ez5qPb8+qryAvZc+s76IPq69zL4vPaG7mj0Jvg+/qTwfvv09sj2zvls+4b7Tvai+Lz5CPr09mD7avbA+/70Wvg8+gD6/Pn49yz4FPQg+3b1BPvC+rb0yPqG9DL7IvBC+kj2jvvs8iD3jPB0+gT1zPmG+2b0JPn++8T29PRK9jT7Jvf+9nj73Pl0+bL46vks+Cr5rPmQ+4D2FvQm+uT4JvW6+oT5HPoM9wrzPvQS9Hr6Mvp8+dj7yPVS+NT5fPeu+Ar5Rvhu+WL6vvYQ+973iPTK+ODyvPrq9ij7gPei7+D2SPq+95j6cvgU6ib5oPhG+Nb7iPCS+hL23PcU+wj7jPYY+2D76vj893T3UvXE957xBPpu+br4hPeS9x74NPgm+gb5zvv48i70WPXG9aj2kv1m+Fz6APY0+P76NvoO+hD45viq9jLxwPtm9pj6uvrm95T0cv3S+Ur+tvWe9yj3mOsw9iD1jvpg+tT6jPi6+Qr6mvIg+Wb7dvBq+bT6aviu+qzxLPr6+Hz2BvlQ+lTz3PWi+r76Mvh8+ZjqzPmY+yrwzvgS+CD7IvYu+8ryqvbe+Aj/VvIw+776NPAw/C72Pvqo+Hj7ePdU8cD78vBq+nj2yvUA+ML5hvg4+PL67PpK+kj6QvtU93j3wvEg+qT0EPls+tz31PTm+0jwivha8kb3VvdS+2TzMPjs/6b4wPoQ+ar1nPiY9Mz4RvrC9IL1xvdy+bz0pPT674r21vlq+Kj67vc6+lT4pvQq+4rwPPVw+a71uPkA97b4YvqM9BT3ZPjK+m72nPuY72z4SPki8CL60Phu+Mb5zvtU9CL76vg2+jb4Nvi6+k74TPuG94j0IvWu+oD6rvgo+EL57vDO+GT3XPX0+Fj6uvTa+3D4ePxy+ED7RPiW+yDomvsU+kL6NvdS+izwQPgm+Xb77PY8+oD3pvV094jsaPpK+tr2+vSW+AL7IvmA+jT6rvvw9UL3lvPM8o73vPG6+jT7vvPI9Ib0cvUC+Oj4TPrq9wb7ePYa9Lj46vo49N77KvXc82L1LvoS98j5PPS4+bb22vcK+ij3vvXo+hL5uva8+lr2UvTa+9z5hPkO+JDr3Puu9FL5/PXg+0D6Tvqo+u71FPkE+J7xQPmQ9RD5FvZk9Cb42Pg6+kL5WPr67c7zWPQo+mD2DPSa+vL2VPZK9TT7UOwQ/Tz1VvoC+tD6Pvhe+OD5bPiO+XrwavUk+XLz2PtS+ez3Dvtk+8L2FPso+a74RPo4+5T6BvYc+YD7gvkK88b1cvpg+Jb7QPLa+P71wPY8+5T64Ptu+ib4fPom+NLzjvfe9gT60Pts9h74cvpc9iT5vvZs++zwqviO+uj5KPi++Ib7Ivum+6z2NPKa+Jr4Evvc+i76lvbS9AD4YPo++Zb6cPrk+7Ly0vXu8OL3APvi9QTz6Pve9OD5dPnm9Pr40PPu9hD7PvCO+j76tPZQ9Db5rPvm9ST5Nvnc9o74FPvo8gT3ovZa+kr7aPcC+vTzAPHe+LL1iPts9cr6tPiy+Cz+bPsg8mr5avVg+8b33vSC+BD7+u6U+VD5RPVW+HD3UPl2++b11vjA+oD2yPZQ+SDzmvQO+HT5jPhS8o7zqvOc9v70APi29oLwCPno+wz5GPRQ+Bz9lPS0+qL6bvqS+Sr25PIc+MD00vgc+Vb1JvQ2+X76tvmu+0r5QPiI+lrwLPSa+kL55O2Q+pb7BvYO+B79PvQK/2T46vos9FT5Hviu9jD59vic+1T0svlM+hruFvZ2+Ob6KPjI+kT66PVA+Ab+zvii+lD5nPs++Nj2FvRe+mb3WPms+EDqBO5i9lz54vg2+njz3vYe+pb5zvlI9JL4EPVq9AT4ovD++TT1EPoy9LT4gO+q8XT2fPda97D01v86+pb10Paa9CL6Dvkq9iL0jPm68XjxYvoO8gb5NPds9jLxHPi8+YrwdPiC+Ib64Poi+R77aPbO8NL84Phc+lT1jvrm+gj0iv6s+ML6rPo29Bj6Hvp893jxNvii9Zj2mPpY+KD1FPkm+/D1YPZm8Sb14PqG+EL2XPpc7ub6NvuI8kD12vvu+A774vXy94L6QvoG+Xj7HPgu8xL1SvQA6Ub4qPkU+yD3oPhc+lL7aPAS+c74TvgW+o762Ptq9RT4HP0G+oj2mPEE+8D04PaM9Nz4fPqi+k7yhPXi+zT2rvpC9sb72vWc+pb5Ivja+I77QPVK+77wKvgw6Oz6dPuc+p76WPZa9Iz7vPQw+tT39Pio9Hz5Kuw4/bj7CvgK9+jwWPis+CL5wvgQ+5D1EPiM9D76CvG48CT5Lvle+ir3hvQI+5L6APaU+Xr7bPQU9GT30vSC9s75Yujw+G8BmPus+5r0rPZe+6r0bPKW9Qb4kvlC8372bPm88rL5IOw8+CL7MOg493b20PEG95z6tvWu9GT1nPiS+RD7evT2+z73ZPQ88Kb4gPZc+7b6UPUu9nz4tPpc9Jj7KPgI+0DpYvh8+ir6QPUY9E74QP6M9qL18vrW9y76Yvsg+2j4iPpE+PD6aPkc+qL7sPJg9Bb7pvSC+lT7IvR6+h773vS09W75Avr0+Bzw4PR4+KLyDPXs9Qz40vs69mjz9PaC+iD6avJm+uDvzPhW+SL4dveI92L4ovH6+J72/Pd4+lL0Cvr++jz2JPYG9sj06PqQ91b4wPhQ9ob2zvsO9tj6SvJA7PTxQP8q8fD6nveg9AjzLPtg9L71rPtI9RL4Hvo4+lr7+vb09uL6aPSI+i756Pag+Ub36vrQ+xT7xPRE+YT6lPqw+QL7VPok+yb12vio9aD5kvmu+Xr4QPj6+vD0kvvM7Rz7mvO++tL2fPrS+Lz2BPTE+nz3uvRg9JT7VvTe+Wjzfvg2+ED8PPZG++Lv7PEs9676lPpQ+zr0ZvgE7TL2xPgE+LT4zvgc+ED6dPdK+zb2Tvqa+Vr42vU++sb5YPQi+a72XPq294D2wvgM/Dr+iPYm9pL25PB0+972ivQM/Rj4EP6G+2L3aPbU6Aj6gvqy97b4XvpQ+IL7pvjG+aj5kPqc+Zb7bvkY+bD25PqC8qr7ivRa9SL6Pvb288b05PQw+Jj3JP22+JD6tPgY+hT64voA8vL0mvtQ+s757PtY+fD1mvp89gb0Fvpy9wb5FPjq+Az4FPk6+Cj+gvas+8z31PZi9BL0RvSS+8j0Cvwe+RT6/PYw+hj6TvQQ+R76YPZa+Wr6svYa8ED70vSO9iD7KvCO8xb5QPga+ob30Pa8+Ar/IvQ4+RL6tPtw+pj2zPls+JD6XvrM+Nr6RPo++QD6jvk2+n76OvqC+lj49Pbg9nz1Yvso9rz2QvZy9VT0hPkC+CrzMvXC+kj2GvTU/p725Pv48Q76SPhk96LyKOxQ+hz7YPqo82Tuoukg8Fr0rv4u+HT6tPp++Gb6FPIs+MT4KPui+hb7kPe28db1Gvii65b0WvZS9xDwcPhq+oz5zvoI9Ij7XPp+9RD7nvKS9lr4ePtu8M743PW69Lz6YvrI+4L5aPJ6+0L5MPau+6b0uPri+GL3mvfc9oz42Pp4+Hr6UPVE9lD3sPQY9br3PPYq+hz2fvEO+Vz4HvaG9FL9CPAg9Kj7BPZO+fL7NPlM9Tz4kPoy9Sr4EvJy+9TuhvQ8+5r7Nveo9ij62Pk09iL6mPZs+2z3fPQe+tb4SPmi+zD2xPmW+Ij4OPsK9cT15vvk9Pb0+vsU9dr5kvn49xr0xPqW+Bz8Svga/mz5bvQG/Fr6Gvpg8eb6HPVo8Mb6XPYI9xbzOO6W8A7+kPcG9iT67vVy/2rwDPmS+Uz5FvSW9Ir69vXI9t76VPoy9drvVPly++LwqPlM+BLyPPo+9tL4YvSS+9b1Svhm+PzyaPcc9FD6KPoe8kr3/PUO+nTytPkg+fL4kvcC8Bz5Mvq8+h76xPtC9iz0VvsA9uz0Mvd09lT6APqs+Vz6jvHW+Bb5uvuq8GD4uvBA+Zj4RPu69fj6OPrw8LD5SPc2+mj2zPqu+P74+PX6+kr0/vhi+arwevjk/WD3LvXi+ab3AvYY+e70PPUI+yz42v3S9dj6WvIC+zb7lPQ6+fz4yvkK+T75TPeQ8rD2MPoo+6L1zu1o+pb7vvue+Tz7rPYU+1zxDPvQ9Ar+PPkI+2r0UPvY+vT4FvoS+i75Kvf879r0BPWG+JL57vrQ9gj3DvVU+ub1+vlg+Tj6MPu49cb04voM+Yr5VvlK8oD64viW+gL7LvtA+BD6LveA7J76bPnQ9t7wCPoq9XD6RvkM+kL6UOrq9Zz0ePdC+yj47vno+TDz8vdy8CT1bPi8+qT34PWa8lD2dvpy9Mr6fPVq+Tz2uPBE/PT7zPdU9bb5CPmi+uD5UPgm9/r1OPvO9jb5/Pj4+gj7rvbo+k76LvZ29PL6wvii9jT63vTi+Ar7lvgM9+r3YPUS+Kz6HPUg9q774PcG91bx8vV+74L1pvYI+xb4yPVw7t72AvAG8c767Pgk+Cb6QPbQ+9r0TvvQ9FD4Gvre9tDwTPTc+EL5zvqo9kT1dvqy+Wrwevu0+ab5dvFQ+NT4DPjq9Bj7QvVo+/b1tPny9CL2IPl29HL0vvZ49Lj7uPa6+2z7LPX0+Xr6mPsG9272CvRq8YT6WPgA/Qr0CPiu+CL2JvoY8JD/+vjO+6r65PYs+iD7ePIa+Bj4qP3m+Ab6sPly9o77JPgw+QL7XvgE+j724PbY+Bj/zvc09Ajp8vt68Dr7APoQ+cLzgvQy+Br6cPrA8Yb51Pt29Kb7hvIk+kj6uvhK+Fj6Tvr2+FTyEvoM+8r21vjY+lz6YvfI+5j2zvSa+2r6gPoS+yb2oPuc+Hjo3PWQ+Lb7SPCI9Xr1uPZC+qj3gPqE9bj5XPo2+L75JPdQ+Bj41va29y724vvk8Ar6hPoE+7b2PvZy+bb7ku5k+XD6rvpK+s72BPj+91j4QPqu+T76DPnG+YL7wPig+tz4SPes+tL6ovZs+mj2PPbq+i7xsvtA6LD4UPpm+Pr2CPmG+kD01PmA+DL4Ovru+X723vsO+Fj6TPow+hj7NPZC8RL4GP9S9kT6kvrO8Hj7DvvI+Er4+PZS+mz6RPiq8HL6jvsq9Nb7UPUm+Tb0MPMQ7Ir7ZPoy+Rz6BviO+VL4Mvhs/Ur4UPoI8FD7MPQ++G76oviK8fD4gPgi9asBmOgg/gT4yvvy9KzyBPgQ8kL4ZPp09kzzIPR0+nr70Pfa9I726PGC+9j2+vv67P76Ovn696z1ovmw+pr6TviG9Ub6BvoA9ODr9Ply7Sj5CPCk+7z14PX6+Nz6dPt28RLsUvg6+Vr4SPm09rb6bPda8or7PPZi+DL4ovdg7hr70u7k+Ab1Ivjk+cb3oPGK+7r3iPbu9kz6SO4U+i77EvfI8Y7xgvic+Sz2mPcC+Sz05PVQ+GL6CvDO+gj2qvqG95TyZPXq95r57vqK+5r6Avkw++D6WvZM+1TxRvTc+jr46PkY9Zj7QPc69YD0fvuE9ub6evTY+g732veG8xD1TvjO+gr6KPie+2DycPie+qL3LPRu+dDxavIe+qj4DP7s+nb7avm27k77gPoM+Jz4aPlW+T75RPlW+FT6MvZ6+uz4wvr+8Bj6kvtO9Pj59vCA+br4WPRC+rT7pPsY9Zr4Uvku9Oz7BvAY+Oz5kPUE9Tr5aPbI9CT4wPpe9Ar+pPsa9yr5pPpa+6L3EPio+7D6Avqc7i74sPSS+Cr6BvP88ub0hvs2+CT4lPue9s7yfPZU+qb0PPvI+Ib6bPvQ9Tj5ePiK+sT05vlm+Kz7YPYo+Pz5CPl0+A7+wPPK85z0zvsi9XbtkvY47yz6hPp8+kb4bvl6+/j2GPY89uLvxveU9kT6Nvty9iz6NPQA/Z76yvq4+1T1+PpA+ND+8PSA+bz5XPkK+djzSPaO+aTwaPs6+Pb4vPao8FD5DPsU+fz0hvlO9LL0Lvvk+TT6LPjc+Ej7Ivuw8w75MPA++CL+nu1k9L75VPb894L58PRU9Fz8bvbM8gD2vPsM9ZD6OvdQ9fL4FPpG+rD3OPUM96j28vYe+1z0Zvoa9Pb0zPV6+1T0wPvg41j3YPrq9ej27PQy+LL2VPpA+8jytvcC+V73UvRG+bD6FvcO8MD5avqW9pL6CvQI+b74SPho+nb7IPAg+YL7jPfK9kztFPLQ8sT6VvmC+8T3gOl48EL4FPz4+KDzYPcU9Az+WPaI7yL53PS4+ar3KPvm+GL7JPBE+3z4HPsM+aj4uPrK+8b1WPk69PT6TvFK8DT5Rvd894T3DPaK+DL44Pkm+GLtavbG+ej5BPeU9X72+PgY/170oPsq+FT5pPuC9TL4OvsY+TL3uPV0+Yz0svsO9OD22PQU9Ar0FPm88nD4XPYQ+pb43vqU+Ej6hPkq+Rj0MPqK9yT7qvpY+2L31u6y+g77ePNy8iD4Rvik+uL17Pja+Br5gvqa9Ir5hPju+5r21vjE+gj4Nvie+gj6ivVW+gj0nvoc+RD4dPuA7i74OPp4+OL6XvlW+wr3uvWm+UT6wPYS+P76ZvhS+1b2ZvlY+tL1rvoa9tb0Svq89HL2IvM66GT47vps98T4SPbW+3T68vi++iz2RvuS9Qb6wvfy9iDyAuKk9fDy1vTW+db7Runo+sT77PV69Ej4fPjG+uj4ePkS+lDxivs2+8r2vvqY+pr0KPjG8zT7fva49Rj7jvSQ+n71tPiu+4r3aPhy9lb24PhE+vj3cPrE8+L1IvLc9gj1Nvoi+Qj57vrq9fD7VPT29qT2hvIg+srynPRy+8r6sPSS/ULy5Pq0+qj5+vBi9jz5HPs29oTyJPpK+crwtPu6+RD2LvVA+hr7fu4i+Er3XPjW+Y72yvo89Nr2vuwg+rb2zvay9Db4Vvoy+lL0pvdE9ED6iPVY+rT47vt69gb0/PlU+kz7Evi+/gb4qvCI9Jz6MPgy9wb3YPSQ+nT62vAk/wz6WPe89vD1iPoQ+yD7DPTA+az52vXq+2L5UPkO+lz7fOxI+172HvRc+t74Wvrw+ir5svkK+ar5HPsM+mz00vla+sL04PqE+CT7TPWC+aj4BPdS+XT6pvWK9Yr3wPjK+HT7UPgk+mz7sPeA8Ur5EPck8KT2zvaq7Wj0KPok+iz6+PTC8hD2NPlg+6T5Avji+2r3UPiU9mD78PCc+tT7lPVu+tDpLvna94726PXo+67yAvp68Gb8Qvio+3j1xvok+Bb5Ovmi9Xb6GPVY9ury0vRs9oj2yvV2+JT5avla+Dz66PkA+6j3dPN29mj6pPQ0+oT4Fvia8ibzAvjK+YjxKPpi7az3tvGS9S74uvY+8Fz7DPcE+NjnQPsa+Jj5JPS6+gz0sPim8hL7evCe9vz0nvkY8ID4kPG09vrzfPjY7yz6GvDo+zL1ZvRS+Gb64PjQ+Jr59PU29zjvRPju+vT7GPdU+er3QPfa+Vr7QvkW+rz6FvA8+Gj7Qvlm+zj6NPfg9gr7iPp2+ez4qvs89BT6NvsK+cj6cPeK9gLwwvkA9uj6mPbe9fL2svg8+zzzovaI8hLyAPbO9X7zRPde8uL4hvpK+HT9wvX6+7z3OvVq+mD5RvSo+4z7rPfu+nb0uPb89j71avW8+EL6ePUq+Uj5ovuE8Ub2PvgG+cT4pPfw9kj72PBE9iTu3PW283b1mPbE9xD7AvdQ+ljz6vbE8lb4kPmQ9gL0Gv1++0b0Dv4w+OT6jvrc7br77PU09570OvoK8tb1JPsI9L77ivaK+Ab6/va6++jwLPlc9DD6HPni+zT2lvQ09P77JPTU8mT4yvg6+kz1WPB8+Rj0cPdw+yr5dPfs9+D1NvvK+Y7x4Poo8vD3yvlu9cL58Pqs94b3oPvW9WT3uPQM7aD4hPXK+PD5gvQK9Tr6EPn29Rj40PUw82byhvhE/8r3rvEc9Cz4QvhU+bj2IPMU8Vb1hPai+kD2hvXm9Yj52Pj++VT7rPbU+kb45PRO+gL1sPgg+TD7kPQm+tD69vey+i72Qvrq7Ej5/PsS9Oj6aPh++8L7rPZa+Kb4ivqK+RT7Hvh2/h745vEo+bz2iviE/TT6qPZy9GL+LvpM+wD2KPog9676yvnG+cr4Lvum9wD5HvYa+zz5IPKG9Lz4wPKa+Xr1EO1s+mr6Yvts+8j0TvhG+/TwIPj4+kb3evoq9EL4lvrK9cD0BPpE9o70vPsA+MLx/PiK+172bvq09oL12vno+hL6/vdU+ij6Ovpg9Kz0+PqA8zz0ZPsa9JD9oPYq8OT6Vu7y9Lj7EPsq+Xz7jPQa+d71Fvsq8oD3EvJQ8TsBivZo9lj4SPmK+Fz7Tvdy+MD6rPeC9hL5CPsO9uj5sPoK84z0yvke8oz5Gvu49z72bvrC+4T7LvQE+tT2WPui+mr0oPlc+UT61PS+90b5mvq2+gD6hvtI9hb4vvjI+gruBvnQ+FL12vuY8Mj5aPci+WL4NP4Y8Pz4OvQE+C704Pdw9Ez6GPoM9ZL6xvBk+N7s5PsC+dD38PXI9mr3xvWi9dD5ovQG+fj5LPlI+jz0DvYg+db6OPpo+ID2ovQA/l7zTvQU+Zj5/PpI+qb4UvXq+9L1hvQs+UT7uvcG+L74Mvag+3j5Xvgi88r0Xvg69CL6HPR8+R74MPkY+Lb4aPne+pT7gvQM+tTz2Pcc9zb1HPYq8hD5Qvv09Dj4MvYq+zT1BPgm/Rr0hPVa+nT72vhe+DT0QvWY+cL64vkC9CD6gPsi9pj6FvpS9kT62vRy/lT0pPjC9lb4Wvga+KT42vuE9gj6cvY6+C761PpC9O746Plm+AT2uvT8+lry0vVQ+Nr2nPAY9Yr6tPra+l74ZPkC+rryEPhW+trsqPoO+trwMPh0+xj5VPkm9r70KPgW+FrwvvW6+ubvhvpq8Sr4ovYC9Bj3yPUM9Jj0gvvA8ub7yvZ6967ySPXQ+cb6zPT8+LTwlPve+mb6Xvnq9ub40voO+Dr7hvf2+bbvIPnw+7D0EvZy8UL4fvvk9db13PRu+qzyRPZo9sb5GvT+9Rb3YPci8lL6Wves82j3YPoc+Az57vpW+b75cvRW+oL2TPf49lD5TPdE9VT62vv8+aLwxvvC8ob2LPQi8uTwTPgi+Cz0/vmI8HT65Pi4+Iz5Qvf29kD66PMA97z3tPdU9gL3SPlc+yz1Ivsw+qL6FP4C9uj5qPr29sb2pPDU+lL69PH++bT1ivLg+sz46vkS8ET6avu29Xb0LPu49k70gPmm+Qb20vi6+kr6tvac+Sj6TPbY+Ab5pvq08Jz2OvV0+o74JPro+Fz2uvIg+mb7QvTg+wD0cPdW+JL50PQE+r75hvmi9mL50vjK+jr4BP1g+izw0v3A+BD52OoM9fb5pPtQ9ob2wO529r73TvTs+MT6SPpC8cb5Vvtu9Nj6wvqi+5r0rPtw+Tr0LvuQ9Az6DvhY+ND7SvLs9MD4/vu+9ab1Ivlo+2L3QPnS+hj7EvVY+6LmYPs49QT6NvtG9dT2qvYI+aL5Jvu496j3avNs9Oj2hvlY9fD2lvsG9P75rvQu+iEC+vFo+XT2QvrK+s74svpU+RL7zPcS9rD65vsW9fj2uvd89tb69vqU+FD+JvOW9Sj6HvcG+B72gvYs+bD5dPtK+773nvsa+RT7SPRM+fj49vRe+CL0QPgo8yL0sPrk9oz05Pji8MD5mu2K+Sb7JvlC+HD4/vpE+Xj2wPrm+/T3RvlU+zz2GPkE9gbx9PWo9Cr5TPoA+bj5GPdK9Rj7EvVs+br11PYQ8xL6CPYm+Az2evqE+rr6lvWy9lD0PvqM+8b2IvlO+kLsvPjW+SD5qPZ6+HD4UPpA9rj1wvYo9EL4Uvk0+1T1fvh0+8byNvE88wD5Vvqo9rz7mvl+9+D4SPAi+Gz6wvQS97TzjPtQ9tz4bvhe8Wz0LPF+9uL5iPsi+xT2VvWW+Fr4xPjq+E76FPba9iL64vUO+P75uvY28Gb77PWs+1L7mPIW+nD4APtO87j77PTA+Ur5+Pog5Lj3VvRa+xLuIPve93b6lPWI+7j6pPYq8cr5aPra887zfPRY+hD52Pl48Wj4BPoq+xz0NPS0+uLyYPgC8ej46Ppk+jT7nPRK+lT0APeU9gz29vZe+cT5JPYA9/j2YPtq7Bz5YvJS9tj6+vo4+G76Bvtk+qb11PZQ+5r40vmC9Gj5IPlg8Nb2tPao+aD/rvIU+cz6xvnC9Eb6oPAy+nT0ivt08RT4qPgA+br2jvjo+oD0YvrW9jTymvvy81T7Svoa+R76zvL+9QD0kPZu8wL1HvQG+8D2MPVs+Eb1GvbM9f76QPSA+Ar7NvvK+DD3TPR8+Bj+oPoI9BL70vjC8e74pPQS+hD1Pvhy80D1CPjG8PT4UPhQ/hz76PQk8Yb0gvt29C74ovgq/kT0WPhw9zb6bPYM+HL5AvmI+jT0zPpI+Qz4mPpE+jz6AvQe+Yj5APne+oz2cPne9zr2jvIi+pjwxvmi8RD40PiU9NbuAvZ6+s7wBPWI8yrw3vom9nr3NvRi+Nj5CvRM+Qj0Pvmq+Gb5QvpC+uryOPcC+dD4bvsE+Q74sPxY9Vz0oPv0+b778PXA81b5dvq2++D0aPn08Oj6bPpa+vr6cPqw90b3KvQC+Gj2gPkq+HL4IPsG7Fb7LvdK92D2RPVw9Pz3gvlK9YD0KvWS+170/Pqs9OD0FPoW+pT0KPsK+rb3bvHO+Jj6UPbg8GL41v8Y+bT7/PN2+1by3Phk+d76VPZo+BL5NvqI9Db7aPU2+/j05vkY9Cj+tPvu9lb2LPZc9Jb09Po89lr6/PCK9Fr54vQM/Sz/ovQu/j76Ivtc+6LyNPYW+Xj0vPvU+oL2RPje9x75XPiS/Vj3LPUg6fz3kPQO+mT7ivrs+h74nvVY+Zj2ivrS7PLw1Psi+sT6WO2k++D2Tu8g9y70GPos+C743v8O+UT6EvE0+hz5tvQS/pT72PPU+lL6ZPk49Vbu6vso8nb4oPqc9Tj7XPgI+C77HPYc9QLzBPR0+Pj0Xvm2+WbzTvTG9WryRPh89IL4TPR2+kD6Jvq+8xLokPte8Gb+MviO+Pb8APm+9Dj7uvuq+MT6hvZQ9ez73vZu++L1GPgW+Wr2evpK+uL5gvf+9Pb6uPb++hzyiPRW81D3YOZk9/rsMvFm+Y7yyPCY99L3WviI+R72SvqK+CT/NvO+9UD4yvne+ib4Svky+9735vTA+nT4/PcI7L779PJq+pTsGvma+pT6HPkw+ir6+PS4+oD7vvEs+eD1fPmy9lz0pvkG/2L0qO/m9XT3QPt87qr2zPl2+XT7XvhK+Zz6vvjw+yr3TPi0+aD6Qvvs9CT4JvU49gz5JPpc+1b7ivWW8kD7PvLe+zL23vYY+zL5yvn6+J72bPlc7gb5BvtY9dz7tPP29PD2OvlY90r16vhQ9uT5GPJO95L0Qvn6+Ar4hPu69CDwTPg++Tb6bPUk+3T0svjE+Eb7tPa89Pz5lvHS+Fj7RvEU8qrwEPuM9gzyHPfw8ML2WPu0+gzwtPgm+i74mvWe9Dz6IPkg9Rb2TvlK+vj6EvYI+V76nviu+xL3fPYs+Cjt8vo49DT5Cvke+sr7JPiW+qLwxPhQ+KT6MPgW+yzy2vky95byCPuq+KT4Ev7q9zz6evDY+BT5EPuy7qD5RvjW+Tb6Cvb49hzu3vE89XD6Cvm2+cD5Hvoi+hT6XvXE+TT6KPpW+2DyqPsW9UD6JPns9fr4vvoG+Bz9SPj69k76kviy9gT5uvnS+xT7ePSU+1T04vmI+PD7Cvlo+KT1Lvuq+Jr3CPg6+vb2wvRG84j20Pqc9jT7wvLC+rr6WPQY8ob2MvSo+pj64PoS+2r04Pde8Bj1zvY0+ij1uvRc+mL0SPlA9N74Hvsw9tb1sPkQ+E74Uvge97D18Pfg9az07Prm9ND2cvb48lb7UPcY9FL9xvS2+qL7jPVg+4zxGPg8+bj6dPnY8DD7Hvpi+iL1HvZM9hb4sPqU+M75XvXm9M73RPW8+Oz+Ovf+9FD20Omg93r2Qvm6+5z5rvqK+nLwLPdQ9gD40Pwk9a76iPcO+zL5SPqs+kL5yvFe+cz7Ovpo9mD53vru8pj5+Pgo+qj4LPtI++L2nPcw9hL5uPUW9OD3evZM+G74DvsO+Rz0/vpO92D4GvT++Ojw2PFG+ij5Jvc69hL6Dvka+AbsgvjI+Kb6Uve295D1BPUa9vr1Pvri+qr0Fv8I+ED3ZPQi/n77KPfA9+r2Vvc69xD1kPni+cD7PPR8+67wxPqo+mb61vTK7j70LPku93L0avQ0+ez5TPRE83j2UPQe/hL5Nvr28AD2cvTg+SL62vqQ8kT3oul++tD63vo49ar7AveM+4b5vvQq+HL58vRO+pr6FPhy/BT4qPAu+Bz5Ivo4+Db3HPoe8rz27vZU+Vj6wvvA+BT6Hva6+Mb6xvSC+9r0qvkG+zL2cvtM9or6Pvp29eb2KvR2+iz0lvPq+hbyDPkO+Oj5VPh8+cz3APnO+0j4oP7M8nb6cvbi+qr6Tvce7t77DPZY+Ir4kPqm+GrxiPDs9tb44PlM+7r2xPrO9Uz4bPhI9rj4WPXi9A76lvrs+Ar9Uvru8A7wYvhE98r1hvk++/7wTPF+9wD5wvcE9Qr6Cvpu9DT7dPqO9q722vP69hT2pPY6+Rj6uvFs+tT4BPmq+QT6Xvh4+kb0Gvko9NT6VvSs9BL5vPjq+8D3qPpA+6b4qPiy+7b7PvkS+wr6Avg2+3j7hPv+9I7z2vfM9Er1MO9e9I728PlA+9T13Pg==", "ffn_scale_f32": "Bhz3PJHrlTwcmx89", "ffn_base_f32": "BPIhQBPEM8C59XLA1KI8wOgPZL/y+Wq/EUWpv5Qsgr8WokZAcqJHwNkEW8AT7C/AhaNTwFOsIEAZ8zLAl2YewLxjMMBQY1DAtu03QIyWSMDG8zDArWIiwK8lfMBUUCNA", "provenance": "Retained mHC stage inputs, no expected outputs. eight_rows_bf16: the eight teacher token embeddings (4x5120 bf16 rows) from ordinary-rms-eight-embeddings-fixture.json. ffn_stream_bf16/ffn_branch_bf16/ffn_scale_f32/ffn_base_f32: token-1 layer FFN hyperconnection stream, branch and real checkpoint hc_ffn scale/base from hc-residual-token1-ffn-proof.json. Base64 payloads are little-endian BF16 or FP32. SHA256: eight=6942a581c6b9d84d stream=9409aee30d5836ff branch=74d00910644e0bd8"} \ No newline at end of file diff --git a/zig/tests/deepseek_v41_reference/math.zig b/zig/tests/deepseek_v41_reference/math.zig index a6dccb2b93..da7bbf89ec 100644 --- a/zig/tests/deepseek_v41_reference/math.zig +++ b/zig/tests/deepseek_v41_reference/math.zig @@ -120,25 +120,89 @@ pub fn rms(x: []const u16, weight: Tensor, eps: f32, out: []u16) void { pub fn sigmoid(x: f32) f32 { return 1 / (1 + @exp(-x)); } -pub const Mix = struct { pre: [4]f32, post: [4]f32, comb: [16]f32 }; -pub fn mixes(x: []const u16, hc: Hc, c: Config) Mix { +/// Element type widened to the reference precision; FP32 is the source contract, FP64 the ideal. +pub fn wide(comptime T: type, v: f32) T { + return if (T == f32) v else @as(f64, v); +} + +/// Round an FP64 value once to BF16 (round-to-nearest-even) with no intermediate FP32 rounding. +pub fn b64(x: f64) u16 { + const bits: u64 = @bitCast(x); + const sign: u16 = @intCast((bits >> 63) << 15); + const magnitude = bits & 0x7fffffffffffffff; + if (magnitude > 0x7ff0000000000000) return sign | 0x7fc0; // NaN keeps the quiet bit. + if (magnitude >= 0x7ff0000000000000) return sign | 0x7f80; + if (magnitude == 0) return sign; + const exponent: i32 = @intCast(magnitude >> 52); + if (exponent == 0) return sign; // FP64 subnormals sit far below any BF16 subnormal. + const unbiased: i32 = exponent - 1023; + const significand: u64 = (magnitude & 0x000fffffffffffff) | (1 << 52); + if (unbiased > 127) return sign | 0x7f80; + if (unbiased >= -126) { + // Normal BF16 keeps seven fraction bits; RNE the remaining 45. + var kept = significand >> 45; + const round_bit: u64 = (significand >> 44) & 1; + const sticky: u64 = significand & ((@as(u64, 1) << 44) - 1); + if (round_bit == 1 and (sticky != 0 or (kept & 1) == 1)) kept += 1; + if (kept == (1 << 8)) return sign | (@as(u16, @intCast(unbiased + 128)) << 7); + return sign | (@as(u16, @intCast(unbiased + 127)) << 7) | @as(u16, @intCast(kept & 0x7f)); + } + // Subnormal BF16: value = significand * 2^(unbiased-52); representable steps of 2^-133. + const drop: i32 = -81 - unbiased; + if (drop >= 54) return sign; // Below half of the smallest subnormal, including the 2^-134 tie. + var k: u64 = significand >> @intCast(drop); + const round_bit: u64 = (significand >> @intCast(drop - 1)) & 1; + const sticky: u64 = significand & ((@as(u64, 1) << @intCast(drop - 1)) - 1); + if (round_bit == 1 and (sticky != 0 or (k & 1) == 1)) k += 1; + if (k >= (1 << 7)) return sign | (1 << 7); // Rounded up into the minimum normal. + return sign | @as(u16, @intCast(k)); +} + +pub fn MixOf(comptime T: type) type { + return struct { pre: [4]T, post: [4]T, comb: [16]T }; +} +pub const Mix = MixOf(f32); +/// The compensated dense dot keeps FP64 accumulation for both reference precisions. +pub fn denseOf(comptime T: type, t: Tensor, x: []const u16, out: []T) void { + for (out, 0..) |*y, row| { + var sum: f64 = 0; + for (x, 0..) |v, k| sum += @as(f64, f(v)) * value(t, row * x.len + k); + y.* = if (T == f32) @floatCast(sum) else sum; + } +} +fn logistic(comptime T: type, x: T) T { + return 1 / (1 + @exp(-x)); +} +/// The HC RMS statistic inverse: the source-contract FP32 projection for f32, the ideal FP64 for f64. +pub fn inverseOf(comptime T: type, x: []const u16, eps: f32) T { @setFloatMode(.strict); - var projected: [24]f32 = undefined; - dense(hc.projection, x, &projected); var squares: f64 = 0; for (x) |v| squares += @as(f64, f(v)) * f(v); - const mean = meanFromWide(squares, x.len); - const r = idealRsqrt(mean + c.rms_norm_eps); - var m: Mix = undefined; + if (T == f32) return idealRsqrt(meanFromWide(squares, x.len) + eps); + return 1 / @sqrt(squares / @as(f64, @floatFromInt(x.len)) + eps); +} +pub fn mixes(x: []const u16, hc: Hc, c: Config) Mix { + return mixesOf(f32, x, hc, c); +} +pub fn mixesOf(comptime T: type, x: []const u16, hc: Hc, c: Config) MixOf(T) { + var projected: [24]T = undefined; + denseOf(T, hc.projection, x, &projected); + const r = inverseOf(T, x, c.rms_norm_eps); + return splitOf(T, &projected, r, hc, c); +} +/// The affine, softmax and Sinkhorn coefficient stage at reference precision T. +pub fn splitOf(comptime T: type, projected: []const T, r: T, hc: Hc, c: Config) MixOf(T) { + @setFloatMode(.strict); + var m: MixOf(T) = undefined; for (0..4) |i| { - m.pre[i] = sigmoid(@mulAdd(f32, projected[i] * r, value(hc.scale, 0), value(hc.base, i))) + c.hc_eps; - m.post[i] = 2 * sigmoid(@mulAdd(f32, projected[i + 4] * r, value(hc.scale, 1), value(hc.base, i + 4))); + m.pre[i] = logistic(T, @mulAdd(T, projected[i] * r, wide(T, value(hc.scale, 0)), wide(T, value(hc.base, i)))) + wide(T, c.hc_eps); + m.post[i] = 2 * logistic(T, @mulAdd(T, projected[i + 4] * r, wide(T, value(hc.scale, 1)), wide(T, value(hc.base, i + 4)))); } for (0..4) |i| { - var high: f32 = -std.math.inf(f32); + var high: T = -std.math.inf(T); for (0..4) |j| { const at = i * 4 + j; - m.comb[at] = @mulAdd(f32, projected[at + 8] * r, value(hc.scale, 2), value(hc.base, at + 8)); + m.comb[at] = @mulAdd(T, projected[at + 8] * r, wide(T, value(hc.scale, 2)), wide(T, value(hc.base, at + 8))); high = @max(high, m.comb[at]); } for (0..4) |j| { @@ -147,45 +211,51 @@ pub fn mixes(x: []const u16, hc: Hc, c: Config) Mix { // TileLang 0.1.8 emits FP32 XOR reductions with offsets 2, then 1. const denominator = (m.comb[i * 4] + m.comb[i * 4 + 2]) + (m.comb[i * 4 + 1] + m.comb[i * 4 + 3]); - for (0..4) |j| m.comb[i * 4 + j] = m.comb[i * 4 + j] / denominator + c.hc_eps; + for (0..4) |j| m.comb[i * 4 + j] = m.comb[i * 4 + j] / denominator + wide(T, c.hc_eps); } for (0..c.hc_sinkhorn_iters) |iteration| { if (iteration != 0) for (0..4) |i| { const denominator = ((m.comb[i * 4] + m.comb[i * 4 + 2]) + - (m.comb[i * 4 + 1] + m.comb[i * 4 + 3])) + c.hc_eps; + (m.comb[i * 4 + 1] + m.comb[i * 4 + 3])) + wide(T, c.hc_eps); for (0..4) |j| m.comb[i * 4 + j] /= denominator; }; for (0..4) |j| { const denominator = ((m.comb[j] + m.comb[8 + j]) + - (m.comb[4 + j] + m.comb[12 + j])) + c.hc_eps; + (m.comb[4 + j] + m.comb[12 + j])) + wide(T, c.hc_eps); for (0..4) |i| m.comb[i * 4 + j] /= denominator; } } return m; } pub fn collapse(h: []const u16, pre: [4]f32, out: []u16) void { + collapseOf(f32, h, pre, out); +} +pub fn collapseOf(comptime T: type, h: []const u16, pre: [4]T, out: []u16) void { @setFloatMode(.strict); for (out, 0..) |*v, d| { - var sum: f32 = 0; + var sum: T = 0; for (0..4) |i| { - const product: f32 = pre[i] * f(h[i * out.len + d]); + const product: T = pre[i] * wide(T, f(h[i * out.len + d])); sum += product; } - v.* = b(sum); + v.* = if (T == f32) b(sum) else b64(sum); } } pub fn expand(h: []u16, x: []const u16, m: Mix) void { + expandOf(f32, h, x, m); +} +pub fn expandOf(comptime T: type, h: []u16, x: []const u16, m: MixOf(T)) void { @setFloatMode(.strict); for (0..x.len) |d| { - var old: [4]f32 = undefined; - for (0..4) |i| old[i] = f(h[i * x.len + d]); + var old: [4]T = undefined; + for (0..4) |i| old[i] = wide(T, f(h[i * x.len + d])); for (0..4) |i| { - var residual: f32 = 0; + var residual: T = 0; for (0..4) |j| { - const product: f32 = m.comb[j * 4 + i] * old[j]; + const product: T = m.comb[j * 4 + i] * old[j]; residual += product; } - h[i * x.len + d] = b(m.post[i] * f(x[d]) + residual); + h[i * x.len + d] = if (T == f32) b(m.post[i] * wide(T, f(x[d])) + residual) else b64(m.post[i] * wide(T, f(x[d])) + residual); } } } @@ -221,6 +291,22 @@ pub fn rope(x: []u16, position: usize, compressed: bool, inverse: bool, c: Confi } } +test "FP64 to BF16 rounds once and agrees with the FP32 boundary on FP32 values" { + // Every FP32 value rounds identically through either path. + const samples = [_]f32{ 0, 1, -1, 3, 0x1p-21, 0x1p-126, 3.4e38, 0.0371947, 26.5, 26.625, 1.4393715858459473 }; + for (samples) |v| try std.testing.expectEqual(b(v), b64(v)); + // Exact tie rounds to even mantissa. + try std.testing.expectEqual(@as(u16, 0x3f80), b64(1.00390625)); + try std.testing.expectEqual(@as(u16, 0x3f81), b64(1.00390625 + 0x1p-30)); + // The double-rounding trap: FP64 -> FP32 -> BF16 would keep 0x3f80 here. + try std.testing.expectEqual(@as(u16, 0x3f80), b(@floatCast(1.00390625 + 0x1p-30))); + // Subnormal BF16 range and the 2^-134 tie. + try std.testing.expectEqual(@as(u16, 0), b64(0x1p-134)); + try std.testing.expectEqual(@as(u16, 2), b64(1.5 * 0x1p-133)); + try std.testing.expectEqual(@as(u16, 0x7f80), b64(std.math.inf(f64))); + try std.testing.expectEqual(@as(u16, 0xff80), b64(-std.math.inf(f64))); +} + test "BF16 probability ties and FP4 ties retain source boundaries" { try std.testing.expectEqual(@as(u16, 0x3f80), b(@bitCast(@as(u32, 0x3f808000)))); try std.testing.expectEqual(@as(f32, 2), roundE2(2.5)); diff --git a/zig/tests/deepseek_v41_reference/operator_hc_fp64.zig b/zig/tests/deepseek_v41_reference/operator_hc_fp64.zig new file mode 100644 index 0000000000..c41381c32e --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_hc_fp64.zig @@ -0,0 +1,1095 @@ +//! FP64 ideal-reference isolation for the mHC stage: rounding level, Metal deltas and 1-ULP +//! input amplification. Receipt lines print with an "HC64" prefix. No tolerance is relaxed and +//! no source coefficient is injected: retained rows and real checkpoint scale/base are inputs. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const math = @import("math.zig"); + +const mhc = tf.deepseek_v41.mhc; +const ops = tf.deepseek_v41.ops; +const View = tf.deepseek_v41.attention.View; +const Tensor = tf.checkpoint_metal.Tensor; +const Hc = tf.deepseek_v41_weights.Hc; +const Config = tf.deepseek_v41_config.Config; + +const width: usize = 5120; +const copies: usize = 4; +const stream_width: usize = width * copies; +const mix_count: usize = 24; + +const released_config = + \\{"model_type":"deepseek_v41","quantization_config":{"quant_method":"exl3","version":"1"}, + \\"text_config":{"vocab_size":129280,"hidden_size":5120,"moe_intermediate_size":2304, + \\"num_hidden_layers":40,"num_attention_heads":64,"num_key_value_heads":1,"head_dim":512, + \\"qk_rope_head_dim":64,"q_lora_rank":1280,"o_lora_rank":1024,"o_groups":8, + \\"swiglu_limit":10,"rms_norm_eps":0.000001,"max_position_embeddings":1048576, + \\"rope_theta":10000,"rope_scaling":{"rope_type":"yarn","factor":16,"beta_fast":32, + \\"beta_slow":1,"original_max_position_embeddings":65536},"n_routed_experts":384, + \\"n_shared_experts":1,"num_experts_per_tok":6,"scoring_func":"sqrtsoftplus", + \\"topk_method":"noaux_tc","norm_topk_prob":true,"routed_scaling_factor":1.5, + \\"sliding_window":128,"compress_ratios":[0,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, + \\1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0],"compress_rope_theta":10000, + \\"kv_source_layer_ids":[2,8,14,20],"index_source_layer_ids":[2,8,14,20,24,28,32,36], + \\"index_n_heads":32,"index_head_dim":128,"index_topk":512,"candidate_source_layer_id":20, + \\"candidate_topk_blocks":2048,"candidate_block_size":8,"hc_mult":4,"hc_sinkhorn_iters":20, + \\"hc_eps":0.000001,"engram_layer_ids":[1,14],"engram_num_embeddings":[384006168,384016682], + \\"engram_max_ngram_size":4,"engram_vocab_size":16000000,"engram_n_heads":8,"engram_head_dim":256, + \\"engram_pad_token_id":2,"engram_compressed_vocab_size":99092,"num_nextn_predict_layers":3}} +; + +/// The 16 retained HC unary arguments (hc-exp-variants-input.json), including 1.4393715858459473. +const retained_exp_argument_bits = [_]u32{ + 3217026610, 1081519143, 1049174916, 1078264130, 1073892416, 1078760804, 1083574034, 1069038932, + 1043563522, 1083490981, 1087210444, 1094911558, 3195364771, 1082252091, 1096635047, 1091866698, +}; + +/// The documented controlling exponent input: Metal FP32 0x4086fa39, ideal 0x4086fa38 (1 ULP). +const controlling_exponent_input: f32 = 1.4393715858459473; + +const fixture_path = "../../src/families/deepseek_v41/fixtures/hc_fp64_rows.json"; + +const Fixture = struct { + eight_rows_bf16: []const u8, + ffn_stream_bf16: []const u8, + ffn_branch_bf16: []const u8, + ffn_scale_f32: []const u8, + ffn_base_f32: []const u8, + provenance: []const u8 = "", +}; + +fn decodeBase64(gpa: std.mem.Allocator, text: []const u8) ![]u8 { + const size = try std.base64.standard.Decoder.calcSizeForSlice(text); + const out = try gpa.alloc(u8, size); + errdefer gpa.free(out); + try std.base64.standard.Decoder.decode(out, text); + return out; +} + +fn decodeU16(gpa: std.mem.Allocator, text: []const u8) ![]u16 { + const bytes = try decodeBase64(gpa, text); + defer gpa.free(bytes); + const out = try gpa.alloc(u16, bytes.len / 2); + for (out, 0..) |*v, i| v.* = std.mem.readInt(u16, bytes[i * 2 ..][0..2], .little); + return out; +} + +fn decodeF32(gpa: std.mem.Allocator, text: []const u8) ![]f32 { + const bytes = try decodeBase64(gpa, text); + defer gpa.free(bytes); + const out = try gpa.alloc(f32, bytes.len / 4); + for (out, 0..) |*v, i| v.* = @bitCast(std.mem.readInt(u32, bytes[i * 4 ..][0..4], .little)); + return out; +} + +const Lcg = struct { + state: u64 = 0x853c49e6748fea9b, + fn next(s: *Lcg) u64 { + s.state = s.state *% 6364136223846793005 +% 1442695040888963407; + return s.state >> 11; + } + /// Uniform [-1, 1) with 24-bit granularity, exactly representable in FP32. + fn uniform(s: *Lcg) f32 { + const v: f64 = @as(f64, @floatFromInt(s.next() & 0xffffff)) / 16777216.0; + return @floatCast(v * 2 - 1); + } +}; + +fn nextUpF32(x: f32) f32 { + if (std.math.isNan(x) or std.math.isInf(x)) return x; + const bits: u32 = @bitCast(x); + if (bits == 0 or bits == 0x80000000) return @bitCast(@as(u32, 1)); + return @bitCast(if (bits >> 31 == 0) bits + 1 else bits - 1); +} + +/// The local FP32 spacing at the FP64 reference value: the ULP unit for kernel deltas. +fn ulp32At(v: f64) f64 { + const f: f32 = @floatCast(v); + if (std.math.isNan(f) or std.math.isInf(f)) return @abs(v); + const a = @abs(f); + if (a == 0) return 0x1p-149; + const up = nextUpF32(a); + if (std.math.isInf(up)) return @as(f64, a) - @as(f64, -nextUpF32(-a)); + return @as(f64, up) - @as(f64, a); +} + +const Delta = struct { + nrms: f64 = 0, + max_abs: f64 = 0, + max_ulp: f64 = 0, + worst: usize = 0, + count: usize = 0, +}; + +fn deltaF32VsF64(actual: []const f32, ref: []const f64) Delta { + var d: Delta = .{ .count = actual.len }; + var squared: f64 = 0; + var magnitude: f64 = 0; + for (actual, ref, 0..) |a, r, i| { + const diff = @as(f64, a) - r; + squared += diff * diff; + magnitude += r * r; + const ad = @abs(diff); + if (ad > d.max_abs) { + d.max_abs = ad; + d.worst = i; + } + const u = ad / ulp32At(r); + if (u > d.max_ulp) d.max_ulp = u; + } + d.nrms = if (magnitude == 0) @sqrt(squared) else @sqrt(squared / magnitude); + return d; +} + +const BfDelta = struct { + nrms: f64 = 0, + unequal_ideal: usize = 0, + unequal_f32_ref: usize = 0, + count: usize = 0, + worst: usize = 0, + worst_gap: f64 = 0, +}; + +/// BF16 outputs: decoded NRMS against the FP64 ideal cells plus ideal/F32-reference cell counts. +fn deltaBf16(actual: []const u16, ideal: []const u16, f32_ref: []const u16) BfDelta { + var d: BfDelta = .{ .count = actual.len }; + var squared: f64 = 0; + var magnitude: f64 = 0; + for (actual, ideal, f32_ref, 0..) |a, id, fr, i| { + const av: f64 = @floatCast(math.f(a)); + const iv: f64 = @floatCast(math.f(id)); + const diff = av - iv; + squared += diff * diff; + magnitude += iv * iv; + if (a != id) { + d.unequal_ideal += 1; + if (@abs(diff) > d.worst_gap) { + d.worst_gap = @abs(diff); + d.worst = i; + } + } + if (a != fr) d.unequal_f32_ref += 1; + } + d.nrms = if (magnitude == 0) @sqrt(squared) else @sqrt(squared / magnitude); + return d; +} + +fn StageRefOf(comptime T: type) type { + return struct { + inv: T, + proj: [mix_count]T, + pre: [4]T, + post: [4]T, + comb: [16]T, + collapsed: [width]u16, + y: [stream_width]u16, + }; +} + +fn referenceRow(comptime T: type, stream: []const u16, hc: Hc, cfg: Config, branch: []const u16, collapsed: *[width]u16, y: *[stream_width]u16) StageRefOf(T) { + var out: StageRefOf(T) = undefined; + out.inv = math.inverseOf(T, stream, cfg.text_config.rms_norm_eps); + math.denseOf(T, hc.projection, stream, &out.proj); + const mix = math.splitOf(T, &out.proj, out.inv, hc, cfg.text_config); + out.pre = mix.pre; + out.post = mix.post; + out.comb = mix.comb; + math.collapseOf(T, stream, mix.pre, collapsed); + var h: [stream_width]u16 = undefined; + @memcpy(&h, stream); + math.expandOf(T, &h, branch, mix); + @memcpy(y, &h); + return out; +} + +const Buffers = struct { + device: mtl.Device, + items: [24]mtl.Buffer = undefined, + count: usize = 0, + + fn get(b: *Buffers, comptime T: type, n: usize) !mtl.Buffer { + const x = try b.device.buffer(try std.math.mul(usize, @sizeOf(T), n), mtl.ResourceOptions.shared); + b.items[b.count] = x; + b.count += 1; + @memset(x.slice(T, n), 0); + return x; + } + fn deinit(b: *Buffers) void { + for (b.items[0..b.count]) |x| x.deinit(); + } +}; + +fn upload(comptime T: type, buffer: mtl.Buffer, data: []const T) void { + @memcpy(buffer.slice(T, data.len), data); +} + +fn view(buffer: mtl.Buffer) View { + return .{ .buffer = buffer }; +} + +fn runStage( + queue: mtl.Queue, + p: *const mhc.Pipelines, + o: *const ops.Pipelines, + rows: u32, + cfg: Config, + stream: View, + weight: View, + inv: View, + mixes: View, + scale: View, + base: View, + pre: View, + post: View, + comb: View, + collapsed: View, + branch: View, + y: View, +) !void { + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + mhc.encodeRms(e, p, stream, inv, .{ .rows = rows, .width = width, .stream_stride = stream_width, .eps = cfg.text_config.rms_norm_eps }); + ops.encodeDense(e, o, stream, weight, mixes, .{ .rows = rows, .k = stream_width, .n = mix_count, .x_stride = stream_width, .w_stride = stream_width, .y_stride = mix_count, .w_dtype = .f32, .y_dtype = .f32 }); + e.barrier(); + mhc.encodeSplit(e, p, mixes, inv, scale, base, pre, post, comb, .{ .rows = rows, .eps = cfg.text_config.hc_eps }); + e.barrier(); + mhc.encodeCollapse(e, p, stream, pre, collapsed, .{ .rows = rows, .width = width, .stream_stride = stream_width, .y_stride = width }); + e.barrier(); + mhc.encodePost(e, p, branch, stream, post, comb, y, .{ .rows = rows, .width = width, .branch_stride = width, .residual_stride = stream_width, .y_stride = stream_width }); + e.end(); + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.MhcFp64GpuFailure; +} + +const Case = struct { + name: []const u8, + rows: usize, + stream: []u16, + weight: []f32, + branch: []u16, + scale: []f32, + base: []f32, +}; + +const StageMetric = struct { + name: []const u8, + metal: Delta = .{}, + f32ref: Delta = .{}, + bf16: ?BfDelta = null, +}; + +fn printMetric(case_name: []const u8, m: StageMetric) void { + if (m.bf16) |bf| { + std.debug.print("HC64 {{\"case\":\"{s}\",\"stage\":\"{s}\",\"elements\":{d},\"nrms_metal_vs_ideal\":{d},\"bf16_unequal_metal_vs_ideal\":{d},\"bf16_unequal_f32ref_vs_ideal\":{d},\"worst_gap\":{d},\"worst_index\":{d}}}\n", .{ case_name, m.name, bf.count, bf.nrms, m.bf16.?.unequal_ideal, m.bf16.?.unequal_f32_ref, m.bf16.?.worst_gap, m.bf16.?.worst }); + } else { + std.debug.print("HC64 {{\"case\":\"{s}\",\"stage\":\"{s}\",\"elements\":{d},\"nrms_metal_vs_f64\":{d},\"max_ulp_metal_vs_f64\":{d:.4},\"max_abs_metal_vs_f64\":{d},\"worst_metal\":{d},\"nrms_f32_vs_f64\":{d},\"max_ulp_f32_vs_f64\":{d:.4}}}\n", .{ case_name, m.name, m.metal.count, m.metal.nrms, m.metal.max_ulp, m.metal.max_abs, m.metal.worst, m.f32ref.nrms, m.f32ref.max_ulp }); + } +} + +/// One full HC stage over `rows` streams: statistic, dense, split, collapse and post-expansion, +/// each side (Metal, FP32 source-contract reference, FP64 ideal reference) on identical inputs. +fn runCase( + gpa: std.mem.Allocator, + device: mtl.Device, + queue: mtl.Queue, + p: *const mhc.Pipelines, + o: *const ops.Pipelines, + cfg: Config, + case: Case, +) !void { + var b = Buffers{ .device = device }; + defer b.deinit(); + const rows: u32 = @intCast(case.rows); + const n = case.rows; + const stream = try b.get(u16, n * stream_width); + upload(u16, stream, case.stream); + const weight = try b.get(f32, mix_count * stream_width); + upload(f32, weight, case.weight); + const inv = try b.get(f32, n); + const mixes = try b.get(f32, n * mix_count); + const scale = try b.get(f32, 3); + upload(f32, scale, case.scale); + const base = try b.get(f32, mix_count); + upload(f32, base, case.base); + const pre = try b.get(f32, n * 4); + const post = try b.get(f32, n * 4); + const comb = try b.get(f32, n * 16); + const collapsed = try b.get(u16, n * width); + const branch = try b.get(u16, n * width); + upload(u16, branch, case.branch); + const y = try b.get(u16, n * stream_width); + try runStage(queue, p, o, rows, cfg, view(stream), view(weight), view(inv), view(mixes), view(scale), view(base), view(pre), view(post), view(comb), view(collapsed), view(branch), view(y)); + + const weight_tensor: Tensor = .{ .buffer = weight, .offset = 0, .bytes = mix_count * stream_width * 4, .dtype = .f32 }; + const scale_tensor: Tensor = .{ .buffer = scale, .offset = 0, .bytes = 12, .dtype = .f32 }; + const base_tensor: Tensor = .{ .buffer = base, .offset = 0, .bytes = mix_count * 4, .dtype = .f32 }; + const hc = Hc{ .projection = weight_tensor, .scale = scale_tensor, .base = base_tensor }; + + // Gathered arrays: Metal (m), FP32 source-contract reference (32), FP64 ideal (64). + const inv_m = try gpa.alloc(f32, n); + defer gpa.free(inv_m); + @memcpy(inv_m, inv.slice(f32, n)); + const proj_m = try gpa.alloc(f32, n * mix_count); + defer gpa.free(proj_m); + @memcpy(proj_m, mixes.slice(f32, n * mix_count)); + const pre_m = try gpa.alloc(f32, n * 4); + defer gpa.free(pre_m); + @memcpy(pre_m, pre.slice(f32, n * 4)); + const post_m = try gpa.alloc(f32, n * 4); + defer gpa.free(post_m); + @memcpy(post_m, post.slice(f32, n * 4)); + const comb_m = try gpa.alloc(f32, n * 16); + defer gpa.free(comb_m); + @memcpy(comb_m, comb.slice(f32, n * 16)); + const collapsed_m = try gpa.alloc(u16, n * width); + defer gpa.free(collapsed_m); + @memcpy(collapsed_m, collapsed.slice(u16, n * width)); + const y_m = try gpa.alloc(u16, n * stream_width); + defer gpa.free(y_m); + @memcpy(y_m, y.slice(u16, n * stream_width)); + + const inv_64 = try gpa.alloc(f64, n); + defer gpa.free(inv_64); + const proj_64 = try gpa.alloc(f64, n * mix_count); + defer gpa.free(proj_64); + const pre_64 = try gpa.alloc(f64, n * 4); + defer gpa.free(pre_64); + const post_64 = try gpa.alloc(f64, n * 4); + defer gpa.free(post_64); + const comb_64 = try gpa.alloc(f64, n * 16); + defer gpa.free(comb_64); + const collapsed_64 = try gpa.alloc(u16, n * width); + defer gpa.free(collapsed_64); + const y_64 = try gpa.alloc(u16, n * stream_width); + defer gpa.free(y_64); + + const inv_32 = try gpa.alloc(f32, n); + defer gpa.free(inv_32); + const proj_32 = try gpa.alloc(f32, n * mix_count); + defer gpa.free(proj_32); + const pre_32 = try gpa.alloc(f32, n * 4); + defer gpa.free(pre_32); + const post_32 = try gpa.alloc(f32, n * 4); + defer gpa.free(post_32); + const comb_32 = try gpa.alloc(f32, n * 16); + defer gpa.free(comb_32); + const collapsed_32 = try gpa.alloc(u16, n * width); + defer gpa.free(collapsed_32); + const y_32 = try gpa.alloc(u16, n * stream_width); + defer gpa.free(y_32); + + for (0..n) |r| { + const stream_row = case.stream[r * stream_width ..][0..stream_width]; + const branch_row = case.branch[r * width ..][0..width]; + var collapsed64: [width]u16 = undefined; + var y64: [stream_width]u16 = undefined; + const ref64 = referenceRow(f64, stream_row, hc, cfg, branch_row, &collapsed64, &y64); + var collapsed32: [width]u16 = undefined; + var y32: [stream_width]u16 = undefined; + const ref32 = referenceRow(f32, stream_row, hc, cfg, branch_row, &collapsed32, &y32); + inv_64[r] = ref64.inv; + inv_32[r] = ref32.inv; + @memcpy(proj_64[r * mix_count ..][0..mix_count], &ref64.proj); + @memcpy(proj_32[r * mix_count ..][0..mix_count], &ref32.proj); + @memcpy(pre_64[r * 4 ..][0..4], &ref64.pre); + @memcpy(pre_32[r * 4 ..][0..4], &ref32.pre); + @memcpy(post_64[r * 4 ..][0..4], &ref64.post); + @memcpy(post_32[r * 4 ..][0..4], &ref32.post); + @memcpy(comb_64[r * 16 ..][0..16], &ref64.comb); + @memcpy(comb_32[r * 16 ..][0..16], &ref32.comb); + @memcpy(collapsed_64[r * width ..][0..width], &collapsed64); + @memcpy(collapsed_32[r * width ..][0..width], &collapsed32); + @memcpy(y_64[r * stream_width ..][0..stream_width], &y64); + @memcpy(y_32[r * stream_width ..][0..stream_width], &y32); + } + + const metrics = [_]StageMetric{ + .{ .name = "inverse", .metal = deltaF32VsF64(inv_m, inv_64), .f32ref = deltaF32VsF64(inv_32, inv_64) }, + .{ .name = "projection", .metal = deltaF32VsF64(proj_m, proj_64), .f32ref = deltaF32VsF64(proj_32, proj_64) }, + .{ .name = "pre", .metal = deltaF32VsF64(pre_m, pre_64), .f32ref = deltaF32VsF64(pre_32, pre_64) }, + .{ .name = "post", .metal = deltaF32VsF64(post_m, post_64), .f32ref = deltaF32VsF64(post_32, post_64) }, + .{ .name = "comb", .metal = deltaF32VsF64(comb_m, comb_64), .f32ref = deltaF32VsF64(comb_32, comb_64) }, + .{ .name = "collapse", .bf16 = deltaBf16(collapsed_m, collapsed_64, collapsed_32) }, + .{ .name = "post_expand", .bf16 = deltaBf16(y_m, y_64, y_32) }, + }; + for (metrics) |m| printMetric(case.name, m); + + // Rounding-level gate: Metal must sit at the FP32 source-contract reference's own distance + // from the FP64 ideal (measured bounds; a real operator defect breaks these by orders). + for (metrics, 0..) |m, i| { + if (i < 5) { + try std.testing.expect(m.metal.nrms < 2e-7); + try std.testing.expect(m.f32ref.nrms < 2e-7); + } + } + try std.testing.expect(metrics[0].metal.max_ulp < 1.0); // statistic tree matches the wide sum here + try std.testing.expect(metrics[1].metal.max_ulp < 1.0); // compensated dense dot + try std.testing.expect(metrics[2].metal.max_ulp < 4.0); // sigmoid affine + exp boundary + try std.testing.expect(metrics[3].metal.max_ulp < 4.0); + // comb ULPs inflate on near-cancelled Sinkhorn cells; the absolute error stays at FP32 level. + try std.testing.expect(metrics[4].metal.max_abs < 5e-7); + try std.testing.expect(metrics[4].f32ref.nrms < 2e-7); + try std.testing.expect(metrics[5].bf16.?.unequal_ideal * 4096 <= metrics[5].bf16.?.count + 4096); + try std.testing.expect(metrics[6].bf16.?.unequal_ideal * 4096 <= metrics[6].bf16.?.count + 4096); + try std.testing.expect(metrics[5].bf16.?.unequal_f32_ref * 4096 <= metrics[5].bf16.?.count + 4096); + try std.testing.expect(metrics[6].bf16.?.unequal_f32_ref * 4096 <= metrics[6].bf16.?.count + 4096); +} + +fn syntheticWeight(gpa: std.mem.Allocator) ![]f32 { + var lcg = Lcg{}; + const w = try gpa.alloc(f32, mix_count * stream_width); + for (w) |*v| v.* = lcg.uniform() * 0.03; + return w; +} + +fn fourCopies(gpa: std.mem.Allocator, row: []const u16) ![]u16 { + var stream = try gpa.alloc(u16, stream_width); + for (0..copies) |c| @memcpy(stream[c * width ..][0..width], row); + return stream; +} + +test "mHC fp64 rounding level and Metal deltas on retained streams" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try mhc.Pipelines.load(device); + defer p.deinit(); + var o = try ops.Pipelines.load(device); + defer o.deinit(); + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const gpa = arena.allocator(); + const cfg = try Config.parse(gpa, released_config); + + const fixture = try std.json.parseFromSlice(Fixture, gpa, @embedFile(fixture_path), .{ .ignore_unknown_fields = true }); + const eight_rows = try decodeU16(gpa, fixture.value.eight_rows_bf16); + const ffn_stream = try decodeU16(gpa, fixture.value.ffn_stream_bf16); + const ffn_branch = try decodeU16(gpa, fixture.value.ffn_branch_bf16); + const ffn_scale = try decodeF32(gpa, fixture.value.ffn_scale_f32); + const ffn_base = try decodeF32(gpa, fixture.value.ffn_base_f32); + const token671 = eight_rows[1 * width ..][0..width]; // hc_attention_rms.json provenance: token 671. + + const weight = try syntheticWeight(gpa); + + // Token 671 embedding expanded to four copies; real FFN scale/base; synthetic branch-free case + // uses the retained branch row. All three sides consume identical inputs. + { + const stream = try fourCopies(gpa, token671); + const branch = try gpa.alloc(u16, width); + @memcpy(branch, ffn_branch); + try runCase(gpa, device, queue, &p, &o, cfg, .{ .name = "token671_attention", .rows = 1, .stream = stream, .weight = weight, .branch = branch, .scale = ffn_scale, .base = ffn_base }); + } + { + const branch = try gpa.alloc(u16, width); + @memcpy(branch, ffn_branch); + try runCase(gpa, device, queue, &p, &o, cfg, .{ .name = "token1_ffn", .rows = 1, .stream = ffn_stream, .weight = weight, .branch = branch, .scale = ffn_scale, .base = ffn_base }); + } + { + const rows = 8; + const stream = try gpa.alloc(u16, rows * stream_width); + for (0..rows) |r| { + const row = eight_rows[r * width ..][0..width]; + for (0..copies) |c| @memcpy(stream[r * stream_width + c * width ..][0..width], row); + } + const branch = try gpa.alloc(u16, rows * width); + for (0..rows) |r| @memcpy(branch[r * width ..][0..width], ffn_branch); + try runCase(gpa, device, queue, &p, &o, cfg, .{ .name = "eight_teacher_rows", .rows = rows, .stream = stream, .weight = weight, .branch = branch, .scale = ffn_scale, .base = ffn_base }); + } + // Wide dynamic range: copy magnitudes 2^24..2^-12 with rotating signs, branch 2^8..2^-4. + { + const stream = try gpa.alloc(u16, stream_width); + const exponents = [_]i32{ 24, 12, 0, -12 }; + for (0..copies) |c| { + for (0..width) |d| { + const sign: f32 = if ((d + c) % 2 == 0) 1 else -1; + stream[c * width + d] = math.b(sign * @as(f32, @floatCast(std.math.pow(f64, 2, @floatFromInt(exponents[c]))))); + } + } + const branch = try gpa.alloc(u16, width); + const branch_exponents = [_]i32{ 8, 4, 0, -4 }; + for (0..width) |d| { + const sign: f32 = if (d % 2 == 0) 1 else -1; + branch[d] = math.b(sign * @as(f32, @floatCast(std.math.pow(f64, 2, @floatFromInt(branch_exponents[d % 4]))))); + } + try runCase(gpa, device, queue, &p, &o, cfg, .{ .name = "wide_dynamic_range", .rows = 1, .stream = stream, .weight = weight, .branch = branch, .scale = ffn_scale, .base = ffn_base }); + } +} + +// The comptime parameterization must keep the FP32 source contract bitwise; this compares +// math.mixesOf(f32) against the pre-parameterization algorithm verbatim on a retained row. +test "mHC fp32 reference path stays bit-identical under the comptime parameterization" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const gpa = arena.allocator(); + const cfg = try Config.parse(gpa, released_config); + const fixture = try std.json.parseFromSlice(Fixture, gpa, @embedFile(fixture_path), .{ .ignore_unknown_fields = true }); + const eight_rows = try decodeU16(gpa, fixture.value.eight_rows_bf16); + const ffn_scale = try decodeF32(gpa, fixture.value.ffn_scale_f32); + const ffn_base = try decodeF32(gpa, fixture.value.ffn_base_f32); + const scale = try device.buffer(12, mtl.ResourceOptions.shared); + defer scale.deinit(); + upload(f32, scale, ffn_scale); + const base = try device.buffer(mix_count * 4, mtl.ResourceOptions.shared); + defer base.deinit(); + upload(f32, base, ffn_base); + var lcg = Lcg{}; + const weight = try device.buffer(mix_count * stream_width * 4, mtl.ResourceOptions.shared); + defer weight.deinit(); + const weight_host = try gpa.alloc(f32, mix_count * stream_width); + for (weight_host) |*v| v.* = lcg.uniform() * 0.03; + upload(f32, weight, weight_host); + const hc = Hc{ + .projection = .{ .buffer = weight, .offset = 0, .bytes = mix_count * stream_width * 4, .dtype = .f32 }, + .scale = .{ .buffer = scale, .offset = 0, .bytes = 12, .dtype = .f32 }, + .base = .{ .buffer = base, .offset = 0, .bytes = mix_count * 4, .dtype = .f32 }, + }; + for (0..8) |r| { + const row = eight_rows[r * width ..][0..width]; + var stream: [stream_width]u16 = undefined; + for (0..copies) |c| @memcpy(stream[c * width ..][0..width], row); + const parameterized = math.mixesOf(f32, &stream, hc, cfg.text_config); + const legacy = mixesLegacy(&stream, hc, cfg); + try std.testing.expect(std.meta.eql(parameterized, legacy)); + } +} + +fn mixesLegacy(x: []const u16, hc: Hc, c: Config) math.Mix { + @setFloatMode(.strict); + var projected: [24]f32 = undefined; + math.denseOf(f32, hc.projection, x, &projected); + var squares: f64 = 0; + for (x) |v| squares += @as(f64, math.f(v)) * math.f(v); + const mean = math.meanFromWide(squares, x.len); + const r = math.idealRsqrt(mean + c.text_config.rms_norm_eps); + var m: math.Mix = undefined; + for (0..4) |i| { + m.pre[i] = math.sigmoid(@mulAdd(f32, projected[i] * r, math.value(hc.scale, 0), math.value(hc.base, i))) + c.text_config.hc_eps; + m.post[i] = 2 * math.sigmoid(@mulAdd(f32, projected[i + 4] * r, math.value(hc.scale, 1), math.value(hc.base, i + 4))); + } + for (0..4) |i| { + var high: f32 = -std.math.inf(f32); + for (0..4) |j| { + const at = i * 4 + j; + m.comb[at] = @mulAdd(f32, projected[at + 8] * r, math.value(hc.scale, 2), math.value(hc.base, at + 8)); + high = @max(high, m.comb[at]); + } + for (0..4) |j| { + m.comb[i * 4 + j] = @exp(m.comb[i * 4 + j] - high); + } + const denominator = (m.comb[i * 4] + m.comb[i * 4 + 2]) + + (m.comb[i * 4 + 1] + m.comb[i * 4 + 3]); + for (0..4) |j| m.comb[i * 4 + j] = m.comb[i * 4 + j] / denominator + c.text_config.hc_eps; + } + for (0..c.text_config.hc_sinkhorn_iters) |iteration| { + if (iteration != 0) for (0..4) |i| { + const denominator = ((m.comb[i * 4] + m.comb[i * 4 + 2]) + + (m.comb[i * 4 + 1] + m.comb[i * 4 + 3])) + c.text_config.hc_eps; + for (0..4) |j| m.comb[i * 4 + j] /= denominator; + }; + for (0..4) |j| { + const denominator = ((m.comb[j] + m.comb[8 + j]) + + (m.comb[4 + j] + m.comb[12 + j])) + c.text_config.hc_eps; + for (0..4) |i| m.comb[i * 4 + j] /= denominator; + } + } + return m; +} + +// Direct split-stage isolation: identical projected mixes and inverse on both sides, so every +// delta is a unary (sigmoid/exp) or reduction boundary, not a projection difference. +test "mHC split unary boundaries against the FP64 ideal on retained arguments" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try mhc.Pipelines.load(device); + defer p.deinit(); + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const gpa = arena.allocator(); + const cfg = try Config.parse(gpa, released_config); + + const rows = 7 + 65; // retained arguments plus every integer exponent input -32..32 + var b = Buffers{ .device = device }; + defer b.deinit(); + const mixes_host = try gpa.alloc(f32, rows * mix_count); + @memset(mixes_host, 0); + // Rows 0..3: the 16 retained unary arguments rotated through pre, post and comb positions. + for (0..4) |i| { + for (0..4) |j| { + mixes_host[i * mix_count + j] = @bitCast(retained_exp_argument_bits[(4 * i + j) % 16]); + mixes_host[i * mix_count + 4 + j] = @bitCast(retained_exp_argument_bits[(4 * i + j) % 16]); + } + for (0..16) |q| mixes_host[i * mix_count + 8 + q] = @bitCast(retained_exp_argument_bits[(i + q) % 16]); + } + // Row 4: the documented controlling exponent input as a post argument (exp(+1.439...) path). + mixes_host[4 * mix_count + 5] = -controlling_exponent_input; + // Row 5: comb affine arguments spanning the exponent range. + mixes_host[5 * mix_count + 8] = -32; + mixes_host[5 * mix_count + 9] = -16; + mixes_host[5 * mix_count + 10] = 16; + mixes_host[5 * mix_count + 11] = 32; + // Row 6: the same span through the sigmoid pre/post arguments. + const span = [_]f32{ -32, -16, 16, 32 }; + for (span, 0..) |v, j| { + mixes_host[6 * mix_count + j] = v; + mixes_host[6 * mix_count + 4 + j] = v; + } + + // Rows 7..69: one integer exponent input per row at a pre, post and comb position each. + for (0..65) |i| { + const r = 7 + i; + const v: f32 = @floatFromInt(@as(i32, @intCast(i)) - 32); + mixes_host[r * mix_count] = v; + mixes_host[r * mix_count + 4] = v; + mixes_host[r * mix_count + 8] = v; + } + + const mixes = try b.get(f32, rows * mix_count); + upload(f32, mixes, mixes_host); + const inv = try b.get(f32, rows); + @memset(inv.slice(f32, rows), 1.0); + const scale = try b.get(f32, 3); + @memset(scale.slice(f32, 3), 1.0); + const base = try b.get(f32, mix_count); + const pre = try b.get(f32, rows * 4); + const post = try b.get(f32, rows * 4); + const comb = try b.get(f32, rows * 16); + + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + mhc.encodeSplit(e, &p, view(mixes), view(inv), view(scale), view(base), view(pre), view(post), view(comb), .{ .rows = rows, .eps = cfg.text_config.hc_eps }); + e.end(); + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.MhcFp64GpuFailure; + + const scale_tensor: Tensor = .{ .buffer = scale, .offset = 0, .bytes = 12, .dtype = .f32 }; + const base_tensor: Tensor = .{ .buffer = base, .offset = 0, .bytes = mix_count * 4, .dtype = .f32 }; + const hc = Hc{ .projection = base_tensor, .scale = scale_tensor, .base = base_tensor }; + + var max_ulp_metal: f64 = 0; + var max_ulp_f32: f64 = 0; + var worst_name: []const u8 = ""; + const stages = [_]struct { name: []const u8, offset: usize, len: usize }{ + .{ .name = "pre", .offset = 0, .len = 4 }, + .{ .name = "post", .offset = 4, .len = 4 }, + .{ .name = "comb", .offset = 8, .len = 16 }, + }; + for (stages) |stage| { + for (0..rows) |r| { + const m: []const f32 = mixes_host[r * mix_count ..][0..mix_count]; + var m64: [mix_count]f64 = undefined; + for (m, 0..) |v, i| m64[i] = v; + const mix64 = math.splitOf(f64, &m64, 1.0, hc, cfg.text_config); + const mix32 = math.splitOf(f32, m, 1.0, hc, cfg.text_config); + const src64: []const f64 = if (std.mem.eql(u8, stage.name, "pre")) &mix64.pre else if (std.mem.eql(u8, stage.name, "post")) &mix64.post else &mix64.comb; + const src32: []const f32 = if (std.mem.eql(u8, stage.name, "pre")) &mix32.pre else if (std.mem.eql(u8, stage.name, "post")) &mix32.post else &mix32.comb; + const metal: []const f32 = if (std.mem.eql(u8, stage.name, "pre")) pre.slice(f32, rows * 4)[r * 4 ..][0..4] else if (std.mem.eql(u8, stage.name, "post")) post.slice(f32, rows * 4)[r * 4 ..][0..4] else comb.slice(f32, rows * 16)[r * 16 ..][0..16]; + const d = deltaF32VsF64(metal[0..stage.len], src64[0..stage.len]); + const d32 = deltaF32VsF64(src32[0..stage.len], src64[0..stage.len]); + if (d.max_ulp > max_ulp_metal) { + max_ulp_metal = d.max_ulp; + worst_name = stage.name; + } + if (d32.max_ulp > max_ulp_f32) max_ulp_f32 = d32.max_ulp; + std.debug.print("HC64 {{\"case\":\"split_args\",\"stage\":\"{s}\",\"row\":{d},\"nrms_metal_vs_f64\":{d},\"max_ulp_metal_vs_f64\":{d:.4},\"max_ulp_f32_vs_f64\":{d:.4}}}\n", .{ stage.name, r, d.nrms, d.max_ulp, d32.max_ulp }); + } + } + std.debug.print("HC64 {{\"case\":\"split_args\",\"summary\":true,\"max_ulp_metal_vs_f64\":{d:.4},\"worst_stage\":\"{s}\",\"max_ulp_f32_vs_f64\":{d:.4}}}\n", .{ max_ulp_metal, worst_name, max_ulp_f32 }); + // The kernel may exceed the FP32 reference by at most the documented Metal exp slack (1 ULP). + // Absolute ULPs inflate on near-cancelled Sinkhorn cells; NRMS stays below 1e-7 everywhere. + try std.testing.expect(max_ulp_metal <= max_ulp_f32 + 2.0); + try std.testing.expect(max_ulp_metal < 48.0); +} + +// The retained cancellation fixtures: the kernel must keep the source FP32 accumulation +// contract exactly; the FP64 ideal column documents the by-design half-ULP divergence. +test "mHC collapse and post keep source FP32 accumulation under cancellation" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try mhc.Pipelines.load(device); + defer p.deinit(); + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const gpa = arena.allocator(); + const cfg = try Config.parse(gpa, released_config); + + var b = Buffers{ .device = device }; + defer b.deinit(); + // Collapse fixture: stream {3,3,0,0} (width 1), previous_pre {-1, 1+2^-23, 0, 0}. + const stream_a = try b.get(u16, 4); + const values_a = [_]u16{ math.b(3), math.b(3), math.b(0), math.b(0) }; + upload(u16, stream_a, &values_a); + const previous_a = try b.get(f32, 4); + const coeff = 1 + 0x1p-23; + const previous_values = [_]f32{ -1, coeff, 0, 0 }; + upload(f32, previous_a, &previous_values); + const collapsed = try b.get(u16, 1); + + // Post fixture: stream {1,1,1,1}, branch {1}, post {-1,0,0,0}, + // comb column 0 = {1, 2^-24, 2^-24, 0} (two half-ULP terms after 1.0). + const stream_b = try b.get(u16, 4); + const values_b = [_]u16{ math.b(1), math.b(1), math.b(1), math.b(1) }; + upload(u16, stream_b, &values_b); + const branch_b = try b.get(u16, 1); + const branch_values = [_]u16{math.b(1)}; + upload(u16, branch_b, &branch_values); + const post_b = try b.get(f32, 4); + const post_values = [_]f32{ -1, 0, 0, 0 }; + upload(f32, post_b, &post_values); + const comb_b = try b.get(f32, 16); + comb_b.slice(f32, 16)[0] = 1; + comb_b.slice(f32, 16)[4] = 0x1p-24; + comb_b.slice(f32, 16)[8] = 0x1p-24; + const y_b = try b.get(u16, 4); + + // Branch-product fixture: stream {3,0,0,0}, post {1+2^-23,...}, comb[0] = -1, branch {3}. + const stream_c = try b.get(u16, 4); + const values_c = [_]u16{ math.b(3), math.b(0), math.b(0), math.b(0) }; + upload(u16, stream_c, &values_c); + const branch_c = try b.get(u16, 1); + const branch_values_c = [_]u16{math.b(3)}; + upload(u16, branch_c, &branch_values_c); + const post_c = try b.get(f32, 4); + const post_values_c = [_]f32{ coeff, 0, 0, 0 }; + upload(f32, post_c, &post_values_c); + const comb_c = try b.get(f32, 16); + comb_c.slice(f32, 16)[0] = -1; + const y_c = try b.get(u16, 4); + + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + mhc.encodeCollapse(e, &p, view(stream_a), view(previous_a), view(collapsed), .{ .rows = 1, .width = 1, .stream_stride = 4, .y_stride = 1 }); + mhc.encodePost(e, &p, view(branch_b), view(stream_b), view(post_b), view(comb_b), view(y_b), .{ .rows = 1, .width = 1, .branch_stride = 1, .residual_stride = 4, .y_stride = 4 }); + mhc.encodePost(e, &p, view(branch_c), view(stream_c), view(post_c), view(comb_c), view(y_c), .{ .rows = 1, .width = 1, .branch_stride = 1, .residual_stride = 4, .y_stride = 4 }); + e.end(); + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.MhcFp64GpuFailure; + + // Source-contract expectations (math.zig retained boundaries). + try std.testing.expectEqual(@as(u16, 0x3500), collapsed.slice(u16, 1)[0]); + try std.testing.expectEqual(@as(u16, 0), y_b.slice(u16, 4)[0]); + try std.testing.expectEqual(@as(u16, 0x3500), y_c.slice(u16, 4)[0]); + + // Reference columns for the same fixtures. + var collapsed32: [1]u16 = undefined; + var collapsed64: [1]u16 = undefined; + math.collapseOf(f32, &values_a, .{ -1, coeff, 0, 0 }, &collapsed32); + math.collapseOf(f64, &values_a, .{ -1, @as(f64, coeff), 0, 0 }, &collapsed64); + std.debug.print("HC64 {{\"case\":\"cancellation\",\"fixture\":\"collapse_half_ulps\",\"metal\":\"0x{x}\",\"f32_ref\":\"0x{x}\",\"f64_ideal\":\"0x{x}\"}}\n", .{ collapsed.slice(u16, 1)[0], collapsed32[0], collapsed64[0] }); + try std.testing.expectEqual(collapsed32[0], collapsed.slice(u16, 1)[0]); + + const mix_b = math.MixOf(f32){ .pre = .{ 1, 0, 0, 0 }, .post = .{ -1, 0, 0, 0 }, .comb = .{ 1, 0, 0, 0, 0x1p-24, 0, 0, 0, 0x1p-24, 0, 0, 0, 0, 0, 0, 0 } }; + var y32: [4]u16 = undefined; + var h32 = values_b; + math.expandOf(f32, &h32, &branch_values, mix_b); + @memcpy(&y32, &h32); + const mix_b64 = math.MixOf(f64){ .pre = .{ 1, 0, 0, 0 }, .post = .{ -1, 0, 0, 0 }, .comb = .{ 1, 0, 0, 0, 0x1p-24, 0, 0, 0, 0x1p-24, 0, 0, 0, 0, 0, 0, 0 } }; + var h64 = values_b; + math.expandOf(f64, &h64, &branch_values, mix_b64); + std.debug.print("HC64 {{\"case\":\"cancellation\",\"fixture\":\"post_half_ulps\",\"metal\":\"0x{x}\",\"f32_ref\":\"0x{x}\",\"f64_ideal\":\"0x{x}\"}}\n", .{ y_b.slice(u16, 4)[0], y32[0], h64[0] }); + try std.testing.expectEqual(y32[0], y_b.slice(u16, 4)[0]); + + const mix_c = math.MixOf(f32){ .pre = .{ 1, 0, 0, 0 }, .post = .{ coeff, 0, 0, 0 }, .comb = .{ -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; + var hc32 = values_c; + math.expandOf(f32, &hc32, &branch_values_c, mix_c); + const mix_c64 = math.MixOf(f64){ .pre = .{ 1, 0, 0, 0 }, .post = .{ @as(f64, coeff), 0, 0, 0 }, .comb = .{ -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; + var hc64 = values_c; + math.expandOf(f64, &hc64, &branch_values_c, mix_c64); + std.debug.print("HC64 {{\"case\":\"cancellation\",\"fixture\":\"branch_product\",\"metal\":\"0x{x}\",\"f32_ref\":\"0x{x}\",\"f64_ideal\":\"0x{x}\"}}\n", .{ y_c.slice(u16, 4)[0], hc32[0], hc64[0] }); + try std.testing.expectEqual(hc32[0], y_c.slice(u16, 4)[0]); + _ = cfg; +} + +// Ideal FP64 stage propagation used by the amplification measurement: statistic, dense, split, +// collapse and post-expansion, with the pre-rounding FP64 outputs kept for analog-gain figures. +fn stageF64(stream: []const u16, hc: Hc, cfg: Config, branch: []const u16, y_pre: []f64, y_bf: *[stream_width]u16, coll_pre: []f64, inv_out: *f64, comb_out: *[16]f64) void { + var proj: [mix_count]f64 = undefined; + math.denseOf(f64, hc.projection, stream, &proj); + const inv = math.inverseOf(f64, stream, cfg.text_config.rms_norm_eps); + const mix = math.splitOf(f64, &proj, inv, hc, cfg.text_config); + inv_out.* = inv; + comb_out.* = mix.comb; + for (0..width) |d| { + var s: f64 = 0; + for (0..copies) |c| s += mix.pre[c] * @as(f64, math.f(stream[c * width + d])); + coll_pre[d] = s; + } + for (0..width) |d| { + for (0..copies) |k| { + var acc: f64 = mix.post[k] * @as(f64, math.f(branch[d])); + for (0..copies) |j| acc += mix.comb[j * 4 + k] * @as(f64, math.f(stream[j * width + d])); + y_pre[k * width + d] = acc; + } + } + var h: [stream_width]u16 = undefined; + @memcpy(&h, stream); + math.expandOf(f64, &h, branch, mix); + @memcpy(y_bf, &h); +} + +fn perturbOneUlp(x: u16) u16 { + return if ((x & 0x8000) == 0) x + 1 else x - 1; +} + +test "mHC 1-ULP input amplification through the hyperconnection mix" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try mhc.Pipelines.load(device); + defer p.deinit(); + var o = try ops.Pipelines.load(device); + defer o.deinit(); + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const gpa = arena.allocator(); + const cfg = try Config.parse(gpa, released_config); + + const fixture = try std.json.parseFromSlice(Fixture, gpa, @embedFile(fixture_path), .{ .ignore_unknown_fields = true }); + const eight_rows = try decodeU16(gpa, fixture.value.eight_rows_bf16); + const ffn_stream = try decodeU16(gpa, fixture.value.ffn_stream_bf16); + const ffn_branch = try decodeU16(gpa, fixture.value.ffn_branch_bf16); + const ffn_scale = try decodeF32(gpa, fixture.value.ffn_scale_f32); + const ffn_base = try decodeF32(gpa, fixture.value.ffn_base_f32); + const weight = try syntheticWeight(gpa); + + const scale = try device.buffer(12, mtl.ResourceOptions.shared); + defer scale.deinit(); + upload(f32, scale, ffn_scale); + const base = try device.buffer(mix_count * 4, mtl.ResourceOptions.shared); + defer base.deinit(); + upload(f32, base, ffn_base); + const weight_buffer = try device.buffer(mix_count * stream_width * 4, mtl.ResourceOptions.shared); + defer weight_buffer.deinit(); + upload(f32, weight_buffer, weight); + const hc = Hc{ + .projection = .{ .buffer = weight_buffer, .offset = 0, .bytes = mix_count * stream_width * 4, .dtype = .f32 }, + .scale = .{ .buffer = scale, .offset = 0, .bytes = 12, .dtype = .f32 }, + .base = .{ .buffer = base, .offset = 0, .bytes = mix_count * 4, .dtype = .f32 }, + }; + + const columns = 64; + const column_stride = width / columns; + const Row = struct { name: []const u8, stream: []u16 }; + var rows: [9]Row = undefined; + for (0..8) |r| { + rows[r] = .{ .name = "teacher", .stream = try fourCopies(gpa, eight_rows[r * width ..][0..width]) }; + } + rows[8] = .{ .name = "ffn_token1", .stream = try gpa.dupe(u16, ffn_stream) }; + + const y0 = try gpa.alloc(f64, stream_width); + const y1 = try gpa.alloc(f64, stream_width); + var y0_bf: [stream_width]u16 = undefined; + var y1_bf: [stream_width]u16 = undefined; + const coll0 = try gpa.alloc(f64, width); + const coll1 = try gpa.alloc(f64, width); + var inv0: f64 = undefined; + var inv1: f64 = undefined; + var comb0: [16]f64 = undefined; + var comb1: [16]f64 = undefined; + + for (rows, 0..) |row, row_index| { + stageF64(row.stream, hc, cfg, ffn_branch, y0, &y0_bf, coll0, &inv0, &comb0); + var sum_y2: f64 = 0; + for (y0) |v| sum_y2 += v * v; + var sum_x2: f64 = 0; + for (row.stream) |v| sum_x2 += blk: { + const f: f64 = @floatCast(math.f(v)); + break :blk f * f; + }; + var sum_c2: f64 = 0; + for (coll0) |v| sum_c2 += v * v; + var sum_b2: f64 = 0; + for (y0_bf) |v| { + const f: f64 = @floatCast(math.f(v)); + sum_b2 += f * f; + } + + var amp_sum: f64 = 0; + var amp_max: f64 = 0; + var amp_n: usize = 0; + var amp_c_sum: f64 = 0; + var amp_c_max: f64 = 0; + var amp_b_sum: f64 = 0; + var amp_b_max: f64 = 0; + var changed_any: usize = 0; + var changed_cells: usize = 0; + var inv_rel_sum: f64 = 0; + var comb_sq: f64 = 0; + var comb_n: usize = 0; + + for (0..copies) |j| { + for (0..columns) |k| { + const col = k * column_stride; + const idx = j * width + col; + const orig = row.stream[idx]; + if (math.f(orig) == 0) continue; + const perturbed = perturbOneUlp(orig); + const dx = @as(f64, @floatCast(math.f(perturbed))) - @as(f64, @floatCast(math.f(orig))); + row.stream[idx] = perturbed; + stageF64(row.stream, hc, cfg, ffn_branch, y1, &y1_bf, coll1, &inv1, &comb1); + row.stream[idx] = orig; + + var dy2: f64 = 0; + for (y0, y1) |a, b| { + const d = a - b; + dy2 += d * d; + } + const amp = @sqrt(dy2 / sum_y2) * @sqrt(sum_x2) / @abs(dx); + amp_sum += amp; + if (amp > amp_max) amp_max = amp; + amp_n += 1; + + var dc2: f64 = 0; + for (coll0, coll1) |a, b| { + const d = a - b; + dc2 += d * d; + } + const amp_c = @sqrt(dc2 / sum_c2) * @sqrt(sum_x2) / @abs(dx); + amp_c_sum += amp_c; + if (amp_c > amp_c_max) amp_c_max = amp_c; + + var changed: usize = 0; + var dyb2: f64 = 0; + for (y0_bf, y1_bf) |a, b| { + if (a != b) changed += 1; + const av: f64 = @floatCast(math.f(a)); + const bv: f64 = @floatCast(math.f(b)); + const d = av - bv; + dyb2 += d * d; + } + const amp_b = @sqrt(dyb2 / sum_b2) * @sqrt(sum_x2) / @abs(dx); + amp_b_sum += amp_b; + if (amp_b > amp_b_max) amp_b_max = amp_b; + if (changed > 0) { + changed_any += 1; + changed_cells += changed; + } + inv_rel_sum += @abs(inv1 - inv0) / inv0; + var comb_d2: f64 = 0; + var comb_m2: f64 = 0; + for (comb0, comb1) |a, b| { + const d = a - b; + comb_d2 += d * d; + comb_m2 += b * b; + } + comb_sq += @sqrt(comb_d2 / comb_m2); + comb_n += 1; + } + } + std.debug.print("HC64 {{\"amplification\":{{\"row\":{d},\"row_name\":\"{s}\",\"perturbations\":{d},\"amp_mean\":{d},\"amp_max\":{d},\"amp_collapse_mean\":{d},\"amp_collapse_max\":{d},\"amp_bf16_mean\":{d},\"amp_bf16_max\":{d},\"bf16_changed_fraction\":{d},\"mean_changed_cells\":{d},\"inv_rel_mean\":{d},\"comb_nrms_mean\":{d}}}}}\n", .{ row_index, row.name, amp_n, amp_sum / @as(f64, @floatFromInt(amp_n)), amp_max, amp_c_sum / @as(f64, @floatFromInt(amp_n)), amp_c_max, amp_b_sum / @as(f64, @floatFromInt(amp_n)), amp_b_max, changed_any, @as(f64, @floatFromInt(changed_cells)) / @as(f64, @floatFromInt(amp_n)), inv_rel_sum / @as(f64, @floatFromInt(amp_n)), comb_sq / @as(f64, @floatFromInt(comb_n)) }); + // The mix attenuates a single-ULP input in FP64 (analog gain < 1) and stays near one + // BF16-visible ULP of growth; the collapse gain is pinned near one half. + try std.testing.expect(amp_sum / @as(f64, @floatFromInt(amp_n)) < 1.0); + try std.testing.expect(amp_max < 1.0); + try std.testing.expect(amp_c_max < 2.5); + try std.testing.expect(amp_b_sum / @as(f64, @floatFromInt(amp_n)) < 3.0); + try std.testing.expect(inv_rel_sum / @as(f64, @floatFromInt(amp_n)) < 1e-5); + } + + // Metal confirmation on the first teacher row: batch the base row plus 16 stream-cell + // perturbations through the real kernels and compare against the FP64 analog gain. + { + const metal_perturbations = 16; + const batch = metal_perturbations + 1; + var batch_stream = try gpa.alloc(u16, batch * stream_width); + var cells: [metal_perturbations]usize = undefined; + var dxs: [metal_perturbations]f64 = undefined; + var filled: usize = 0; + @memcpy(batch_stream[0..stream_width], rows[0].stream); + outer: for (0..copies) |j| { + for (0..columns) |k| { + const idx = j * width + k * column_stride; + const orig = rows[0].stream[idx]; + if (math.f(orig) == 0) continue; + cells[filled] = idx; + dxs[filled] = @as(f64, @floatCast(math.f(perturbOneUlp(orig)))) - @as(f64, @floatCast(math.f(orig))); + filled += 1; + if (filled == metal_perturbations) break :outer; + } + } + for (0..filled) |i| { + @memcpy(batch_stream[(i + 1) * stream_width ..][0..stream_width], rows[0].stream); + batch_stream[(i + 1) * stream_width + cells[i]] = perturbOneUlp(rows[0].stream[cells[i]]); + } + const stream_buf = try device.buffer(batch * stream_width * 2, mtl.ResourceOptions.shared); + defer stream_buf.deinit(); + upload(u16, stream_buf, batch_stream); + const inv_buf = try device.buffer(batch * 4, mtl.ResourceOptions.shared); + defer inv_buf.deinit(); + const mixes_buf = try device.buffer(batch * mix_count * 4, mtl.ResourceOptions.shared); + defer mixes_buf.deinit(); + const pre_buf = try device.buffer(batch * 4 * 4, mtl.ResourceOptions.shared); + defer pre_buf.deinit(); + const post_buf = try device.buffer(batch * 4 * 4, mtl.ResourceOptions.shared); + defer post_buf.deinit(); + const comb_buf = try device.buffer(batch * 16 * 4, mtl.ResourceOptions.shared); + defer comb_buf.deinit(); + const collapsed_buf = try device.buffer(batch * width * 2, mtl.ResourceOptions.shared); + defer collapsed_buf.deinit(); + const branch_buf = try device.buffer(batch * width * 2, mtl.ResourceOptions.shared); + defer branch_buf.deinit(); + for (0..batch) |r| @memcpy(branch_buf.slice(u16, batch * width)[r * width ..][0..width], ffn_branch); + const y_buf = try device.buffer(batch * stream_width * 2, mtl.ResourceOptions.shared); + defer y_buf.deinit(); + try runStage(queue, &p, &o, @intCast(batch), cfg, view(stream_buf), view(weight_buffer), view(inv_buf), view(mixes_buf), view(scale), view(base), view(pre_buf), view(post_buf), view(comb_buf), view(collapsed_buf), view(branch_buf), view(y_buf)); + + const y_metal = y_buf.slice(u16, batch * stream_width); + var nan_count: usize = 0; + var first_nan: usize = 0; + for (y_metal[0..stream_width], 0..) |v, i| { + const f = math.f(v); + if (std.math.isNan(f)) { + nan_count += 1; + if (nan_count == 1) first_nan = i; + } + } + var sum_x2_row0: f64 = 0; + for (rows[0].stream) |v| { + const f: f64 = @floatCast(math.f(v)); + sum_x2_row0 += f * f; + } + var sum_base2: f64 = 0; + for (0..stream_width) |i| { + const f: f64 = @floatCast(math.f(y_metal[i])); + sum_base2 += f * f; + } + var amp_sum: f64 = 0; + var changed_total: usize = 0; + for (0..filled) |i| { + var dy2: f64 = 0; + var changed: usize = 0; + for (0..stream_width) |c| { + const a: f64 = @floatCast(math.f(y_metal[(i + 1) * stream_width + c])); + const b: f64 = @floatCast(math.f(y_metal[c])); + const d = a - b; + if (d != 0) changed += 1; + dy2 += d * d; + } + changed_total += changed; + amp_sum += @sqrt(dy2 / sum_base2) * @sqrt(sum_x2_row0) / @abs(dxs[i]); + } + std.debug.print("HC64 {{\"amplification\":{{\"row\":0,\"metal\":true,\"perturbations\":{d},\"amp_mean\":{d},\"mean_changed_cells\":{d}}}}}\n", .{ filled, amp_sum / @as(f64, @floatFromInt(filled)), @as(f64, @floatFromInt(changed_total)) / @as(f64, @floatFromInt(filled)) }); + const metal_amp = amp_sum / @as(f64, @floatFromInt(filled)); + try std.testing.expect(metal_amp > 0.5 and metal_amp < 3.0); + } +} From 943d93fb8f7358e4361bf4c4aa3316a0104e9559 Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:42:53 -0700 Subject: [PATCH 10/33] CSA2 reference: FP64-ideal parameterization of the pool, RMS and sparse-head boundaries Carries the in-progress edits: comptime Real-style math (wide/d64/b64, rmsOf/rmsOfValues, poolOf, sparseHeadOf + KeyTable) with the FP32 instances kept as the source contract. Two repairs to that draft: model.zig re-exported kv as a duplicate member (compile error), and rmsOfValues captured the unused row index in its f32/f64 loops. --- .gitignore | 1 + zig/src/families/deepseek_v41/model.zig | 5 +- .../deepseek_v41_reference/attention_math.zig | 133 +++++++++++------- zig/tests/deepseek_v41_reference/math.zig | 69 ++++++++- 4 files changed, 153 insertions(+), 55 deletions(-) diff --git a/.gitignore b/.gitignore index 9aa8c78c21..070b8ca186 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ zig-out/ !app/build/ # mHC fp64 qualification receipts. /receipts/ +*.log diff --git a/zig/src/families/deepseek_v41/model.zig b/zig/src/families/deepseek_v41/model.zig index 4ad847bc45..443b3b2747 100644 --- a/zig/src/families/deepseek_v41/model.zig +++ b/zig/src/families/deepseek_v41/model.zig @@ -8,9 +8,12 @@ const load = @import("load.zig"); const weights = @import("weights.zig"); const work = @import("workspace.zig"); pub const attention = @import("attention.zig"); -const kv = @import("kv.zig"); +pub const kv = @import("kv.zig"); pub const ops = @import("ops.zig"); pub const mhc = @import("mhc.zig"); +pub const csa2 = @import("csa2.zig"); +pub const csa2_index = @import("csa2_index.zig"); +pub const rope = @import("rope.zig"); const route = @import("route.zig"); const engram = @import("engram.zig"); const View = @import("view.zig").View; diff --git a/zig/tests/deepseek_v41_reference/attention_math.zig b/zig/tests/deepseek_v41_reference/attention_math.zig index 1adec0172d..dbda8fd907 100644 --- a/zig/tests/deepseek_v41_reference/attention_math.zig +++ b/zig/tests/deepseek_v41_reference/attention_math.zig @@ -7,6 +7,33 @@ const Config = tf.deepseek_v41_config.Text; const Attention = tf.deepseek_v41_weights.Attention; pub const max_capacity = 4096; +/// Kernel-shaped key addressing: window rows at physical ring slots, then compressed rows. +pub const KeyTable = struct { + window: []const u16, + window_stride: usize = 512, + compressed: []const u16 = &[_]u16{}, + compressed_stride: usize = 512, + pub fn keyRow(t: KeyTable, at: usize) []const u16 { + if (at < 128) return t.window[at * t.window_stride ..][0..512]; + const row = at - 128; + return t.compressed[row * t.compressed_stride ..][0..512]; + } +}; +fn narrow(comptime T: type, x: f64) T { + return if (T == f32) @as(f32, @floatCast(x)) else x; +} +/// Compressor softmax pooling at reference precision T: FP32 is the source contract, FP64 the ideal. +pub fn poolOf(comptime T: type, kv: [2][]const T, gate: [2][]const T, out: []u16) void { + @setFloatMode(.strict); + for (out, 0..) |*o, d| { + const high = @max(gate[0][d], gate[1][d]); + const p0 = @exp(gate[0][d] - high); + const p1 = @exp(gate[1][d] - high); + const denominator = p0 + p1; + const pooled = kv[0][d] * (p0 / denominator) + kv[1][d] * (p1 / denominator); + o.* = if (T == f32) m.b(pooled) else m.b64(pooled); + } +} pub const State = struct { gpa: std.mem.Allocator, capacity: usize, @@ -80,16 +107,8 @@ pub const State = struct { try projection.applyF32(compressor.wkv, x, &self.partial_kv[owner][slot], scratch); try projection.applyF32(compressor.wgate.?, x, &self.partial_gate[owner][slot], scratch); if ((position + 1) % ratio != 0) return false; - for (0..512) |d| { - const high = @max(self.partial_gate[owner][0][d], self.partial_gate[owner][1][d]); - const p0 = @exp(self.partial_gate[owner][0][d] - high); - const p1 = @exp(self.partial_gate[owner][1][d] - high); - const denominator = p0 + p1; - const weighted0 = self.partial_kv[owner][0][d] * (p0 / denominator); - const weighted1 = self.partial_kv[owner][1][d] * (p1 / denominator); - const pooled = weighted0 + weighted1; - self.latent[d] = m.b(pooled); - } + poolOf(f32, .{ self.partial_kv[owner][0], self.partial_kv[owner][1] }, + .{ self.partial_gate[owner][0], self.partial_gate[owner][1] }, &self.latent); } m.rms(&self.latent, compressor.norm, c.rms_norm_eps, &self.latent); const indexer = a.indexer.?; @@ -225,51 +244,63 @@ pub const State = struct { if (self.sparse_snapshot) |snapshot| @memcpy(snapshot[head * 512 ..][0..512], self.o[head * 512 ..][0..512]); } - fn keyRow(self: *State, window: []const u16, key_id: i32) []const u16 { - const at: usize = @intCast(key_id); - return if (at < 128) window[at * 512 ..][0..512] else self.mainRow(self.active_owner, at - 128); - } pub fn sparseHead(self: *State, q: []const u16, window: []const u16, indices: []const i32, sink: f32, out: []u16) void { - @setFloatMode(.strict); - var acc: [512]f32 = @splat(0); - var high: f32 = -1e30; - var denominator: f32 = 0; - var base: usize = 0; - while (base < indices.len) : (base += 64) { - var probabilities: [64]f32 = undefined; - const width = @min(64, indices.len - base); - var next_high = high; - for (0..width) |j| { - const index = indices[base + j]; - var dot: f64 = 0; - if (index >= 0) { - const key = self.keyRow(window, index); - for (q, key) |a, v| dot += @as(f64, m.f(a)) * m.f(v); - } - probabilities[j] = if (index < 0) -std.math.inf(f32) else @as(f32, @floatCast(dot)) * (1 / @sqrt(@as(f32, 512))); - next_high = @max(next_high, probabilities[j]); - } - const correction = @exp(high - next_high); - var sum: f64 = 0; - for (0..width) |j| { - probabilities[j] = @exp(probabilities[j] - next_high); - sum += probabilities[j]; - } - denominator = denominator * correction + @as(f32, @floatCast(sum)); - for (0..512) |d| { - var accumulated: f64 = acc[d] * correction; - for (0..width) |j| if (indices[base + j] >= 0) { - // Online 64-column source kernel rounds unnormalized probabilities to BF16 before P*V. - accumulated += @as(f64, m.f(m.b(probabilities[j]))) * m.f(self.keyRow(window, indices[base + j])[d]); - }; - acc[d] = @floatCast(accumulated); + // The source-contract FP32 instance; mainRow addresses compressed rows at the 640 stride. + sparseHeadOf(f32, q, .{ + .window = window, + .compressed = self.sources[self.active_owner * self.capacity * 640 ..], + .compressed_stride = 640, + }, indices, sink, out); + } +}; + +/// The online-softmax sparse head at reference precision T. With T=f32 this reproduces the pinned +/// source contract bit-for-bit (wide FP64 dot, one FP32 narrowing per score, FP32 online softmax, +/// BF16-rounded probabilities before the value product, FP32 per-tile accumulator narrowing). +/// With T=f64 every boundary is the ideal FP64 result: no FP32 narrowing, BF16 appears only at the +/// algorithm's quantized inputs, the documented probability rounding and the final output. +pub fn sparseHeadOf(comptime T: type, q: []const u16, keys: KeyTable, indices: []const i32, sink: f32, out: []u16) void { + @setFloatMode(.strict); + const scale: T = if (T == f32) 1 / @sqrt(@as(f32, 512)) else 1 / @sqrt(@as(f64, 512)); + var acc: [512]T = @splat(0); + var high: T = m.wide(T, -1e30); + var denominator: T = 0; + var base: usize = 0; + while (base < indices.len) : (base += 64) { + var probabilities: [64]T = undefined; + const width = @min(64, indices.len - base); + var next_high = high; + for (0..width) |j| { + const index = indices[base + j]; + var dot: f64 = 0; + if (index >= 0) { + const key = keys.keyRow(@intCast(index)); + for (q, key) |a, v| dot += m.d64(a) * m.d64(v); } - high = next_high; + probabilities[j] = if (index < 0) -std.math.inf(T) else narrow(T, dot) * scale; + next_high = @max(next_high, probabilities[j]); + } + const correction = @exp(high - next_high); + var sum: f64 = 0; + for (0..width) |j| { + probabilities[j] = @exp(probabilities[j] - next_high); + sum += probabilities[j]; } - denominator += @exp(sink - high); - for (out, acc) |*v, a| v.* = m.b(a / denominator); + denominator = denominator * correction + narrow(T, sum); + for (0..512) |d| { + var accumulated: f64 = acc[d] * correction; + for (0..width) |j| if (indices[base + j] >= 0) { + // Online 64-column source kernel rounds unnormalized probabilities to BF16 before P*V. + const p: f64 = if (T == f32) @as(f64, m.f(m.b(probabilities[j]))) else m.d64(m.b64(probabilities[j])); + accumulated += p * m.d64(keys.keyRow(@intCast(indices[base + j]))[d]); + }; + acc[d] = @floatCast(accumulated); + } + high = next_high; } -}; + denominator += @exp(m.wide(T, sink) - high); + for (out, acc) |*v, a| v.* = if (T == f32) m.b(a / denominator) else m.b64(a / denominator); +} test "online 64-column sparse attention rounds probabilities before value GEMM" { var state = try State.init(std.testing.allocator, 2); diff --git a/zig/tests/deepseek_v41_reference/math.zig b/zig/tests/deepseek_v41_reference/math.zig index da7bbf89ec..e060295626 100644 --- a/zig/tests/deepseek_v41_reference/math.zig +++ b/zig/tests/deepseek_v41_reference/math.zig @@ -110,12 +110,75 @@ pub fn idealRsqrt(x: f32) f32 { return @floatCast(1 / @sqrt(@as(f64, x))); } pub fn rms(x: []const u16, weight: Tensor, eps: f32, out: []u16) void { + rmsOf(f32, x, weight, eps, out); +} +/// Element type widened to the reference precision; FP32 is the source contract, FP64 the ideal. +pub fn wide(comptime T: type, v: f32) T { + return if (T == f32) v else @as(f64, v); +} +/// FP64 decode of a BF16 cell, exact at both reference precisions. +pub fn d64(x: u16) f64 { + return @as(f64, f(x)); +} +/// Round an FP64 value once to BF16 (round-to-nearest-even) with no intermediate FP32 rounding. +pub fn b64(x: f64) u16 { + const bits: u64 = @bitCast(x); + const sign: u16 = @intCast((bits >> 63) << 15); + const magnitude = bits & 0x7fffffffffffffff; + if (magnitude > 0x7ff0000000000000) return sign | 0x7fc0; // NaN keeps the quiet bit. + if (magnitude >= 0x7ff0000000000000) return sign | 0x7f80; + if (magnitude == 0) return sign; + const exponent: i32 = @intCast(magnitude >> 52); + if (exponent == 0) return sign; // FP64 subnormals sit far below any BF16 subnormal. + const unbiased: i32 = exponent - 1023; + const significand: u64 = (magnitude & 0x000fffffffffffff) | (1 << 52); + if (unbiased > 127) return sign | 0x7f80; + if (unbiased >= -126) { + // Normal BF16 keeps seven fraction bits; RNE the remaining 45. + var kept = significand >> 45; + const round_bit: u64 = (significand >> 44) & 1; + const sticky: u64 = significand & ((@as(u64, 1) << 44) - 1); + if (round_bit == 1 and (sticky != 0 or (kept & 1) == 1)) kept += 1; + if (kept == (1 << 8)) return sign | (@as(u16, @intCast(unbiased + 128)) << 7); + return sign | (@as(u16, @intCast(unbiased + 127)) << 7) | @as(u16, @intCast(kept & 0x7f)); + } + // Subnormal BF16: value = significand * 2^(unbiased-52); representable steps of 2^-133. + const drop: i32 = -81 - unbiased; + if (drop >= 54) return sign; // Below half of the smallest subnormal, including the 2^-134 tie. + var k: u64 = significand >> @intCast(drop); + const round_bit: u64 = (significand >> @intCast(drop - 1)) & 1; + const sticky: u64 = significand & ((@as(u64, 1) << @intCast(drop - 1)) - 1); + if (round_bit == 1 and (sticky != 0 or (k & 1) == 1)) k += 1; + if (k >= (1 << 7)) return sign | (1 << 7); // Rounded up into the minimum normal. + return sign | @as(u16, @intCast(k)); +} +/// The RMS boundary at reference precision T: the source-contract FP32 mean projection for f32, +/// a wide FP64 mean, exact division and reciprocal square root for f64. +pub fn rmsOf(comptime T: type, x: []const u16, weight: Tensor, eps: f32, out: []u16) void { @setFloatMode(.strict); var sum: f64 = 0; for (x) |v| sum += @as(f64, f(v)) * f(v); - const mean = meanFromWide(sum, x.len); - const r = idealRsqrt(mean + eps); - for (x, out, 0..) |v, *y, i| y.* = b(f(v) * r * value(weight, i)); + if (T == f32) { + const r = idealRsqrt(meanFromWide(sum, x.len) + eps); + for (x, out, 0..) |v, *y, i| y.* = b(f(v) * r * value(weight, i)); + return; + } + const r = 1 / @sqrt(sum / @as(f64, @floatFromInt(x.len)) + @as(f64, eps)); + for (x, out, 0..) |v, *y, i| y.* = b64(@as(f64, f(v)) * r * @as(f64, value(weight, i))); +} +/// The same boundary with weight cells given explicitly (decoded BF16 or native FP32), so a +/// harness needs no checkpoint tensor. +pub fn rmsOfValues(comptime T: type, x: []const u16, weight: []const f32, eps: f32, out: []u16) void { + @setFloatMode(.strict); + var sum: f64 = 0; + for (x) |v| sum += @as(f64, f(v)) * f(v); + if (T == f32) { + const r = idealRsqrt(meanFromWide(sum, x.len) + eps); + for (x, weight, 0..) |v, w, i| out[i] = b(f(v) * r * w); + return; + } + const r = 1 / @sqrt(sum / @as(f64, @floatFromInt(x.len)) + @as(f64, eps)); + for (x, weight, 0..) |v, w, i| out[i] = b64(@as(f64, f(v)) * r * @as(f64, w)); } pub fn sigmoid(x: f32) f32 { return 1 / (1 + @exp(-x)); From 1cf442a3c91a57c46e09a355e03b05260fdf6e04 Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:43:19 -0700 Subject: [PATCH 11/33] csa2: FP64 stage harness, compressor pool + RMS, six adversarial input families --- zig/deepseek_v41_reference_checks.zig | 1 + .../operator_csa2_fp64.zig | 304 ++++++++++++++++++ 2 files changed, 305 insertions(+) create mode 100644 zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig diff --git a/zig/deepseek_v41_reference_checks.zig b/zig/deepseek_v41_reference_checks.zig index d1f17838e1..cc6d35f278 100644 --- a/zig/deepseek_v41_reference_checks.zig +++ b/zig/deepseek_v41_reference_checks.zig @@ -5,4 +5,5 @@ test { _ = @import("tests/deepseek_v41_reference/attention_math.zig"); _ = @import("tests/deepseek_v41_reference/qualify.zig"); _ = @import("tests/deepseek_v41_reference/operator_hc_fp64.zig"); + _ = @import("tests/deepseek_v41_reference/operator_csa2_fp64.zig"); } diff --git a/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig b/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig new file mode 100644 index 0000000000..bfd37b59e7 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig @@ -0,0 +1,304 @@ +//! FP64 ideal-reference isolation for the CSA2 compressed/sparse-attention stages: rounding level, +//! Metal deltas and cancellation amplification. Receipt lines print with a "CSA64" prefix. +//! No tolerance is relaxed and no source coefficient is injected: synthetic tensors only, the +//! Metal kernel, the FP32 source-contract reference and the FP64 ideal reference on identical rows. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const math = @import("math.zig"); +const attention_math = @import("attention_math.zig"); + +const kv = tf.deepseek_v41.kv; +const View = tf.deepseek_v41.attention.View; + +const width: usize = 512; +const ratio: usize = 2; +const eps: f32 = 0.000001; + +/// The local FP32 spacing at the FP64 reference value: the ULP unit for kernel deltas. +fn nextUpF32(x: f32) f32 { + if (std.math.isNan(x) or std.math.isInf(x)) return x; + const bits: u32 = @bitCast(x); + if (bits == 0 or bits == 0x80000000) return @bitCast(@as(u32, 1)); + return @bitCast(if (bits >> 31 == 0) bits + 1 else bits - 1); +} + +fn ulp32At(v: f64) f64 { + const f: f32 = @floatCast(v); + if (std.math.isNan(f) or std.math.isInf(f)) return @abs(v); + const a = @abs(f); + if (a == 0) return 0x1p-149; + const up = nextUpF32(a); + if (std.math.isInf(up)) return @as(f64, a) - @as(f64, -nextUpF32(-a)); + return @as(f64, up) - @as(f64, a); +} + +/// A BF16-cell comparison: NRMS against the ideal, and the count of cells the two sides disagree on. +const BfDelta = struct { + count: usize = 0, + nrms: f64 = 0, + unequal: usize = 0, + worst: usize = 0, + worst_relative: f64 = 0, +}; + +fn deltaBf16(actual: []const u16, ideal: []const f64) BfDelta { + var d: BfDelta = .{ .count = actual.len }; + var squared: f64 = 0; + var magnitude: f64 = 0; + for (actual, ideal, 0..) |cell, r, i| { + const a: f64 = math.f(cell); + const diff = a - r; + squared += diff * diff; + magnitude += r * r; + const rel = @abs(diff) / @max(@abs(r), 0x1p-126); + if (cell != math.b64(r)) { + d.unequal += 1; + if (rel > d.worst_relative) { + d.worst_relative = rel; + d.worst = i; + } + } + } + d.nrms = if (magnitude == 0) @sqrt(squared) else @sqrt(squared / magnitude); + return d; +} + +/// An FP32-cell comparison, used for pre-rounding stage boundaries that stay in FP32. +const Delta = struct { + count: usize = 0, + nrms: f64 = 0, + max_abs: f64 = 0, + max_ulp: f64 = 0, + worst: usize = 0, +}; + +fn deltaF32VsF64(actual: []const f32, ideal: []const f64) Delta { + var d: Delta = .{ .count = actual.len }; + var squared: f64 = 0; + var magnitude: f64 = 0; + for (actual, ideal, 0..) |a, r, i| { + const diff = @as(f64, a) - r; + squared += diff * diff; + magnitude += r * r; + const ad = @abs(diff); + if (ad > d.max_abs) { + d.max_abs = ad; + d.worst = i; + } + const u = ad / ulp32At(r); + if (u > d.max_ulp) d.max_ulp = u; + } + d.nrms = if (magnitude == 0) @sqrt(squared) else @sqrt(squared / magnitude); + return d; +} + +fn printBf(case_name: []const u8, stage: []const u8, label: []const u8, d: BfDelta) void { + std.debug.print( + \\CSA64 {{"case":"{s}","stage":"{s}","side":"{s}","cells":{d},"nrms_vs_ideal":{d},"bf16_unequal_vs_ideal":{d},"worst_index":{d},"worst_relative":{d}}} + , .{ case_name, stage, label, d.count, d.nrms, d.unequal, d.worst, d.worst_relative }); +} + +fn printF32(case_name: []const u8, stage: []const u8, label: []const u8, d: Delta) void { + std.debug.print("CSA64 {{\"case\":\"{s}\",\"stage\":\"{s}\",\"side\":\"{s}\",\"cells\":{d}," ++ + "\"nrms_vs_ideal\":{d},\"max_abs_vs_ideal\":{d},\"max_ulp_vs_ideal\":{d:.3}," ++ + "\"worst_index\":{d}}}\n", .{ case_name, stage, label, d.count, d.nrms, d.max_abs, d.max_ulp, d.worst }); +} + +const Lcg = struct { + state: u64 = 0x853c49e6748fea9b, + fn next(s: *Lcg) u64 { + s.state = s.state *% 6364136223846793005 +% 1442695040888963407; + return s.state >> 11; + } + /// Uniform [-1, 1) with 24-bit granularity, exactly representable in FP32. + fn uniform(s: *Lcg) f32 { + const v: f64 = @as(f64, @floatFromInt(s.next() & 0xffffff)) / 16777216.0; + return @floatCast(v * 2 - 1); + } + fn scale(s: *Lcg, amplitude: f32) f32 { + return s.uniform() * amplitude; + } +}; + +const Buffers = struct { + device: mtl.Device, + items: [16]mtl.Buffer = undefined, + count: usize = 0, + fn get(b: *Buffers, comptime T: type, n: usize) !mtl.Buffer { + const x = try b.device.buffer(try std.math.mul(usize, @sizeOf(T), n), mtl.ResourceOptions.shared); + b.items[b.count] = x; + b.count += 1; + @memset(x.slice(T, n), 0); + return x; + } + fn deinit(b: *Buffers) void { + for (b.items[0..b.count]) |x| x.deinit(); + } +}; + +fn view(buffer: mtl.Buffer) View { + return .{ .buffer = buffer }; +} + +/// One compressor-pool group: the two projected positions of a ratio-2 group, pooled then RMS-normed. +/// `values[i][d]`/`gates[i][d]` are the FP32 outputs of the compressor's wkv/wgate projections; the +/// kernel and both references read exactly these numbers. +fn poolCase(device: mtl.Device, queue: mtl.Queue, p: *const kv.Pipelines, + case_name: []const u8, values: []const f32, gates: []const f32, weight: []const u16) !void +{ + var b = Buffers{ .device = device }; + defer b.deinit(); + const values_gpu = try b.get(f32, 2 * width); + const gates_gpu = try b.get(f32, 2 * width); + const partial_gpu = try b.get(f32, 2 * ratio * width); + const weight_gpu = try b.get(u16, width); + const latent_gpu = try b.get(u16, width); + @memcpy(values_gpu.slice(f32, 2 * width), values); + @memcpy(gates_gpu.slice(f32, 2 * width), gates); + @memcpy(weight_gpu.slice(u16, width), weight); + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encodePool(enc, view(values_gpu), view(gates_gpu), view(partial_gpu), view(weight_gpu), view(latent_gpu), + .{ .rows = 2, .width = @intCast(width), .ratio = @intCast(ratio), .position = 2, .stride = @intCast(width), .input_f32 = 1, .eps = eps }); + enc.end(); + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.Csa2Fp64GpuFailure; + const metal = latent_gpu.slice(u16, width); + + // Both references: the same pool, then the same RMS boundary, each at its own precision. + var ref32: [width]u16 = undefined; + var ref64: [width]u16 = undefined; + var pooled32: [width]u16 = undefined; + var pooled64: [width]u16 = undefined; + attention_math.poolOf(f32, .{ values[0..width], values[width .. 2 * width] }, + .{ gates[0..width], gates[width .. 2 * width] }, &pooled32); + var wide_values: [2][width]f64 = undefined; + var wide_gates: [2][width]f64 = undefined; + for (0..2) |slot| wide_values[slot] = widen(values[slot * width ..][0..width]); + for (0..2) |slot| wide_gates[slot] = widen(gates[slot * width ..][0..width]); + attention_math.poolOf(f64, .{ &wide_values[0], &wide_values[1] }, .{ &wide_gates[0], &wide_gates[1] }, &pooled64); + var cells: [width]f32 = undefined; + for (weight, 0..) |cell, i| cells[i] = math.f(cell); + math.rmsOfValues(f32, &pooled32, &cells, eps, &ref32); + math.rmsOfValues(f64, &pooled64, &cells, eps, &ref64); + + const ideal = widenCells(&ref64); + var from_f32 = widenCells(&ref32); + var magnitude: f64 = 0; + for (ideal) |v| magnitude += v * v; + std.debug.print( + \\CSA64 {{"case":"{s}","stage":"pool","side":"scale","ideal_rms":{d}}} + , .{ case_name, @sqrt(magnitude / @as(f64, width)) }); + printBf(case_name, "pool", "metal", deltaBf16(metal, &ideal)); + printBf(case_name, "pool", "f32_reference", deltaBf16(&ref32, &ideal)); + printBf(case_name, "pool", "metal_vs_f32_reference", deltaBf16(metal, &from_f32)); +} + +fn widen(cells: []const f32) [width]f64 { + var out: [width]f64 = undefined; + for (cells, 0..) |v, i| out[i] = @as(f64, v); + return out; +} + +fn widenCells(cells: *const [width]u16) [width]f64 { + var out: [width]f64 = undefined; + for (cells, 0..) |cell, i| out[i] = math.f(cell); + return out; +} + +test "compressor pool stage sits at rounding level against the FP64 ideal" { + var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_instance.deinit(); + const gpa = arena_instance.allocator(); + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try kv.Pipelines.load(device); + defer p.deinit(); + + var rng = Lcg{ .state = 0x9e3779b97f4a7c15 }; + // Case 1: typical magnitudes; gates straddle each other so the pool weights split. + { + const values = try gpa.alloc(f32, 2 * width); + const gates = try gpa.alloc(f32, 2 * width); + for (0..2 * width) |i| values[i] = rng.scale(1.0); + for (0..2 * width) |i| gates[i] = rng.scale(8.0); + try poolCase(device, queue, &p, "typical", values, gates, try weightCells(gpa, &rng)); + } + // Case 2: cancellation - the two pooled terms cancel to a few ULPs of the larger one. + { + const values = try gpa.alloc(f32, 2 * width); + const gates = try gpa.alloc(f32, 2 * width); + for (0..width) |d| { + const large = rng.scale(64.0); + const small = rng.scale(0.001); + values[d] = large + small; + values[width + d] = -large + rng.scale(0.001); + } + for (0..2 * width) |i| gates[i] = rng.scale(0.5); + try poolCase(device, queue, &p, "cancellation", values, gates, try weightCells(gpa, &rng)); + } + // Case 3: wide dynamic range, so one slot dominates and the other is far below it. + { + const values = try gpa.alloc(f32, 2 * width); + const gates = try gpa.alloc(f32, 2 * width); + for (0..2 * width) |i| { + const exponent = @as(f32, @floatFromInt(rng.next() % 60)) - 30; + values[i] = std.math.pow(f32, 2, exponent) * rng.uniform(); + gates[i] = rng.scale(3.0); + } + try poolCase(device, queue, &p, "wide_range", values, gates, try weightCells(gpa, &rng)); + } + // Case 4: deep cancellation, 1e-7 of the terms it cancels: FP32 error is a visible fraction. + { + const values = try gpa.alloc(f32, 2 * width); + const gates = try gpa.alloc(f32, 2 * width); + for (0..width) |d| { + const large = rng.scale(64.0); + values[d] = large * (1 + rng.scale(1e-7)); + values[width + d] = -large * (1 + rng.scale(1e-7)); + } + for (0..2 * width) |i| gates[i] = rng.scale(0.5); + try poolCase(device, queue, &p, "cancel_1e7", values, gates, try weightCells(gpa, &rng)); + } + // Case 5: gate skew that pushes one softmax weight into FP32 subnormal territory. + { + const values = try gpa.alloc(f32, 2 * width); + const gates = try gpa.alloc(f32, 2 * width); + for (0..2 * width) |i| values[i] = rng.scale(1.0); + for (0..width) |d| { + const base = rng.scale(4.0); + gates[d] = base; + gates[width + d] = base - 85 - @as(f32, @floatFromInt(rng.next() % 8)); + } + try poolCase(device, queue, &p, "gate_skew", values, gates, try weightCells(gpa, &rng)); + } + // Case 6: near-equal gates so the weights are both near one half, with large cancelling values. + { + const values = try gpa.alloc(f32, 2 * width); + const gates = try gpa.alloc(f32, 2 * width); + const shared = try gpa.alloc(f32, width); + for (0..width) |d| shared[d] = rng.scale(32.0); + for (0..width) |d| { + values[d] = shared[d] + rng.scale(1e-4); + values[width + d] = -shared[d] + rng.scale(1e-4); + gates[d] = rng.scale(0.01); + gates[width + d] = rng.scale(0.01); + } + try poolCase(device, queue, &p, "balanced_cancel", values, gates, try weightCells(gpa, &rng)); + } +} + +fn weightCells(gpa: std.mem.Allocator, rng: *Lcg) ![]u16 { + const out = try gpa.alloc(u16, width); + for (out) |*cell| { + const amplitude: f32 = 0.5 + 0.25 * @as(f32, @floatFromInt(rng.next() % 1000)) / 1000; + cell.* = math.b(amplitude); + } + return out; +} From 630f5d2b943b8cba4895d5825bf299de460b4539 Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:43:28 -0700 Subject: [PATCH 12/33] csa2: FP64 stage harness over the sparse attention kernel, with kernel-faithful key addressing --- .../operator_csa2_fp64.zig | 426 +++++++++++++++++- 1 file changed, 425 insertions(+), 1 deletion(-) diff --git a/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig b/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig index bfd37b59e7..9418ca7fac 100644 --- a/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig +++ b/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig @@ -9,6 +9,7 @@ const math = @import("math.zig"); const attention_math = @import("attention_math.zig"); const kv = tf.deepseek_v41.kv; +const csa2 = tf.deepseek_v41.csa2; const View = tf.deepseek_v41.attention.View; const width: usize = 512; @@ -126,7 +127,8 @@ const Buffers = struct { items: [16]mtl.Buffer = undefined, count: usize = 0, fn get(b: *Buffers, comptime T: type, n: usize) !mtl.Buffer { - const x = try b.device.buffer(try std.math.mul(usize, @sizeOf(T), n), mtl.ResourceOptions.shared); + const count = @max(n, 1); + const x = try b.device.buffer(try std.math.mul(usize, @sizeOf(T), count), mtl.ResourceOptions.shared); b.items[b.count] = x; b.count += 1; @memset(x.slice(T, n), 0); @@ -294,6 +296,428 @@ test "compressor pool stage sits at rounding level against the FP64 ideal" { } } +/// The packed-cache code tables. Codes are generated directly, so the FP64 ideal and the kernel +/// read bit-identical BF16 keys and the stage delta isolates the attention arithmetic itself. +fn e4(code: u8) f32 { + const magnitude = code & 127; + if (magnitude == 127) return std.math.nan(f32); + const exponent: i32 = @intCast(magnitude >> 3); + const mantissa = magnitude & 7; + const base: f32 = if (exponent == 0) @as(f32, @floatFromInt(mantissa)) / 512 else std.math.ldexp(1 + @as(f32, @floatFromInt(mantissa)) / 8, exponent - 7); + return if ((code & 128) != 0) -base else base; +} + +fn e2(code: u8) f32 { + const levels = [_]f32{ 0, 0.5, 1, 1.5, 2, 3, 4, 6 }; + const v = levels[code & 7]; + return if ((code & 8) != 0) -v else v; +} + +/// One ring row: 512 FP8 codes plus the FP32 scale of each 32-column block, exactly as dsv41_unpack reads it. +fn decodeWindowRow(bytes: []const u8) [width]u16 { + var row: [width]u16 = undefined; + for (0..width) |d| { + const scale: f32 = @bitCast(@as(u32, bytes[width + d / 32]) << 23); + row[d] = math.b(e4(bytes[d]) * scale); + } + return row; +} + +/// One compressed row: 256 FP4 nibble bytes plus the FP8 scale of each 16-column block. +fn decodeMainRow(bytes: []const u8) [width]u16 { + var row: [width]u16 = undefined; + for (0..width) |d| { + const nibble: u8 = if ((d & 1) != 0) bytes[d / 2] >> 4 else bytes[d / 2] & 15; + row[d] = math.b(e2(nibble) * e4(bytes[width / 2 + d / 16])); + } + return row; +} + +/// The FP8 code of a value under the FP8 scale the pack kernel would choose for its block. +/// Metal's `rint`: nearest integer, ties to even. The exponent-only code rounding needs it exactly. +fn roundHalfEven(x: f32) f32 { + const toward: f32 = @trunc(x); + const magnitude: f32 = if (@abs(x - toward) > 0.5) toward + std.math.sign(x) else toward; + if (@abs(x - magnitude) != 0.5) return magnitude; + const as_int: i64 = @intFromFloat(magnitude); + if (@mod(as_int, 2) == 0) return magnitude; + return if (x < 0) magnitude + 1 else magnitude - 1; +} + +fn e4Code(x: f32) u8 { + const a = @min(@abs(x), 448.0); + if (a == 0) return if (std.math.signbit(x)) 128 else 0; + const bits: u32 = @bitCast(a); + const unbiased: i32 = @as(i32, @intCast((bits >> 23) & 255)) - 127; + const e = @max(unbiased, -6); + const q: i32 = @intFromFloat(roundHalfEven(std.math.ldexp(a, 3 - @as(i32, @intCast(e))))); + const code: u32 = if (e == -6 and q < 8) @intCast(q) else @intCast((e + 7) * 8 + q - 8); + return @as(u8, @intCast(@min(code, 126))) | @as(u8, if (std.math.signbit(x)) 128 else 0); +} + +fn e4Scale(x: f32) u8 { + const bits: u32 = @bitCast(x); + const unbiased: i32 = @as(i32, @intCast((bits >> 23) & 255)) - 127 + @intFromBool((bits & 0x7fffff) != 0); + return @intCast(unbiased + 127); +} + +fn e2Code(x: f32) u8 { + const a = @min(@abs(x), 6.0); + const pick: u8 = if (a <= 0.25) 0 else if (a < 0.75) 1 else if (a <= 1.25) 2 else if (a < 1.75) 3 + else if (a <= 2.5) 4 else if (a < 3.5) 5 else if (a <= 5.0) 6 else 7; + return pick | @as(u8, if (std.math.signbit(x)) 8 else 0); +} + +/// Pack a window row exactly as dsv41_pack kind 0 would: per 32-column block, a power-of-two FP8 +/// scale and FP8 codes. The decoded cells are what both the kernel and the references will read. +fn packWindowRow(values: []const f32, out: []u8) void { + @memset(out[0..width + width / 32], 0); + for (0..width / 32) |block| { + var a: f32 = 1e-4; + for (values[block * 32 ..][0..32]) |v| a = @max(a, @abs(v)); + const z = a / 448; + const zbits: u32 = @bitCast(z); + const scale = std.math.ldexp(@as(f32, 1.0), @as(i32, @intCast((zbits >> 23) & 255)) - 127 + @intFromBool((zbits & 0x7fffff) != 0)); + out[width + block] = e4Scale(scale); + for (0..32) |j| out[block * 32 + j] = e4Code(values[block * 32 + j] / scale); + } +} + +/// Pack a compressed row exactly as dsv41_pack kind 1 would: per 16-column block, an FP8 scale and +/// FP4 codes; the decoded cells are what both sides read. +fn packMainRow(values: []const f32, out: []u8) void { + @memset(out[0 .. width / 2 + width / 16], 0); + for (0..width / 16) |block| { + var a: f32 = 6.0 * 0x1p-9; + for (values[block * 16 ..][0..16]) |v| a = @max(a, @abs(v)); + const scale = e4(e4Code(@max(a, 6.0 * 0x1p-9) / 6.0)); + out[width / 2 + block] = e4Code(@max(a, 6.0 * 0x1p-9) / 6.0); + for (0..16) |j| { + const code = e2Code(values[block * 16 + j] / scale); + const at = block * 8 + j / 2; + if ((j & 1) != 0) out[at] |= code << 4 else out[at] |= code; + } + } +} + +const SparseSpec = struct { + name: []const u8, + heads: usize, + rows: usize, + position: usize, + ratio: usize, + compressed_rows: usize, + topk: usize, +}; + +/// The kernel's own column addressing, so the reference reads the identical key per column slot. +fn columnKey(spec: SparseSpec, row: usize, slot: usize, indices: []const i32) ?KeySource { + const query = spec.position + row; + const window_columns: usize = if (spec.position + row == 0) 1 else 128; + if (slot < window_columns) { + const pos: usize = if (query == 0) 0 else if (query + 1 < 128) (query + 1 + slot) % 128 else blk: { + const begin = if (query + 1 > 128) query + 1 - 128 else 0; + break :blk begin + slot; + }; + if (pos > query) return null; + return .{ .row = pos % 128 }; + } + if (spec.ratio == 0) return null; + const pos_i = indices[row * spec.topk + slot - window_columns]; + const limit = @min(spec.compressed_rows, (query + 1) / spec.ratio); + if (pos_i < 0 or @as(usize, @intCast(pos_i)) >= limit) return null; + return .{ .compressed = true, .row = @intCast(pos_i) }; +} + +const KeySource = struct { compressed: bool = false, row: usize }; + +fn sparseCase(device: mtl.Device, queue: mtl.Queue, p: *const csa2.Pipelines, spec: SparseSpec, + q: []const u16, window_rows: []const []const f32, compressed_rows: []const []const f32, + indices: []const i32, sinks: []const f32, +) !void { + var b = Buffers{ .device = device }; + defer b.deinit(); + const q_stride = spec.heads * width; + const q_gpu = try b.get(u16, spec.rows * q_stride); + const ring_gpu = try b.get(u8, 128 * 528); + const chunk_gpu = try b.get(u8, @max(spec.rows, 1) * 528); + const main_gpu = try b.get(u8, @max(spec.compressed_rows, 1) * 288); + const positions_gpu = try b.get(i32, spec.rows * spec.topk); + const sink_gpu = try b.get(f32, spec.heads); + const out_gpu = try b.get(u16, spec.rows * q_stride); + @memcpy(q_gpu.slice(u16, spec.rows * q_stride), q); + @memcpy(positions_gpu.slice(i32, spec.rows * spec.topk), indices); + @memcpy(sink_gpu.slice(f32, spec.heads), sinks); + + // The reference's key table is the kernel's decoded cache: window columns by slot, then the + // selected compressed rows in the order the index array selects them. + var window_table = try std.testing.allocator.alloc(u16, 128 * width); + defer std.testing.allocator.free(window_table); + @memset(window_table, 0); + // The reference table is indexed 128 + row, so it needs one row per compressed row. + const compressed_cells = @max(compressed_rows.len, 1); + var compressed_table = try std.testing.allocator.alloc(u16, compressed_cells * width); + defer std.testing.allocator.free(compressed_table); + @memset(compressed_table, 0); + // Rows are laid out by ring slot: a logical position reads ring[(pos % 128)] unless it belongs to + // the chunk this call publishes, which the kernel reads from `chunk` instead. + for (window_rows, 0..) |values, logical| { + const bytes = try std.testing.allocator.alloc(u8, 528); + defer std.testing.allocator.free(bytes); + packWindowRow(values, bytes); + const slot = logical % 128; + @memcpy(ring_gpu.slice(u8, 128 * 528)[slot * 528 ..][0..528], bytes); + const decoded = decodeWindowRow(bytes); + @memcpy(window_table[slot * width ..][0..width], decoded[0..width]); + if (logical >= spec.position and logical < spec.position + spec.rows) + @memcpy(chunk_gpu.slice(u8, @max(spec.rows, 1) * 528)[(logical - spec.position) * 528 ..][0..528], bytes); + } + for (compressed_rows, 0..) |values, row| { + const bytes = try std.testing.allocator.alloc(u8, 288); + defer std.testing.allocator.free(bytes); + packMainRow(values, bytes); + @memcpy(main_gpu.slice(u8, @max(spec.compressed_rows, 1) * 288)[row * 288 ..][0..288], bytes); + const decoded = decodeMainRow(bytes); + @memcpy(compressed_table[row * width ..][0..width], decoded[0..width]); + } + + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encode(enc, view(q_gpu), view(ring_gpu), view(chunk_gpu), view(main_gpu), view(positions_gpu), + view(sink_gpu), view(out_gpu), .{ + .rows = @intCast(spec.rows), + .heads = @intCast(spec.heads), + .q_stride = @intCast(q_stride), + .position = @intCast(spec.position), + .chunk_rows = @intCast(spec.rows), + .ratio = @intCast(spec.ratio), + .compressed_rows = @intCast(spec.compressed_rows), + .topk = @intCast(spec.topk), + .index_stride = @intCast(spec.topk), + }); + enc.end(); + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.Csa2Fp64GpuFailure; + const metal = out_gpu.slice(u16, spec.rows * q_stride); + + var column_indices = try std.testing.allocator.alloc(i32, 640); + defer std.testing.allocator.free(column_indices); + var wide32: [width]u16 = undefined; + var wide64: [width]u16 = undefined; + for (0..spec.rows) |row| { + const window_columns: usize = if (spec.position + row == 0) 1 else 128; + const compressed_columns: usize = if (spec.ratio == 0) 0 else @min(spec.topk, @min(spec.compressed_rows, (spec.position + row + 1) / spec.ratio)); + const count = window_columns + compressed_columns; + for (0..count) |slot| { + if (columnKey(spec, row, slot, indices)) |key| { + column_indices[slot] = if (key.compressed) @intCast(128 + key.row) else @intCast(key.row); + } else column_indices[slot] = -1; + } + const keys = attention_math.KeyTable{ + .window = window_table, + .window_stride = width, + .compressed = compressed_table, + .compressed_stride = width, + }; + for (0..spec.heads) |head| { + const head_q = q[row * q_stride + head * width ..][0..width]; + attention_math.sparseHeadOf(f32, head_q, keys, column_indices[0..count], sinks[head], wide32[0..width]); + attention_math.sparseHeadOf(f64, head_q, keys, column_indices[0..count], sinks[head], wide64[0..width]); + var ideal: [width]f64 = undefined; + for (wide64, 0..) |cell, d| ideal[d] = math.f(cell); + var energy: f64 = 0; + var peak: f64 = 0; + var live: usize = 0; + for (column_indices[0..count]) |at| live += @intFromBool(at >= 0); + for (ideal) |v| { + energy += v * v; + peak = @max(peak, @abs(v)); + } + std.debug.print( + \\CSA64 {{"case":"{s}","stage":"scale","side":"{s}","columns":{d},"live":{d},"ideal_rms":{d},"ideal_peak":{d}}} + , .{ spec.name, "sparse", count, live, @sqrt(energy / @as(f64, width)), peak }); + var stage_buffer: [32]u8 = undefined; + const stage = std.fmt.bufPrint(&stage_buffer, "row{d}_head{d}", .{ row, head }) catch unreachable; + printBf(spec.name, stage, "metal", deltaBf16(metal[row * q_stride + head * width ..][0..width], &ideal)); + printBf(spec.name, stage, "f32_reference", deltaBf16(wide32[0..width], &ideal)); + } + } +} + +const KeyPattern = enum { random, cancelling, near_tie, one_sided }; + +fn sparseCaseFrom(gpa: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, p: *const csa2.Pipelines, + spec: SparseSpec, rng: *Lcg, key_amplitude: f32, scale_exponent: i32, indices_mode: usize, + sink: f32, pattern: KeyPattern, +) !void { + const q_stride = spec.heads * width; + const q = try gpa.alloc(u16, spec.rows * q_stride); + defer gpa.free(q); + const q_scale = std.math.pow(f32, 2, @as(f32, @floatFromInt(scale_exponent))); + for (q) |*cell| cell.* = math.b(rng.uniform() * q_scale); + const logical_window = spec.position + spec.rows; + const window_rows = try gpa.alloc([]const f32, logical_window); + defer gpa.free(window_rows); + for (window_rows, 0..) |*row, i| { + const cells = try gpa.alloc(f32, width); + fillPatternRow(cells, rng, i, key_amplitude, scale_exponent, pattern); + row.* = cells; + } + const compressed_rows = try gpa.alloc([]const f32, spec.compressed_rows); + defer gpa.free(compressed_rows); + for (compressed_rows, 0..) |*row, i| { + const cells = try gpa.alloc(f32, width); + fillPatternRow(cells, rng, i, key_amplitude, scale_exponent, pattern); + row.* = cells; + } + const indices = try gpa.alloc(i32, spec.rows * spec.topk); + defer gpa.free(indices); + const limit = @max(spec.compressed_rows, 1); + for (0..spec.rows) |r| for (0..spec.topk) |k| { + indices[r * spec.topk + k] = switch (indices_mode) { + 0 => @intCast(k * limit / @max(spec.topk, 1)), // spread and ascending + 1 => @intCast((limit - 1) - k), // descending (the reference sorts; the kernel does not) + else => @as(i32, @intCast((r *% 7 +% 13 * k) % limit)), // arbitrary order, unsorted + }; + }; + const sinks = try gpa.alloc(f32, spec.heads); + defer gpa.free(sinks); + for (sinks) |*v| v.* = sink; + try sparseCase(device, queue, p, spec, q, window_rows, compressed_rows, indices, sinks); +} + +fn fillPatternRow(row: []f32, rng: *Lcg, index: usize, amplitude: f32, exponent: i32, pattern: KeyPattern) void { + const scale = std.math.pow(f32, 2, @as(f32, @floatFromInt(exponent))); + if (pattern == .random) { + for (row) |*v| v.* = rng.uniform() * amplitude * scale; + return; + } + for (row, 0..) |*v, d| { + if (pattern == .cancelling) { + // Same shape in every key: all scores tie, and the value sum cancels to the ULPs of + // the large terms that make it up. + const flip: f32 = if ((d & 1) == 0) 1 else -1; + v.* = flip * amplitude * scale + rng.uniform() * amplitude * scale * 1e-3; + } else if (pattern == .near_tie) { + // Keys differ only far above the score rounding boundary: probabilities tie at the top. + v.* = @as(f32, @floatFromInt(index)) * amplitude * scale + rng.uniform() * amplitude * scale * 1e-7; + } else { + // One dominant key against keys orders of magnitude smaller. + v.* = rng.uniform() * amplitude * scale * @as(f32, if (index == 0) @as(f32, 1e4) else @as(f32, 1e-4)); + } + } +} + +test "sparse attention stages sit at rounding level against the FP64 ideal" { + var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_instance.deinit(); + const gpa = arena_instance.allocator(); + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try csa2.Pipelines.load(device); + defer p.deinit(); + + var rng = Lcg{ .state = 0x2545f4914f6cdd1d }; + + // Window only, the decode-step boundary at a wrapped ring. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "window_decode", .heads = 4, .rows = 1, + .position = 255, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); + // Compressed rows with a small selection, ascending indices. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "compressed_asc", .heads = 4, .rows = 1, + .position = 255, .ratio = 2, .compressed_rows = 128, .topk = 16 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); + // Unsorted indices: the kernel consumes the array in the given order; the reference must too. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "compressed_unsorted", .heads = 2, .rows = 1, + .position = 9, .ratio = 2, .compressed_rows = 40, .topk = 12 }, &rng, 1.0, 0, 2, 0, KeyPattern.random); + // Prefill rows, multi-row chunk with a wrapped ring. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "prefill_rows", .heads = 2, .rows = 4, + .position = 250, .ratio = 2, .compressed_rows = 100, .topk = 8 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); + // Wide dynamic range keys: the score sum runs across many orders of magnitude. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "wide_range", .heads = 1, .rows = 1, + .position = 200, .ratio = 2, .compressed_rows = 32, .topk = 8 }, &rng, 1.0, -20, 0, 0, KeyPattern.random); + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "wide_range_hi", .heads = 1, .rows = 1, + .position = 200, .ratio = 2, .compressed_rows = 32, .topk = 8 }, &rng, 1.0, 20, 0, 0, KeyPattern.random); + // Denormal-scale sink: the sink mass underflows FP32 while it is exact in FP64. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "sink_denormal", .heads = 1, .rows = 1, + .position = 100, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, -16.0, KeyPattern.random); + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "sink_large", .heads = 1, .rows = 1, + .position = 100, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, 12.0, KeyPattern.random); + // Value-product cancellation: identical-shaped keys, so every score ties and the P·V sum + // cancels to the ULPs of the terms that make it up. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "cancel_values", .heads = 1, .rows = 1, + .position = 300, .ratio = 2, .compressed_rows = 64, .topk = 64 }, &rng, 32.0, 0, 0, 0, + KeyPattern.cancelling); + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "cancel_wide", .heads = 1, .rows = 1, + .position = 300, .ratio = 2, .compressed_rows = 64, .topk = 64 }, &rng, 32.0, 6, 0, 0, + KeyPattern.cancelling); + // Scores that tie: the softmax denominator is the sum of equal exponentials, and the + // accumulation order decides which probability lands on which side of its rounding boundary. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "score_tie", .heads = 1, .rows = 1, + .position = 300, .ratio = 2, .compressed_rows = 32, .topk = 32 }, &rng, 1.0, 0, 0, 0, + KeyPattern.near_tie); + // One dominant key against keys orders of magnitude smaller: denominator and accumulator + // run at the extreme ratio. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "one_dominant", .heads = 1, .rows = 1, + .position = 300, .ratio = 2, .compressed_rows = 16, .topk = 16 }, &rng, 1.0, 0, 0, 0, + KeyPattern.one_sided); + // Many tiles, so the online correction multiplies the accumulator once per 64 columns. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "many_tiles", .heads = 1, .rows = 1, + .position = 511, .ratio = 2, .compressed_rows = 512, .topk = 384 }, &rng, 1.0, 0, 0, 0, + KeyPattern.random); + // Cancellation in the value product: keys of alternating sign, so the P·V sum is small next + // to the terms that make it up. + { + const spec = SparseSpec{ .name = "pv_cancellation", .heads = 1, .rows = 1, .position = 64, + .ratio = 0, .compressed_rows = 0, .topk = 0 }; + const q = try gpa.alloc(u16, width); + defer gpa.free(q); + for (q) |*cell| cell.* = math.b(rng.scale(1.0)); + const window_rows = try gpa.alloc([]const f32, spec.position + spec.rows); + defer gpa.free(window_rows); + for (window_rows) |*row| { + const cells = try gpa.alloc(f32, width); + for (cells, 0..) |*v, d| v.* = if ((d & 1) == 0) rng.scale(64.0) else rng.scale(64.0) * 0.987654321; + row.* = cells; + } + const sinks = try gpa.alloc(f32, 1); + defer gpa.free(sinks); + sinks[0] = 0; + const empty = try gpa.alloc([]const f32, 0); + defer gpa.free(empty); + const indices = try gpa.alloc(i32, 0); + defer gpa.free(indices); + try sparseCase(device, queue, &p, spec, q, window_rows, empty, indices, sinks); + } + // Large cancelling keys beside small ones: score dot products run through near-cancelling terms. + { + const spec = SparseSpec{ .name = "score_cancellation", .heads = 1, .rows = 1, .position = 96, + .ratio = 0, .compressed_rows = 0, .topk = 0 }; + const q = try gpa.alloc(u16, width); + defer gpa.free(q); + for (q, 0..) |*cell, d| cell.* = if ((d & 1) == 0) math.b(1.0) else math.b(-1.0); + const window_rows = try gpa.alloc([]const f32, spec.position + spec.rows); + defer gpa.free(window_rows); + for (window_rows, 0..) |*row, i| { + const cells = try gpa.alloc(f32, width); + const bias: f32 = @as(f32, @floatFromInt(@rem(@as(i32, @intCast(i)), 8))) * 4.0; + for (cells, 0..) |*v, d| v.* = bias + (if ((d & 1) == 0) rng.scale(0.001) else rng.scale(0.001) * 0.99999); + row.* = cells; + } + const sinks = try gpa.alloc(f32, 1); + defer gpa.free(sinks); + sinks[0] = 0; + const empty = try gpa.alloc([]const f32, 0); + defer gpa.free(empty); + const indices = try gpa.alloc(i32, 0); + defer gpa.free(indices); + try sparseCase(device, queue, &p, spec, q, window_rows, empty, indices, sinks); + } +} + fn weightCells(gpa: std.mem.Allocator, rng: *Lcg) ![]u16 { const out = try gpa.alloc(u16, width); for (out) |*cell| { From 0be18333927d069c033b4b0f63c01365a7ce4924 Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:43:42 -0700 Subject: [PATCH 13/33] =?UTF-8?q?csa2:=20indexer=20stage=20harness=20?= =?UTF-8?q?=E2=80=94=20FP32/FP64=20score=20comparison=20and=20top-k=20sele?= =?UTF-8?q?ction=20agreement?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../operator_csa2_fp64.zig | 191 ++++++++++++++++++ 1 file changed, 191 insertions(+) diff --git a/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig b/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig index 9418ca7fac..63ce39b60a 100644 --- a/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig +++ b/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig @@ -718,6 +718,197 @@ test "sparse attention stages sit at rounding level against the FP64 ideal" { } } +/// One indexer-score row set: 32 heads of 128 FP4-index columns each, scored against packed keys. +const IndexSpec = struct { name: []const u8, rows: usize, keys: usize, topk: usize }; + +fn packIndexRow(values: []const f32, out: []u8) void { + @memset(out[0 .. 128 / 2 + 128 / 32], 0); + for (0..128 / 32) |block| { + var a: f32 = 6.0 * 0x1p-126; + for (values[block * 32 ..][0..32]) |v| a = @max(a, @abs(v)); + const bits: u32 = @bitCast(a / 6.0); + const unbiased: i32 = @as(i32, @intCast((bits >> 23) & 255)) - 127 + @intFromBool((bits & 0x7fffff) != 0); + const scale = std.math.ldexp(@as(f32, 1.0), unbiased); + out[64 + block] = @intCast(unbiased + 127); + for (0..32) |j| { + const code = e2Code(values[block * 32 + j] / scale); + const at = block * 16 + j / 2; + if ((j & 1) != 0) out[at] |= code << 4 else out[at] |= code; + } + } +} + +fn decodeIndexRow(bytes: []const u8) [128]u16 { + var row: [128]u16 = undefined; + for (0..128) |d| { + const nibble: u8 = if ((d & 1) != 0) bytes[d / 2] >> 4 else bytes[d / 2] & 15; + const scale: f32 = @bitCast(@as(u32, bytes[64 + d / 32]) << 23); + row[d] = math.b(e2(nibble) * scale); + } + return row; +} + +fn indexCase(device: mtl.Device, queue: mtl.Queue, p: *const tf.deepseek_v41.csa2_index.Pipelines, + spec: IndexSpec, q: []const u16, weights: []const u16, key_values: []const []const f32, +) !void { + var b = Buffers{ .device = device }; + defer b.deinit(); + var scratch = try tf.deepseek_v41.csa2_index.Scratch.init(device, @intCast(spec.rows)); + defer scratch.deinit(); + const q_gpu = try b.get(u16, spec.rows * 32 * 128); + const weights_gpu = try b.get(u16, spec.rows * 32); + const keys_gpu = try b.get(u8, spec.keys * 68); + @memcpy(q_gpu.slice(u16, spec.rows * 32 * 128), q); + @memcpy(weights_gpu.slice(u16, spec.rows * 32), weights); + + // Both sides read the same decoded keys; the references differ only in their arithmetic. + const decoded = try std.testing.allocator.alloc(u16, spec.keys * 128); + defer std.testing.allocator.free(decoded); + for (key_values, 0..) |values, row| { + const bytes = try std.testing.allocator.alloc(u8, 68); + defer std.testing.allocator.free(bytes); + packIndexRow(values, bytes); + @memcpy(keys_gpu.slice(u8, spec.keys * 68)[row * 68 ..][0..68], bytes); + const decoded_row = decodeIndexRow(bytes); + @memcpy(decoded[row * 128 ..][0..128], &decoded_row); + } + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encode(enc, view(q_gpu), view(weights_gpu), view(keys_gpu), scratch, .{ + .rows = @intCast(spec.rows), + .keys = @intCast(spec.keys), + .q_stride = 32 * 128, + .weight_stride = 32, + .position = @intCast(spec.keys), + .ratio = 1, + }); + enc.end(); + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.Csa2Fp64GpuFailure; + + const kernel_scores = scratch.scores().buffer.slice(f32, spec.rows * 4096); + const kernel_positions = scratch.positions().buffer.slice(i32, spec.rows * 512); + var ref32 = try std.testing.allocator.alloc(f32, spec.keys); + defer std.testing.allocator.free(ref32); + var ref64 = try std.testing.allocator.alloc(f64, spec.keys); + defer std.testing.allocator.free(ref64); + const w_scale: f32 = 1 / @sqrt(@as(f32, 128 * 32)); + for (0..spec.keys) |row| { + const key = decoded[row * 128 ..][0..128]; + var ideal: f64 = 0; + var narrow32: f32 = 0; + for (0..32) |head| { + var dot: f64 = 0; + for (0..128) |d| dot += @as(f64, math.f(key[d])) * @as(f64, math.f(q[head * 128 + d])); + const rectified: f64 = @max(@min(dot, 1e38), 0); + const weight: f32 = math.f(weights[head]); + const scaled_weight = math.f(math.b(weight * w_scale)); + const weighted = math.f(math.b(@as(f32, @floatCast(rectified)) * scaled_weight)); + ideal += weighted; + narrow32 += @as(f32, @floatCast(weighted)); + } + ref64[row] = ideal; + ref32[row] = math.f(math.b(narrow32)); + } + var ideal_cells: [1024]f64 = undefined; + const shown = @min(spec.keys, ideal_cells.len); + for (0..shown) |row| ideal_cells[row] = ref64[row]; + printF32(spec.name, "index_score", "metal_vs_ideal", deltaF32VsF64( + kernel_scores[0..shown], ideal_cells[0..shown])); + printF32(spec.name, "index_score", "f32_vs_ideal", deltaF32VsF64(ref32[0..shown], ideal_cells[0..shown])); + + // Selection: which keys land in the top-k. Ties are decided by the merge's comparator on the + // kernel side and by ascending index in the reference; the sets can therefore differ. + var chosen = try std.testing.allocator.alloc(bool, spec.keys); + defer std.testing.allocator.free(chosen); + @memset(chosen, false); + var scores_copy = try std.testing.allocator.alloc(f32, spec.keys); + defer std.testing.allocator.free(scores_copy); + @memcpy(scores_copy, ref32[0..spec.keys]); + var reference_picks: [512]u32 = undefined; + const picks = @min(spec.topk, spec.keys); + for (0..picks) |pick| { + var best: usize = 0; + for (0..spec.keys) |i| { + if (scores_copy[i] > scores_copy[best]) best = i; + } + reference_picks[pick] = @intCast(best); + scores_copy[best] = -std.math.inf(f32); + } + std.mem.sort(u32, reference_picks[0..picks], {}, std.sort.asc(u32)); + var overlap: usize = 0; + for (reference_picks[0..picks]) |at| chosen[at] = true; + var min_selected: f32 = std.math.inf(f32); + var max_unselected: f32 = -std.math.inf(f32); + for (0..spec.keys) |row| { + if (chosen[row]) { + overlap += 1; + min_selected = @min(min_selected, ref32[row]); + } else max_unselected = @max(max_unselected, ref32[row]); + } + var kernel_only: usize = 0; + for (0..picks) |i| { + const slot = kernel_positions[i]; + if (slot < 0 or slot >= @as(i32, @intCast(spec.keys))) continue; + if (!chosen[@intCast(slot)]) kernel_only += 1; + } + std.debug.print( + \\CSA64 {{"case":"{s}","stage":"index_select","side":"metal","keys":{d},"picks":{d},"reference_only":{d},"kernel_only":{d},"lowest_selected":{d},"highest_unselected":{d}}} + , .{ spec.name, spec.keys, picks, picks - overlap, kernel_only, min_selected, max_unselected }); +} + +test "indexer scoring and selection sit at rounding level against the FP64 ideal" { + var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_instance.deinit(); + const gpa = arena_instance.allocator(); + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try tf.deepseek_v41.csa2_index.Pipelines.load(device); + defer p.deinit(); + + var rng = Lcg{ .state = 0x1234_5678_9abc_def0 }; + var amplitude: f32 = 1; + for ([_]IndexSpec{ + .{ .name = "index_random", .rows = 1, .keys = 1024, .topk = 512 }, + .{ .name = "index_random_4k", .rows = 1, .keys = 4096, .topk = 512 }, + .{ .name = "index_ties", .rows = 1, .keys = 1024, .topk = 512 }, + .{ .name = "index_boundary", .rows = 1, .keys = 1024, .topk = 512 }, + }, 0..) |spec, case_index| { + _ = case_index; + const q = try gpa.alloc(u16, 32 * 128); + defer gpa.free(q); + const weights = try gpa.alloc(u16, 32); + defer gpa.free(weights); + const cells = try gpa.alloc(f32, spec.keys * 128); + defer gpa.free(cells); + const keys = try gpa.alloc([]const f32, spec.keys); + defer gpa.free(keys); + for (q) |*cell| cell.* = math.b(rng.scale(1.0)); + for (weights) |*cell| cell.* = math.b(rng.scale(1.0)); + for (cells, 0..) |*v, i| { + v.* = rng.scale(1.0); + keys[i / 128] = cells[i / 128 * 128 ..][0..128]; + } + // Identical keys: every score is the same value, so the whole selection is tie-driven. + if (std.mem.eql(u8, spec.name, "index_ties")) { + for (cells[128..], cells[0..128]) |*dst, src| dst.* = src; + } + // A score cliff just inside the selection boundary: half the keys sit on the boundary. + if (std.mem.eql(u8, spec.name, "index_boundary")) { + for (cells, 0..) |*v, i| { + if ((i % 128) == 0) amplitude = (if (i / 128 < 512) @as(f32, 1.000001) else @as(f32, 1.0)); + v.* = rng.scale(amplitude); + } + } + try indexCase(device, queue, &p, spec, q, weights, keys); + } +} + fn weightCells(gpa: std.mem.Allocator, rng: *Lcg) ![]u16 { const out = try gpa.alloc(u16, width); for (out) |*cell| { From 2e56ff2b24d28ec1f3ce03d0d1ef652f2c9e03df Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:43:42 -0700 Subject: [PATCH 14/33] =?UTF-8?q?csa2:=20indexer=20stage=20receipts=20?= =?UTF-8?q?=E2=80=94=20carry=20the=20BF16=20einsum/score=20boundaries=20in?= =?UTF-8?q?=20the=20FP64=20ideal,=20read=20positions=20through=20the=20vie?= =?UTF-8?q?w=20offset?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The written-but-unrun indexer harness reported a 0.37% NRMS metal-vs-ideal delta and a vacuous selection check. Both were reference-side: * the ideal skipped the pinned source's per-head einsum BF16 output and the score's BF16 storage, so it compared an unrounded equation against a quantized contract; * the selection check sliced scratch.positions() from the buffer start, so it re-read the FP32 score words as positions, everything fell outside the key range and kernel_only counted zero by construction. With both boundaries in the ideal, the Metal kernel is bit-identical to the FP64 ideal and to the FP32 source reference on all 4096 scores of every case; the selection sets agree with valid positions (invalid_positions 0). The remaining 0.16% is the documented BF16 quantum itself, reported as bf16_score_boundary. --- .../operator_csa2_fp64.zig | 44 ++++++++++++++----- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig b/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig index 63ce39b60a..c96f0da2a5 100644 --- a/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig +++ b/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig @@ -787,12 +787,19 @@ fn indexCase(device: mtl.Device, queue: mtl.Queue, p: *const tf.deepseek_v41.csa cb.wait(); if (cb.failure() != null) return error.Csa2Fp64GpuFailure; + // Positions live behind the scores/heap/candidates regions; read them through the view's offset, + // otherwise this compares the score words against themselves and the check is vacuous. const kernel_scores = scratch.scores().buffer.slice(f32, spec.rows * 4096); - const kernel_positions = scratch.positions().buffer.slice(i32, spec.rows * 512); + const positions_offset = scratch.positions().offset / @sizeOf(i32); + const kernel_positions = scratch.positions().buffer.slice(i32, positions_offset + spec.rows * 512); var ref32 = try std.testing.allocator.alloc(f32, spec.keys); defer std.testing.allocator.free(ref32); var ref64 = try std.testing.allocator.alloc(f64, spec.keys); defer std.testing.allocator.free(ref64); + // The same scores with only the documented quantizations below the final one: the BF16 quantum + // that no change of arithmetic can remove. + var exact = try std.testing.allocator.alloc(f64, spec.keys); + defer std.testing.allocator.free(exact); const w_scale: f32 = 1 / @sqrt(@as(f32, 128 * 32)); for (0..spec.keys) |row| { const key = decoded[row * 128 ..][0..128]; @@ -801,25 +808,38 @@ fn indexCase(device: mtl.Device, queue: mtl.Queue, p: *const tf.deepseek_v41.csa for (0..32) |head| { var dot: f64 = 0; for (0..128) |d| dot += @as(f64, math.f(key[d])) * @as(f64, math.f(q[head * 128 + d])); - const rectified: f64 = @max(@min(dot, 1e38), 0); + // The pinned source's head einsum lands in BF16, so the rectified score is a quantized + // algorithm boundary like the cache codes are; only the reduction above it is widened. + const rectified: f64 = @max(math.f(math.b(@min(@as(f32, @floatCast(dot)), 1e38))), 0); const weight: f32 = math.f(weights[head]); const scaled_weight = math.f(math.b(weight * w_scale)); const weighted = math.f(math.b(@as(f32, @floatCast(rectified)) * scaled_weight)); ideal += weighted; narrow32 += @as(f32, @floatCast(weighted)); } - ref64[row] = ideal; + exact[row] = ideal; + // The score is written back through BF16 by the kernel and by the source contract, so the + // ideal carries that final boundary too: the stage then differs from its output by nothing + // but the arithmetic being compared. + ref64[row] = math.f(math.b64(ideal)); ref32[row] = math.f(math.b(narrow32)); } var ideal_cells: [1024]f64 = undefined; const shown = @min(spec.keys, ideal_cells.len); for (0..shown) |row| ideal_cells[row] = ref64[row]; + printF32(spec.name, "index_score", "bf16_score_boundary", deltaF32VsF64(ref32[0..shown], exact[0..shown])); printF32(spec.name, "index_score", "metal_vs_ideal", deltaF32VsF64( kernel_scores[0..shown], ideal_cells[0..shown])); printF32(spec.name, "index_score", "f32_vs_ideal", deltaF32VsF64(ref32[0..shown], ideal_cells[0..shown])); - - // Selection: which keys land in the top-k. Ties are decided by the merge's comparator on the - // kernel side and by ascending index in the reference; the sets can therefore differ. + // The kernel against the source-contract FP32 reference on identical keys: the stage delta that + // selection can actually see, independent of how far the reference itself sits from the ideal. + var contract: [1024]f64 = undefined; + for (0..shown) |row| contract[row] = @as(f64, ref32[row]); + printF32(spec.name, "index_score", "metal_vs_f32_reference", deltaF32VsF64( + kernel_scores[0..shown], contract[0..shown])); + + // Selection: which keys land in the top-k. Both sides break ties by the lowest position, the + // kernel's heap comparator and the reference's strict-greater scan agree on that. var chosen = try std.testing.allocator.alloc(bool, spec.keys); defer std.testing.allocator.free(chosen); @memset(chosen, false); @@ -848,14 +868,18 @@ fn indexCase(device: mtl.Device, queue: mtl.Queue, p: *const tf.deepseek_v41.csa } else max_unselected = @max(max_unselected, ref32[row]); } var kernel_only: usize = 0; + var out_of_range: usize = 0; for (0..picks) |i| { - const slot = kernel_positions[i]; - if (slot < 0 or slot >= @as(i32, @intCast(spec.keys))) continue; + const slot = kernel_positions[positions_offset + i]; + if (slot < 0 or slot >= @as(i32, @intCast(spec.keys))) { + out_of_range += 1; + continue; + } if (!chosen[@intCast(slot)]) kernel_only += 1; } std.debug.print( - \\CSA64 {{"case":"{s}","stage":"index_select","side":"metal","keys":{d},"picks":{d},"reference_only":{d},"kernel_only":{d},"lowest_selected":{d},"highest_unselected":{d}}} - , .{ spec.name, spec.keys, picks, picks - overlap, kernel_only, min_selected, max_unselected }); + \\CSA64 {{"case":"{s}","stage":"index_select","side":"metal","keys":{d},"picks":{d},"reference_only":{d},"kernel_only":{d},"invalid_positions":{d},"lowest_selected":{d},"highest_unselected":{d}}} + , .{ spec.name, spec.keys, picks, picks - overlap, kernel_only, out_of_range, min_selected, max_unselected }); } test "indexer scoring and selection sit at rounding level against the FP64 ideal" { From 79e59ac961c07d7cf0aa685e4a31453ae6a42ae2 Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:43:42 -0700 Subject: [PATCH 15/33] csa2: sparse stage reads its own FP32 rounding level before the BF16 output quantum MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sparseHeadPre captures acc/denominator before the output BF16 rounding, so the FP32-versus-FP64 distance is measurable instead of hidden by the 2^-9 output grid. Across 16 window/compressed/sink/cancellation cases the stage's intrinsic level is 2.5e-8..3.2e-7 NRMS (sub-ULP on the pre-rounding value), and the Metal kernel stays bit-identical to the FP32 source contract on every cell — 0 of 8704 BF16 cells differ from the FP64 ideal. Receipts: receipts/sparse-stage.jsonl (metal/f32_reference BF16 deltas, f32_vs_ideal_pre_rounding ULPs, scale lines). --- .../deepseek_v41_reference/attention_math.zig | 15 ++++++++++++++- .../deepseek_v41_reference/operator_csa2_fp64.zig | 11 +++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/zig/tests/deepseek_v41_reference/attention_math.zig b/zig/tests/deepseek_v41_reference/attention_math.zig index dbda8fd907..1a7a98883e 100644 --- a/zig/tests/deepseek_v41_reference/attention_math.zig +++ b/zig/tests/deepseek_v41_reference/attention_math.zig @@ -260,6 +260,15 @@ pub const State = struct { /// With T=f64 every boundary is the ideal FP64 result: no FP32 narrowing, BF16 appears only at the /// algorithm's quantized inputs, the documented probability rounding and the final output. pub fn sparseHeadOf(comptime T: type, q: []const u16, keys: KeyTable, indices: []const i32, sink: f32, out: []u16) void { + return sparseHeadPre(T, q, keys, indices, sink, out, null); +} + +/// The same head, with `pre` optionally capturing acc/denominator before the output rounding: the +/// same values in FP32 versus FP64 then differ only by the stage's arithmetic, so their ULP distance +/// is the stage's true rounding level instead of the BF16 output quantum that hides it. +pub fn sparseHeadPre(comptime T: type, q: []const u16, keys: KeyTable, indices: []const i32, sink: f32, out: []u16, + pre: ?[]T, +) void { @setFloatMode(.strict); const scale: T = if (T == f32) 1 / @sqrt(@as(f32, 512)) else 1 / @sqrt(@as(f64, 512)); var acc: [512]T = @splat(0); @@ -299,7 +308,11 @@ pub fn sparseHeadOf(comptime T: type, q: []const u16, keys: KeyTable, indices: [ high = next_high; } denominator += @exp(m.wide(T, sink) - high); - for (out, acc) |*v, a| v.* = if (T == f32) m.b(a / denominator) else m.b64(a / denominator); + for (out, acc, 0..) |*v, a, i| { + const quotient = a / denominator; + if (pre) |cells| cells[i] = quotient; + v.* = if (T == f32) m.b(quotient) else m.b64(quotient); + } } test "online 64-column sparse attention rounds probabilities before value GEMM" { diff --git a/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig b/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig index c96f0da2a5..eefb5b7acb 100644 --- a/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig +++ b/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig @@ -522,8 +522,12 @@ fn sparseCase(device: mtl.Device, queue: mtl.Queue, p: *const csa2.Pipelines, sp }; for (0..spec.heads) |head| { const head_q = q[row * q_stride + head * width ..][0..width]; - attention_math.sparseHeadOf(f32, head_q, keys, column_indices[0..count], sinks[head], wide32[0..width]); - attention_math.sparseHeadOf(f64, head_q, keys, column_indices[0..count], sinks[head], wide64[0..width]); + var pre32: [width]f32 = undefined; + var pre64: [width]f64 = undefined; + attention_math.sparseHeadPre(f32, head_q, keys, column_indices[0..count], sinks[head], wide32[0..width], + &pre32); + attention_math.sparseHeadPre(f64, head_q, keys, column_indices[0..count], sinks[head], wide64[0..width], + &pre64); var ideal: [width]f64 = undefined; for (wide64, 0..) |cell, d| ideal[d] = math.f(cell); var energy: f64 = 0; @@ -539,6 +543,9 @@ fn sparseCase(device: mtl.Device, queue: mtl.Queue, p: *const csa2.Pipelines, sp , .{ spec.name, "sparse", count, live, @sqrt(energy / @as(f64, width)), peak }); var stage_buffer: [32]u8 = undefined; const stage = std.fmt.bufPrint(&stage_buffer, "row{d}_head{d}", .{ row, head }) catch unreachable; + // The intrinsic FP32-versus-FP64 level of the stage, before the BF16 output quantum hides + // it. This is the rounding level the stage verdicts are measured against. + printF32(spec.name, stage, "f32_vs_ideal_pre_rounding", deltaF32VsF64(&pre32, &pre64)); printBf(spec.name, stage, "metal", deltaBf16(metal[row * q_stride + head * width ..][0..width], &ideal)); printBf(spec.name, stage, "f32_reference", deltaBf16(wide32[0..width], &ideal)); } From bba3e7df099105f963e85285bb73acac39c8f89e Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:43:43 -0700 Subject: [PATCH 16/33] csa2: per-stage harnesses for sinks, tile denominators, output mix and RoPE Each stage boundary now has its own adversarial case family, and every case reports the kernel against BOTH the FP64 ideal and the FP32 source-contract reference, so a kernel-only delta is distinguishable from a contract delta. * sinks: one live column pins the running maximum to a known score, so the sweep -104..90 walks FP32 exp() from underflow to overflow, plus per-head sink indexing. * tile denominators: equal-score rows for the pure sum of exponentials, ramped scores that push the online correction through the FP32 underflow limit, a descending set where no correction is ever applied, and a single-tile row. * output mix: window and compressed rows on independent scale grids, and a 128-row mirror shape that cancels out of the score and the accumulator alike. * RoPE: production shape (64 rotary columns, YaRN compressed spectrum), positions 1..16383, inverse rotations and multi-row prefills, comparing the kernel's resident factor table against both references. Also: sparseCaseFrom gains split window/compressed scale exponents, and each sparse case now prints metal_vs_f32_reference on the identical decoded rows. Receipts: receipts/sinks-stage.jsonl, tile-denominators-stage.jsonl, output-mix-stage.jsonl, rope-stage.jsonl. Both qualification suites green. --- .../operator_csa2_fp64.zig | 341 +++++++++++++++++- 1 file changed, 338 insertions(+), 3 deletions(-) diff --git a/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig b/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig index eefb5b7acb..b129e93719 100644 --- a/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig +++ b/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig @@ -548,6 +548,12 @@ fn sparseCase(device: mtl.Device, queue: mtl.Queue, p: *const csa2.Pipelines, sp printF32(spec.name, stage, "f32_vs_ideal_pre_rounding", deltaF32VsF64(&pre32, &pre64)); printBf(spec.name, stage, "metal", deltaBf16(metal[row * q_stride + head * width ..][0..width], &ideal)); printBf(spec.name, stage, "f32_reference", deltaBf16(wide32[0..width], &ideal)); + // Decisive for the stage verdict: the kernel and the source-contract reference on the + // identical decoded rows. Anything nonzero here is the kernel, not the contract. + var contract_cells: [width]f64 = undefined; + for (wide32, 0..) |cell, d| contract_cells[d] = math.f(cell); + printBf(spec.name, stage, "metal_vs_f32_reference", deltaBf16( + metal[row * q_stride + head * width ..][0..width], &contract_cells)); } } } @@ -557,25 +563,36 @@ const KeyPattern = enum { random, cancelling, near_tie, one_sided }; fn sparseCaseFrom(gpa: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, p: *const csa2.Pipelines, spec: SparseSpec, rng: *Lcg, key_amplitude: f32, scale_exponent: i32, indices_mode: usize, sink: f32, pattern: KeyPattern, +) !void { + try sparseCaseScaled(gpa, device, queue, p, spec, rng, key_amplitude, scale_exponent, scale_exponent, + indices_mode, sink, pattern); +} + +/// The same rows with the window and the compressed cache on independent scale grids: the stage +/// output then carries a large and a tiny contribution in one accumulator, which is what the +/// mixing step actually rounds. +fn sparseCaseScaled(gpa: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, p: *const csa2.Pipelines, + spec: SparseSpec, rng: *Lcg, key_amplitude: f32, window_exponent: i32, compressed_exponent: i32, + indices_mode: usize, sink: f32, pattern: KeyPattern, ) !void { const q_stride = spec.heads * width; const q = try gpa.alloc(u16, spec.rows * q_stride); defer gpa.free(q); - const q_scale = std.math.pow(f32, 2, @as(f32, @floatFromInt(scale_exponent))); + const q_scale = std.math.pow(f32, 2, @as(f32, @floatFromInt(window_exponent))); for (q) |*cell| cell.* = math.b(rng.uniform() * q_scale); const logical_window = spec.position + spec.rows; const window_rows = try gpa.alloc([]const f32, logical_window); defer gpa.free(window_rows); for (window_rows, 0..) |*row, i| { const cells = try gpa.alloc(f32, width); - fillPatternRow(cells, rng, i, key_amplitude, scale_exponent, pattern); + fillPatternRow(cells, rng, i, key_amplitude, window_exponent, pattern); row.* = cells; } const compressed_rows = try gpa.alloc([]const f32, spec.compressed_rows); defer gpa.free(compressed_rows); for (compressed_rows, 0..) |*row, i| { const cells = try gpa.alloc(f32, width); - fillPatternRow(cells, rng, i, key_amplitude, scale_exponent, pattern); + fillPatternRow(cells, rng, i, key_amplitude, compressed_exponent, pattern); row.* = cells; } const indices = try gpa.alloc(i32, spec.rows * spec.topk); @@ -725,6 +742,324 @@ test "sparse attention stages sit at rounding level against the FP64 ideal" { } } +/// One live column pins the running maximum to a single known score, so `sink - max` is the whole +/// normalization. Sweeping it across the FP32 exp() overflow and underflow limits is the only path +/// by which the sink term can move the stage output, and it is also where an FP32 exp() would leave +/// the FP64 equation behind. +fn sinkCase(gpa: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, p: *const csa2.Pipelines, + name: []const u8, rng: *Lcg, sink: f32, +) !void { + const spec = SparseSpec{ .name = name, .heads = 1, .rows = 1, .position = 0, .ratio = 0, + .compressed_rows = 0, .topk = 0 }; + const q = try gpa.alloc(u16, width); + defer gpa.free(q); + for (q) |*cell| cell.* = math.b(rng.scale(1.0)); + const window_rows = try gpa.alloc([]const f32, 1); + defer gpa.free(window_rows); + const cells = try gpa.alloc(f32, width); + defer gpa.free(cells); + for (cells) |*v| v.* = rng.scale(1.0); + window_rows[0] = cells; + const sinks = try gpa.alloc(f32, 1); + defer gpa.free(sinks); + sinks[0] = sink; + const empty = try gpa.alloc([]const f32, 0); + defer gpa.free(empty); + const indices = try gpa.alloc(i32, 0); + defer gpa.free(indices); + try sparseCase(device, queue, p, spec, q, window_rows, empty, indices, sinks); +} + +test "sink normalization sits at rounding level across the exp overflow boundaries" { + var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_instance.deinit(); + const gpa = arena_instance.allocator(); + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var pipelines = try csa2.Pipelines.load(device); + defer pipelines.deinit(); + var rng = Lcg{ .state = 0x51c0de51c0de51c0 }; + + // One case per boundary FP32 exp() has: underflow to zero, the overflow edge, and the + // mass-balanced middle where the sink decides the output on its own. + const sweep = [_]struct { sink: f32, name: []const u8 }{ + .{ .sink = -104, .name = "sink_underflow" }, + .{ .sink = -90, .name = "sink_near_underflow" }, + .{ .sink = -60, .name = "sink_tiny" }, + .{ .sink = -12, .name = "sink_small" }, + .{ .sink = -1, .name = "sink_minus_one" }, + .{ .sink = 0, .name = "sink_zero" }, + .{ .sink = 1, .name = "sink_one" }, + .{ .sink = 12, .name = "sink_medium" }, + .{ .sink = 60, .name = "sink_large" }, + .{ .sink = 86, .name = "sink_over_edge" }, + .{ .sink = 90, .name = "sink_overflow" }, + }; + for (sweep) |entry| try sinkCase(gpa, device, queue, &pipelines, entry.name, &rng, entry.sink); + + // Per-head sinks: the kernel indexes the sink array by head, so heads whose sink masses differ + // by orders of magnitude check that indexing as well as the arithmetic. + { + const spec = SparseSpec{ .name = "sink_per_head", .heads = 4, .rows = 1, .position = 0, + .ratio = 0, .compressed_rows = 0, .topk = 0 }; + var head_rng = Lcg{ .state = 0xabcdef0123456789 }; + const q = try gpa.alloc(u16, spec.heads * width); + defer gpa.free(q); + for (q) |*cell| cell.* = math.b(head_rng.scale(1.0)); + const window_rows = try gpa.alloc([]const f32, 1); + defer gpa.free(window_rows); + const cells = try gpa.alloc(f32, width); + defer gpa.free(cells); + for (cells) |*v| v.* = head_rng.scale(1.0); + window_rows[0] = cells; + const sinks = try gpa.alloc(f32, spec.heads); + defer gpa.free(sinks); + for (0..spec.heads) |h| sinks[h] = @as(f32, @floatFromInt(@as(i32, @intCast(h)) * 24 - 36)); + const empty = try gpa.alloc([]const f32, 0); + defer gpa.free(empty); + const indices = try gpa.alloc(i32, 0); + defer gpa.free(indices); + try sparseCase(device, queue, &pipelines, spec, q, window_rows, empty, indices, sinks); + } +} + +// Tile denominators: the online correction, the per-tile sum and the running maximum. A flat +// score set makes the denominator a pure sum of equal exponentials; a ramped score set drives the +// correction through the FP32 underflow limit; a descending set never applies a correction at all. +test "tile denominators sit at rounding level for every tile shape" { + var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_instance.deinit(); + const gpa = arena_instance.allocator(); + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var pipelines = try csa2.Pipelines.load(device); + defer pipelines.deinit(); + var rng = Lcg{ .state = 0x1a2b3c4d5e6f7081 }; + + // Ten tiles, so the denominator accumulates 639 exponentials and the accumulator takes 639 + // products. Identical key shapes make every score tie: the denominator is exactly the count. + try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = "tiles_equal_scores", .heads = 1, .rows = 1, + .position = 1022, .ratio = 2, .compressed_rows = 512, .topk = 512 }, &rng, 1.0, 0, 0, 0, KeyPattern.cancelling); + // Ascending scores by tile: the correction falls one power of two per step until FP32 flushes + // it, and the dropped mass is the whole question. + for ([_]i32{ -2, 0, 1, 2 }) |step| { + var name_buf: [24]u8 = undefined; + const label = std.fmt.bufPrint(&name_buf, "tiles_ramp{d}", .{step}) catch unreachable; + try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = label, .heads = 1, .rows = 1, + .position = 1000, .ratio = 2, .compressed_rows = 512, .topk = 512 }, &rng, 1.0, step, 0, 0, + KeyPattern.near_tie); + } + // Descending scores: the running maximum never moves, so every tile lands at full weight and + // the denominator is a plain sum. + try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = "tiles_descending", .heads = 1, .rows = 1, + .position = 1000, .ratio = 2, .compressed_rows = 512, .topk = 512 }, &rng, 1.0, 0, 1, 0, + KeyPattern.near_tie); + // One tile only, the boundary case of the online loop: the correction is applied zero times. + try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = "tiles_single", .heads = 1, .rows = 1, + .position = 20, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); +} + +// The output mix: window rows and compressed rows meeting in one accumulator on different scale +// grids, and two mirror rows whose shared large shape cancels out of the weighted sum. The mirror +// case is the stage's real amplification probe - whatever the score arithmetic gets wrong shows up +// multiplied by the shape that the sum is cancelling. +test "the output mix sits at rounding level when window and compressed rows cancel" { + var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_instance.deinit(); + const gpa = arena_instance.allocator(); + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var pipelines = try csa2.Pipelines.load(device); + defer pipelines.deinit(); + var rng = Lcg{ .state = 0x77a1c0de_5b5e1a2b }; + + // Large window rows against tiny compressed rows, then the reverse: one accumulator holding + // terms six orders of magnitude apart. + try sparseCaseScaled(gpa, device, queue, &pipelines, .{ .name = "mix_window_dominates", .heads = 1, .rows = 1, + .position = 100, .ratio = 2, .compressed_rows = 100, .topk = 64 }, &rng, 1.0, 14, -12, 0, 0, KeyPattern.random); + try sparseCaseScaled(gpa, device, queue, &pipelines, .{ .name = "mix_compressed_dominates", .heads = 1, .rows = 1, + .position = 100, .ratio = 2, .compressed_rows = 100, .topk = 64 }, &rng, 1.0, -12, 14, 0, 0, KeyPattern.random); + try sparseCaseScaled(gpa, device, queue, &pipelines, .{ .name = "mix_balanced_magnitudes", .heads = 1, .rows = 1, + .position = 100, .ratio = 2, .compressed_rows = 100, .topk = 64 }, &rng, 1.0, 6, 6, 0, 0, + KeyPattern.cancelling); + + // The stage's real amplification probe: 128 window rows that carry a large alternating shape + // which cancels out of both the score (a uniform query against a sign-alternating shape) and + // the weighted sum, so every accumulator term is large and their total is small. Whatever the + // arithmetic gets wrong arrives multiplied by that shape. + for ([_]i32{ 8, 16, 23 }) |shape_exponent| { + var name_buf: [24]u8 = undefined; + const label = std.fmt.bufPrint(&name_buf, "mirror_shape_e{d}", .{shape_exponent}) catch unreachable; + const spec = SparseSpec{ .name = label, .heads = 1, .rows = 1, .position = 127, .ratio = 0, + .compressed_rows = 0, .topk = 0 }; + const q = try gpa.alloc(u16, width); + defer gpa.free(q); + // q dot shape == 0 exactly: a uniform query against a shape that alternates sign per + // column, so the large part drops out of both scores and the accumulator alike. + for (q) |*cell| cell.* = math.b(1.0); + const window_rows = try gpa.alloc([]const f32, 128); + defer gpa.free(window_rows); + const shape = std.math.pow(f32, 2, @floatFromInt(shape_exponent)); + const shared = try gpa.alloc(f32, width); + defer gpa.free(shared); + // One 64th of the cancelling shape: near the FP8 window grid's floor, so the accumulator's + // partial sums run 2^16 times the sum they end at. Below that the code grid cancels the + // small part away and FP32 needs no help to be exact. + for (shared) |*v| v.* = rng.scale(shape * 0x1p-16); + for (window_rows, 0..) |*row, i| { + const cells = try gpa.alloc(f32, width); + const side: f32 = if ((i & 1) == 0) 1 else -1; + for (0..width) |d| { + const flip: f32 = if ((d & 1) == 0) 1 else -1; + cells[d] = side * flip * shape + shared[d]; + } + row.* = cells; + } + const sinks = try gpa.alloc(f32, 1); + defer gpa.free(sinks); + sinks[0] = 0; + const empty = try gpa.alloc([]const f32, 0); + defer gpa.free(empty); + const indices = try gpa.alloc(i32, 0); + defer gpa.free(indices); + try sparseCase(device, queue, &pipelines, spec, q, window_rows, empty, indices, sinks); + } +} + +/// Both sides of one rotary call: `contract` is the pinned source's FP32 path (FP32 frequency, FP32 +/// angle, FP32 trig), `ideal` keeps frequency, angle, both unaries and the products in FP64. The +/// kernel differs from the contract by the factor table's own unaries, which is what this checks. +fn ropeFrequency64(index: usize, theta: f64, original: f64, factor: f64, beta_fast: f64, + beta_slow: f64, +) f64 { + const frequency: f64 = 1 / std.math.pow(f64, theta, @as(f64, @floatFromInt(index * 2)) / 64); + if (original <= 0) return frequency; + const denominator = 2 * @log(theta); + const low = @max(@floor(64 * @log(original / (beta_fast * 2 * std.math.pi)) / denominator), 0); + const high = @min(@ceil(64 * @log(original / (beta_slow * 2 * std.math.pi)) / denominator), 63); + const ramp = std.math.clamp((@as(f64, @floatFromInt(index)) - low) / @max(high - low, 1e-3), 0, 1); + return frequency / factor * ramp + frequency * (1 - ramp); +} + +fn ropeCase(device: mtl.Device, queue: mtl.Queue, p: *const tf.deepseek_v41.rope.Pipelines, + name: []const u8, rng: *Lcg, position: usize, rows: usize, heads: usize, + spectrum: tf.deepseek_v41.rope.Spectrum, inverse: bool, theta: f32, original: u32, factor: f32, +) !void { + const stride = heads * width; + const cells = rows * stride; + var b = Buffers{ .device = device }; + defer b.deinit(); + const x_gpu = try b.get(u16, cells); + const x = x_gpu.slice(u16, cells); + const before = try std.testing.allocator.alloc(u16, cells); + defer std.testing.allocator.free(before); + for (x, 0..) |*cell, i| { + cell.* = math.b(rng.scale(1.0)); + before[i] = cell.*; + } + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encode(enc, view(x_gpu), .{ .rows = @intCast(rows), .heads = @intCast(heads), + .width = width, .rotary = 64, .stride = @intCast(stride), .position = @intCast(position), + .step = 1, .inverse = @intFromBool(inverse), .spectrum = spectrum }); + enc.end(); + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.Csa2Fp64GpuFailure; + + const theta64: f64 = @as(f64, theta); + const original64: f64 = if (original == 0) 0 else @floatFromInt(original); + const ideal_cells = try std.testing.allocator.alloc(u16, cells); + defer std.testing.allocator.free(ideal_cells); + const contract_cells = try std.testing.allocator.alloc(u16, cells); + defer std.testing.allocator.free(contract_cells); + for (0..rows) |row| for (0..heads) |head| { + const base = row * stride + head * width; + // Everything outside the rotary window is copied through, so both sides must agree with the + // input there too. + for (0..width - 64) |d| { + ideal_cells[base + d] = before[base + d]; + contract_cells[base + d] = before[base + d]; + } + for (0..32) |i| { + const at = base + width - 64 + 2 * i; + const a = math.f(before[at]); + const z = math.f(before[at + 1]); + const frequency = math.ropeFrequency(i, theta, original, factor, 32, 1); + const angle: f32 = @as(f32, @floatFromInt(position + row)) * frequency; + const cosine32 = @cos(angle); + const sine32 = @sin(angle) * (if (inverse) @as(f32, -1) else 1); + contract_cells[at] = math.b(a * cosine32 - z * sine32); + contract_cells[at + 1] = math.b(a * sine32 + z * cosine32); + const frequency64 = ropeFrequency64(i, theta64, original64, @as(f64, factor), 32, 1); + const angle64: f64 = @as(f64, @floatFromInt(position + row)) * frequency64; + const cosine64 = @cos(angle64); + const sine64 = @sin(angle64) * (if (inverse) @as(f64, -1) else @as(f64, 1)); + ideal_cells[at] = math.b64(@as(f64, a) * cosine64 - @as(f64, z) * sine64); + ideal_cells[at + 1] = math.b64(@as(f64, a) * sine64 + @as(f64, z) * cosine64); + } + }; + + var stage_buffer: [32]u8 = undefined; + for (0..rows) |row| for (0..heads) |head| { + const stage = std.fmt.bufPrint(&stage_buffer, "row{d}_head{d}", .{ row, head }) catch unreachable; + const base = row * stride + head * width; + var ideal: [width]f64 = undefined; + var contract: [width]f64 = undefined; + for (0..width) |d| { + ideal[d] = math.f(ideal_cells[base + d]); + contract[d] = math.f(contract_cells[base + d]); + } + printBf(name, stage, "metal", deltaBf16(x[base..][0..width], &ideal)); + printBf(name, stage, "f32_reference", deltaBf16(contract_cells[base..][0..width], &ideal)); + printBf(name, stage, "metal_vs_f32_reference", deltaBf16(x[base..][0..width], &contract)); + }; +} + +test "the rotary stage sits at rounding level for both spectra and at long positions" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + // The production shape: 64 rotary columns, YaRN on the compressed spectrum only, positions out + // to the resident capacity. + var p = try tf.deepseek_v41.rope.Pipelines.load(device, 16384, 64, + .{ .theta = 10000, .original = 0 }, + .{ .theta = 10000, .factor = 16, .original = 65536, .beta_fast = 32, .beta_slow = 1 }); + defer p.deinit(); + + const positions = [_]usize{ 1, 7, 63, 64, 4095, 8191, 16383 }; + for (positions) |position| { + var name_buf: [32]u8 = undefined; + var scoped = Lcg{ .state = 0x9e3779b97f4a7c15 ^ @as(u64, position) }; + const label = std.fmt.bufPrint(&name_buf, "rope_normal_p{d}", .{position}) catch unreachable; + try ropeCase(device, queue, &p, label, &scoped, position, 1, 2, .normal, false, 10000, 0, 16); + var scoped2 = Lcg{ .state = 0x85ebca77c0b98731 ^ @as(u64, position) }; + const label2 = std.fmt.bufPrint(&name_buf, "rope_compressed_p{d}", .{position}) catch unreachable; + try ropeCase(device, queue, &p, label2, &scoped2, position, 1, 2, .compressed, false, 10000, 65536, 16); + var scoped3 = Lcg{ .state = 0xc2b2ae3d27d4eb4f ^ @as(u64, position) }; + const label3 = std.fmt.bufPrint(&name_buf, "rope_inverse_p{d}", .{position}) catch unreachable; + try ropeCase(device, queue, &p, label3, &scoped3, position, 1, 1, .compressed, true, 10000, 65536, 16); + } + // A prefill row set: the kernel indexes the factor table by absolute position per row. + var multi = Lcg{ .state = 0x2545f4914f6cdd1d }; + try ropeCase(device, queue, &p, "rope_prefill_rows", &multi, 8100, 4, 2, .compressed, false, 10000, 65536, 16); +} + /// One indexer-score row set: 32 heads of 128 FP4-index columns each, scored against packed keys. const IndexSpec = struct { name: []const u8, rows: usize, keys: usize, topk: usize }; From 5c27e3d6f92de978d083dea6406dfd07f5105294 Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:43:58 -0700 Subject: [PATCH 17/33] csa2: measure how much score noise the top-k selection absorbs The stage-by-stage verdict is that every stage is bit-identical to the FP32 source contract, which leaves the possibility that what this operator contributes to the whole-model gate is not arithmetic but a knife-edge: a pick that sits closer to the boundary than the score's own rounding noise. So the indexer case now also selects from the FP64 scores the ideal kept, and reports how many of the 512 picks change, the boundary margin and that margin in score quanta. Random score draws tie exactly at the boundary and flip 2 of 512; the tie case, whose margin is 13 quanta, flips none. Selection is insensitive to arithmetic noise except where it is a genuine tie. Also records the per-stage verdict table in RESUME.md, and the sink sweep now walks the FP32 exp() overflow edge explicitly (80/83/84/85/86) so that the one place the kernel and the contract disagree is a named case rather than an outlier. Receipts regenerated from a single run: receipts/raw/final-run.log plus {pool,indexer,sparse,tiles,mix,rope,sinks}-stage.jsonl. Both suites green. --- .../operator_csa2_fp64.zig | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig b/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig index b129e93719..f67a6726de 100644 --- a/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig +++ b/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig @@ -785,7 +785,10 @@ test "sink normalization sits at rounding level across the exp overflow boundari var rng = Lcg{ .state = 0x51c0de51c0de51c0 }; // One case per boundary FP32 exp() has: underflow to zero, the overflow edge, and the - // mass-balanced middle where the sink decides the output on its own. + // mass-balanced middle where the sink decides the output on its own. From about 83 upward the + // sink exponent reaches the FP32 overflow boundary, so the cells that differ from the reference + // sit near 1e-38 and a one-ULP difference in the running maximum decides whether either side + // sees a finite denominator at all. const sweep = [_]struct { sink: f32, name: []const u8 }{ .{ .sink = -104, .name = "sink_underflow" }, .{ .sink = -90, .name = "sink_near_underflow" }, @@ -796,6 +799,10 @@ test "sink normalization sits at rounding level across the exp overflow boundari .{ .sink = 1, .name = "sink_one" }, .{ .sink = 12, .name = "sink_medium" }, .{ .sink = 60, .name = "sink_large" }, + .{ .sink = 80, .name = "sink_edge80" }, + .{ .sink = 83, .name = "sink_edge83" }, + .{ .sink = 84, .name = "sink_edge84" }, + .{ .sink = 85, .name = "sink_edge85" }, .{ .sink = 86, .name = "sink_over_edge" }, .{ .sink = 90, .name = "sink_overflow" }, }; @@ -1219,6 +1226,40 @@ fn indexCase(device: mtl.Device, queue: mtl.Queue, p: *const tf.deepseek_v41.csa } if (!chosen[@intCast(slot)]) kernel_only += 1; } + // How much arithmetic noise the selection can absorb before the *set* changes. The score is only + // worth what the storage keeps, so a pick is knife-edged when the boundary margin is comparable + // to one score quantum; if it is not, no 1-ULP arithmetic difference can flip this row. + var exact_picks: [512]u32 = undefined; + var exact_scores = try std.testing.allocator.alloc(f64, spec.keys); + defer std.testing.allocator.free(exact_scores); + @memcpy(exact_scores, exact[0..spec.keys]); + for (0..picks) |pick| { + var best: usize = 0; + for (0..spec.keys) |i| { + if (exact_scores[i] > exact_scores[best]) best = i; + } + exact_picks[pick] = @intCast(best); + exact_scores[best] = -std.math.inf(f64); + } + var flips: usize = 0; + var exact_chosen = try std.testing.allocator.alloc(bool, spec.keys); + defer std.testing.allocator.free(exact_chosen); + @memset(exact_chosen, false); + for (exact_picks[0..picks]) |at| exact_chosen[at] = true; + for (0..spec.keys) |at| { + if (chosen[at] != exact_chosen[at]) flips += 1; + } + var quantum: f32 = 0; + var margin_in_quanta: f64 = 0; + if (min_selected != std.math.inf(f32) and max_unselected != -std.math.inf(f32)) { + const boundary = min_selected * 0.5 + max_unselected * 0.5; + const exponent: i32 = if (@abs(boundary) > 0) std.math.ilogb(@abs(boundary)) - 7 else -149; + quantum = std.math.pow(f32, 2.0, @as(f32, @floatFromInt(exponent))); + margin_in_quanta = @as(f64, min_selected - max_unselected) / @as(f64, quantum); + } + std.debug.print( + \\CSA64 {{"case":"{s}","stage":"index_select","side":"noise_tolerance","keys":{d},"picks":{d},"precision_flips":{d},"selection_margin":{d},"score_quantum":{d},"margin_in_quanta":{d:.1}}} + , .{ spec.name, spec.keys, picks, flips, min_selected - max_unselected, quantum, margin_in_quanta }); std.debug.print( \\CSA64 {{"case":"{s}","stage":"index_select","side":"metal","keys":{d},"picks":{d},"reference_only":{d},"kernel_only":{d},"invalid_positions":{d},"lowest_selected":{d},"highest_unselected":{d}}} , .{ spec.name, spec.keys, picks, picks - overlap, kernel_only, out_of_range, min_selected, max_unselected }); From 2f9e5064ce98741a9b4d5686d0e930141ab8a948 Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:48:22 -0700 Subject: [PATCH 18/33] The fp64 harness ports: dedupe the shared math helpers, fix the shard-plan test calls math.zig's wide and b64 arrived twice through the two worktree lines; the shard-plan tests now pass the index-ownership argument the loader gained. --- zig/tests/deepseek_v41_reference/math.zig | 36 ++--------------------- 1 file changed, 2 insertions(+), 34 deletions(-) diff --git a/zig/tests/deepseek_v41_reference/math.zig b/zig/tests/deepseek_v41_reference/math.zig index e060295626..daa277c564 100644 --- a/zig/tests/deepseek_v41_reference/math.zig +++ b/zig/tests/deepseek_v41_reference/math.zig @@ -113,45 +113,13 @@ pub fn rms(x: []const u16, weight: Tensor, eps: f32, out: []u16) void { rmsOf(f32, x, weight, eps, out); } /// Element type widened to the reference precision; FP32 is the source contract, FP64 the ideal. -pub fn wide(comptime T: type, v: f32) T { - return if (T == f32) v else @as(f64, v); -} + /// FP64 decode of a BF16 cell, exact at both reference precisions. pub fn d64(x: u16) f64 { return @as(f64, f(x)); } /// Round an FP64 value once to BF16 (round-to-nearest-even) with no intermediate FP32 rounding. -pub fn b64(x: f64) u16 { - const bits: u64 = @bitCast(x); - const sign: u16 = @intCast((bits >> 63) << 15); - const magnitude = bits & 0x7fffffffffffffff; - if (magnitude > 0x7ff0000000000000) return sign | 0x7fc0; // NaN keeps the quiet bit. - if (magnitude >= 0x7ff0000000000000) return sign | 0x7f80; - if (magnitude == 0) return sign; - const exponent: i32 = @intCast(magnitude >> 52); - if (exponent == 0) return sign; // FP64 subnormals sit far below any BF16 subnormal. - const unbiased: i32 = exponent - 1023; - const significand: u64 = (magnitude & 0x000fffffffffffff) | (1 << 52); - if (unbiased > 127) return sign | 0x7f80; - if (unbiased >= -126) { - // Normal BF16 keeps seven fraction bits; RNE the remaining 45. - var kept = significand >> 45; - const round_bit: u64 = (significand >> 44) & 1; - const sticky: u64 = significand & ((@as(u64, 1) << 44) - 1); - if (round_bit == 1 and (sticky != 0 or (kept & 1) == 1)) kept += 1; - if (kept == (1 << 8)) return sign | (@as(u16, @intCast(unbiased + 128)) << 7); - return sign | (@as(u16, @intCast(unbiased + 127)) << 7) | @as(u16, @intCast(kept & 0x7f)); - } - // Subnormal BF16: value = significand * 2^(unbiased-52); representable steps of 2^-133. - const drop: i32 = -81 - unbiased; - if (drop >= 54) return sign; // Below half of the smallest subnormal, including the 2^-134 tie. - var k: u64 = significand >> @intCast(drop); - const round_bit: u64 = (significand >> @intCast(drop - 1)) & 1; - const sticky: u64 = significand & ((@as(u64, 1) << @intCast(drop - 1)) - 1); - if (round_bit == 1 and (sticky != 0 or (k & 1) == 1)) k += 1; - if (k >= (1 << 7)) return sign | (1 << 7); // Rounded up into the minimum normal. - return sign | @as(u16, @intCast(k)); -} + /// The RMS boundary at reference precision T: the source-contract FP32 mean projection for f32, /// a wide FP64 mean, exact division and reciprocal square root for f64. pub fn rmsOf(comptime T: type, x: []const u16, weight: Tensor, eps: f32, out: []u16) void { From ffc7028db8ac642c9d5dcbb60ace6807cf805bd9 Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:15:15 -0700 Subject: [PATCH 19/33] The oracle records the indexer's selection margins while it teaches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reference state now probes every scored window: the contract's boundary margin (BF16-rounded FP32 scores), the FP64-ideal margin on the same inputs, and the top-k pick flips between them — the real-data support-flip measurement. The poolOf call takes the partials by address; the oracle path had never been compiled since the runner landed, which the probe's first build exposed. --- .../deepseek_v41_reference/attention_math.zig | 113 +++++++++++++++++- zig/tests/deepseek_v41_reference/qualify.zig | 65 ++++++++++ 2 files changed, 175 insertions(+), 3 deletions(-) diff --git a/zig/tests/deepseek_v41_reference/attention_math.zig b/zig/tests/deepseek_v41_reference/attention_math.zig index 1a7a98883e..52abfd00f2 100644 --- a/zig/tests/deepseek_v41_reference/attention_math.zig +++ b/zig/tests/deepseek_v41_reference/attention_math.zig @@ -58,6 +58,18 @@ pub const State = struct { grouped: [8192]u16 = undefined, observer: ?Observer = null, sparse_snapshot: ?[]u16 = null, + /// The indexer-selection probe: fires once per scored window with the contract's + /// boundary margin, the FP64-ideal boundary margin, and the pick flips between them. + score_probe: ?ScoreProbe = null, + score_copy: []f32 = &.{}, + ideal_scores: []f32 = &.{}, + ideal_copy: []f32 = &.{}, + pick_work: []f32 = &.{}, + selected_ideal: []u32 = &.{}, + pub const ScoreProbe = struct { + ctx: *anyopaque, + record: *const fn (*anyopaque, layer: usize, position: usize, count: usize, selected_len: usize, margin: f32, margin_ideal: f32, flips: usize, lowest: f32, highest_unselected: f32) void, + }; pub const Observer = struct { ctx: *anyopaque, capture: *const fn (*anyopaque, tf.deepseek_v41.attention.DiagnosticStage, []const u16) void, @@ -77,8 +89,19 @@ pub const State = struct { const candidates = try gpa.alloc(bool, capacity); errdefer gpa.free(candidates); const blocks = try gpa.alloc(f32, (capacity + 7) / 8); + errdefer gpa.free(blocks); + const score_copy = try gpa.alloc(f32, capacity); + errdefer gpa.free(score_copy); + const ideal_scores = try gpa.alloc(f32, capacity); + errdefer gpa.free(ideal_scores); + const ideal_copy = try gpa.alloc(f32, capacity); + errdefer gpa.free(ideal_copy); + const pick_work = try gpa.alloc(f32, capacity); + errdefer gpa.free(pick_work); + const selected_ideal = try gpa.alloc(u32, capacity); + errdefer gpa.free(selected_ideal); @memset(window, 0); - return .{ .gpa = gpa, .capacity = capacity, .window = window, .sources = sources, .scores = scores, .candidates = candidates, .blocks = blocks }; + return .{ .gpa = gpa, .capacity = capacity, .window = window, .sources = sources, .scores = scores, .candidates = candidates, .blocks = blocks, .score_copy = score_copy, .ideal_scores = ideal_scores, .ideal_copy = ideal_copy, .pick_work = pick_work, .selected_ideal = selected_ideal }; } pub fn deinit(self: *State) void { self.gpa.free(self.window); @@ -86,6 +109,11 @@ pub const State = struct { self.gpa.free(self.scores); self.gpa.free(self.candidates); self.gpa.free(self.blocks); + self.gpa.free(self.score_copy); + self.gpa.free(self.ideal_scores); + self.gpa.free(self.ideal_copy); + self.gpa.free(self.pick_work); + self.gpa.free(self.selected_ideal); self.* = undefined; } fn mainRow(self: *State, owner: usize, row: usize) []u16 { @@ -107,8 +135,8 @@ pub const State = struct { try projection.applyF32(compressor.wkv, x, &self.partial_kv[owner][slot], scratch); try projection.applyF32(compressor.wgate.?, x, &self.partial_gate[owner][slot], scratch); if ((position + 1) % ratio != 0) return false; - poolOf(f32, .{ self.partial_kv[owner][0], self.partial_kv[owner][1] }, - .{ self.partial_gate[owner][0], self.partial_gate[owner][1] }, &self.latent); + poolOf(f32, .{ &self.partial_kv[owner][0], &self.partial_kv[owner][1] }, + .{ &self.partial_gate[owner][0], &self.partial_gate[owner][1] }, &self.latent); } m.rms(&self.latent, compressor.norm, c.rms_norm_eps, &self.latent); const indexer = a.indexer.?; @@ -175,6 +203,8 @@ pub const State = struct { if (layer > c.candidate_source_layer_id) for (0..count) |row| { if (!self.candidates[row]) self.scores[row] = -std.math.inf(f32); }; + @memcpy(self.score_copy[0..count], self.scores[0..count]); + self.idealScores(a, count); for (0..self.selected_len) |pick| { var best: usize = 0; for (0..count) |i| if (self.scores[i] > self.scores[best]) { @@ -184,6 +214,83 @@ pub const State = struct { self.scores[best] = -std.math.inf(f32); } std.mem.sort(u32, self.selected[0..self.selected_len], {}, std.sort.asc(u32)); + if (self.score_probe) |probe| self.reportSelection(probe, layer, position, count); + } + /// The same selection against the FP64-ideal scores: same BF16 inputs, no + /// intermediate roundings. Flips between the two picks are the support-flip signal. + fn idealScores(self: *State, a: Attention, count: usize) void { + @setFloatMode(.strict); + if (a.indexer == null) return; + for (0..count) |row| { + if (self.score_copy[row] == -std.math.inf(f32)) { + self.ideal_scores[row] = -std.math.inf(f32); + continue; + } + const key = self.indexRow(self.active_owner, row); + var score: f64 = 0; + for (0..32) |head| { + var dot: f64 = 0; + for (key, 0..) |v, d| dot += @as(f64, m.f(v)) * m.f(self.index_q[head * 128 + d]); + score += @max(dot, 0) * @as(f64, self.index_weights[head]); + } + self.ideal_scores[row] = @floatCast(score); + } + } + fn pickTop(self: *State, scores: []const f32, into: []u32) void { + @memcpy(self.pick_work[0..scores.len], scores); + const work = self.pick_work[0..scores.len]; + for (into) |*pick| { + var best: usize = 0; + for (work, 0..) |v, i| if (v > work[best]) { + best = i; + }; + pick.* = @intCast(best); + work[best] = -std.math.inf(f32); + } + std.mem.sort(u32, into, {}, std.sort.asc(u32)); + } + fn pickedAt(selection: []const u32, row: usize) bool { + var lo: usize = 0; + var hi: usize = selection.len; + while (lo < hi) { + const mid = (lo + hi) / 2; + if (selection[mid] == row) return true; + if (selection[mid] < row) lo = mid + 1 else hi = mid; + } + return false; + } + fn reportSelection(self: *State, probe: ScoreProbe, layer: usize, position: usize, count: usize) void { + var lowest: f32 = std.math.inf(f32); + var highest_unselected: f32 = -std.math.inf(f32); + for (self.selected[0..self.selected_len]) |row| { + lowest = @min(lowest, self.score_copy[row]); + } + for (0..count) |row| { + if (!pickedAt(self.selected[0..self.selected_len], row) and self.score_copy[row] > -std.math.inf(f32)) + highest_unselected = @max(highest_unselected, self.score_copy[row]); + } + const margin: f32 = if (self.selected_len == 0 or lowest == std.math.inf(f32)) 0 else lowest - highest_unselected; + // The ideal picks, on a copy so the probe's ideal scores survive for callers. + @memcpy(self.ideal_copy[0..count], self.ideal_scores[0..count]); + const ideal_len = @min(self.selected_len, self.selected_ideal.len); + self.pickTop(self.ideal_copy[0..count], self.selected_ideal[0..ideal_len]); + var flips: usize = 0; + if (self.selected_len == ideal_len) { + for (self.selected[0..self.selected_len], self.selected_ideal[0..ideal_len]) |s, i| { + if (s != i) flips += 1; + } + } else flips = self.selected_len; + var ideal_lowest: f32 = std.math.inf(f32); + var ideal_highest_unselected: f32 = -std.math.inf(f32); + for (self.selected_ideal[0..ideal_len]) |row| { + ideal_lowest = @min(ideal_lowest, self.ideal_scores[row]); + } + for (0..count) |row| { + if (!pickedAt(self.selected_ideal[0..ideal_len], row) and self.ideal_scores[row] > -std.math.inf(f32)) + ideal_highest_unselected = @max(ideal_highest_unselected, self.ideal_scores[row]); + } + const margin_ideal: f32 = if (ideal_len == 0 or ideal_lowest == std.math.inf(f32)) 0 else ideal_lowest - ideal_highest_unselected; + probe.record(probe.ctx, layer, position, count, self.selected_len, margin, margin_ideal, flips, lowest, highest_unselected); } pub fn apply(self: *State, a: Attention, layer: usize, x: []const u16, position: usize, c: Config, out: []u16, scratch: *projection.Scratch) !void { const compressed = c.compress_ratios[layer] != 0; diff --git a/zig/tests/deepseek_v41_reference/qualify.zig b/zig/tests/deepseek_v41_reference/qualify.zig index e5190ee546..2b3c975976 100644 --- a/zig/tests/deepseek_v41_reference/qualify.zig +++ b/zig/tests/deepseek_v41_reference/qualify.zig @@ -18,6 +18,68 @@ pub const Result = struct { greedy_gate_passed: bool = false, }; +/// One indexer-selection boundary per scored window: the contract's margin +/// (BF16-rounded FP32 scores), the FP64-ideal margin on the same inputs, and how +/// many top-k picks differ between them — the real-data support-flip measurement. +const MarginLog = struct { + records: [1024]MarginRecord = undefined, + len: usize = 0, + + const MarginRecord = struct { + layer: u32, + position: u32, + count: u32, + selected: u32, + margin: f32, + margin_ideal: f32, + flips: u32, + lowest_selected: f32, + highest_unselected: f32, + }; + + fn capture(ctx: *anyopaque, layer: usize, position: usize, count: usize, selected_len: usize, margin: f32, margin_ideal: f32, flips: usize, lowest: f32, highest_unselected: f32) void { + const self: *MarginLog = @ptrCast(@alignCast(ctx)); + if (self.len == self.records.len) return; + self.records[self.len] = .{ + .layer = @intCast(layer), + .position = @intCast(position), + .count = @intCast(count), + .selected = @intCast(selected_len), + .margin = margin, + .margin_ideal = margin_ideal, + .flips = @intCast(flips), + .lowest_selected = lowest, + .highest_unselected = highest_unselected, + }; + self.len += 1; + } + + fn write(self: *MarginLog, gpa: std.mem.Allocator, io: std.Io, dir: []const u8) !void { + var text: std.ArrayList(u8) = .empty; + defer text.deinit(gpa); + var flips: usize = 0; + var tight: usize = 0; + var total: usize = 0; + for (self.records[0..self.len]) |r| { + total += 1; + flips += r.flips; + if (r.margin <= 0) tight += 1; + var buf: [320]u8 = undefined; + const line = try std.fmt.bufPrint(&buf, "{{\"layer\":{d},\"position\":{d},\"count\":{d},\"selected\":{d},\"margin\":{d},\"margin_ideal\":{d},\"flips\":{d},\"lowest_selected\":{d},\"highest_unselected\":{d}}}\n", .{ r.layer, r.position, r.count, r.selected, r.margin, r.margin_ideal, r.flips, r.lowest_selected, r.highest_unselected }); + try text.appendSlice(gpa, line); + } + var head_buf: [192]u8 = undefined; + const head = try std.fmt.bufPrint(&head_buf, "{{\"records\":{d},\"total_flips\":{d},\"zero_or_negative_margin\":{d}}}\n", .{ total, flips, tight }); + const path = try std.fs.path.join(gpa, &.{ dir, "indexer-margins.jsonl" }); + defer gpa.free(path); + var file = try std.Io.Dir.cwd().createFile(io, path, .{}); + defer file.close(io); + try file.writeStreamingAll(io, head); + try file.writeStreamingAll(io, text.items); + std.debug.print("indexer margins: {d} windows, {d} flips contract-vs-ideal, {d} zero-or-negative margins\n", .{ total, flips, tight }); + } +}; + fn maximum(row: []const f32) !u32 { var id: u32 = 0; var best = -std.math.inf(f32); @@ -35,6 +97,8 @@ pub fn run(gpa: std.mem.Allocator, io: std.Io, model: *Model, ids: []const u32, if (ids.len == 0 or ids.len > 16) return error.InvalidOracleWindow; var reference = try Reference.init(gpa, model.cfg.text_config, &model.weights, &model.checkpoint, token_map, @intCast(ids.len)); defer reference.deinit(); + var margins = MarginLog{}; + reference.attention_state.score_probe = .{ .ctx = &margins, .record = &MarginLog.capture }; const cpu = try gpa.alloc(f32, ids.len * 129280); defer gpa.free(cpu); const gpu = try gpa.alloc(f32, cpu.len); @@ -76,6 +140,7 @@ pub fn run(gpa: std.mem.Allocator, io: std.Io, model: *Model, ids: []const u32, const json = try std.json.Stringify.valueAlloc(gpa, result, .{}); defer gpa.free(json); try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = report, .data = json }); + try margins.write(gpa, io, dir); std.debug.print("{s}\n", .{json}); try enforceGate(result.normalized_rms_error, result.greedy_disagreements); return result; From 6e2a7cee8c00ca9090bbdbb00fff4187866dd0a2 Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:56:01 -0700 Subject: [PATCH 20/33] The indexer-margin capture at depth: one diagnostic token's selection scratch, verbatim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The margins mode prefills a long prompt, then runs a single diagnostic token with the observer hooking each indexer layer, copying the selection scratch (score tiles, heap, positions) out verbatim. The offline decode reads the score tile directly: at 2600 tokens the boundary of the 512-pick selection sits inside an exact tie of the BF16-quantized scores at every writable layer — the support-flip condition the synthetic harness predicted, now on real weights. The candidate-masked layers' tiles are unwritten; their boundaries need the candidate-region decode. --- zig/deepseek_v41_run.zig | 8 +- zig/tests/deepseek_v41_reference/margins.zig | 96 ++++++++++++++++++++ 2 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 zig/tests/deepseek_v41_reference/margins.zig diff --git a/zig/deepseek_v41_run.zig b/zig/deepseek_v41_run.zig index e6ca81e783..10cae27f87 100644 --- a/zig/deepseek_v41_run.zig +++ b/zig/deepseek_v41_run.zig @@ -8,6 +8,7 @@ const load = tf.deepseek_v41_load; const weights = tf.deepseek_v41_weights; const process_memory = @import("src/server/process_memory.zig"); const oracle = @import("tests/deepseek_v41_reference/qualify.zig"); +const margins = @import("tests/deepseek_v41_reference/margins.zig"); const profile = @import("tests/deepseek_v41_reference/profile.zig"); const trace = @import("tests/deepseek_v41_reference/trace.zig"); const operators = @import("tests/deepseek_v41_reference/operators.zig"); @@ -21,6 +22,7 @@ fn usage(code: u8) noreturn { " tf-dsv41-run teacher MODEL TABLES IDS_JSON OUT_DIR [CHUNK]\n" ++ " tf-dsv41-run seams MODEL TABLES IDS_JSON OUT_DIR\n" ++ " tf-dsv41-run oracle MODEL TABLES IDS_JSON OUT_DIR\n" ++ + " tf-dsv41-run margins MODEL TABLES IDS_JSON OUT_DIR [DEEP]\n" ++ " tf-dsv41-run profile MODEL TABLES IDS_JSON OUT_DIR\n" ++ " tf-dsv41-run trace MODEL TABLES IDS_JSON OUT_DIR\n" ++ " tf-dsv41-run operators MODEL TABLES OUT_DIR\n" ++ @@ -44,7 +46,7 @@ fn validArguments(args: []const [:0]const u8) bool { for ([_][]const u8{ "plan", "operators" }, [_]usize{ 4, 5 }) |name, count| if (std.mem.eql(u8, mode, name)) return args.len == count; if (std.mem.eql(u8, mode, "attention-history-sameinput")) return args.len == 8; - for ([_][]const u8{ "teacher", "generate", "attention-sameinput", "hc-sameinput", "moe-sameinput" }) |name| + for ([_][]const u8{ "teacher", "generate", "attention-sameinput", "hc-sameinput", "moe-sameinput", "margins" }) |name| if (std.mem.eql(u8, mode, name)) return args.len == 6 or args.len == 7; for ([_][]const u8{ "seams", "oracle", "profile", "trace" }) |name| if (std.mem.eql(u8, mode, name)) return args.len == 6; @@ -144,7 +146,7 @@ pub fn main(init: std.process.Init) !void { const is_attention_history = std.mem.eql(u8, mode, "attention-history-sameinput"); const is_hc = std.mem.eql(u8, mode, "hc-sameinput"); const is_moe = std.mem.eql(u8, mode, "moe-sameinput"); - const max_ids: usize = if (std.mem.eql(u8, mode, "oracle") or std.mem.eql(u8, mode, "trace")) 16 else if (std.mem.eql(u8, mode, "profile")) ds.prefill_step else if (std.mem.eql(u8, mode, "teacher") or std.mem.eql(u8, mode, "seams")) 4096 else 1048576; + const max_ids: usize = if (std.mem.eql(u8, mode, "margins")) 1048576 else if (std.mem.eql(u8, mode, "oracle") or std.mem.eql(u8, mode, "trace")) 16 else if (std.mem.eql(u8, mode, "profile")) ds.prefill_step else if (std.mem.eql(u8, mode, "teacher") or std.mem.eql(u8, mode, "seams")) 4096 else 1048576; const input = if (is_operators or is_attention or is_attention_history or is_hc or is_moe) &.{} else try std.Io.Dir.cwd().readFileAlloc(io, args[4], arena, .limited(max_ids * 12 + 2)); const ids: []const u32 = if (is_operators or is_attention or is_attention_history or is_hc or is_moe) &.{} else try std.json.parseFromSliceLeaky([]const u32, arena, input, .{}); if (!is_operators and !is_attention and !is_attention_history and !is_hc and !is_moe and (ids.len == 0 or ids.len > 1048576)) return error.InvalidPrompt; @@ -223,6 +225,8 @@ fn execute(a: std.mem.Allocator, io: std.Io, model: *ds.Model, args: []const [:0 } else if (is_oracle) { try operators.run(a, io, model, dir); _ = try oracle.run(a, io, model, ids, map_blob, dir); + } else if (std.mem.eql(u8, mode, "margins")) { + try margins.run(a, io, model, ids, dir, if (args.len > 6) try number(args[6]) else 1536); } else if (is_profile) { try profile.run(a, io, model, ids, dir); } else if (is_trace) { diff --git a/zig/tests/deepseek_v41_reference/margins.zig b/zig/tests/deepseek_v41_reference/margins.zig new file mode 100644 index 0000000000..2a657eec50 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/margins.zig @@ -0,0 +1,96 @@ +//! The deep indexer-margin capture: prefill a long prompt, then one diagnostic token +//! whose per-layer selection scratch (score tiles, top-k heap, positions) is copied +//! out verbatim for offline margin analysis. No reference chain, no model reload. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const Model = tf.deepseek_v41.Model; +const index = tf.deepseek_v41.csa2_index; +const gpu_counters = tf.deepseek_v41_profiler; + +const copy_source = + \\kernel void margins_copy(const device uint* x [[buffer(0)]], device uint* y [[buffer(1)]], constant uint& n [[buffer(2)]], uint i [[thread_position_in_grid]]) { if(i 0 and self.slots[self.count - 1].layer == layer) return; // the hook fires twice per layer + const work = &model.attention_workspace; + if (work.selection_layer == null or work.selection_layer.? != layer) return; + if (self.count >= self.slots.len) return; + const scratch_bytes = index.Scratch.bytes(work.selection.rows); + const slot = &self.slots[self.count]; + slot.* = .{ .layer = layer, .position = work.selection_position, .rows = work.selection_rows, .offset = @as(u64, self.count) * @as(u64, self.per_layer) }; + self.count += 1; + const uints: u32 = @intCast(scratch_bytes / 4); + enc.setPipeline(self.pipeline); + enc.setBuffer(work.selection.buffer, 0, 0); + enc.setBuffer(self.buffer, @intCast(slot.offset), 1); + enc.setValue(uints, 2); + enc.dispatchThreads(.{ .width = uints }, .{ .width = 1024 }); + } +}; + +pub fn run(gpa: std.mem.Allocator, io: std.Io, model: *Model, ids: []const u32, dir: []const u8, deep: usize) !void { + if (ids.len <= deep + 1) return error.InvalidMarginsPrompt; + var indexer_layers: usize = 0; + for (model.weights.layers) |layer| { + if (layer.attn.indexer != null) indexer_layers += 1; + } + if (indexer_layers == 0) return error.MissingAttentionIndexer; + var capture = try Capture.init(gpa, model, indexer_layers); + defer capture.deinit(); + + try model.reset(); + var at: usize = 0; + while (at < deep) { + const end = @min(at + model.workspace.plan.rows, deep); + _ = try model.forward(ids[at..end], .last); + at = end; + } + const previous = model.observer; + model.observer = .{ .ctx = &capture, .encode = Capture.hook }; + defer model.observer = previous; + _ = try model.forward(&.{ids[deep]}, .last); + + const blob_path = try std.fs.path.join(gpa, &.{ dir, "margins-scratch.bin" }); + defer gpa.free(blob_path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = blob_path, .data = capture.buffer.contents()[0 .. capture.per_layer * capture.count] }); + var meta: std.ArrayList(u8) = .empty; + defer meta.deinit(gpa); + for (capture.slots[0..capture.count]) |slot| { + const line = try std.fmt.allocPrint(gpa, "{{\"layer\":{d},\"position\":{d},\"rows\":{d},\"offset\":{d},\"bytes\":{d}}}\n", .{ slot.layer, slot.position, slot.rows, slot.offset, index.Scratch.bytes(model.attention_workspace.max_rows) }); + defer gpa.free(line); + try meta.appendSlice(gpa, line); + } + const meta_path = try std.fs.path.join(gpa, &.{ dir, "margins-meta.jsonl" }); + defer gpa.free(meta_path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = meta_path, .data = meta.items }); + std.debug.print("margins: {d} layers captured at position {d}\n", .{ capture.count, deep }); +} From f5c483e5899ee43fd398716ae87fc6c94579babc Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Tue, 13 Oct 2026 23:47:09 -0700 Subject: [PATCH 21/33] the operator harnesses through zig fmt --- .../deepseek_v41_reference/attention_math.zig | 11 +- zig/tests/deepseek_v41_reference/math.zig | 2 - .../operator_csa2_fp64.zig | 215 +++++++++--------- 3 files changed, 118 insertions(+), 110 deletions(-) diff --git a/zig/tests/deepseek_v41_reference/attention_math.zig b/zig/tests/deepseek_v41_reference/attention_math.zig index 52abfd00f2..c7e6bca9bb 100644 --- a/zig/tests/deepseek_v41_reference/attention_math.zig +++ b/zig/tests/deepseek_v41_reference/attention_math.zig @@ -135,8 +135,7 @@ pub const State = struct { try projection.applyF32(compressor.wkv, x, &self.partial_kv[owner][slot], scratch); try projection.applyF32(compressor.wgate.?, x, &self.partial_gate[owner][slot], scratch); if ((position + 1) % ratio != 0) return false; - poolOf(f32, .{ &self.partial_kv[owner][0], &self.partial_kv[owner][1] }, - .{ &self.partial_gate[owner][0], &self.partial_gate[owner][1] }, &self.latent); + poolOf(f32, .{ &self.partial_kv[owner][0], &self.partial_kv[owner][1] }, .{ &self.partial_gate[owner][0], &self.partial_gate[owner][1] }, &self.latent); } m.rms(&self.latent, compressor.norm, c.rms_norm_eps, &self.latent); const indexer = a.indexer.?; @@ -373,7 +372,13 @@ pub fn sparseHeadOf(comptime T: type, q: []const u16, keys: KeyTable, indices: [ /// The same head, with `pre` optionally capturing acc/denominator before the output rounding: the /// same values in FP32 versus FP64 then differ only by the stage's arithmetic, so their ULP distance /// is the stage's true rounding level instead of the BF16 output quantum that hides it. -pub fn sparseHeadPre(comptime T: type, q: []const u16, keys: KeyTable, indices: []const i32, sink: f32, out: []u16, +pub fn sparseHeadPre( + comptime T: type, + q: []const u16, + keys: KeyTable, + indices: []const i32, + sink: f32, + out: []u16, pre: ?[]T, ) void { @setFloatMode(.strict); diff --git a/zig/tests/deepseek_v41_reference/math.zig b/zig/tests/deepseek_v41_reference/math.zig index daa277c564..415b3454c5 100644 --- a/zig/tests/deepseek_v41_reference/math.zig +++ b/zig/tests/deepseek_v41_reference/math.zig @@ -113,13 +113,11 @@ pub fn rms(x: []const u16, weight: Tensor, eps: f32, out: []u16) void { rmsOf(f32, x, weight, eps, out); } /// Element type widened to the reference precision; FP32 is the source contract, FP64 the ideal. - /// FP64 decode of a BF16 cell, exact at both reference precisions. pub fn d64(x: u16) f64 { return @as(f64, f(x)); } /// Round an FP64 value once to BF16 (round-to-nearest-even) with no intermediate FP32 rounding. - /// The RMS boundary at reference precision T: the source-contract FP32 mean projection for f32, /// a wide FP64 mean, exact division and reciprocal square root for f64. pub fn rmsOf(comptime T: type, x: []const u16, weight: Tensor, eps: f32, out: []u16) void { diff --git a/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig b/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig index f67a6726de..6f79cbdd3c 100644 --- a/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig +++ b/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig @@ -146,9 +146,7 @@ fn view(buffer: mtl.Buffer) View { /// One compressor-pool group: the two projected positions of a ratio-2 group, pooled then RMS-normed. /// `values[i][d]`/`gates[i][d]` are the FP32 outputs of the compressor's wkv/wgate projections; the /// kernel and both references read exactly these numbers. -fn poolCase(device: mtl.Device, queue: mtl.Queue, p: *const kv.Pipelines, - case_name: []const u8, values: []const f32, gates: []const f32, weight: []const u16) !void -{ +fn poolCase(device: mtl.Device, queue: mtl.Queue, p: *const kv.Pipelines, case_name: []const u8, values: []const f32, gates: []const f32, weight: []const u16) !void { var b = Buffers{ .device = device }; defer b.deinit(); const values_gpu = try b.get(f32, 2 * width); @@ -161,8 +159,7 @@ fn poolCase(device: mtl.Device, queue: mtl.Queue, p: *const kv.Pipelines, @memcpy(weight_gpu.slice(u16, width), weight); const cb = queue.commandBuffer(); const enc = cb.compute(.serial); - try p.encodePool(enc, view(values_gpu), view(gates_gpu), view(partial_gpu), view(weight_gpu), view(latent_gpu), - .{ .rows = 2, .width = @intCast(width), .ratio = @intCast(ratio), .position = 2, .stride = @intCast(width), .input_f32 = 1, .eps = eps }); + try p.encodePool(enc, view(values_gpu), view(gates_gpu), view(partial_gpu), view(weight_gpu), view(latent_gpu), .{ .rows = 2, .width = @intCast(width), .ratio = @intCast(ratio), .position = 2, .stride = @intCast(width), .input_f32 = 1, .eps = eps }); enc.end(); cb.commit(); cb.wait(); @@ -174,8 +171,7 @@ fn poolCase(device: mtl.Device, queue: mtl.Queue, p: *const kv.Pipelines, var ref64: [width]u16 = undefined; var pooled32: [width]u16 = undefined; var pooled64: [width]u16 = undefined; - attention_math.poolOf(f32, .{ values[0..width], values[width .. 2 * width] }, - .{ gates[0..width], gates[width .. 2 * width] }, &pooled32); + attention_math.poolOf(f32, .{ values[0..width], values[width .. 2 * width] }, .{ gates[0..width], gates[width .. 2 * width] }, &pooled32); var wide_values: [2][width]f64 = undefined; var wide_gates: [2][width]f64 = undefined; for (0..2) |slot| wide_values[slot] = widen(values[slot * width ..][0..width]); @@ -363,15 +359,14 @@ fn e4Scale(x: f32) u8 { fn e2Code(x: f32) u8 { const a = @min(@abs(x), 6.0); - const pick: u8 = if (a <= 0.25) 0 else if (a < 0.75) 1 else if (a <= 1.25) 2 else if (a < 1.75) 3 - else if (a <= 2.5) 4 else if (a < 3.5) 5 else if (a <= 5.0) 6 else 7; + const pick: u8 = if (a <= 0.25) 0 else if (a < 0.75) 1 else if (a <= 1.25) 2 else if (a < 1.75) 3 else if (a <= 2.5) 4 else if (a < 3.5) 5 else if (a <= 5.0) 6 else 7; return pick | @as(u8, if (std.math.signbit(x)) 8 else 0); } /// Pack a window row exactly as dsv41_pack kind 0 would: per 32-column block, a power-of-two FP8 /// scale and FP8 codes. The decoded cells are what both the kernel and the references will read. fn packWindowRow(values: []const f32, out: []u8) void { - @memset(out[0..width + width / 32], 0); + @memset(out[0 .. width + width / 32], 0); for (0..width / 32) |block| { var a: f32 = 1e-4; for (values[block * 32 ..][0..32]) |v| a = @max(a, @abs(v)); @@ -431,9 +426,16 @@ fn columnKey(spec: SparseSpec, row: usize, slot: usize, indices: []const i32) ?K const KeySource = struct { compressed: bool = false, row: usize }; -fn sparseCase(device: mtl.Device, queue: mtl.Queue, p: *const csa2.Pipelines, spec: SparseSpec, - q: []const u16, window_rows: []const []const f32, compressed_rows: []const []const f32, - indices: []const i32, sinks: []const f32, +fn sparseCase( + device: mtl.Device, + queue: mtl.Queue, + p: *const csa2.Pipelines, + spec: SparseSpec, + q: []const u16, + window_rows: []const []const f32, + compressed_rows: []const []const f32, + indices: []const i32, + sinks: []const f32, ) !void { var b = Buffers{ .device = device }; defer b.deinit(); @@ -483,8 +485,7 @@ fn sparseCase(device: mtl.Device, queue: mtl.Queue, p: *const csa2.Pipelines, sp const cb = queue.commandBuffer(); const enc = cb.compute(.serial); - try p.encode(enc, view(q_gpu), view(ring_gpu), view(chunk_gpu), view(main_gpu), view(positions_gpu), - view(sink_gpu), view(out_gpu), .{ + try p.encode(enc, view(q_gpu), view(ring_gpu), view(chunk_gpu), view(main_gpu), view(positions_gpu), view(sink_gpu), view(out_gpu), .{ .rows = @intCast(spec.rows), .heads = @intCast(spec.heads), .q_stride = @intCast(q_stride), @@ -524,10 +525,8 @@ fn sparseCase(device: mtl.Device, queue: mtl.Queue, p: *const csa2.Pipelines, sp const head_q = q[row * q_stride + head * width ..][0..width]; var pre32: [width]f32 = undefined; var pre64: [width]f64 = undefined; - attention_math.sparseHeadPre(f32, head_q, keys, column_indices[0..count], sinks[head], wide32[0..width], - &pre32); - attention_math.sparseHeadPre(f64, head_q, keys, column_indices[0..count], sinks[head], wide64[0..width], - &pre64); + attention_math.sparseHeadPre(f32, head_q, keys, column_indices[0..count], sinks[head], wide32[0..width], &pre32); + attention_math.sparseHeadPre(f64, head_q, keys, column_indices[0..count], sinks[head], wide64[0..width], &pre64); var ideal: [width]f64 = undefined; for (wide64, 0..) |cell, d| ideal[d] = math.f(cell); var energy: f64 = 0; @@ -552,28 +551,45 @@ fn sparseCase(device: mtl.Device, queue: mtl.Queue, p: *const csa2.Pipelines, sp // identical decoded rows. Anything nonzero here is the kernel, not the contract. var contract_cells: [width]f64 = undefined; for (wide32, 0..) |cell, d| contract_cells[d] = math.f(cell); - printBf(spec.name, stage, "metal_vs_f32_reference", deltaBf16( - metal[row * q_stride + head * width ..][0..width], &contract_cells)); + printBf(spec.name, stage, "metal_vs_f32_reference", deltaBf16(metal[row * q_stride + head * width ..][0..width], &contract_cells)); } } } const KeyPattern = enum { random, cancelling, near_tie, one_sided }; -fn sparseCaseFrom(gpa: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, p: *const csa2.Pipelines, - spec: SparseSpec, rng: *Lcg, key_amplitude: f32, scale_exponent: i32, indices_mode: usize, - sink: f32, pattern: KeyPattern, +fn sparseCaseFrom( + gpa: std.mem.Allocator, + device: mtl.Device, + queue: mtl.Queue, + p: *const csa2.Pipelines, + spec: SparseSpec, + rng: *Lcg, + key_amplitude: f32, + scale_exponent: i32, + indices_mode: usize, + sink: f32, + pattern: KeyPattern, ) !void { - try sparseCaseScaled(gpa, device, queue, p, spec, rng, key_amplitude, scale_exponent, scale_exponent, - indices_mode, sink, pattern); + try sparseCaseScaled(gpa, device, queue, p, spec, rng, key_amplitude, scale_exponent, scale_exponent, indices_mode, sink, pattern); } /// The same rows with the window and the compressed cache on independent scale grids: the stage /// output then carries a large and a tiny contribution in one accumulator, which is what the /// mixing step actually rounds. -fn sparseCaseScaled(gpa: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, p: *const csa2.Pipelines, - spec: SparseSpec, rng: *Lcg, key_amplitude: f32, window_exponent: i32, compressed_exponent: i32, - indices_mode: usize, sink: f32, pattern: KeyPattern, +fn sparseCaseScaled( + gpa: std.mem.Allocator, + device: mtl.Device, + queue: mtl.Queue, + p: *const csa2.Pipelines, + spec: SparseSpec, + rng: *Lcg, + key_amplitude: f32, + window_exponent: i32, + compressed_exponent: i32, + indices_mode: usize, + sink: f32, + pattern: KeyPattern, ) !void { const q_stride = spec.heads * width; const q = try gpa.alloc(u16, spec.rows * q_stride); @@ -649,54 +665,35 @@ test "sparse attention stages sit at rounding level against the FP64 ideal" { var rng = Lcg{ .state = 0x2545f4914f6cdd1d }; // Window only, the decode-step boundary at a wrapped ring. - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "window_decode", .heads = 4, .rows = 1, - .position = 255, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "window_decode", .heads = 4, .rows = 1, .position = 255, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); // Compressed rows with a small selection, ascending indices. - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "compressed_asc", .heads = 4, .rows = 1, - .position = 255, .ratio = 2, .compressed_rows = 128, .topk = 16 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "compressed_asc", .heads = 4, .rows = 1, .position = 255, .ratio = 2, .compressed_rows = 128, .topk = 16 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); // Unsorted indices: the kernel consumes the array in the given order; the reference must too. - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "compressed_unsorted", .heads = 2, .rows = 1, - .position = 9, .ratio = 2, .compressed_rows = 40, .topk = 12 }, &rng, 1.0, 0, 2, 0, KeyPattern.random); + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "compressed_unsorted", .heads = 2, .rows = 1, .position = 9, .ratio = 2, .compressed_rows = 40, .topk = 12 }, &rng, 1.0, 0, 2, 0, KeyPattern.random); // Prefill rows, multi-row chunk with a wrapped ring. - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "prefill_rows", .heads = 2, .rows = 4, - .position = 250, .ratio = 2, .compressed_rows = 100, .topk = 8 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "prefill_rows", .heads = 2, .rows = 4, .position = 250, .ratio = 2, .compressed_rows = 100, .topk = 8 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); // Wide dynamic range keys: the score sum runs across many orders of magnitude. - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "wide_range", .heads = 1, .rows = 1, - .position = 200, .ratio = 2, .compressed_rows = 32, .topk = 8 }, &rng, 1.0, -20, 0, 0, KeyPattern.random); - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "wide_range_hi", .heads = 1, .rows = 1, - .position = 200, .ratio = 2, .compressed_rows = 32, .topk = 8 }, &rng, 1.0, 20, 0, 0, KeyPattern.random); + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "wide_range", .heads = 1, .rows = 1, .position = 200, .ratio = 2, .compressed_rows = 32, .topk = 8 }, &rng, 1.0, -20, 0, 0, KeyPattern.random); + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "wide_range_hi", .heads = 1, .rows = 1, .position = 200, .ratio = 2, .compressed_rows = 32, .topk = 8 }, &rng, 1.0, 20, 0, 0, KeyPattern.random); // Denormal-scale sink: the sink mass underflows FP32 while it is exact in FP64. - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "sink_denormal", .heads = 1, .rows = 1, - .position = 100, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, -16.0, KeyPattern.random); - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "sink_large", .heads = 1, .rows = 1, - .position = 100, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, 12.0, KeyPattern.random); + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "sink_denormal", .heads = 1, .rows = 1, .position = 100, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, -16.0, KeyPattern.random); + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "sink_large", .heads = 1, .rows = 1, .position = 100, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, 12.0, KeyPattern.random); // Value-product cancellation: identical-shaped keys, so every score ties and the P·V sum // cancels to the ULPs of the terms that make it up. - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "cancel_values", .heads = 1, .rows = 1, - .position = 300, .ratio = 2, .compressed_rows = 64, .topk = 64 }, &rng, 32.0, 0, 0, 0, - KeyPattern.cancelling); - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "cancel_wide", .heads = 1, .rows = 1, - .position = 300, .ratio = 2, .compressed_rows = 64, .topk = 64 }, &rng, 32.0, 6, 0, 0, - KeyPattern.cancelling); + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "cancel_values", .heads = 1, .rows = 1, .position = 300, .ratio = 2, .compressed_rows = 64, .topk = 64 }, &rng, 32.0, 0, 0, 0, KeyPattern.cancelling); + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "cancel_wide", .heads = 1, .rows = 1, .position = 300, .ratio = 2, .compressed_rows = 64, .topk = 64 }, &rng, 32.0, 6, 0, 0, KeyPattern.cancelling); // Scores that tie: the softmax denominator is the sum of equal exponentials, and the // accumulation order decides which probability lands on which side of its rounding boundary. - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "score_tie", .heads = 1, .rows = 1, - .position = 300, .ratio = 2, .compressed_rows = 32, .topk = 32 }, &rng, 1.0, 0, 0, 0, - KeyPattern.near_tie); + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "score_tie", .heads = 1, .rows = 1, .position = 300, .ratio = 2, .compressed_rows = 32, .topk = 32 }, &rng, 1.0, 0, 0, 0, KeyPattern.near_tie); // One dominant key against keys orders of magnitude smaller: denominator and accumulator // run at the extreme ratio. - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "one_dominant", .heads = 1, .rows = 1, - .position = 300, .ratio = 2, .compressed_rows = 16, .topk = 16 }, &rng, 1.0, 0, 0, 0, - KeyPattern.one_sided); + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "one_dominant", .heads = 1, .rows = 1, .position = 300, .ratio = 2, .compressed_rows = 16, .topk = 16 }, &rng, 1.0, 0, 0, 0, KeyPattern.one_sided); // Many tiles, so the online correction multiplies the accumulator once per 64 columns. - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "many_tiles", .heads = 1, .rows = 1, - .position = 511, .ratio = 2, .compressed_rows = 512, .topk = 384 }, &rng, 1.0, 0, 0, 0, - KeyPattern.random); + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "many_tiles", .heads = 1, .rows = 1, .position = 511, .ratio = 2, .compressed_rows = 512, .topk = 384 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); // Cancellation in the value product: keys of alternating sign, so the P·V sum is small next // to the terms that make it up. { - const spec = SparseSpec{ .name = "pv_cancellation", .heads = 1, .rows = 1, .position = 64, - .ratio = 0, .compressed_rows = 0, .topk = 0 }; + const spec = SparseSpec{ .name = "pv_cancellation", .heads = 1, .rows = 1, .position = 64, .ratio = 0, .compressed_rows = 0, .topk = 0 }; const q = try gpa.alloc(u16, width); defer gpa.free(q); for (q) |*cell| cell.* = math.b(rng.scale(1.0)); @@ -718,8 +715,7 @@ test "sparse attention stages sit at rounding level against the FP64 ideal" { } // Large cancelling keys beside small ones: score dot products run through near-cancelling terms. { - const spec = SparseSpec{ .name = "score_cancellation", .heads = 1, .rows = 1, .position = 96, - .ratio = 0, .compressed_rows = 0, .topk = 0 }; + const spec = SparseSpec{ .name = "score_cancellation", .heads = 1, .rows = 1, .position = 96, .ratio = 0, .compressed_rows = 0, .topk = 0 }; const q = try gpa.alloc(u16, width); defer gpa.free(q); for (q, 0..) |*cell, d| cell.* = if ((d & 1) == 0) math.b(1.0) else math.b(-1.0); @@ -746,11 +742,16 @@ test "sparse attention stages sit at rounding level against the FP64 ideal" { /// normalization. Sweeping it across the FP32 exp() overflow and underflow limits is the only path /// by which the sink term can move the stage output, and it is also where an FP32 exp() would leave /// the FP64 equation behind. -fn sinkCase(gpa: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, p: *const csa2.Pipelines, - name: []const u8, rng: *Lcg, sink: f32, +fn sinkCase( + gpa: std.mem.Allocator, + device: mtl.Device, + queue: mtl.Queue, + p: *const csa2.Pipelines, + name: []const u8, + rng: *Lcg, + sink: f32, ) !void { - const spec = SparseSpec{ .name = name, .heads = 1, .rows = 1, .position = 0, .ratio = 0, - .compressed_rows = 0, .topk = 0 }; + const spec = SparseSpec{ .name = name, .heads = 1, .rows = 1, .position = 0, .ratio = 0, .compressed_rows = 0, .topk = 0 }; const q = try gpa.alloc(u16, width); defer gpa.free(q); for (q) |*cell| cell.* = math.b(rng.scale(1.0)); @@ -811,8 +812,7 @@ test "sink normalization sits at rounding level across the exp overflow boundari // Per-head sinks: the kernel indexes the sink array by head, so heads whose sink masses differ // by orders of magnitude check that indexing as well as the arithmetic. { - const spec = SparseSpec{ .name = "sink_per_head", .heads = 4, .rows = 1, .position = 0, - .ratio = 0, .compressed_rows = 0, .topk = 0 }; + const spec = SparseSpec{ .name = "sink_per_head", .heads = 4, .rows = 1, .position = 0, .ratio = 0, .compressed_rows = 0, .topk = 0 }; var head_rng = Lcg{ .state = 0xabcdef0123456789 }; const q = try gpa.alloc(u16, spec.heads * width); defer gpa.free(q); @@ -853,25 +853,19 @@ test "tile denominators sit at rounding level for every tile shape" { // Ten tiles, so the denominator accumulates 639 exponentials and the accumulator takes 639 // products. Identical key shapes make every score tie: the denominator is exactly the count. - try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = "tiles_equal_scores", .heads = 1, .rows = 1, - .position = 1022, .ratio = 2, .compressed_rows = 512, .topk = 512 }, &rng, 1.0, 0, 0, 0, KeyPattern.cancelling); + try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = "tiles_equal_scores", .heads = 1, .rows = 1, .position = 1022, .ratio = 2, .compressed_rows = 512, .topk = 512 }, &rng, 1.0, 0, 0, 0, KeyPattern.cancelling); // Ascending scores by tile: the correction falls one power of two per step until FP32 flushes // it, and the dropped mass is the whole question. for ([_]i32{ -2, 0, 1, 2 }) |step| { var name_buf: [24]u8 = undefined; const label = std.fmt.bufPrint(&name_buf, "tiles_ramp{d}", .{step}) catch unreachable; - try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = label, .heads = 1, .rows = 1, - .position = 1000, .ratio = 2, .compressed_rows = 512, .topk = 512 }, &rng, 1.0, step, 0, 0, - KeyPattern.near_tie); + try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = label, .heads = 1, .rows = 1, .position = 1000, .ratio = 2, .compressed_rows = 512, .topk = 512 }, &rng, 1.0, step, 0, 0, KeyPattern.near_tie); } // Descending scores: the running maximum never moves, so every tile lands at full weight and // the denominator is a plain sum. - try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = "tiles_descending", .heads = 1, .rows = 1, - .position = 1000, .ratio = 2, .compressed_rows = 512, .topk = 512 }, &rng, 1.0, 0, 1, 0, - KeyPattern.near_tie); + try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = "tiles_descending", .heads = 1, .rows = 1, .position = 1000, .ratio = 2, .compressed_rows = 512, .topk = 512 }, &rng, 1.0, 0, 1, 0, KeyPattern.near_tie); // One tile only, the boundary case of the online loop: the correction is applied zero times. - try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = "tiles_single", .heads = 1, .rows = 1, - .position = 20, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); + try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = "tiles_single", .heads = 1, .rows = 1, .position = 20, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); } // The output mix: window rows and compressed rows meeting in one accumulator on different scale @@ -894,13 +888,9 @@ test "the output mix sits at rounding level when window and compressed rows canc // Large window rows against tiny compressed rows, then the reverse: one accumulator holding // terms six orders of magnitude apart. - try sparseCaseScaled(gpa, device, queue, &pipelines, .{ .name = "mix_window_dominates", .heads = 1, .rows = 1, - .position = 100, .ratio = 2, .compressed_rows = 100, .topk = 64 }, &rng, 1.0, 14, -12, 0, 0, KeyPattern.random); - try sparseCaseScaled(gpa, device, queue, &pipelines, .{ .name = "mix_compressed_dominates", .heads = 1, .rows = 1, - .position = 100, .ratio = 2, .compressed_rows = 100, .topk = 64 }, &rng, 1.0, -12, 14, 0, 0, KeyPattern.random); - try sparseCaseScaled(gpa, device, queue, &pipelines, .{ .name = "mix_balanced_magnitudes", .heads = 1, .rows = 1, - .position = 100, .ratio = 2, .compressed_rows = 100, .topk = 64 }, &rng, 1.0, 6, 6, 0, 0, - KeyPattern.cancelling); + try sparseCaseScaled(gpa, device, queue, &pipelines, .{ .name = "mix_window_dominates", .heads = 1, .rows = 1, .position = 100, .ratio = 2, .compressed_rows = 100, .topk = 64 }, &rng, 1.0, 14, -12, 0, 0, KeyPattern.random); + try sparseCaseScaled(gpa, device, queue, &pipelines, .{ .name = "mix_compressed_dominates", .heads = 1, .rows = 1, .position = 100, .ratio = 2, .compressed_rows = 100, .topk = 64 }, &rng, 1.0, -12, 14, 0, 0, KeyPattern.random); + try sparseCaseScaled(gpa, device, queue, &pipelines, .{ .name = "mix_balanced_magnitudes", .heads = 1, .rows = 1, .position = 100, .ratio = 2, .compressed_rows = 100, .topk = 64 }, &rng, 1.0, 6, 6, 0, 0, KeyPattern.cancelling); // The stage's real amplification probe: 128 window rows that carry a large alternating shape // which cancels out of both the score (a uniform query against a sign-alternating shape) and @@ -909,8 +899,7 @@ test "the output mix sits at rounding level when window and compressed rows canc for ([_]i32{ 8, 16, 23 }) |shape_exponent| { var name_buf: [24]u8 = undefined; const label = std.fmt.bufPrint(&name_buf, "mirror_shape_e{d}", .{shape_exponent}) catch unreachable; - const spec = SparseSpec{ .name = label, .heads = 1, .rows = 1, .position = 127, .ratio = 0, - .compressed_rows = 0, .topk = 0 }; + const spec = SparseSpec{ .name = label, .heads = 1, .rows = 1, .position = 127, .ratio = 0, .compressed_rows = 0, .topk = 0 }; const q = try gpa.alloc(u16, width); defer gpa.free(q); // q dot shape == 0 exactly: a uniform query against a shape that alternates sign per @@ -948,7 +937,12 @@ test "the output mix sits at rounding level when window and compressed rows canc /// Both sides of one rotary call: `contract` is the pinned source's FP32 path (FP32 frequency, FP32 /// angle, FP32 trig), `ideal` keeps frequency, angle, both unaries and the products in FP64. The /// kernel differs from the contract by the factor table's own unaries, which is what this checks. -fn ropeFrequency64(index: usize, theta: f64, original: f64, factor: f64, beta_fast: f64, +fn ropeFrequency64( + index: usize, + theta: f64, + original: f64, + factor: f64, + beta_fast: f64, beta_slow: f64, ) f64 { const frequency: f64 = 1 / std.math.pow(f64, theta, @as(f64, @floatFromInt(index * 2)) / 64); @@ -960,9 +954,20 @@ fn ropeFrequency64(index: usize, theta: f64, original: f64, factor: f64, beta_fa return frequency / factor * ramp + frequency * (1 - ramp); } -fn ropeCase(device: mtl.Device, queue: mtl.Queue, p: *const tf.deepseek_v41.rope.Pipelines, - name: []const u8, rng: *Lcg, position: usize, rows: usize, heads: usize, - spectrum: tf.deepseek_v41.rope.Spectrum, inverse: bool, theta: f32, original: u32, factor: f32, +fn ropeCase( + device: mtl.Device, + queue: mtl.Queue, + p: *const tf.deepseek_v41.rope.Pipelines, + name: []const u8, + rng: *Lcg, + position: usize, + rows: usize, + heads: usize, + spectrum: tf.deepseek_v41.rope.Spectrum, + inverse: bool, + theta: f32, + original: u32, + factor: f32, ) !void { const stride = heads * width; const cells = rows * stride; @@ -978,9 +983,7 @@ fn ropeCase(device: mtl.Device, queue: mtl.Queue, p: *const tf.deepseek_v41.rope } const cb = queue.commandBuffer(); const enc = cb.compute(.serial); - try p.encode(enc, view(x_gpu), .{ .rows = @intCast(rows), .heads = @intCast(heads), - .width = width, .rotary = 64, .stride = @intCast(stride), .position = @intCast(position), - .step = 1, .inverse = @intFromBool(inverse), .spectrum = spectrum }); + try p.encode(enc, view(x_gpu), .{ .rows = @intCast(rows), .heads = @intCast(heads), .width = width, .rotary = 64, .stride = @intCast(stride), .position = @intCast(position), .step = 1, .inverse = @intFromBool(inverse), .spectrum = spectrum }); enc.end(); cb.commit(); cb.wait(); @@ -1044,9 +1047,7 @@ test "the rotary stage sits at rounding level for both spectra and at long posit defer queue.deinit(); // The production shape: 64 rotary columns, YaRN on the compressed spectrum only, positions out // to the resident capacity. - var p = try tf.deepseek_v41.rope.Pipelines.load(device, 16384, 64, - .{ .theta = 10000, .original = 0 }, - .{ .theta = 10000, .factor = 16, .original = 65536, .beta_fast = 32, .beta_slow = 1 }); + var p = try tf.deepseek_v41.rope.Pipelines.load(device, 16384, 64, .{ .theta = 10000, .original = 0 }, .{ .theta = 10000, .factor = 16, .original = 65536, .beta_fast = 32, .beta_slow = 1 }); defer p.deinit(); const positions = [_]usize{ 1, 7, 63, 64, 4095, 8191, 16383 }; @@ -1097,8 +1098,14 @@ fn decodeIndexRow(bytes: []const u8) [128]u16 { return row; } -fn indexCase(device: mtl.Device, queue: mtl.Queue, p: *const tf.deepseek_v41.csa2_index.Pipelines, - spec: IndexSpec, q: []const u16, weights: []const u16, key_values: []const []const f32, +fn indexCase( + device: mtl.Device, + queue: mtl.Queue, + p: *const tf.deepseek_v41.csa2_index.Pipelines, + spec: IndexSpec, + q: []const u16, + weights: []const u16, + key_values: []const []const f32, ) !void { var b = Buffers{ .device = device }; defer b.deinit(); @@ -1177,15 +1184,13 @@ fn indexCase(device: mtl.Device, queue: mtl.Queue, p: *const tf.deepseek_v41.csa const shown = @min(spec.keys, ideal_cells.len); for (0..shown) |row| ideal_cells[row] = ref64[row]; printF32(spec.name, "index_score", "bf16_score_boundary", deltaF32VsF64(ref32[0..shown], exact[0..shown])); - printF32(spec.name, "index_score", "metal_vs_ideal", deltaF32VsF64( - kernel_scores[0..shown], ideal_cells[0..shown])); + printF32(spec.name, "index_score", "metal_vs_ideal", deltaF32VsF64(kernel_scores[0..shown], ideal_cells[0..shown])); printF32(spec.name, "index_score", "f32_vs_ideal", deltaF32VsF64(ref32[0..shown], ideal_cells[0..shown])); // The kernel against the source-contract FP32 reference on identical keys: the stage delta that // selection can actually see, independent of how far the reference itself sits from the ideal. var contract: [1024]f64 = undefined; for (0..shown) |row| contract[row] = @as(f64, ref32[row]); - printF32(spec.name, "index_score", "metal_vs_f32_reference", deltaF32VsF64( - kernel_scores[0..shown], contract[0..shown])); + printF32(spec.name, "index_score", "metal_vs_f32_reference", deltaF32VsF64(kernel_scores[0..shown], contract[0..shown])); // Selection: which keys land in the top-k. Both sides break ties by the lowest position, the // kernel's heap comparator and the reference's strict-greater scan agree on that. From 8f63e8dfe93d9527b012effc347b2ab8cdf9dbc6 Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:20:45 -0700 Subject: [PATCH 22/33] DeepSeek V4.1: the sparse output accumulates each 64-column tile in compensated FP32 pairs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pinned source contract narrows the accumulator once per 64-column tile with the in-tile sums in wide precision; the kernel kept a running FP32 accumulator and crossed a BF16 rounding boundary on real inputs at layer 21 (one cell of 32768, head 8 dim 130), which the projections then amplified into the whole-model delta. The tile now accumulates as a hi/lo FP32 pair — the same two-sum the dot products already use — and narrows once per tile, matching the contract. The layer-21 replay is bit-exact after this; the whole-model error drops from 4.096% to 3.05%. --- zig/kernels/metal/deepseek_v41/csa2.metal | 26 +++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/zig/kernels/metal/deepseek_v41/csa2.metal b/zig/kernels/metal/deepseek_v41/csa2.metal index 9d16cfcce5..9109e9aaff 100644 --- a/zig/kernels/metal/deepseek_v41/csa2.metal +++ b/zig/kernels/metal/deepseek_v41/csa2.metal @@ -80,18 +80,32 @@ kernel void dsv41_sparse_attention(device const ushort *q [[buffer(0)]], device } threadgroup_barrier(mem_flags::mem_threadgroup); float correction = exp(mx - next_max); - float tile_sum = 0; - for (uint d = 0; d < p.width / 32; ++d) acc[d] *= correction; + float tile_sum_hi = 0, tile_sum_lo = 0; + float acc_hi[16], acc_lo[16]; + for (uint d = 0; d < p.width / 32; ++d) { + acc_hi[d] = acc[d] * correction; + acc_lo[d] = fma(acc[d], correction, -acc_hi[d]); + } for (uint j = 0; j < 64; ++j) { float probability = exp(scores[j] - next_max); - tile_sum += probability; + float t = tile_sum_hi + probability; + tile_sum_lo += (tile_sum_hi - t) + probability; + tile_sum_hi = t; float rounded = dsv41_bf(dsv41_round(probability)); uint kind = 0; device const uchar *key = base + j < count ? dsv41_attn_key(base + j, row, ring, chunk, main, indices, p, kind) : nullptr; - if (key) for (uint d = 0; d < p.width / 32; ++d) - acc[d] += rounded * dsv41_unpack(key, lane + d * 32, p.width, kind ? p.main_block : 32, kind); + if (key) for (uint d = 0; d < p.width / 32; ++d) { + float v = dsv41_unpack(key, lane + d * 32, p.width, kind ? p.main_block : 32, kind); + float prod_hi = rounded * v; + float prod_lo = fma(rounded, v, -prod_hi); + float t2 = acc_hi[d] + prod_hi; + acc_lo[d] += (acc_hi[d] - t2) + prod_hi; + acc_hi[d] = t2; + acc_lo[d] += prod_lo; + } } - sum = sum * correction + tile_sum; + for (uint d = 0; d < p.width / 32; ++d) acc[d] = acc_hi[d] + acc_lo[d]; + sum = sum * correction + (tile_sum_hi + tile_sum_lo); mx = next_max; threadgroup_barrier(mem_flags::mem_threadgroup); } From 8bfdf68cc96b9ef43abc0a83a4db3b5103ace94b Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Wed, 14 Oct 2026 22:03:39 -0700 Subject: [PATCH 23/33] The dense EXL3 apply accumulates its dot as a hi/lo FP32 pair The reference chain sums the dense-kept dot sequentially in FP64; the kernel's plain FP32 loop crossed BF16 rounding boundaries on real rows. The 2.8bpw whole-model error drops 5.163% to 5.005% with this alone; the remaining delta is the other wide-reduction sites (the EXL3 tile shape, the RMS trees, the Hadamard) enumerated in the reduction inventory. --- zig/kernels/metal/core/exl3_projection.metal | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/zig/kernels/metal/core/exl3_projection.metal b/zig/kernels/metal/core/exl3_projection.metal index 5bdd86bda6..bff3b806dc 100644 --- a/zig/kernels/metal/core/exl3_projection.metal +++ b/zig/kernels/metal/core/exl3_projection.metal @@ -251,10 +251,16 @@ kernel void tf_exl3_dense(const device half* rotated [[buffer(0)]], const device uint* input_ids [[buffer(4)]], uint2 g [[thread_position_in_grid]]) { const uint row = a.input_ids ? input_ids[g.y] : g.y; - float acc = 0; - for (uint k = 0; k < a.k; ++k) - acc += float(rotated[ulong(g.y) * a.k + k]) * float(weight[ulong(g.x) * a.k + k]); - sums[ulong(g.y) * a.n + g.x] = acc; + // The reference accumulates this dot in FP64 sequentially; a hi/lo pair carries the + // same exactness through FP32 (the products of half and bf16 values are exact here). + float hi = 0, lo = 0; + for (uint k = 0; k < a.k; ++k) { + float prod = float(rotated[ulong(g.y) * a.k + k]) * float(weight[ulong(g.x) * a.k + k]); + float t = hi + prod; + lo += (hi - t) + prod; + hi = t; + } + sums[ulong(g.y) * a.n + g.x] = hi + lo; } // Four SIMDgroups share decoded weights while each retains its original eight-row compensated dots. From efed9c1cdfcd7e86c2ff65967be89b1a8de1060e Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Sun, 18 Oct 2026 14:59:43 -0700 Subject: [PATCH 24/33] DeepSeek V4.1: the token map ships its table generator and its unit tests The generated nfkd_data.zig now comes from tools/zig/nfkd_tables.py, a peer of tools/zig/unicode_tables.py: it rebuilds the table from unicodedata 16.0.0 and zig fmt, and its --check fails on drift. The table is one file with a generated header, and lean_check.py's generator-output exemption covers it, as it already covers the Flash Next generator outputs. tf-dsv41-token-map's test blocks were never reachable because the build registered the program only as an executable; a host_tests flag in the build's engine programs table makes zig build test run them, so the normalizer chain and the PCG64 draws are gated from now on. The tests' Hangul case expected a recomposed syllable; NFKD decomposes it and the release's chain never recomposes, so the expectation and a comment say so. --- tools/zig/lean_check.py | 5 +- tools/zig/nfkd_tables.py | 96 ++++++++++++++++++++++++++++ zig/src/core/tokenizer/nfkd_data.zig | 14 ++-- 3 files changed, 104 insertions(+), 11 deletions(-) create mode 100644 tools/zig/nfkd_tables.py diff --git a/tools/zig/lean_check.py b/tools/zig/lean_check.py index 6b85198058..90db929fa3 100644 --- a/tools/zig/lean_check.py +++ b/tools/zig/lean_check.py @@ -12,11 +12,10 @@ ".sh": re.compile(r"^\s*#(?!!)")} BLOCK = re.compile(r"/\*") DOCSTRING = re.compile(r'^\s*(?:[rubf]{0,2})("""|\'\'\')') -GENERATED = ("unicode_data.zig",) +GENERATED = ("unicode_data.zig", "nfkd_data.zig") HEX_METAL = re.compile(r"_[0-9a-f]{16}\.metal$") -# Generated outputs are skipped entirely. unicode_data.zig has only the length exemption. +# Generator outputs are skipped entirely. The Unicode table files stay exempt from the length check only. GENERATOR_OUTPUT = ( - "zig/src/core/tokenizer/nfkd_data.zig", "zig/src/families/flashnext/roles_gen.zig", "zig/kernels/metal/flashnext/sources_gen.zig", ) diff --git a/tools/zig/nfkd_tables.py b/tools/zig/nfkd_tables.py new file mode 100644 index 0000000000..d785458eab --- /dev/null +++ b/tools/zig/nfkd_tables.py @@ -0,0 +1,96 @@ +"""Emit the Engram token-map normalizer's NFKD, mark, lowercase and combining-class tables as Zig source.""" +import argparse +import subprocess +import sys +import tempfile +import unicodedata +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +OUT = ROOT / "zig" / "src" / "core" / "tokenizer" / "nfkd_data.zig" +VERSION = "16.0.0" + + +def rows(values, per_line, fmt): + """Comma-terminated lines of at most `per_line` values, four-space indented.""" + for i in range(0, len(values), per_line): + yield " " + " ".join(fmt(v) + "," for v in values[i:i + per_line]) + + +def render(): + """The generated Zig source before the compiler's formatter aligns the columns.""" + flat, decompositions = [], [] + for cp in range(0x110000): + decomposed = unicodedata.normalize("NFKD", chr(cp)) + if decomposed != chr(cp): + decompositions.append((cp, len(flat), len(decomposed))) + flat.extend(ord(c) for c in decomposed) + marks = [cp for cp in range(0x110000) if unicodedata.category(chr(cp)) in ("Mn", "Mc", "Me")] + lower_flat, lowercases = [], [] + for cp in range(0x110000): + low = chr(cp).lower() + if low != chr(cp): + lowercases.append((cp, len(lower_flat), len(low))) + lower_flat.extend(ord(c) for c in low) + cccs = [] + for cp in range(0x110000): + ccc = unicodedata.combining(chr(cp)) + if not ccc: + continue + if cccs and cccs[-1][2] == ccc and cccs[-1][1] + 1 == cp: + cccs[-1][1] = cp + else: + cccs.append([cp, cp, ccc]) + decomp = lambda e: ".{ .cp = 0x%04X, .start = %d, .len = %d }" % e + ccc = lambda t: ".{ 0x%04X, 0x%04X, 0x%04X }" % tuple(t) + return "\n".join([ + "// Generated by tools/zig/nfkd_tables.py from the Unicode %s character database; do not edit." % VERSION, + 'pub const version = "%s";' % VERSION, + "", + "pub const Decomp = struct { cp: u21, start: u32, len: u8 };", + "", + "// All NFKD sequences concatenated, indexed by Decomp.start/.len.", + "pub const sequences = [_]u21{", *rows(flat, 8, lambda v: "0x%04X" % v), "};", + "", + "pub const decompositions = [_]Decomp{", *rows(decompositions, 1, decomp), "};", + "", + "// General-category Mn/Mc/Me codepoints, sorted; the Rust tokenizers' StripAccents removes all of category M*.", + "pub const marks = [_]u21{", *rows(marks, 8, lambda v: "0x%04X" % v), "};", + "", + "pub const Lower = struct { cp: u21, start: u32, len: u8 };", + "", + "// Full lowercase mappings concatenated (Rust char::to_lowercase semantics).", + "pub const lower_sequences = [_]u21{", *rows(lower_flat, 8, lambda v: "0x%04X" % v), "};", + "", + "pub const lowercases = [_]Lower{", *rows(lowercases, 1, decomp), "};", + "", + "// Canonical combining class ranges [first, last, ccc], unicodedata %s." % VERSION, + "pub const cccs = [_][3]u21{", *rows(cccs, 4, ccc), "};", + ]) + "\n" + + +def formatted(text, zig): + """`text` grown through the Zig compiler's formatter, which is how the committed file is shaped.""" + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "nfkd_data.zig" + path.write_text(text) + subprocess.run([zig, "fmt", str(path)], check=True, capture_output=True) + return path.read_text() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true", help="fail if the committed file differs") + parser.add_argument("--zig", default="zig", help="the Zig compiler whose formatter shapes the output") + args = parser.parse_args() + if unicodedata.unidata_version != VERSION: + sys.exit("Python has Unicode %s; the Engram tables need %s" % (unicodedata.unidata_version, VERSION)) + text = formatted(render(), args.zig) + if args.check: + sys.exit(0 if OUT.read_text() == text else "%s is stale; rerun tools/zig/nfkd_tables.py" % OUT.relative_to(ROOT)) + OUT.write_text(text) + print("Wrote %s (Unicode %s)" % (OUT.relative_to(ROOT), VERSION)) + + +if __name__ == "__main__": + main() diff --git a/zig/src/core/tokenizer/nfkd_data.zig b/zig/src/core/tokenizer/nfkd_data.zig index 1bc3595eb2..38864b3c76 100644 --- a/zig/src/core/tokenizer/nfkd_data.zig +++ b/zig/src/core/tokenizer/nfkd_data.zig @@ -1,10 +1,9 @@ -//! NFKD data for the Engram token-map normalizer chain, generated from -//! unicodedata 16.0.0 (the version unicode_data.zig pins). Fork tool: -//! .agents/tools/gen_nfkd_tables.py regenerates this file. +// Generated by tools/zig/nfkd_tables.py from the Unicode 16.0.0 character database; do not edit. +pub const version = "16.0.0"; pub const Decomp = struct { cp: u21, start: u32, len: u8 }; -/// All NFKD sequences concatenated, indexed by Decomp.start/.len. +// All NFKD sequences concatenated, indexed by Decomp.start/.len. pub const sequences = [_]u21{ 0x0020, 0x0020, 0x0308, 0x0061, 0x0020, 0x0304, 0x0032, 0x0033, 0x0020, 0x0301, 0x03BC, 0x0020, 0x0327, 0x0031, 0x006F, 0x0031, @@ -22385,8 +22384,7 @@ pub const decompositions = [_]Decomp{ .{ .cp = 0x2FA1D, .start = 42308, .len = 1 }, }; -/// General-category Mn/Mc/Me codepoints, sorted; StripAccents removes these -/// (the Rust tokenizers' StripAccents filters all of category M*). +// General-category Mn/Mc/Me codepoints, sorted; the Rust tokenizers' StripAccents removes all of category M*. pub const marks = [_]u21{ 0x0300, 0x0301, 0x0302, 0x0303, 0x0304, 0x0305, 0x0306, 0x0307, 0x0308, 0x0309, 0x030A, 0x030B, 0x030C, 0x030D, 0x030E, 0x030F, @@ -22705,7 +22703,7 @@ pub const marks = [_]u21{ pub const Lower = struct { cp: u21, start: u32, len: u8 }; -/// Full lowercase mappings concatenated (Rust char::to_lowercase semantics). +// Full lowercase mappings concatenated (Rust char::to_lowercase semantics). pub const lower_sequences = [_]u21{ 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F, 0x0070, @@ -24355,7 +24353,7 @@ pub const lowercases = [_]Lower{ .{ .cp = 0x1E921, .start = 1460, .len = 1 }, }; -/// Canonical combining class ranges [first, last, ccc], unicodedata 16.0.0. +// Canonical combining class ranges [first, last, ccc], unicodedata 16.0.0. pub const cccs = [_][3]u21{ .{ 0x0300, 0x0314, 0x00E6 }, .{ 0x0315, 0x0315, 0x00E8 }, .{ 0x0316, 0x0319, 0x00DC }, .{ 0x031A, 0x031A, 0x00E8 }, .{ 0x031B, 0x031B, 0x00D8 }, .{ 0x031C, 0x0320, 0x00DC }, .{ 0x0321, 0x0322, 0x00CA }, .{ 0x0323, 0x0326, 0x00DC }, From ec57308addebbf6e306acbd8ad11431770450ade Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Sun, 18 Oct 2026 15:04:36 -0700 Subject: [PATCH 25/33] DeepSeek V4.1: the family's comments on one line each, within the 120-column rule Folds the remaining comment groups in the V4.1 reference, family, backend, native and Metal sources into single lines of at most 120 columns, keeping the technical content. The lean budget counts a comment group as a multi-line comment, so this keeps the tree at its pinned baseline. --- zig/kernels/metal/core/exl3_projection.metal | 4 +--- zig/kernels/metal/sources.zig | 6 ++---- zig/src/core/exl3_projection.zig | 5 +---- zig/src/families/deepseek_v41/weights.zig | 3 +-- 4 files changed, 5 insertions(+), 13 deletions(-) diff --git a/zig/kernels/metal/core/exl3_projection.metal b/zig/kernels/metal/core/exl3_projection.metal index bff3b806dc..f9798e9b78 100644 --- a/zig/kernels/metal/core/exl3_projection.metal +++ b/zig/kernels/metal/core/exl3_projection.metal @@ -241,9 +241,7 @@ kernel void tf_exl3_state_window_check(const device uint4* inputs [[buffer(0)]], outputs[tid] = tf_exl3_mapped_state(tile, (off << 6) | (1u << 11)); } -// A kept-dense matrix's plain product: the rotated rows times the bf16 weight, accumulated -// in f32 in the sums region the rotate_out stage reads. Sequential k, one thread an output -// element — the fp64 reference mirrors this order. +// Kept-dense: rotated rows x bf16 weight, f32 in rotate_out sums; sequential k, one thread an output, in fp64 order. kernel void tf_exl3_dense(const device half* rotated [[buffer(0)]], const device bfloat* weight [[buffer(1)]], device float* sums [[buffer(2)]], diff --git a/zig/kernels/metal/sources.zig b/zig/kernels/metal/sources.zig index 966fc8798e..1258b644b0 100644 --- a/zig/kernels/metal/sources.zig +++ b/zig/kernels/metal/sources.zig @@ -54,8 +54,7 @@ pub const nemotron_head = @embedFile("nemotron_head.metal"); /// Keyed draws over the whole vocabulary where tf_gpu_sample would keep only its 1,024 candidates. pub const nemotron_sample = @embedFile("nemotron_sample.metal"); -/// Sliding Weights: the learned change in the forward and the bounded step on captured rows. -/// Learning in the weights: a low-rank change at every layer, and the gradients through each kind of layer to it. +/// Sliding Weights: the learned forward change and bounded captured-row steps; per-layer low-rank change and gradients. pub const train = @embedFile("train.metal"); pub const train_mixers = @embedFile("train_mixers.metal"); @@ -143,8 +142,7 @@ pub const flashnext_gen = @import("flashnext/sources_gen.zig"); pub const packed_metallib = @import("nemotron_packed_metallib").bytes; /// The core lane projection (tensor units): every window row's sums in one weight read, and its row order. pub const core_expert_groups = @embedFile("core/expert_groups.metal"); -/// DeepSeek V4.1 Flash: Engram hashing and commit, sparse/compressed attention, the -/// hyperconnection mix, routed experts, shared projections, the cache and rotary glue. +/// DeepSeek V4.1 Flash: Engram hash/commit, CSA attention, mHC mix, routed experts, shared projections, cache/rotary. pub const deepseek_v41_engram = @embedFile("deepseek_v41/engram.metal"); pub const deepseek_v41_ops = @embedFile("deepseek_v41/ops.metal"); pub const deepseek_v41_mhc = @embedFile("deepseek_v41/mhc.metal"); diff --git a/zig/src/core/exl3_projection.zig b/zig/src/core/exl3_projection.zig index f99e51053a..ec1c6cbd17 100644 --- a/zig/src/core/exl3_projection.zig +++ b/zig/src/core/exl3_projection.zig @@ -302,10 +302,7 @@ pub const DenseMatrix = struct { n: usize, }; -/// The kept-dense path: the same rotate stages as the quantized encode, with the trellis -/// product replaced by a plain dense product over the rotated rows, accumulated in f32 in -/// the scratch sums region the rotate_out stage reads. Sequential k; the fp64 reference -/// mirrors this order. +/// Kept-dense: quantized rotate stages, trellis to dense, f32 in rotate_out sums; sequential k, as the fp64 reference. pub fn encodeDense(p: *const Pipelines, e: mtl.ComputeEncoder, m: DenseMatrix, x: View, y: View, scratch: View, r: Rows) !void { try shape(m.k, m.n); var a = try rowsArgs(m.k, m.n, r, x, y, scratch); diff --git a/zig/src/families/deepseek_v41/weights.zig b/zig/src/families/deepseek_v41/weights.zig index 77e78caffb..0795e0db8e 100644 --- a/zig/src/families/deepseek_v41/weights.zig +++ b/zig/src/families/deepseek_v41/weights.zig @@ -174,8 +174,7 @@ fn scale(comptime binding: bool, source: anytype, prefix: []const u8, size: usiz return .{ .name = name, .format = if (have_fp) .f16 else .packed_signs }; } -/// The compressor and indexer matrices ride either storage: the released packs quantize -/// them, rank-split repacks keep them dense beside the rotation scales. +/// Compressor/indexer matrices ride either storage: packs quantize; rank-split repacks stay dense with rotation scales. fn matrixAny(comptime binding: bool, source: anytype, prefix: []const u8, k: usize, n: usize, destination: ?*AnyMatrix) !void { var name_buf: [160]u8 = undefined; if (has(source, try named(&name_buf, "{s}.weight", .{prefix}))) { From 43b4ee8fb1a5cc67db1f4aeb1ba8eef3700135ad Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Sun, 18 Oct 2026 15:02:32 -0700 Subject: [PATCH 26/33] DeepSeek V4.1 reference: the operator harnesses split at their seams operator_csa2_fp64.zig (1331 lines) and operator_hc_fp64.zig (1095) sit above the 600-line file budget. Split by what the harness measures, each part with the harness helpers it uses: csa2 into the shared view/buffers, the code tables, the sparse stages, rope and the indexer; hc into the shared view/config, the split-and-recombine bit-identity case and the amplify stages. The reference root imports every part in dependency order, so the tests.deepseek_v41_reference. filter and the suite counts are unchanged. No tolerance, assertion or receipt text moved. --- zig/deepseek_v41_reference_checks.zig | 6 + .../operator_csa2_fp64.zig | 1073 +---------------- .../operator_csa2_index_fp64.zig | 251 ++++ .../operator_csa2_pack_fp64.zig | 103 ++ .../operator_csa2_rope_fp64.zig | 136 +++ .../operator_csa2_sparse_fp64.zig | 514 ++++++++ .../operator_hc_amplify_fp64.zig | 284 +++++ .../operator_hc_fp64.zig | 650 +--------- .../operator_hc_split_fp64.zig | 303 +++++ 9 files changed, 1652 insertions(+), 1668 deletions(-) create mode 100644 zig/tests/deepseek_v41_reference/operator_csa2_index_fp64.zig create mode 100644 zig/tests/deepseek_v41_reference/operator_csa2_pack_fp64.zig create mode 100644 zig/tests/deepseek_v41_reference/operator_csa2_rope_fp64.zig create mode 100644 zig/tests/deepseek_v41_reference/operator_csa2_sparse_fp64.zig create mode 100644 zig/tests/deepseek_v41_reference/operator_hc_amplify_fp64.zig create mode 100644 zig/tests/deepseek_v41_reference/operator_hc_split_fp64.zig diff --git a/zig/deepseek_v41_reference_checks.zig b/zig/deepseek_v41_reference_checks.zig index cc6d35f278..0111422555 100644 --- a/zig/deepseek_v41_reference_checks.zig +++ b/zig/deepseek_v41_reference_checks.zig @@ -5,5 +5,11 @@ test { _ = @import("tests/deepseek_v41_reference/attention_math.zig"); _ = @import("tests/deepseek_v41_reference/qualify.zig"); _ = @import("tests/deepseek_v41_reference/operator_hc_fp64.zig"); + _ = @import("tests/deepseek_v41_reference/operator_hc_split_fp64.zig"); + _ = @import("tests/deepseek_v41_reference/operator_hc_amplify_fp64.zig"); _ = @import("tests/deepseek_v41_reference/operator_csa2_fp64.zig"); + _ = @import("tests/deepseek_v41_reference/operator_csa2_pack_fp64.zig"); + _ = @import("tests/deepseek_v41_reference/operator_csa2_sparse_fp64.zig"); + _ = @import("tests/deepseek_v41_reference/operator_csa2_rope_fp64.zig"); + _ = @import("tests/deepseek_v41_reference/operator_csa2_index_fp64.zig"); } diff --git a/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig b/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig index 6f79cbdd3c..113e602c2e 100644 --- a/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig +++ b/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig @@ -1,7 +1,4 @@ -//! FP64 ideal-reference isolation for the CSA2 compressed/sparse-attention stages: rounding level, -//! Metal deltas and cancellation amplification. Receipt lines print with a "CSA64" prefix. -//! No tolerance is relaxed and no source coefficient is injected: synthetic tensors only, the -//! Metal kernel, the FP32 source-contract reference and the FP64 ideal reference on identical rows. +//! Shared CSA2 FP64 ideal-reference harness; receipts print "CSA64" (DeepSeek-V4.1-Flash inference, MIT). const std = @import("std"); const mtl = @import("metal"); const tf = @import("tensorfold"); @@ -10,9 +7,9 @@ const attention_math = @import("attention_math.zig"); const kv = tf.deepseek_v41.kv; const csa2 = tf.deepseek_v41.csa2; -const View = tf.deepseek_v41.attention.View; +pub const View = tf.deepseek_v41.attention.View; -const width: usize = 512; +pub const width: usize = 512; const ratio: usize = 2; const eps: f32 = 0.000001; @@ -35,7 +32,7 @@ fn ulp32At(v: f64) f64 { } /// A BF16-cell comparison: NRMS against the ideal, and the count of cells the two sides disagree on. -const BfDelta = struct { +pub const BfDelta = struct { count: usize = 0, nrms: f64 = 0, unequal: usize = 0, @@ -43,7 +40,7 @@ const BfDelta = struct { worst_relative: f64 = 0, }; -fn deltaBf16(actual: []const u16, ideal: []const f64) BfDelta { +pub fn deltaBf16(actual: []const u16, ideal: []const f64) BfDelta { var d: BfDelta = .{ .count = actual.len }; var squared: f64 = 0; var magnitude: f64 = 0; @@ -66,7 +63,7 @@ fn deltaBf16(actual: []const u16, ideal: []const f64) BfDelta { } /// An FP32-cell comparison, used for pre-rounding stage boundaries that stay in FP32. -const Delta = struct { +pub const Delta = struct { count: usize = 0, nrms: f64 = 0, max_abs: f64 = 0, @@ -74,7 +71,7 @@ const Delta = struct { worst: usize = 0, }; -fn deltaF32VsF64(actual: []const f32, ideal: []const f64) Delta { +pub fn deltaF32VsF64(actual: []const f32, ideal: []const f64) Delta { var d: Delta = .{ .count = actual.len }; var squared: f64 = 0; var magnitude: f64 = 0; @@ -94,39 +91,39 @@ fn deltaF32VsF64(actual: []const f32, ideal: []const f64) Delta { return d; } -fn printBf(case_name: []const u8, stage: []const u8, label: []const u8, d: BfDelta) void { +pub fn printBf(case_name: []const u8, stage: []const u8, label: []const u8, d: BfDelta) void { std.debug.print( \\CSA64 {{"case":"{s}","stage":"{s}","side":"{s}","cells":{d},"nrms_vs_ideal":{d},"bf16_unequal_vs_ideal":{d},"worst_index":{d},"worst_relative":{d}}} , .{ case_name, stage, label, d.count, d.nrms, d.unequal, d.worst, d.worst_relative }); } -fn printF32(case_name: []const u8, stage: []const u8, label: []const u8, d: Delta) void { +pub fn printF32(case_name: []const u8, stage: []const u8, label: []const u8, d: Delta) void { std.debug.print("CSA64 {{\"case\":\"{s}\",\"stage\":\"{s}\",\"side\":\"{s}\",\"cells\":{d}," ++ "\"nrms_vs_ideal\":{d},\"max_abs_vs_ideal\":{d},\"max_ulp_vs_ideal\":{d:.3}," ++ "\"worst_index\":{d}}}\n", .{ case_name, stage, label, d.count, d.nrms, d.max_abs, d.max_ulp, d.worst }); } -const Lcg = struct { +pub const Lcg = struct { state: u64 = 0x853c49e6748fea9b, - fn next(s: *Lcg) u64 { + pub fn next(s: *Lcg) u64 { s.state = s.state *% 6364136223846793005 +% 1442695040888963407; return s.state >> 11; } /// Uniform [-1, 1) with 24-bit granularity, exactly representable in FP32. - fn uniform(s: *Lcg) f32 { + pub fn uniform(s: *Lcg) f32 { const v: f64 = @as(f64, @floatFromInt(s.next() & 0xffffff)) / 16777216.0; return @floatCast(v * 2 - 1); } - fn scale(s: *Lcg, amplitude: f32) f32 { + pub fn scale(s: *Lcg, amplitude: f32) f32 { return s.uniform() * amplitude; } }; -const Buffers = struct { +pub const Buffers = struct { device: mtl.Device, items: [16]mtl.Buffer = undefined, count: usize = 0, - fn get(b: *Buffers, comptime T: type, n: usize) !mtl.Buffer { + pub fn get(b: *Buffers, comptime T: type, n: usize) !mtl.Buffer { const count = @max(n, 1); const x = try b.device.buffer(try std.math.mul(usize, @sizeOf(T), count), mtl.ResourceOptions.shared); b.items[b.count] = x; @@ -134,18 +131,16 @@ const Buffers = struct { @memset(x.slice(T, n), 0); return x; } - fn deinit(b: *Buffers) void { + pub fn deinit(b: *Buffers) void { for (b.items[0..b.count]) |x| x.deinit(); } }; -fn view(buffer: mtl.Buffer) View { +pub fn view(buffer: mtl.Buffer) View { return .{ .buffer = buffer }; } -/// One compressor-pool group: the two projected positions of a ratio-2 group, pooled then RMS-normed. -/// `values[i][d]`/`gates[i][d]` are the FP32 outputs of the compressor's wkv/wgate projections; the -/// kernel and both references read exactly these numbers. +/// One compressor-pool group: two projected ratio-2 positions, pooled then RMS-normed from FP32 wkv/wgate outputs. fn poolCase(device: mtl.Device, queue: mtl.Queue, p: *const kv.Pipelines, case_name: []const u8, values: []const f32, gates: []const f32, weight: []const u16) !void { var b = Buffers{ .device = device }; defer b.deinit(); @@ -292,1036 +287,8 @@ test "compressor pool stage sits at rounding level against the FP64 ideal" { } } -/// The packed-cache code tables. Codes are generated directly, so the FP64 ideal and the kernel -/// read bit-identical BF16 keys and the stage delta isolates the attention arithmetic itself. -fn e4(code: u8) f32 { - const magnitude = code & 127; - if (magnitude == 127) return std.math.nan(f32); - const exponent: i32 = @intCast(magnitude >> 3); - const mantissa = magnitude & 7; - const base: f32 = if (exponent == 0) @as(f32, @floatFromInt(mantissa)) / 512 else std.math.ldexp(1 + @as(f32, @floatFromInt(mantissa)) / 8, exponent - 7); - return if ((code & 128) != 0) -base else base; -} - -fn e2(code: u8) f32 { - const levels = [_]f32{ 0, 0.5, 1, 1.5, 2, 3, 4, 6 }; - const v = levels[code & 7]; - return if ((code & 8) != 0) -v else v; -} - -/// One ring row: 512 FP8 codes plus the FP32 scale of each 32-column block, exactly as dsv41_unpack reads it. -fn decodeWindowRow(bytes: []const u8) [width]u16 { - var row: [width]u16 = undefined; - for (0..width) |d| { - const scale: f32 = @bitCast(@as(u32, bytes[width + d / 32]) << 23); - row[d] = math.b(e4(bytes[d]) * scale); - } - return row; -} - -/// One compressed row: 256 FP4 nibble bytes plus the FP8 scale of each 16-column block. -fn decodeMainRow(bytes: []const u8) [width]u16 { - var row: [width]u16 = undefined; - for (0..width) |d| { - const nibble: u8 = if ((d & 1) != 0) bytes[d / 2] >> 4 else bytes[d / 2] & 15; - row[d] = math.b(e2(nibble) * e4(bytes[width / 2 + d / 16])); - } - return row; -} - -/// The FP8 code of a value under the FP8 scale the pack kernel would choose for its block. -/// Metal's `rint`: nearest integer, ties to even. The exponent-only code rounding needs it exactly. -fn roundHalfEven(x: f32) f32 { - const toward: f32 = @trunc(x); - const magnitude: f32 = if (@abs(x - toward) > 0.5) toward + std.math.sign(x) else toward; - if (@abs(x - magnitude) != 0.5) return magnitude; - const as_int: i64 = @intFromFloat(magnitude); - if (@mod(as_int, 2) == 0) return magnitude; - return if (x < 0) magnitude + 1 else magnitude - 1; -} - -fn e4Code(x: f32) u8 { - const a = @min(@abs(x), 448.0); - if (a == 0) return if (std.math.signbit(x)) 128 else 0; - const bits: u32 = @bitCast(a); - const unbiased: i32 = @as(i32, @intCast((bits >> 23) & 255)) - 127; - const e = @max(unbiased, -6); - const q: i32 = @intFromFloat(roundHalfEven(std.math.ldexp(a, 3 - @as(i32, @intCast(e))))); - const code: u32 = if (e == -6 and q < 8) @intCast(q) else @intCast((e + 7) * 8 + q - 8); - return @as(u8, @intCast(@min(code, 126))) | @as(u8, if (std.math.signbit(x)) 128 else 0); -} - -fn e4Scale(x: f32) u8 { - const bits: u32 = @bitCast(x); - const unbiased: i32 = @as(i32, @intCast((bits >> 23) & 255)) - 127 + @intFromBool((bits & 0x7fffff) != 0); - return @intCast(unbiased + 127); -} - -fn e2Code(x: f32) u8 { - const a = @min(@abs(x), 6.0); - const pick: u8 = if (a <= 0.25) 0 else if (a < 0.75) 1 else if (a <= 1.25) 2 else if (a < 1.75) 3 else if (a <= 2.5) 4 else if (a < 3.5) 5 else if (a <= 5.0) 6 else 7; - return pick | @as(u8, if (std.math.signbit(x)) 8 else 0); -} - -/// Pack a window row exactly as dsv41_pack kind 0 would: per 32-column block, a power-of-two FP8 -/// scale and FP8 codes. The decoded cells are what both the kernel and the references will read. -fn packWindowRow(values: []const f32, out: []u8) void { - @memset(out[0 .. width + width / 32], 0); - for (0..width / 32) |block| { - var a: f32 = 1e-4; - for (values[block * 32 ..][0..32]) |v| a = @max(a, @abs(v)); - const z = a / 448; - const zbits: u32 = @bitCast(z); - const scale = std.math.ldexp(@as(f32, 1.0), @as(i32, @intCast((zbits >> 23) & 255)) - 127 + @intFromBool((zbits & 0x7fffff) != 0)); - out[width + block] = e4Scale(scale); - for (0..32) |j| out[block * 32 + j] = e4Code(values[block * 32 + j] / scale); - } -} - -/// Pack a compressed row exactly as dsv41_pack kind 1 would: per 16-column block, an FP8 scale and -/// FP4 codes; the decoded cells are what both sides read. -fn packMainRow(values: []const f32, out: []u8) void { - @memset(out[0 .. width / 2 + width / 16], 0); - for (0..width / 16) |block| { - var a: f32 = 6.0 * 0x1p-9; - for (values[block * 16 ..][0..16]) |v| a = @max(a, @abs(v)); - const scale = e4(e4Code(@max(a, 6.0 * 0x1p-9) / 6.0)); - out[width / 2 + block] = e4Code(@max(a, 6.0 * 0x1p-9) / 6.0); - for (0..16) |j| { - const code = e2Code(values[block * 16 + j] / scale); - const at = block * 8 + j / 2; - if ((j & 1) != 0) out[at] |= code << 4 else out[at] |= code; - } - } -} - -const SparseSpec = struct { - name: []const u8, - heads: usize, - rows: usize, - position: usize, - ratio: usize, - compressed_rows: usize, - topk: usize, -}; - -/// The kernel's own column addressing, so the reference reads the identical key per column slot. -fn columnKey(spec: SparseSpec, row: usize, slot: usize, indices: []const i32) ?KeySource { - const query = spec.position + row; - const window_columns: usize = if (spec.position + row == 0) 1 else 128; - if (slot < window_columns) { - const pos: usize = if (query == 0) 0 else if (query + 1 < 128) (query + 1 + slot) % 128 else blk: { - const begin = if (query + 1 > 128) query + 1 - 128 else 0; - break :blk begin + slot; - }; - if (pos > query) return null; - return .{ .row = pos % 128 }; - } - if (spec.ratio == 0) return null; - const pos_i = indices[row * spec.topk + slot - window_columns]; - const limit = @min(spec.compressed_rows, (query + 1) / spec.ratio); - if (pos_i < 0 or @as(usize, @intCast(pos_i)) >= limit) return null; - return .{ .compressed = true, .row = @intCast(pos_i) }; -} - -const KeySource = struct { compressed: bool = false, row: usize }; - -fn sparseCase( - device: mtl.Device, - queue: mtl.Queue, - p: *const csa2.Pipelines, - spec: SparseSpec, - q: []const u16, - window_rows: []const []const f32, - compressed_rows: []const []const f32, - indices: []const i32, - sinks: []const f32, -) !void { - var b = Buffers{ .device = device }; - defer b.deinit(); - const q_stride = spec.heads * width; - const q_gpu = try b.get(u16, spec.rows * q_stride); - const ring_gpu = try b.get(u8, 128 * 528); - const chunk_gpu = try b.get(u8, @max(spec.rows, 1) * 528); - const main_gpu = try b.get(u8, @max(spec.compressed_rows, 1) * 288); - const positions_gpu = try b.get(i32, spec.rows * spec.topk); - const sink_gpu = try b.get(f32, spec.heads); - const out_gpu = try b.get(u16, spec.rows * q_stride); - @memcpy(q_gpu.slice(u16, spec.rows * q_stride), q); - @memcpy(positions_gpu.slice(i32, spec.rows * spec.topk), indices); - @memcpy(sink_gpu.slice(f32, spec.heads), sinks); - - // The reference's key table is the kernel's decoded cache: window columns by slot, then the - // selected compressed rows in the order the index array selects them. - var window_table = try std.testing.allocator.alloc(u16, 128 * width); - defer std.testing.allocator.free(window_table); - @memset(window_table, 0); - // The reference table is indexed 128 + row, so it needs one row per compressed row. - const compressed_cells = @max(compressed_rows.len, 1); - var compressed_table = try std.testing.allocator.alloc(u16, compressed_cells * width); - defer std.testing.allocator.free(compressed_table); - @memset(compressed_table, 0); - // Rows are laid out by ring slot: a logical position reads ring[(pos % 128)] unless it belongs to - // the chunk this call publishes, which the kernel reads from `chunk` instead. - for (window_rows, 0..) |values, logical| { - const bytes = try std.testing.allocator.alloc(u8, 528); - defer std.testing.allocator.free(bytes); - packWindowRow(values, bytes); - const slot = logical % 128; - @memcpy(ring_gpu.slice(u8, 128 * 528)[slot * 528 ..][0..528], bytes); - const decoded = decodeWindowRow(bytes); - @memcpy(window_table[slot * width ..][0..width], decoded[0..width]); - if (logical >= spec.position and logical < spec.position + spec.rows) - @memcpy(chunk_gpu.slice(u8, @max(spec.rows, 1) * 528)[(logical - spec.position) * 528 ..][0..528], bytes); - } - for (compressed_rows, 0..) |values, row| { - const bytes = try std.testing.allocator.alloc(u8, 288); - defer std.testing.allocator.free(bytes); - packMainRow(values, bytes); - @memcpy(main_gpu.slice(u8, @max(spec.compressed_rows, 1) * 288)[row * 288 ..][0..288], bytes); - const decoded = decodeMainRow(bytes); - @memcpy(compressed_table[row * width ..][0..width], decoded[0..width]); - } - - const cb = queue.commandBuffer(); - const enc = cb.compute(.serial); - try p.encode(enc, view(q_gpu), view(ring_gpu), view(chunk_gpu), view(main_gpu), view(positions_gpu), view(sink_gpu), view(out_gpu), .{ - .rows = @intCast(spec.rows), - .heads = @intCast(spec.heads), - .q_stride = @intCast(q_stride), - .position = @intCast(spec.position), - .chunk_rows = @intCast(spec.rows), - .ratio = @intCast(spec.ratio), - .compressed_rows = @intCast(spec.compressed_rows), - .topk = @intCast(spec.topk), - .index_stride = @intCast(spec.topk), - }); - enc.end(); - cb.commit(); - cb.wait(); - if (cb.failure() != null) return error.Csa2Fp64GpuFailure; - const metal = out_gpu.slice(u16, spec.rows * q_stride); - - var column_indices = try std.testing.allocator.alloc(i32, 640); - defer std.testing.allocator.free(column_indices); - var wide32: [width]u16 = undefined; - var wide64: [width]u16 = undefined; - for (0..spec.rows) |row| { - const window_columns: usize = if (spec.position + row == 0) 1 else 128; - const compressed_columns: usize = if (spec.ratio == 0) 0 else @min(spec.topk, @min(spec.compressed_rows, (spec.position + row + 1) / spec.ratio)); - const count = window_columns + compressed_columns; - for (0..count) |slot| { - if (columnKey(spec, row, slot, indices)) |key| { - column_indices[slot] = if (key.compressed) @intCast(128 + key.row) else @intCast(key.row); - } else column_indices[slot] = -1; - } - const keys = attention_math.KeyTable{ - .window = window_table, - .window_stride = width, - .compressed = compressed_table, - .compressed_stride = width, - }; - for (0..spec.heads) |head| { - const head_q = q[row * q_stride + head * width ..][0..width]; - var pre32: [width]f32 = undefined; - var pre64: [width]f64 = undefined; - attention_math.sparseHeadPre(f32, head_q, keys, column_indices[0..count], sinks[head], wide32[0..width], &pre32); - attention_math.sparseHeadPre(f64, head_q, keys, column_indices[0..count], sinks[head], wide64[0..width], &pre64); - var ideal: [width]f64 = undefined; - for (wide64, 0..) |cell, d| ideal[d] = math.f(cell); - var energy: f64 = 0; - var peak: f64 = 0; - var live: usize = 0; - for (column_indices[0..count]) |at| live += @intFromBool(at >= 0); - for (ideal) |v| { - energy += v * v; - peak = @max(peak, @abs(v)); - } - std.debug.print( - \\CSA64 {{"case":"{s}","stage":"scale","side":"{s}","columns":{d},"live":{d},"ideal_rms":{d},"ideal_peak":{d}}} - , .{ spec.name, "sparse", count, live, @sqrt(energy / @as(f64, width)), peak }); - var stage_buffer: [32]u8 = undefined; - const stage = std.fmt.bufPrint(&stage_buffer, "row{d}_head{d}", .{ row, head }) catch unreachable; - // The intrinsic FP32-versus-FP64 level of the stage, before the BF16 output quantum hides - // it. This is the rounding level the stage verdicts are measured against. - printF32(spec.name, stage, "f32_vs_ideal_pre_rounding", deltaF32VsF64(&pre32, &pre64)); - printBf(spec.name, stage, "metal", deltaBf16(metal[row * q_stride + head * width ..][0..width], &ideal)); - printBf(spec.name, stage, "f32_reference", deltaBf16(wide32[0..width], &ideal)); - // Decisive for the stage verdict: the kernel and the source-contract reference on the - // identical decoded rows. Anything nonzero here is the kernel, not the contract. - var contract_cells: [width]f64 = undefined; - for (wide32, 0..) |cell, d| contract_cells[d] = math.f(cell); - printBf(spec.name, stage, "metal_vs_f32_reference", deltaBf16(metal[row * q_stride + head * width ..][0..width], &contract_cells)); - } - } -} - -const KeyPattern = enum { random, cancelling, near_tie, one_sided }; - -fn sparseCaseFrom( - gpa: std.mem.Allocator, - device: mtl.Device, - queue: mtl.Queue, - p: *const csa2.Pipelines, - spec: SparseSpec, - rng: *Lcg, - key_amplitude: f32, - scale_exponent: i32, - indices_mode: usize, - sink: f32, - pattern: KeyPattern, -) !void { - try sparseCaseScaled(gpa, device, queue, p, spec, rng, key_amplitude, scale_exponent, scale_exponent, indices_mode, sink, pattern); -} - -/// The same rows with the window and the compressed cache on independent scale grids: the stage -/// output then carries a large and a tiny contribution in one accumulator, which is what the -/// mixing step actually rounds. -fn sparseCaseScaled( - gpa: std.mem.Allocator, - device: mtl.Device, - queue: mtl.Queue, - p: *const csa2.Pipelines, - spec: SparseSpec, - rng: *Lcg, - key_amplitude: f32, - window_exponent: i32, - compressed_exponent: i32, - indices_mode: usize, - sink: f32, - pattern: KeyPattern, -) !void { - const q_stride = spec.heads * width; - const q = try gpa.alloc(u16, spec.rows * q_stride); - defer gpa.free(q); - const q_scale = std.math.pow(f32, 2, @as(f32, @floatFromInt(window_exponent))); - for (q) |*cell| cell.* = math.b(rng.uniform() * q_scale); - const logical_window = spec.position + spec.rows; - const window_rows = try gpa.alloc([]const f32, logical_window); - defer gpa.free(window_rows); - for (window_rows, 0..) |*row, i| { - const cells = try gpa.alloc(f32, width); - fillPatternRow(cells, rng, i, key_amplitude, window_exponent, pattern); - row.* = cells; - } - const compressed_rows = try gpa.alloc([]const f32, spec.compressed_rows); - defer gpa.free(compressed_rows); - for (compressed_rows, 0..) |*row, i| { - const cells = try gpa.alloc(f32, width); - fillPatternRow(cells, rng, i, key_amplitude, compressed_exponent, pattern); - row.* = cells; - } - const indices = try gpa.alloc(i32, spec.rows * spec.topk); - defer gpa.free(indices); - const limit = @max(spec.compressed_rows, 1); - for (0..spec.rows) |r| for (0..spec.topk) |k| { - indices[r * spec.topk + k] = switch (indices_mode) { - 0 => @intCast(k * limit / @max(spec.topk, 1)), // spread and ascending - 1 => @intCast((limit - 1) - k), // descending (the reference sorts; the kernel does not) - else => @as(i32, @intCast((r *% 7 +% 13 * k) % limit)), // arbitrary order, unsorted - }; - }; - const sinks = try gpa.alloc(f32, spec.heads); - defer gpa.free(sinks); - for (sinks) |*v| v.* = sink; - try sparseCase(device, queue, p, spec, q, window_rows, compressed_rows, indices, sinks); -} - -fn fillPatternRow(row: []f32, rng: *Lcg, index: usize, amplitude: f32, exponent: i32, pattern: KeyPattern) void { - const scale = std.math.pow(f32, 2, @as(f32, @floatFromInt(exponent))); - if (pattern == .random) { - for (row) |*v| v.* = rng.uniform() * amplitude * scale; - return; - } - for (row, 0..) |*v, d| { - if (pattern == .cancelling) { - // Same shape in every key: all scores tie, and the value sum cancels to the ULPs of - // the large terms that make it up. - const flip: f32 = if ((d & 1) == 0) 1 else -1; - v.* = flip * amplitude * scale + rng.uniform() * amplitude * scale * 1e-3; - } else if (pattern == .near_tie) { - // Keys differ only far above the score rounding boundary: probabilities tie at the top. - v.* = @as(f32, @floatFromInt(index)) * amplitude * scale + rng.uniform() * amplitude * scale * 1e-7; - } else { - // One dominant key against keys orders of magnitude smaller. - v.* = rng.uniform() * amplitude * scale * @as(f32, if (index == 0) @as(f32, 1e4) else @as(f32, 1e-4)); - } - } -} - -test "sparse attention stages sit at rounding level against the FP64 ideal" { - var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena_instance.deinit(); - const gpa = arena_instance.allocator(); - const device = mtl.Device.init() catch return error.SkipZigTest; - defer device.deinit(); - const pool = mtl.objc.Pool.push(); - defer pool.pop(); - const queue = try device.queue(); - defer queue.deinit(); - var p = try csa2.Pipelines.load(device); - defer p.deinit(); - - var rng = Lcg{ .state = 0x2545f4914f6cdd1d }; - - // Window only, the decode-step boundary at a wrapped ring. - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "window_decode", .heads = 4, .rows = 1, .position = 255, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); - // Compressed rows with a small selection, ascending indices. - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "compressed_asc", .heads = 4, .rows = 1, .position = 255, .ratio = 2, .compressed_rows = 128, .topk = 16 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); - // Unsorted indices: the kernel consumes the array in the given order; the reference must too. - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "compressed_unsorted", .heads = 2, .rows = 1, .position = 9, .ratio = 2, .compressed_rows = 40, .topk = 12 }, &rng, 1.0, 0, 2, 0, KeyPattern.random); - // Prefill rows, multi-row chunk with a wrapped ring. - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "prefill_rows", .heads = 2, .rows = 4, .position = 250, .ratio = 2, .compressed_rows = 100, .topk = 8 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); - // Wide dynamic range keys: the score sum runs across many orders of magnitude. - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "wide_range", .heads = 1, .rows = 1, .position = 200, .ratio = 2, .compressed_rows = 32, .topk = 8 }, &rng, 1.0, -20, 0, 0, KeyPattern.random); - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "wide_range_hi", .heads = 1, .rows = 1, .position = 200, .ratio = 2, .compressed_rows = 32, .topk = 8 }, &rng, 1.0, 20, 0, 0, KeyPattern.random); - // Denormal-scale sink: the sink mass underflows FP32 while it is exact in FP64. - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "sink_denormal", .heads = 1, .rows = 1, .position = 100, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, -16.0, KeyPattern.random); - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "sink_large", .heads = 1, .rows = 1, .position = 100, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, 12.0, KeyPattern.random); - // Value-product cancellation: identical-shaped keys, so every score ties and the P·V sum - // cancels to the ULPs of the terms that make it up. - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "cancel_values", .heads = 1, .rows = 1, .position = 300, .ratio = 2, .compressed_rows = 64, .topk = 64 }, &rng, 32.0, 0, 0, 0, KeyPattern.cancelling); - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "cancel_wide", .heads = 1, .rows = 1, .position = 300, .ratio = 2, .compressed_rows = 64, .topk = 64 }, &rng, 32.0, 6, 0, 0, KeyPattern.cancelling); - // Scores that tie: the softmax denominator is the sum of equal exponentials, and the - // accumulation order decides which probability lands on which side of its rounding boundary. - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "score_tie", .heads = 1, .rows = 1, .position = 300, .ratio = 2, .compressed_rows = 32, .topk = 32 }, &rng, 1.0, 0, 0, 0, KeyPattern.near_tie); - // One dominant key against keys orders of magnitude smaller: denominator and accumulator - // run at the extreme ratio. - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "one_dominant", .heads = 1, .rows = 1, .position = 300, .ratio = 2, .compressed_rows = 16, .topk = 16 }, &rng, 1.0, 0, 0, 0, KeyPattern.one_sided); - // Many tiles, so the online correction multiplies the accumulator once per 64 columns. - try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "many_tiles", .heads = 1, .rows = 1, .position = 511, .ratio = 2, .compressed_rows = 512, .topk = 384 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); - // Cancellation in the value product: keys of alternating sign, so the P·V sum is small next - // to the terms that make it up. - { - const spec = SparseSpec{ .name = "pv_cancellation", .heads = 1, .rows = 1, .position = 64, .ratio = 0, .compressed_rows = 0, .topk = 0 }; - const q = try gpa.alloc(u16, width); - defer gpa.free(q); - for (q) |*cell| cell.* = math.b(rng.scale(1.0)); - const window_rows = try gpa.alloc([]const f32, spec.position + spec.rows); - defer gpa.free(window_rows); - for (window_rows) |*row| { - const cells = try gpa.alloc(f32, width); - for (cells, 0..) |*v, d| v.* = if ((d & 1) == 0) rng.scale(64.0) else rng.scale(64.0) * 0.987654321; - row.* = cells; - } - const sinks = try gpa.alloc(f32, 1); - defer gpa.free(sinks); - sinks[0] = 0; - const empty = try gpa.alloc([]const f32, 0); - defer gpa.free(empty); - const indices = try gpa.alloc(i32, 0); - defer gpa.free(indices); - try sparseCase(device, queue, &p, spec, q, window_rows, empty, indices, sinks); - } - // Large cancelling keys beside small ones: score dot products run through near-cancelling terms. - { - const spec = SparseSpec{ .name = "score_cancellation", .heads = 1, .rows = 1, .position = 96, .ratio = 0, .compressed_rows = 0, .topk = 0 }; - const q = try gpa.alloc(u16, width); - defer gpa.free(q); - for (q, 0..) |*cell, d| cell.* = if ((d & 1) == 0) math.b(1.0) else math.b(-1.0); - const window_rows = try gpa.alloc([]const f32, spec.position + spec.rows); - defer gpa.free(window_rows); - for (window_rows, 0..) |*row, i| { - const cells = try gpa.alloc(f32, width); - const bias: f32 = @as(f32, @floatFromInt(@rem(@as(i32, @intCast(i)), 8))) * 4.0; - for (cells, 0..) |*v, d| v.* = bias + (if ((d & 1) == 0) rng.scale(0.001) else rng.scale(0.001) * 0.99999); - row.* = cells; - } - const sinks = try gpa.alloc(f32, 1); - defer gpa.free(sinks); - sinks[0] = 0; - const empty = try gpa.alloc([]const f32, 0); - defer gpa.free(empty); - const indices = try gpa.alloc(i32, 0); - defer gpa.free(indices); - try sparseCase(device, queue, &p, spec, q, window_rows, empty, indices, sinks); - } -} - -/// One live column pins the running maximum to a single known score, so `sink - max` is the whole -/// normalization. Sweeping it across the FP32 exp() overflow and underflow limits is the only path -/// by which the sink term can move the stage output, and it is also where an FP32 exp() would leave -/// the FP64 equation behind. -fn sinkCase( - gpa: std.mem.Allocator, - device: mtl.Device, - queue: mtl.Queue, - p: *const csa2.Pipelines, - name: []const u8, - rng: *Lcg, - sink: f32, -) !void { - const spec = SparseSpec{ .name = name, .heads = 1, .rows = 1, .position = 0, .ratio = 0, .compressed_rows = 0, .topk = 0 }; - const q = try gpa.alloc(u16, width); - defer gpa.free(q); - for (q) |*cell| cell.* = math.b(rng.scale(1.0)); - const window_rows = try gpa.alloc([]const f32, 1); - defer gpa.free(window_rows); - const cells = try gpa.alloc(f32, width); - defer gpa.free(cells); - for (cells) |*v| v.* = rng.scale(1.0); - window_rows[0] = cells; - const sinks = try gpa.alloc(f32, 1); - defer gpa.free(sinks); - sinks[0] = sink; - const empty = try gpa.alloc([]const f32, 0); - defer gpa.free(empty); - const indices = try gpa.alloc(i32, 0); - defer gpa.free(indices); - try sparseCase(device, queue, p, spec, q, window_rows, empty, indices, sinks); -} - -test "sink normalization sits at rounding level across the exp overflow boundaries" { - var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena_instance.deinit(); - const gpa = arena_instance.allocator(); - const device = mtl.Device.init() catch return error.SkipZigTest; - defer device.deinit(); - const pool = mtl.objc.Pool.push(); - defer pool.pop(); - const queue = try device.queue(); - defer queue.deinit(); - var pipelines = try csa2.Pipelines.load(device); - defer pipelines.deinit(); - var rng = Lcg{ .state = 0x51c0de51c0de51c0 }; - - // One case per boundary FP32 exp() has: underflow to zero, the overflow edge, and the - // mass-balanced middle where the sink decides the output on its own. From about 83 upward the - // sink exponent reaches the FP32 overflow boundary, so the cells that differ from the reference - // sit near 1e-38 and a one-ULP difference in the running maximum decides whether either side - // sees a finite denominator at all. - const sweep = [_]struct { sink: f32, name: []const u8 }{ - .{ .sink = -104, .name = "sink_underflow" }, - .{ .sink = -90, .name = "sink_near_underflow" }, - .{ .sink = -60, .name = "sink_tiny" }, - .{ .sink = -12, .name = "sink_small" }, - .{ .sink = -1, .name = "sink_minus_one" }, - .{ .sink = 0, .name = "sink_zero" }, - .{ .sink = 1, .name = "sink_one" }, - .{ .sink = 12, .name = "sink_medium" }, - .{ .sink = 60, .name = "sink_large" }, - .{ .sink = 80, .name = "sink_edge80" }, - .{ .sink = 83, .name = "sink_edge83" }, - .{ .sink = 84, .name = "sink_edge84" }, - .{ .sink = 85, .name = "sink_edge85" }, - .{ .sink = 86, .name = "sink_over_edge" }, - .{ .sink = 90, .name = "sink_overflow" }, - }; - for (sweep) |entry| try sinkCase(gpa, device, queue, &pipelines, entry.name, &rng, entry.sink); - - // Per-head sinks: the kernel indexes the sink array by head, so heads whose sink masses differ - // by orders of magnitude check that indexing as well as the arithmetic. - { - const spec = SparseSpec{ .name = "sink_per_head", .heads = 4, .rows = 1, .position = 0, .ratio = 0, .compressed_rows = 0, .topk = 0 }; - var head_rng = Lcg{ .state = 0xabcdef0123456789 }; - const q = try gpa.alloc(u16, spec.heads * width); - defer gpa.free(q); - for (q) |*cell| cell.* = math.b(head_rng.scale(1.0)); - const window_rows = try gpa.alloc([]const f32, 1); - defer gpa.free(window_rows); - const cells = try gpa.alloc(f32, width); - defer gpa.free(cells); - for (cells) |*v| v.* = head_rng.scale(1.0); - window_rows[0] = cells; - const sinks = try gpa.alloc(f32, spec.heads); - defer gpa.free(sinks); - for (0..spec.heads) |h| sinks[h] = @as(f32, @floatFromInt(@as(i32, @intCast(h)) * 24 - 36)); - const empty = try gpa.alloc([]const f32, 0); - defer gpa.free(empty); - const indices = try gpa.alloc(i32, 0); - defer gpa.free(indices); - try sparseCase(device, queue, &pipelines, spec, q, window_rows, empty, indices, sinks); - } -} - -// Tile denominators: the online correction, the per-tile sum and the running maximum. A flat -// score set makes the denominator a pure sum of equal exponentials; a ramped score set drives the -// correction through the FP32 underflow limit; a descending set never applies a correction at all. -test "tile denominators sit at rounding level for every tile shape" { - var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena_instance.deinit(); - const gpa = arena_instance.allocator(); - const device = mtl.Device.init() catch return error.SkipZigTest; - defer device.deinit(); - const pool = mtl.objc.Pool.push(); - defer pool.pop(); - const queue = try device.queue(); - defer queue.deinit(); - var pipelines = try csa2.Pipelines.load(device); - defer pipelines.deinit(); - var rng = Lcg{ .state = 0x1a2b3c4d5e6f7081 }; - - // Ten tiles, so the denominator accumulates 639 exponentials and the accumulator takes 639 - // products. Identical key shapes make every score tie: the denominator is exactly the count. - try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = "tiles_equal_scores", .heads = 1, .rows = 1, .position = 1022, .ratio = 2, .compressed_rows = 512, .topk = 512 }, &rng, 1.0, 0, 0, 0, KeyPattern.cancelling); - // Ascending scores by tile: the correction falls one power of two per step until FP32 flushes - // it, and the dropped mass is the whole question. - for ([_]i32{ -2, 0, 1, 2 }) |step| { - var name_buf: [24]u8 = undefined; - const label = std.fmt.bufPrint(&name_buf, "tiles_ramp{d}", .{step}) catch unreachable; - try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = label, .heads = 1, .rows = 1, .position = 1000, .ratio = 2, .compressed_rows = 512, .topk = 512 }, &rng, 1.0, step, 0, 0, KeyPattern.near_tie); - } - // Descending scores: the running maximum never moves, so every tile lands at full weight and - // the denominator is a plain sum. - try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = "tiles_descending", .heads = 1, .rows = 1, .position = 1000, .ratio = 2, .compressed_rows = 512, .topk = 512 }, &rng, 1.0, 0, 1, 0, KeyPattern.near_tie); - // One tile only, the boundary case of the online loop: the correction is applied zero times. - try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = "tiles_single", .heads = 1, .rows = 1, .position = 20, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); -} - -// The output mix: window rows and compressed rows meeting in one accumulator on different scale -// grids, and two mirror rows whose shared large shape cancels out of the weighted sum. The mirror -// case is the stage's real amplification probe - whatever the score arithmetic gets wrong shows up -// multiplied by the shape that the sum is cancelling. -test "the output mix sits at rounding level when window and compressed rows cancel" { - var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena_instance.deinit(); - const gpa = arena_instance.allocator(); - const device = mtl.Device.init() catch return error.SkipZigTest; - defer device.deinit(); - const pool = mtl.objc.Pool.push(); - defer pool.pop(); - const queue = try device.queue(); - defer queue.deinit(); - var pipelines = try csa2.Pipelines.load(device); - defer pipelines.deinit(); - var rng = Lcg{ .state = 0x77a1c0de_5b5e1a2b }; - - // Large window rows against tiny compressed rows, then the reverse: one accumulator holding - // terms six orders of magnitude apart. - try sparseCaseScaled(gpa, device, queue, &pipelines, .{ .name = "mix_window_dominates", .heads = 1, .rows = 1, .position = 100, .ratio = 2, .compressed_rows = 100, .topk = 64 }, &rng, 1.0, 14, -12, 0, 0, KeyPattern.random); - try sparseCaseScaled(gpa, device, queue, &pipelines, .{ .name = "mix_compressed_dominates", .heads = 1, .rows = 1, .position = 100, .ratio = 2, .compressed_rows = 100, .topk = 64 }, &rng, 1.0, -12, 14, 0, 0, KeyPattern.random); - try sparseCaseScaled(gpa, device, queue, &pipelines, .{ .name = "mix_balanced_magnitudes", .heads = 1, .rows = 1, .position = 100, .ratio = 2, .compressed_rows = 100, .topk = 64 }, &rng, 1.0, 6, 6, 0, 0, KeyPattern.cancelling); - - // The stage's real amplification probe: 128 window rows that carry a large alternating shape - // which cancels out of both the score (a uniform query against a sign-alternating shape) and - // the weighted sum, so every accumulator term is large and their total is small. Whatever the - // arithmetic gets wrong arrives multiplied by that shape. - for ([_]i32{ 8, 16, 23 }) |shape_exponent| { - var name_buf: [24]u8 = undefined; - const label = std.fmt.bufPrint(&name_buf, "mirror_shape_e{d}", .{shape_exponent}) catch unreachable; - const spec = SparseSpec{ .name = label, .heads = 1, .rows = 1, .position = 127, .ratio = 0, .compressed_rows = 0, .topk = 0 }; - const q = try gpa.alloc(u16, width); - defer gpa.free(q); - // q dot shape == 0 exactly: a uniform query against a shape that alternates sign per - // column, so the large part drops out of both scores and the accumulator alike. - for (q) |*cell| cell.* = math.b(1.0); - const window_rows = try gpa.alloc([]const f32, 128); - defer gpa.free(window_rows); - const shape = std.math.pow(f32, 2, @floatFromInt(shape_exponent)); - const shared = try gpa.alloc(f32, width); - defer gpa.free(shared); - // One 64th of the cancelling shape: near the FP8 window grid's floor, so the accumulator's - // partial sums run 2^16 times the sum they end at. Below that the code grid cancels the - // small part away and FP32 needs no help to be exact. - for (shared) |*v| v.* = rng.scale(shape * 0x1p-16); - for (window_rows, 0..) |*row, i| { - const cells = try gpa.alloc(f32, width); - const side: f32 = if ((i & 1) == 0) 1 else -1; - for (0..width) |d| { - const flip: f32 = if ((d & 1) == 0) 1 else -1; - cells[d] = side * flip * shape + shared[d]; - } - row.* = cells; - } - const sinks = try gpa.alloc(f32, 1); - defer gpa.free(sinks); - sinks[0] = 0; - const empty = try gpa.alloc([]const f32, 0); - defer gpa.free(empty); - const indices = try gpa.alloc(i32, 0); - defer gpa.free(indices); - try sparseCase(device, queue, &pipelines, spec, q, window_rows, empty, indices, sinks); - } -} - -/// Both sides of one rotary call: `contract` is the pinned source's FP32 path (FP32 frequency, FP32 -/// angle, FP32 trig), `ideal` keeps frequency, angle, both unaries and the products in FP64. The -/// kernel differs from the contract by the factor table's own unaries, which is what this checks. -fn ropeFrequency64( - index: usize, - theta: f64, - original: f64, - factor: f64, - beta_fast: f64, - beta_slow: f64, -) f64 { - const frequency: f64 = 1 / std.math.pow(f64, theta, @as(f64, @floatFromInt(index * 2)) / 64); - if (original <= 0) return frequency; - const denominator = 2 * @log(theta); - const low = @max(@floor(64 * @log(original / (beta_fast * 2 * std.math.pi)) / denominator), 0); - const high = @min(@ceil(64 * @log(original / (beta_slow * 2 * std.math.pi)) / denominator), 63); - const ramp = std.math.clamp((@as(f64, @floatFromInt(index)) - low) / @max(high - low, 1e-3), 0, 1); - return frequency / factor * ramp + frequency * (1 - ramp); -} - -fn ropeCase( - device: mtl.Device, - queue: mtl.Queue, - p: *const tf.deepseek_v41.rope.Pipelines, - name: []const u8, - rng: *Lcg, - position: usize, - rows: usize, - heads: usize, - spectrum: tf.deepseek_v41.rope.Spectrum, - inverse: bool, - theta: f32, - original: u32, - factor: f32, -) !void { - const stride = heads * width; - const cells = rows * stride; - var b = Buffers{ .device = device }; - defer b.deinit(); - const x_gpu = try b.get(u16, cells); - const x = x_gpu.slice(u16, cells); - const before = try std.testing.allocator.alloc(u16, cells); - defer std.testing.allocator.free(before); - for (x, 0..) |*cell, i| { - cell.* = math.b(rng.scale(1.0)); - before[i] = cell.*; - } - const cb = queue.commandBuffer(); - const enc = cb.compute(.serial); - try p.encode(enc, view(x_gpu), .{ .rows = @intCast(rows), .heads = @intCast(heads), .width = width, .rotary = 64, .stride = @intCast(stride), .position = @intCast(position), .step = 1, .inverse = @intFromBool(inverse), .spectrum = spectrum }); - enc.end(); - cb.commit(); - cb.wait(); - if (cb.failure() != null) return error.Csa2Fp64GpuFailure; - - const theta64: f64 = @as(f64, theta); - const original64: f64 = if (original == 0) 0 else @floatFromInt(original); - const ideal_cells = try std.testing.allocator.alloc(u16, cells); - defer std.testing.allocator.free(ideal_cells); - const contract_cells = try std.testing.allocator.alloc(u16, cells); - defer std.testing.allocator.free(contract_cells); - for (0..rows) |row| for (0..heads) |head| { - const base = row * stride + head * width; - // Everything outside the rotary window is copied through, so both sides must agree with the - // input there too. - for (0..width - 64) |d| { - ideal_cells[base + d] = before[base + d]; - contract_cells[base + d] = before[base + d]; - } - for (0..32) |i| { - const at = base + width - 64 + 2 * i; - const a = math.f(before[at]); - const z = math.f(before[at + 1]); - const frequency = math.ropeFrequency(i, theta, original, factor, 32, 1); - const angle: f32 = @as(f32, @floatFromInt(position + row)) * frequency; - const cosine32 = @cos(angle); - const sine32 = @sin(angle) * (if (inverse) @as(f32, -1) else 1); - contract_cells[at] = math.b(a * cosine32 - z * sine32); - contract_cells[at + 1] = math.b(a * sine32 + z * cosine32); - const frequency64 = ropeFrequency64(i, theta64, original64, @as(f64, factor), 32, 1); - const angle64: f64 = @as(f64, @floatFromInt(position + row)) * frequency64; - const cosine64 = @cos(angle64); - const sine64 = @sin(angle64) * (if (inverse) @as(f64, -1) else @as(f64, 1)); - ideal_cells[at] = math.b64(@as(f64, a) * cosine64 - @as(f64, z) * sine64); - ideal_cells[at + 1] = math.b64(@as(f64, a) * sine64 + @as(f64, z) * cosine64); - } - }; - - var stage_buffer: [32]u8 = undefined; - for (0..rows) |row| for (0..heads) |head| { - const stage = std.fmt.bufPrint(&stage_buffer, "row{d}_head{d}", .{ row, head }) catch unreachable; - const base = row * stride + head * width; - var ideal: [width]f64 = undefined; - var contract: [width]f64 = undefined; - for (0..width) |d| { - ideal[d] = math.f(ideal_cells[base + d]); - contract[d] = math.f(contract_cells[base + d]); - } - printBf(name, stage, "metal", deltaBf16(x[base..][0..width], &ideal)); - printBf(name, stage, "f32_reference", deltaBf16(contract_cells[base..][0..width], &ideal)); - printBf(name, stage, "metal_vs_f32_reference", deltaBf16(x[base..][0..width], &contract)); - }; -} - -test "the rotary stage sits at rounding level for both spectra and at long positions" { - const device = mtl.Device.init() catch return error.SkipZigTest; - defer device.deinit(); - const pool = mtl.objc.Pool.push(); - defer pool.pop(); - const queue = try device.queue(); - defer queue.deinit(); - // The production shape: 64 rotary columns, YaRN on the compressed spectrum only, positions out - // to the resident capacity. - var p = try tf.deepseek_v41.rope.Pipelines.load(device, 16384, 64, .{ .theta = 10000, .original = 0 }, .{ .theta = 10000, .factor = 16, .original = 65536, .beta_fast = 32, .beta_slow = 1 }); - defer p.deinit(); - - const positions = [_]usize{ 1, 7, 63, 64, 4095, 8191, 16383 }; - for (positions) |position| { - var name_buf: [32]u8 = undefined; - var scoped = Lcg{ .state = 0x9e3779b97f4a7c15 ^ @as(u64, position) }; - const label = std.fmt.bufPrint(&name_buf, "rope_normal_p{d}", .{position}) catch unreachable; - try ropeCase(device, queue, &p, label, &scoped, position, 1, 2, .normal, false, 10000, 0, 16); - var scoped2 = Lcg{ .state = 0x85ebca77c0b98731 ^ @as(u64, position) }; - const label2 = std.fmt.bufPrint(&name_buf, "rope_compressed_p{d}", .{position}) catch unreachable; - try ropeCase(device, queue, &p, label2, &scoped2, position, 1, 2, .compressed, false, 10000, 65536, 16); - var scoped3 = Lcg{ .state = 0xc2b2ae3d27d4eb4f ^ @as(u64, position) }; - const label3 = std.fmt.bufPrint(&name_buf, "rope_inverse_p{d}", .{position}) catch unreachable; - try ropeCase(device, queue, &p, label3, &scoped3, position, 1, 1, .compressed, true, 10000, 65536, 16); - } - // A prefill row set: the kernel indexes the factor table by absolute position per row. - var multi = Lcg{ .state = 0x2545f4914f6cdd1d }; - try ropeCase(device, queue, &p, "rope_prefill_rows", &multi, 8100, 4, 2, .compressed, false, 10000, 65536, 16); -} - -/// One indexer-score row set: 32 heads of 128 FP4-index columns each, scored against packed keys. -const IndexSpec = struct { name: []const u8, rows: usize, keys: usize, topk: usize }; - -fn packIndexRow(values: []const f32, out: []u8) void { - @memset(out[0 .. 128 / 2 + 128 / 32], 0); - for (0..128 / 32) |block| { - var a: f32 = 6.0 * 0x1p-126; - for (values[block * 32 ..][0..32]) |v| a = @max(a, @abs(v)); - const bits: u32 = @bitCast(a / 6.0); - const unbiased: i32 = @as(i32, @intCast((bits >> 23) & 255)) - 127 + @intFromBool((bits & 0x7fffff) != 0); - const scale = std.math.ldexp(@as(f32, 1.0), unbiased); - out[64 + block] = @intCast(unbiased + 127); - for (0..32) |j| { - const code = e2Code(values[block * 32 + j] / scale); - const at = block * 16 + j / 2; - if ((j & 1) != 0) out[at] |= code << 4 else out[at] |= code; - } - } -} - -fn decodeIndexRow(bytes: []const u8) [128]u16 { - var row: [128]u16 = undefined; - for (0..128) |d| { - const nibble: u8 = if ((d & 1) != 0) bytes[d / 2] >> 4 else bytes[d / 2] & 15; - const scale: f32 = @bitCast(@as(u32, bytes[64 + d / 32]) << 23); - row[d] = math.b(e2(nibble) * scale); - } - return row; -} - -fn indexCase( - device: mtl.Device, - queue: mtl.Queue, - p: *const tf.deepseek_v41.csa2_index.Pipelines, - spec: IndexSpec, - q: []const u16, - weights: []const u16, - key_values: []const []const f32, -) !void { - var b = Buffers{ .device = device }; - defer b.deinit(); - var scratch = try tf.deepseek_v41.csa2_index.Scratch.init(device, @intCast(spec.rows)); - defer scratch.deinit(); - const q_gpu = try b.get(u16, spec.rows * 32 * 128); - const weights_gpu = try b.get(u16, spec.rows * 32); - const keys_gpu = try b.get(u8, spec.keys * 68); - @memcpy(q_gpu.slice(u16, spec.rows * 32 * 128), q); - @memcpy(weights_gpu.slice(u16, spec.rows * 32), weights); - - // Both sides read the same decoded keys; the references differ only in their arithmetic. - const decoded = try std.testing.allocator.alloc(u16, spec.keys * 128); - defer std.testing.allocator.free(decoded); - for (key_values, 0..) |values, row| { - const bytes = try std.testing.allocator.alloc(u8, 68); - defer std.testing.allocator.free(bytes); - packIndexRow(values, bytes); - @memcpy(keys_gpu.slice(u8, spec.keys * 68)[row * 68 ..][0..68], bytes); - const decoded_row = decodeIndexRow(bytes); - @memcpy(decoded[row * 128 ..][0..128], &decoded_row); - } - const cb = queue.commandBuffer(); - const enc = cb.compute(.serial); - try p.encode(enc, view(q_gpu), view(weights_gpu), view(keys_gpu), scratch, .{ - .rows = @intCast(spec.rows), - .keys = @intCast(spec.keys), - .q_stride = 32 * 128, - .weight_stride = 32, - .position = @intCast(spec.keys), - .ratio = 1, - }); - enc.end(); - cb.commit(); - cb.wait(); - if (cb.failure() != null) return error.Csa2Fp64GpuFailure; - - // Positions live behind the scores/heap/candidates regions; read them through the view's offset, - // otherwise this compares the score words against themselves and the check is vacuous. - const kernel_scores = scratch.scores().buffer.slice(f32, spec.rows * 4096); - const positions_offset = scratch.positions().offset / @sizeOf(i32); - const kernel_positions = scratch.positions().buffer.slice(i32, positions_offset + spec.rows * 512); - var ref32 = try std.testing.allocator.alloc(f32, spec.keys); - defer std.testing.allocator.free(ref32); - var ref64 = try std.testing.allocator.alloc(f64, spec.keys); - defer std.testing.allocator.free(ref64); - // The same scores with only the documented quantizations below the final one: the BF16 quantum - // that no change of arithmetic can remove. - var exact = try std.testing.allocator.alloc(f64, spec.keys); - defer std.testing.allocator.free(exact); - const w_scale: f32 = 1 / @sqrt(@as(f32, 128 * 32)); - for (0..spec.keys) |row| { - const key = decoded[row * 128 ..][0..128]; - var ideal: f64 = 0; - var narrow32: f32 = 0; - for (0..32) |head| { - var dot: f64 = 0; - for (0..128) |d| dot += @as(f64, math.f(key[d])) * @as(f64, math.f(q[head * 128 + d])); - // The pinned source's head einsum lands in BF16, so the rectified score is a quantized - // algorithm boundary like the cache codes are; only the reduction above it is widened. - const rectified: f64 = @max(math.f(math.b(@min(@as(f32, @floatCast(dot)), 1e38))), 0); - const weight: f32 = math.f(weights[head]); - const scaled_weight = math.f(math.b(weight * w_scale)); - const weighted = math.f(math.b(@as(f32, @floatCast(rectified)) * scaled_weight)); - ideal += weighted; - narrow32 += @as(f32, @floatCast(weighted)); - } - exact[row] = ideal; - // The score is written back through BF16 by the kernel and by the source contract, so the - // ideal carries that final boundary too: the stage then differs from its output by nothing - // but the arithmetic being compared. - ref64[row] = math.f(math.b64(ideal)); - ref32[row] = math.f(math.b(narrow32)); - } - var ideal_cells: [1024]f64 = undefined; - const shown = @min(spec.keys, ideal_cells.len); - for (0..shown) |row| ideal_cells[row] = ref64[row]; - printF32(spec.name, "index_score", "bf16_score_boundary", deltaF32VsF64(ref32[0..shown], exact[0..shown])); - printF32(spec.name, "index_score", "metal_vs_ideal", deltaF32VsF64(kernel_scores[0..shown], ideal_cells[0..shown])); - printF32(spec.name, "index_score", "f32_vs_ideal", deltaF32VsF64(ref32[0..shown], ideal_cells[0..shown])); - // The kernel against the source-contract FP32 reference on identical keys: the stage delta that - // selection can actually see, independent of how far the reference itself sits from the ideal. - var contract: [1024]f64 = undefined; - for (0..shown) |row| contract[row] = @as(f64, ref32[row]); - printF32(spec.name, "index_score", "metal_vs_f32_reference", deltaF32VsF64(kernel_scores[0..shown], contract[0..shown])); - - // Selection: which keys land in the top-k. Both sides break ties by the lowest position, the - // kernel's heap comparator and the reference's strict-greater scan agree on that. - var chosen = try std.testing.allocator.alloc(bool, spec.keys); - defer std.testing.allocator.free(chosen); - @memset(chosen, false); - var scores_copy = try std.testing.allocator.alloc(f32, spec.keys); - defer std.testing.allocator.free(scores_copy); - @memcpy(scores_copy, ref32[0..spec.keys]); - var reference_picks: [512]u32 = undefined; - const picks = @min(spec.topk, spec.keys); - for (0..picks) |pick| { - var best: usize = 0; - for (0..spec.keys) |i| { - if (scores_copy[i] > scores_copy[best]) best = i; - } - reference_picks[pick] = @intCast(best); - scores_copy[best] = -std.math.inf(f32); - } - std.mem.sort(u32, reference_picks[0..picks], {}, std.sort.asc(u32)); - var overlap: usize = 0; - for (reference_picks[0..picks]) |at| chosen[at] = true; - var min_selected: f32 = std.math.inf(f32); - var max_unselected: f32 = -std.math.inf(f32); - for (0..spec.keys) |row| { - if (chosen[row]) { - overlap += 1; - min_selected = @min(min_selected, ref32[row]); - } else max_unselected = @max(max_unselected, ref32[row]); - } - var kernel_only: usize = 0; - var out_of_range: usize = 0; - for (0..picks) |i| { - const slot = kernel_positions[positions_offset + i]; - if (slot < 0 or slot >= @as(i32, @intCast(spec.keys))) { - out_of_range += 1; - continue; - } - if (!chosen[@intCast(slot)]) kernel_only += 1; - } - // How much arithmetic noise the selection can absorb before the *set* changes. The score is only - // worth what the storage keeps, so a pick is knife-edged when the boundary margin is comparable - // to one score quantum; if it is not, no 1-ULP arithmetic difference can flip this row. - var exact_picks: [512]u32 = undefined; - var exact_scores = try std.testing.allocator.alloc(f64, spec.keys); - defer std.testing.allocator.free(exact_scores); - @memcpy(exact_scores, exact[0..spec.keys]); - for (0..picks) |pick| { - var best: usize = 0; - for (0..spec.keys) |i| { - if (exact_scores[i] > exact_scores[best]) best = i; - } - exact_picks[pick] = @intCast(best); - exact_scores[best] = -std.math.inf(f64); - } - var flips: usize = 0; - var exact_chosen = try std.testing.allocator.alloc(bool, spec.keys); - defer std.testing.allocator.free(exact_chosen); - @memset(exact_chosen, false); - for (exact_picks[0..picks]) |at| exact_chosen[at] = true; - for (0..spec.keys) |at| { - if (chosen[at] != exact_chosen[at]) flips += 1; - } - var quantum: f32 = 0; - var margin_in_quanta: f64 = 0; - if (min_selected != std.math.inf(f32) and max_unselected != -std.math.inf(f32)) { - const boundary = min_selected * 0.5 + max_unselected * 0.5; - const exponent: i32 = if (@abs(boundary) > 0) std.math.ilogb(@abs(boundary)) - 7 else -149; - quantum = std.math.pow(f32, 2.0, @as(f32, @floatFromInt(exponent))); - margin_in_quanta = @as(f64, min_selected - max_unselected) / @as(f64, quantum); - } - std.debug.print( - \\CSA64 {{"case":"{s}","stage":"index_select","side":"noise_tolerance","keys":{d},"picks":{d},"precision_flips":{d},"selection_margin":{d},"score_quantum":{d},"margin_in_quanta":{d:.1}}} - , .{ spec.name, spec.keys, picks, flips, min_selected - max_unselected, quantum, margin_in_quanta }); - std.debug.print( - \\CSA64 {{"case":"{s}","stage":"index_select","side":"metal","keys":{d},"picks":{d},"reference_only":{d},"kernel_only":{d},"invalid_positions":{d},"lowest_selected":{d},"highest_unselected":{d}}} - , .{ spec.name, spec.keys, picks, picks - overlap, kernel_only, out_of_range, min_selected, max_unselected }); -} - -test "indexer scoring and selection sit at rounding level against the FP64 ideal" { - var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena_instance.deinit(); - const gpa = arena_instance.allocator(); - const device = mtl.Device.init() catch return error.SkipZigTest; - defer device.deinit(); - const pool = mtl.objc.Pool.push(); - defer pool.pop(); - const queue = try device.queue(); - defer queue.deinit(); - var p = try tf.deepseek_v41.csa2_index.Pipelines.load(device); - defer p.deinit(); - - var rng = Lcg{ .state = 0x1234_5678_9abc_def0 }; - var amplitude: f32 = 1; - for ([_]IndexSpec{ - .{ .name = "index_random", .rows = 1, .keys = 1024, .topk = 512 }, - .{ .name = "index_random_4k", .rows = 1, .keys = 4096, .topk = 512 }, - .{ .name = "index_ties", .rows = 1, .keys = 1024, .topk = 512 }, - .{ .name = "index_boundary", .rows = 1, .keys = 1024, .topk = 512 }, - }, 0..) |spec, case_index| { - _ = case_index; - const q = try gpa.alloc(u16, 32 * 128); - defer gpa.free(q); - const weights = try gpa.alloc(u16, 32); - defer gpa.free(weights); - const cells = try gpa.alloc(f32, spec.keys * 128); - defer gpa.free(cells); - const keys = try gpa.alloc([]const f32, spec.keys); - defer gpa.free(keys); - for (q) |*cell| cell.* = math.b(rng.scale(1.0)); - for (weights) |*cell| cell.* = math.b(rng.scale(1.0)); - for (cells, 0..) |*v, i| { - v.* = rng.scale(1.0); - keys[i / 128] = cells[i / 128 * 128 ..][0..128]; - } - // Identical keys: every score is the same value, so the whole selection is tie-driven. - if (std.mem.eql(u8, spec.name, "index_ties")) { - for (cells[128..], cells[0..128]) |*dst, src| dst.* = src; - } - // A score cliff just inside the selection boundary: half the keys sit on the boundary. - if (std.mem.eql(u8, spec.name, "index_boundary")) { - for (cells, 0..) |*v, i| { - if ((i % 128) == 0) amplitude = (if (i / 128 < 512) @as(f32, 1.000001) else @as(f32, 1.0)); - v.* = rng.scale(amplitude); - } - } - try indexCase(device, queue, &p, spec, q, weights, keys); - } -} - -fn weightCells(gpa: std.mem.Allocator, rng: *Lcg) ![]u16 { +// Weight cells for the compressor-pool stage: near-unit magnitudes with no exponent extremes. +pub fn weightCells(gpa: std.mem.Allocator, rng: *Lcg) ![]u16 { const out = try gpa.alloc(u16, width); for (out) |*cell| { const amplitude: f32 = 0.5 + 0.25 * @as(f32, @floatFromInt(rng.next() % 1000)) / 1000; diff --git a/zig/tests/deepseek_v41_reference/operator_csa2_index_fp64.zig b/zig/tests/deepseek_v41_reference/operator_csa2_index_fp64.zig new file mode 100644 index 0000000000..410a0115fc --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_csa2_index_fp64.zig @@ -0,0 +1,251 @@ +//! CSA2 indexer-scoring FP64 ideal-reference test (DeepSeek-V4.1-Flash inference, MIT). +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const math = @import("math.zig"); +const h = @import("operator_csa2_fp64.zig"); +const pack = @import("operator_csa2_pack_fp64.zig"); + +/// One indexer-score row set: 32 heads of 128 FP4-index columns each, scored against packed keys. +const IndexSpec = struct { name: []const u8, rows: usize, keys: usize, topk: usize }; + +fn packIndexRow(values: []const f32, out: []u8) void { + @memset(out[0 .. 128 / 2 + 128 / 32], 0); + for (0..128 / 32) |block| { + var a: f32 = 6.0 * 0x1p-126; + for (values[block * 32 ..][0..32]) |v| a = @max(a, @abs(v)); + const bits: u32 = @bitCast(a / 6.0); + const unbiased: i32 = @as(i32, @intCast((bits >> 23) & 255)) - 127 + @intFromBool((bits & 0x7fffff) != 0); + const scale = std.math.ldexp(@as(f32, 1.0), unbiased); + out[64 + block] = @intCast(unbiased + 127); + for (0..32) |j| { + const code = pack.e2Code(values[block * 32 + j] / scale); + const at = block * 16 + j / 2; + if ((j & 1) != 0) out[at] |= code << 4 else out[at] |= code; + } + } +} + +fn decodeIndexRow(bytes: []const u8) [128]u16 { + var row: [128]u16 = undefined; + for (0..128) |d| { + const nibble: u8 = if ((d & 1) != 0) bytes[d / 2] >> 4 else bytes[d / 2] & 15; + const scale: f32 = @bitCast(@as(u32, bytes[64 + d / 32]) << 23); + row[d] = math.b(pack.e2(nibble) * scale); + } + return row; +} + +fn indexCase( + device: mtl.Device, + queue: mtl.Queue, + p: *const tf.deepseek_v41.csa2_index.Pipelines, + spec: IndexSpec, + q: []const u16, + weights: []const u16, + key_values: []const []const f32, +) !void { + var b = h.Buffers{ .device = device }; + defer b.deinit(); + var scratch = try tf.deepseek_v41.csa2_index.Scratch.init(device, @intCast(spec.rows)); + defer scratch.deinit(); + const q_gpu = try b.get(u16, spec.rows * 32 * 128); + const weights_gpu = try b.get(u16, spec.rows * 32); + const keys_gpu = try b.get(u8, spec.keys * 68); + @memcpy(q_gpu.slice(u16, spec.rows * 32 * 128), q); + @memcpy(weights_gpu.slice(u16, spec.rows * 32), weights); + + // Both sides read the same decoded keys; the references differ only in their arithmetic. + const decoded = try std.testing.allocator.alloc(u16, spec.keys * 128); + defer std.testing.allocator.free(decoded); + for (key_values, 0..) |values, row| { + const bytes = try std.testing.allocator.alloc(u8, 68); + defer std.testing.allocator.free(bytes); + packIndexRow(values, bytes); + @memcpy(keys_gpu.slice(u8, spec.keys * 68)[row * 68 ..][0..68], bytes); + const decoded_row = decodeIndexRow(bytes); + @memcpy(decoded[row * 128 ..][0..128], &decoded_row); + } + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encode(enc, h.view(q_gpu), h.view(weights_gpu), h.view(keys_gpu), scratch, .{ + .rows = @intCast(spec.rows), + .keys = @intCast(spec.keys), + .q_stride = 32 * 128, + .weight_stride = 32, + .position = @intCast(spec.keys), + .ratio = 1, + }); + enc.end(); + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.Csa2Fp64GpuFailure; + + // Positions live behind the scores/heap/candidates regions; read them through the view offset. + const kernel_scores = scratch.scores().buffer.slice(f32, spec.rows * 4096); + const positions_offset = scratch.positions().offset / @sizeOf(i32); + const kernel_positions = scratch.positions().buffer.slice(i32, positions_offset + spec.rows * 512); + var ref32 = try std.testing.allocator.alloc(f32, spec.keys); + defer std.testing.allocator.free(ref32); + var ref64 = try std.testing.allocator.alloc(f64, spec.keys); + defer std.testing.allocator.free(ref64); + // The same scores with only the documented quantizations below the final one: the BF16 quantum stays. + var exact = try std.testing.allocator.alloc(f64, spec.keys); + defer std.testing.allocator.free(exact); + const w_scale: f32 = 1 / @sqrt(@as(f32, 128 * 32)); + for (0..spec.keys) |row| { + const key = decoded[row * 128 ..][0..128]; + var ideal: f64 = 0; + var narrow32: f32 = 0; + for (0..32) |head| { + var dot: f64 = 0; + for (0..128) |d| dot += @as(f64, math.f(key[d])) * @as(f64, math.f(q[head * 128 + d])); + // The source head einsum lands in BF16, so the score is a quantized boundary like the cache codes. + const rectified: f64 = @max(math.f(math.b(@min(@as(f32, @floatCast(dot)), 1e38))), 0); + const weight: f32 = math.f(weights[head]); + const scaled_weight = math.f(math.b(weight * w_scale)); + const weighted = math.f(math.b(@as(f32, @floatCast(rectified)) * scaled_weight)); + ideal += weighted; + narrow32 += @as(f32, @floatCast(weighted)); + } + exact[row] = ideal; + // The kernel and the source contract write the score back through BF16, so the ideal carries it too. + ref64[row] = math.f(math.b64(ideal)); + ref32[row] = math.f(math.b(narrow32)); + } + var ideal_cells: [1024]f64 = undefined; + const shown = @min(spec.keys, ideal_cells.len); + for (0..shown) |row| ideal_cells[row] = ref64[row]; + h.printF32(spec.name, "index_score", "bf16_score_boundary", h.deltaF32VsF64(ref32[0..shown], exact[0..shown])); + h.printF32(spec.name, "index_score", "metal_vs_ideal", h.deltaF32VsF64(kernel_scores[0..shown], ideal_cells[0..shown])); + h.printF32(spec.name, "index_score", "f32_vs_ideal", h.deltaF32VsF64(ref32[0..shown], ideal_cells[0..shown])); + // The kernel against the source-contract FP32 reference on identical keys: the delta selection can see. + var contract: [1024]f64 = undefined; + for (0..shown) |row| contract[row] = @as(f64, ref32[row]); + h.printF32(spec.name, "index_score", "metal_vs_f32_reference", h.deltaF32VsF64(kernel_scores[0..shown], contract[0..shown])); + + // Selection: which keys land in the top-k; the kernel heap and strict-greater scan break ties by position. + var chosen = try std.testing.allocator.alloc(bool, spec.keys); + defer std.testing.allocator.free(chosen); + @memset(chosen, false); + var scores_copy = try std.testing.allocator.alloc(f32, spec.keys); + defer std.testing.allocator.free(scores_copy); + @memcpy(scores_copy, ref32[0..spec.keys]); + var reference_picks: [512]u32 = undefined; + const picks = @min(spec.topk, spec.keys); + for (0..picks) |pick| { + var best: usize = 0; + for (0..spec.keys) |i| { + if (scores_copy[i] > scores_copy[best]) best = i; + } + reference_picks[pick] = @intCast(best); + scores_copy[best] = -std.math.inf(f32); + } + std.mem.sort(u32, reference_picks[0..picks], {}, std.sort.asc(u32)); + var overlap: usize = 0; + for (reference_picks[0..picks]) |at| chosen[at] = true; + var min_selected: f32 = std.math.inf(f32); + var max_unselected: f32 = -std.math.inf(f32); + for (0..spec.keys) |row| { + if (chosen[row]) { + overlap += 1; + min_selected = @min(min_selected, ref32[row]); + } else max_unselected = @max(max_unselected, ref32[row]); + } + var kernel_only: usize = 0; + var out_of_range: usize = 0; + for (0..picks) |i| { + const slot = kernel_positions[positions_offset + i]; + if (slot < 0 or slot >= @as(i32, @intCast(spec.keys))) { + out_of_range += 1; + continue; + } + if (!chosen[@intCast(slot)]) kernel_only += 1; + } + // Arithmetic noise selection absorbs: a pick is knife-edged at one score quantum of margin; else no 1-ULP flip. + var exact_picks: [512]u32 = undefined; + var exact_scores = try std.testing.allocator.alloc(f64, spec.keys); + defer std.testing.allocator.free(exact_scores); + @memcpy(exact_scores, exact[0..spec.keys]); + for (0..picks) |pick| { + var best: usize = 0; + for (0..spec.keys) |i| { + if (exact_scores[i] > exact_scores[best]) best = i; + } + exact_picks[pick] = @intCast(best); + exact_scores[best] = -std.math.inf(f64); + } + var flips: usize = 0; + var exact_chosen = try std.testing.allocator.alloc(bool, spec.keys); + defer std.testing.allocator.free(exact_chosen); + @memset(exact_chosen, false); + for (exact_picks[0..picks]) |at| exact_chosen[at] = true; + for (0..spec.keys) |at| { + if (chosen[at] != exact_chosen[at]) flips += 1; + } + var quantum: f32 = 0; + var margin_in_quanta: f64 = 0; + if (min_selected != std.math.inf(f32) and max_unselected != -std.math.inf(f32)) { + const boundary = min_selected * 0.5 + max_unselected * 0.5; + const exponent: i32 = if (@abs(boundary) > 0) std.math.ilogb(@abs(boundary)) - 7 else -149; + quantum = std.math.pow(f32, 2.0, @as(f32, @floatFromInt(exponent))); + margin_in_quanta = @as(f64, min_selected - max_unselected) / @as(f64, quantum); + } + std.debug.print( + \\CSA64 {{"case":"{s}","stage":"index_select","side":"noise_tolerance","keys":{d},"picks":{d},"precision_flips":{d},"selection_margin":{d},"score_quantum":{d},"margin_in_quanta":{d:.1}}} + , .{ spec.name, spec.keys, picks, flips, min_selected - max_unselected, quantum, margin_in_quanta }); + std.debug.print( + \\CSA64 {{"case":"{s}","stage":"index_select","side":"metal","keys":{d},"picks":{d},"reference_only":{d},"kernel_only":{d},"invalid_positions":{d},"lowest_selected":{d},"highest_unselected":{d}}} + , .{ spec.name, spec.keys, picks, picks - overlap, kernel_only, out_of_range, min_selected, max_unselected }); +} + +test "indexer scoring and selection sit at rounding level against the FP64 ideal" { + var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_instance.deinit(); + const gpa = arena_instance.allocator(); + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try tf.deepseek_v41.csa2_index.Pipelines.load(device); + defer p.deinit(); + + var rng = h.Lcg{ .state = 0x1234_5678_9abc_def0 }; + var amplitude: f32 = 1; + for ([_]IndexSpec{ + .{ .name = "index_random", .rows = 1, .keys = 1024, .topk = 512 }, + .{ .name = "index_random_4k", .rows = 1, .keys = 4096, .topk = 512 }, + .{ .name = "index_ties", .rows = 1, .keys = 1024, .topk = 512 }, + .{ .name = "index_boundary", .rows = 1, .keys = 1024, .topk = 512 }, + }, 0..) |spec, case_index| { + _ = case_index; + const q = try gpa.alloc(u16, 32 * 128); + defer gpa.free(q); + const weights = try gpa.alloc(u16, 32); + defer gpa.free(weights); + const cells = try gpa.alloc(f32, spec.keys * 128); + defer gpa.free(cells); + const keys = try gpa.alloc([]const f32, spec.keys); + defer gpa.free(keys); + for (q) |*cell| cell.* = math.b(rng.scale(1.0)); + for (weights) |*cell| cell.* = math.b(rng.scale(1.0)); + for (cells, 0..) |*v, i| { + v.* = rng.scale(1.0); + keys[i / 128] = cells[i / 128 * 128 ..][0..128]; + } + // Identical keys: every score is the same value, so the whole selection is tie-driven. + if (std.mem.eql(u8, spec.name, "index_ties")) { + for (cells[128..], cells[0..128]) |*dst, src| dst.* = src; + } + // A score cliff just inside the selection boundary: half the keys sit on the boundary. + if (std.mem.eql(u8, spec.name, "index_boundary")) { + for (cells, 0..) |*v, i| { + if ((i % 128) == 0) amplitude = (if (i / 128 < 512) @as(f32, 1.000001) else @as(f32, 1.0)); + v.* = rng.scale(amplitude); + } + } + try indexCase(device, queue, &p, spec, q, weights, keys); + } +} diff --git a/zig/tests/deepseek_v41_reference/operator_csa2_pack_fp64.zig b/zig/tests/deepseek_v41_reference/operator_csa2_pack_fp64.zig new file mode 100644 index 0000000000..9aed886414 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_csa2_pack_fp64.zig @@ -0,0 +1,103 @@ +//! CSA2 packed-cache code tables for the FP64 ideal-reference stages (DeepSeek-V4.1-Flash inference, MIT). +const std = @import("std"); +const math = @import("math.zig"); +const h = @import("operator_csa2_fp64.zig"); + +/// Packed-cache tables: generated codes give both sides bit-identical BF16 keys, so only attention arithmetic differs. +fn e4(code: u8) f32 { + const magnitude = code & 127; + if (magnitude == 127) return std.math.nan(f32); + const exponent: i32 = @intCast(magnitude >> 3); + const mantissa = magnitude & 7; + const base: f32 = if (exponent == 0) @as(f32, @floatFromInt(mantissa)) / 512 else std.math.ldexp(1 + @as(f32, @floatFromInt(mantissa)) / 8, exponent - 7); + return if ((code & 128) != 0) -base else base; +} + +pub fn e2(code: u8) f32 { + const levels = [_]f32{ 0, 0.5, 1, 1.5, 2, 3, 4, 6 }; + const v = levels[code & 7]; + return if ((code & 8) != 0) -v else v; +} + +/// One ring row: 512 FP8 codes plus the FP32 scale of each 32-column block, exactly as dsv41_unpack reads it. +pub fn decodeWindowRow(bytes: []const u8) [h.width]u16 { + var row: [h.width]u16 = undefined; + for (0..h.width) |d| { + const scale: f32 = @bitCast(@as(u32, bytes[h.width + d / 32]) << 23); + row[d] = math.b(e4(bytes[d]) * scale); + } + return row; +} + +/// One compressed row: 256 FP4 nibble bytes plus the FP8 scale of each 16-column block. +pub fn decodeMainRow(bytes: []const u8) [h.width]u16 { + var row: [h.width]u16 = undefined; + for (0..h.width) |d| { + const nibble: u8 = if ((d & 1) != 0) bytes[d / 2] >> 4 else bytes[d / 2] & 15; + row[d] = math.b(e2(nibble) * e4(bytes[h.width / 2 + d / 16])); + } + return row; +} + +/// FP8 code under the scale the pack kernel picks for the block: Metal rint rounds ties to even. +fn roundHalfEven(x: f32) f32 { + const toward: f32 = @trunc(x); + const magnitude: f32 = if (@abs(x - toward) > 0.5) toward + std.math.sign(x) else toward; + if (@abs(x - magnitude) != 0.5) return magnitude; + const as_int: i64 = @intFromFloat(magnitude); + if (@mod(as_int, 2) == 0) return magnitude; + return if (x < 0) magnitude + 1 else magnitude - 1; +} + +fn e4Code(x: f32) u8 { + const a = @min(@abs(x), 448.0); + if (a == 0) return if (std.math.signbit(x)) 128 else 0; + const bits: u32 = @bitCast(a); + const unbiased: i32 = @as(i32, @intCast((bits >> 23) & 255)) - 127; + const e = @max(unbiased, -6); + const q: i32 = @intFromFloat(roundHalfEven(std.math.ldexp(a, 3 - @as(i32, @intCast(e))))); + const code: u32 = if (e == -6 and q < 8) @intCast(q) else @intCast((e + 7) * 8 + q - 8); + return @as(u8, @intCast(@min(code, 126))) | @as(u8, if (std.math.signbit(x)) 128 else 0); +} + +fn e4Scale(x: f32) u8 { + const bits: u32 = @bitCast(x); + const unbiased: i32 = @as(i32, @intCast((bits >> 23) & 255)) - 127 + @intFromBool((bits & 0x7fffff) != 0); + return @intCast(unbiased + 127); +} + +pub fn e2Code(x: f32) u8 { + const a = @min(@abs(x), 6.0); + const pick: u8 = if (a <= 0.25) 0 else if (a < 0.75) 1 else if (a <= 1.25) 2 else if (a < 1.75) 3 else if (a <= 2.5) 4 else if (a < 3.5) 5 else if (a <= 5.0) 6 else 7; + return pick | @as(u8, if (std.math.signbit(x)) 8 else 0); +} + +/// Pack a window row as dsv41_pack kind 0: per 32-column block, a power-of-two FP8 scale and FP8 codes. +pub fn packWindowRow(values: []const f32, out: []u8) void { + @memset(out[0 .. h.width + h.width / 32], 0); + for (0..h.width / 32) |block| { + var a: f32 = 1e-4; + for (values[block * 32 ..][0..32]) |v| a = @max(a, @abs(v)); + const z = a / 448; + const zbits: u32 = @bitCast(z); + const scale = std.math.ldexp(@as(f32, 1.0), @as(i32, @intCast((zbits >> 23) & 255)) - 127 + @intFromBool((zbits & 0x7fffff) != 0)); + out[h.width + block] = e4Scale(scale); + for (0..32) |j| out[block * 32 + j] = e4Code(values[block * 32 + j] / scale); + } +} + +/// Pack a compressed row as dsv41_pack kind 1: per 16-column block, an FP8 scale and FP4 codes. +pub fn packMainRow(values: []const f32, out: []u8) void { + @memset(out[0 .. h.width / 2 + h.width / 16], 0); + for (0..h.width / 16) |block| { + var a: f32 = 6.0 * 0x1p-9; + for (values[block * 16 ..][0..16]) |v| a = @max(a, @abs(v)); + const scale = e4(e4Code(@max(a, 6.0 * 0x1p-9) / 6.0)); + out[h.width / 2 + block] = e4Code(@max(a, 6.0 * 0x1p-9) / 6.0); + for (0..16) |j| { + const code = e2Code(values[block * 16 + j] / scale); + const at = block * 8 + j / 2; + if ((j & 1) != 0) out[at] |= code << 4 else out[at] |= code; + } + } +} diff --git a/zig/tests/deepseek_v41_reference/operator_csa2_rope_fp64.zig b/zig/tests/deepseek_v41_reference/operator_csa2_rope_fp64.zig new file mode 100644 index 0000000000..5f78bd425b --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_csa2_rope_fp64.zig @@ -0,0 +1,136 @@ +//! CSA2 rotary-stage FP64 ideal-reference test (DeepSeek-V4.1-Flash inference, MIT). +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const math = @import("math.zig"); +const h = @import("operator_csa2_fp64.zig"); + +/// Both sides of one rotary call: the source's FP32 contract against the FP64 ideal at every step. +fn ropeFrequency64( + index: usize, + theta: f64, + original: f64, + factor: f64, + beta_fast: f64, + beta_slow: f64, +) f64 { + const frequency: f64 = 1 / std.math.pow(f64, theta, @as(f64, @floatFromInt(index * 2)) / 64); + if (original <= 0) return frequency; + const denominator = 2 * @log(theta); + const low = @max(@floor(64 * @log(original / (beta_fast * 2 * std.math.pi)) / denominator), 0); + const high = @min(@ceil(64 * @log(original / (beta_slow * 2 * std.math.pi)) / denominator), 63); + const ramp = std.math.clamp((@as(f64, @floatFromInt(index)) - low) / @max(high - low, 1e-3), 0, 1); + return frequency / factor * ramp + frequency * (1 - ramp); +} + +fn ropeCase( + device: mtl.Device, + queue: mtl.Queue, + p: *const tf.deepseek_v41.rope.Pipelines, + name: []const u8, + rng: *h.Lcg, + position: usize, + rows: usize, + heads: usize, + spectrum: tf.deepseek_v41.rope.Spectrum, + inverse: bool, + theta: f32, + original: u32, + factor: f32, +) !void { + const stride = heads * h.width; + const cells = rows * stride; + var b = h.Buffers{ .device = device }; + defer b.deinit(); + const x_gpu = try b.get(u16, cells); + const x = x_gpu.slice(u16, cells); + const before = try std.testing.allocator.alloc(u16, cells); + defer std.testing.allocator.free(before); + for (x, 0..) |*cell, i| { + cell.* = math.b(rng.scale(1.0)); + before[i] = cell.*; + } + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encode(enc, h.view(x_gpu), .{ .rows = @intCast(rows), .heads = @intCast(heads), .width = h.width, .rotary = 64, .stride = @intCast(stride), .position = @intCast(position), .step = 1, .inverse = @intFromBool(inverse), .spectrum = spectrum }); + enc.end(); + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.Csa2Fp64GpuFailure; + + const theta64: f64 = @as(f64, theta); + const original64: f64 = if (original == 0) 0 else @floatFromInt(original); + const ideal_cells = try std.testing.allocator.alloc(u16, cells); + defer std.testing.allocator.free(ideal_cells); + const contract_cells = try std.testing.allocator.alloc(u16, cells); + defer std.testing.allocator.free(contract_cells); + for (0..rows) |row| for (0..heads) |head| { + const base = row * stride + head * h.width; + // Everything outside the rotary window is copied through, so both sides must agree there. + for (0..h.width - 64) |d| { + ideal_cells[base + d] = before[base + d]; + contract_cells[base + d] = before[base + d]; + } + for (0..32) |i| { + const at = base + h.width - 64 + 2 * i; + const a = math.f(before[at]); + const z = math.f(before[at + 1]); + const frequency = math.ropeFrequency(i, theta, original, factor, 32, 1); + const angle: f32 = @as(f32, @floatFromInt(position + row)) * frequency; + const cosine32 = @cos(angle); + const sine32 = @sin(angle) * (if (inverse) @as(f32, -1) else 1); + contract_cells[at] = math.b(a * cosine32 - z * sine32); + contract_cells[at + 1] = math.b(a * sine32 + z * cosine32); + const frequency64 = ropeFrequency64(i, theta64, original64, @as(f64, factor), 32, 1); + const angle64: f64 = @as(f64, @floatFromInt(position + row)) * frequency64; + const cosine64 = @cos(angle64); + const sine64 = @sin(angle64) * (if (inverse) @as(f64, -1) else @as(f64, 1)); + ideal_cells[at] = math.b64(@as(f64, a) * cosine64 - @as(f64, z) * sine64); + ideal_cells[at + 1] = math.b64(@as(f64, a) * sine64 + @as(f64, z) * cosine64); + } + }; + + var stage_buffer: [32]u8 = undefined; + for (0..rows) |row| for (0..heads) |head| { + const stage = std.fmt.bufPrint(&stage_buffer, "row{d}_head{d}", .{ row, head }) catch unreachable; + const base = row * stride + head * h.width; + var ideal: [h.width]f64 = undefined; + var contract: [h.width]f64 = undefined; + for (0..h.width) |d| { + ideal[d] = math.f(ideal_cells[base + d]); + contract[d] = math.f(contract_cells[base + d]); + } + h.printBf(name, stage, "metal", h.deltaBf16(x[base..][0..h.width], &ideal)); + h.printBf(name, stage, "f32_reference", h.deltaBf16(contract_cells[base..][0..h.width], &ideal)); + h.printBf(name, stage, "metal_vs_f32_reference", h.deltaBf16(x[base..][0..h.width], &contract)); + }; +} + +test "the rotary stage sits at rounding level for both spectra and at long positions" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + // The production shape: 64 rotary columns, YaRN on the compressed spectrum only, positions to capacity. + var p = try tf.deepseek_v41.rope.Pipelines.load(device, 16384, 64, .{ .theta = 10000, .original = 0 }, .{ .theta = 10000, .factor = 16, .original = 65536, .beta_fast = 32, .beta_slow = 1 }); + defer p.deinit(); + + const positions = [_]usize{ 1, 7, 63, 64, 4095, 8191, 16383 }; + for (positions) |position| { + var name_buf: [32]u8 = undefined; + var scoped = h.Lcg{ .state = 0x9e3779b97f4a7c15 ^ @as(u64, position) }; + const label = std.fmt.bufPrint(&name_buf, "rope_normal_p{d}", .{position}) catch unreachable; + try ropeCase(device, queue, &p, label, &scoped, position, 1, 2, .normal, false, 10000, 0, 16); + var scoped2 = h.Lcg{ .state = 0x85ebca77c0b98731 ^ @as(u64, position) }; + const label2 = std.fmt.bufPrint(&name_buf, "rope_compressed_p{d}", .{position}) catch unreachable; + try ropeCase(device, queue, &p, label2, &scoped2, position, 1, 2, .compressed, false, 10000, 65536, 16); + var scoped3 = h.Lcg{ .state = 0xc2b2ae3d27d4eb4f ^ @as(u64, position) }; + const label3 = std.fmt.bufPrint(&name_buf, "rope_inverse_p{d}", .{position}) catch unreachable; + try ropeCase(device, queue, &p, label3, &scoped3, position, 1, 1, .compressed, true, 10000, 65536, 16); + } + // A prefill row set: the kernel indexes the factor table by absolute position per row. + var multi = h.Lcg{ .state = 0x2545f4914f6cdd1d }; + try ropeCase(device, queue, &p, "rope_prefill_rows", &multi, 8100, 4, 2, .compressed, false, 10000, 65536, 16); +} diff --git a/zig/tests/deepseek_v41_reference/operator_csa2_sparse_fp64.zig b/zig/tests/deepseek_v41_reference/operator_csa2_sparse_fp64.zig new file mode 100644 index 0000000000..dbc7618646 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_csa2_sparse_fp64.zig @@ -0,0 +1,514 @@ +//! CSA2 sparse-attention FP64 ideal-reference tests (DeepSeek-V4.1-Flash inference, MIT). +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const math = @import("math.zig"); +const attention_math = @import("attention_math.zig"); +const h = @import("operator_csa2_fp64.zig"); +const pack = @import("operator_csa2_pack_fp64.zig"); +const csa2 = tf.deepseek_v41.csa2; + +const SparseSpec = struct { + name: []const u8, + heads: usize, + rows: usize, + position: usize, + ratio: usize, + compressed_rows: usize, + topk: usize, +}; + +/// The kernel's own column addressing, so the reference reads the identical key per column slot. +fn columnKey(spec: SparseSpec, row: usize, slot: usize, indices: []const i32) ?KeySource { + const query = spec.position + row; + const window_columns: usize = if (spec.position + row == 0) 1 else 128; + if (slot < window_columns) { + const pos: usize = if (query == 0) 0 else if (query + 1 < 128) (query + 1 + slot) % 128 else blk: { + const begin = if (query + 1 > 128) query + 1 - 128 else 0; + break :blk begin + slot; + }; + if (pos > query) return null; + return .{ .row = pos % 128 }; + } + if (spec.ratio == 0) return null; + const pos_i = indices[row * spec.topk + slot - window_columns]; + const limit = @min(spec.compressed_rows, (query + 1) / spec.ratio); + if (pos_i < 0 or @as(usize, @intCast(pos_i)) >= limit) return null; + return .{ .compressed = true, .row = @intCast(pos_i) }; +} + +const KeySource = struct { compressed: bool = false, row: usize }; + +fn sparseCase( + device: mtl.Device, + queue: mtl.Queue, + p: *const csa2.Pipelines, + spec: SparseSpec, + q: []const u16, + window_rows: []const []const f32, + compressed_rows: []const []const f32, + indices: []const i32, + sinks: []const f32, +) !void { + var b = h.Buffers{ .device = device }; + defer b.deinit(); + const q_stride = spec.heads * h.width; + const q_gpu = try b.get(u16, spec.rows * q_stride); + const ring_gpu = try b.get(u8, 128 * 528); + const chunk_gpu = try b.get(u8, @max(spec.rows, 1) * 528); + const main_gpu = try b.get(u8, @max(spec.compressed_rows, 1) * 288); + const positions_gpu = try b.get(i32, spec.rows * spec.topk); + const sink_gpu = try b.get(f32, spec.heads); + const out_gpu = try b.get(u16, spec.rows * q_stride); + @memcpy(q_gpu.slice(u16, spec.rows * q_stride), q); + @memcpy(positions_gpu.slice(i32, spec.rows * spec.topk), indices); + @memcpy(sink_gpu.slice(f32, spec.heads), sinks); + + // The reference key table is the kernel's decoded cache: window columns by slot, then selected compressed rows. + var window_table = try std.testing.allocator.alloc(u16, 128 * h.width); + defer std.testing.allocator.free(window_table); + @memset(window_table, 0); + // The reference table is indexed 128 + row, so it needs one row per compressed row. + const compressed_cells = @max(compressed_rows.len, 1); + var compressed_table = try std.testing.allocator.alloc(u16, compressed_cells * h.width); + defer std.testing.allocator.free(compressed_table); + @memset(compressed_table, 0); + // Rows are ring slots: a logical position reads ring[pos % 128] unless this call publishes it. + for (window_rows, 0..) |values, logical| { + const bytes = try std.testing.allocator.alloc(u8, 528); + defer std.testing.allocator.free(bytes); + pack.packWindowRow(values, bytes); + const slot = logical % 128; + @memcpy(ring_gpu.slice(u8, 128 * 528)[slot * 528 ..][0..528], bytes); + const decoded = pack.decodeWindowRow(bytes); + @memcpy(window_table[slot * h.width ..][0..h.width], decoded[0..h.width]); + if (logical >= spec.position and logical < spec.position + spec.rows) + @memcpy(chunk_gpu.slice(u8, @max(spec.rows, 1) * 528)[(logical - spec.position) * 528 ..][0..528], bytes); + } + for (compressed_rows, 0..) |values, row| { + const bytes = try std.testing.allocator.alloc(u8, 288); + defer std.testing.allocator.free(bytes); + pack.packMainRow(values, bytes); + @memcpy(main_gpu.slice(u8, @max(spec.compressed_rows, 1) * 288)[row * 288 ..][0..288], bytes); + const decoded = pack.decodeMainRow(bytes); + @memcpy(compressed_table[row * h.width ..][0..h.width], decoded[0..h.width]); + } + + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encode(enc, h.view(q_gpu), h.view(ring_gpu), h.view(chunk_gpu), h.view(main_gpu), h.view(positions_gpu), h.view(sink_gpu), h.view(out_gpu), .{ + .rows = @intCast(spec.rows), + .heads = @intCast(spec.heads), + .q_stride = @intCast(q_stride), + .position = @intCast(spec.position), + .chunk_rows = @intCast(spec.rows), + .ratio = @intCast(spec.ratio), + .compressed_rows = @intCast(spec.compressed_rows), + .topk = @intCast(spec.topk), + .index_stride = @intCast(spec.topk), + }); + enc.end(); + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.Csa2Fp64GpuFailure; + const metal = out_gpu.slice(u16, spec.rows * q_stride); + + var column_indices = try std.testing.allocator.alloc(i32, 640); + defer std.testing.allocator.free(column_indices); + var wide32: [h.width]u16 = undefined; + var wide64: [h.width]u16 = undefined; + for (0..spec.rows) |row| { + const window_columns: usize = if (spec.position + row == 0) 1 else 128; + const compressed_columns: usize = if (spec.ratio == 0) 0 else @min(spec.topk, @min(spec.compressed_rows, (spec.position + row + 1) / spec.ratio)); + const count = window_columns + compressed_columns; + for (0..count) |slot| { + if (columnKey(spec, row, slot, indices)) |key| { + column_indices[slot] = if (key.compressed) @intCast(128 + key.row) else @intCast(key.row); + } else column_indices[slot] = -1; + } + const keys = attention_math.KeyTable{ + .window = window_table, + .window_stride = h.width, + .compressed = compressed_table, + .compressed_stride = h.width, + }; + for (0..spec.heads) |head| { + const head_q = q[row * q_stride + head * h.width ..][0..h.width]; + var pre32: [h.width]f32 = undefined; + var pre64: [h.width]f64 = undefined; + attention_math.sparseHeadPre(f32, head_q, keys, column_indices[0..count], sinks[head], wide32[0..h.width], &pre32); + attention_math.sparseHeadPre(f64, head_q, keys, column_indices[0..count], sinks[head], wide64[0..h.width], &pre64); + var ideal: [h.width]f64 = undefined; + for (wide64, 0..) |cell, d| ideal[d] = math.f(cell); + var energy: f64 = 0; + var peak: f64 = 0; + var live: usize = 0; + for (column_indices[0..count]) |at| live += @intFromBool(at >= 0); + for (ideal) |v| { + energy += v * v; + peak = @max(peak, @abs(v)); + } + std.debug.print( + \\CSA64 {{"case":"{s}","stage":"scale","side":"{s}","columns":{d},"live":{d},"ideal_rms":{d},"ideal_peak":{d}}} + , .{ spec.name, "sparse", count, live, @sqrt(energy / @as(f64, h.width)), peak }); + var stage_buffer: [32]u8 = undefined; + const stage = std.fmt.bufPrint(&stage_buffer, "row{d}_head{d}", .{ row, head }) catch unreachable; + // The intrinsic FP32-versus-FP64 level of the stage, before the BF16 quantum hides it; verdicts use it. + h.printF32(spec.name, stage, "f32_vs_ideal_pre_rounding", h.deltaF32VsF64(&pre32, &pre64)); + h.printBf(spec.name, stage, "metal", h.deltaBf16(metal[row * q_stride + head * h.width ..][0..h.width], &ideal)); + h.printBf(spec.name, stage, "f32_reference", h.deltaBf16(wide32[0..h.width], &ideal)); + // Decisive for the verdict: kernel and source-contract reference on identical decoded rows. + var contract_cells: [h.width]f64 = undefined; + for (wide32, 0..) |cell, d| contract_cells[d] = math.f(cell); + h.printBf(spec.name, stage, "metal_vs_f32_reference", h.deltaBf16(metal[row * q_stride + head * h.width ..][0..h.width], &contract_cells)); + } + } +} + +const KeyPattern = enum { random, cancelling, near_tie, one_sided }; + +fn sparseCaseFrom( + gpa: std.mem.Allocator, + device: mtl.Device, + queue: mtl.Queue, + p: *const csa2.Pipelines, + spec: SparseSpec, + rng: *h.Lcg, + key_amplitude: f32, + scale_exponent: i32, + indices_mode: usize, + sink: f32, + pattern: KeyPattern, +) !void { + try sparseCaseScaled(gpa, device, queue, p, spec, rng, key_amplitude, scale_exponent, scale_exponent, indices_mode, sink, pattern); +} + +/// Rows with window and compressed cache on independent scale grids: one accumulator mixes large and tiny terms. +fn sparseCaseScaled( + gpa: std.mem.Allocator, + device: mtl.Device, + queue: mtl.Queue, + p: *const csa2.Pipelines, + spec: SparseSpec, + rng: *h.Lcg, + key_amplitude: f32, + window_exponent: i32, + compressed_exponent: i32, + indices_mode: usize, + sink: f32, + pattern: KeyPattern, +) !void { + const q_stride = spec.heads * h.width; + const q = try gpa.alloc(u16, spec.rows * q_stride); + defer gpa.free(q); + const q_scale = std.math.pow(f32, 2, @as(f32, @floatFromInt(window_exponent))); + for (q) |*cell| cell.* = math.b(rng.uniform() * q_scale); + const logical_window = spec.position + spec.rows; + const window_rows = try gpa.alloc([]const f32, logical_window); + defer gpa.free(window_rows); + for (window_rows, 0..) |*row, i| { + const cells = try gpa.alloc(f32, h.width); + fillPatternRow(cells, rng, i, key_amplitude, window_exponent, pattern); + row.* = cells; + } + const compressed_rows = try gpa.alloc([]const f32, spec.compressed_rows); + defer gpa.free(compressed_rows); + for (compressed_rows, 0..) |*row, i| { + const cells = try gpa.alloc(f32, h.width); + fillPatternRow(cells, rng, i, key_amplitude, compressed_exponent, pattern); + row.* = cells; + } + const indices = try gpa.alloc(i32, spec.rows * spec.topk); + defer gpa.free(indices); + const limit = @max(spec.compressed_rows, 1); + for (0..spec.rows) |r| for (0..spec.topk) |k| { + indices[r * spec.topk + k] = switch (indices_mode) { + 0 => @intCast(k * limit / @max(spec.topk, 1)), // spread and ascending + 1 => @intCast((limit - 1) - k), // descending (the reference sorts; the kernel does not) + else => @as(i32, @intCast((r *% 7 +% 13 * k) % limit)), // arbitrary order, unsorted + }; + }; + const sinks = try gpa.alloc(f32, spec.heads); + defer gpa.free(sinks); + for (sinks) |*v| v.* = sink; + try sparseCase(device, queue, p, spec, q, window_rows, compressed_rows, indices, sinks); +} + +fn fillPatternRow(row: []f32, rng: *h.Lcg, index: usize, amplitude: f32, exponent: i32, pattern: KeyPattern) void { + const scale = std.math.pow(f32, 2, @as(f32, @floatFromInt(exponent))); + if (pattern == .random) { + for (row) |*v| v.* = rng.uniform() * amplitude * scale; + return; + } + for (row, 0..) |*v, d| { + if (pattern == .cancelling) { + // Same key shape: every score ties and the value sum cancels to the ULPs of the terms. + const flip: f32 = if ((d & 1) == 0) 1 else -1; + v.* = flip * amplitude * scale + rng.uniform() * amplitude * scale * 1e-3; + } else if (pattern == .near_tie) { + // Keys differ only far above the score rounding boundary: probabilities tie at the top. + v.* = @as(f32, @floatFromInt(index)) * amplitude * scale + rng.uniform() * amplitude * scale * 1e-7; + } else { + // One dominant key against keys orders of magnitude smaller. + v.* = rng.uniform() * amplitude * scale * @as(f32, if (index == 0) @as(f32, 1e4) else @as(f32, 1e-4)); + } + } +} + +test "sparse attention stages sit at rounding level against the FP64 ideal" { + var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_instance.deinit(); + const gpa = arena_instance.allocator(); + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try csa2.Pipelines.load(device); + defer p.deinit(); + + var rng = h.Lcg{ .state = 0x2545f4914f6cdd1d }; + + // Window only, the decode-step boundary at a wrapped ring. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "window_decode", .heads = 4, .rows = 1, .position = 255, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); + // Compressed rows with a small selection, ascending indices. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "compressed_asc", .heads = 4, .rows = 1, .position = 255, .ratio = 2, .compressed_rows = 128, .topk = 16 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); + // Unsorted indices: the kernel consumes the array in the given order; the reference must too. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "compressed_unsorted", .heads = 2, .rows = 1, .position = 9, .ratio = 2, .compressed_rows = 40, .topk = 12 }, &rng, 1.0, 0, 2, 0, KeyPattern.random); + // Prefill rows, multi-row chunk with a wrapped ring. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "prefill_rows", .heads = 2, .rows = 4, .position = 250, .ratio = 2, .compressed_rows = 100, .topk = 8 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); + // Wide dynamic range keys: the score sum runs across many orders of magnitude. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "wide_range", .heads = 1, .rows = 1, .position = 200, .ratio = 2, .compressed_rows = 32, .topk = 8 }, &rng, 1.0, -20, 0, 0, KeyPattern.random); + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "wide_range_hi", .heads = 1, .rows = 1, .position = 200, .ratio = 2, .compressed_rows = 32, .topk = 8 }, &rng, 1.0, 20, 0, 0, KeyPattern.random); + // Denormal-scale sink: the sink mass underflows FP32 while it is exact in FP64. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "sink_denormal", .heads = 1, .rows = 1, .position = 100, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, -16.0, KeyPattern.random); + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "sink_large", .heads = 1, .rows = 1, .position = 100, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, 12.0, KeyPattern.random); + // Value-product cancellation: identical key shapes tie every score and the P.V sum cancels to the ULPs. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "cancel_values", .heads = 1, .rows = 1, .position = 300, .ratio = 2, .compressed_rows = 64, .topk = 64 }, &rng, 32.0, 0, 0, 0, KeyPattern.cancelling); + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "cancel_wide", .heads = 1, .rows = 1, .position = 300, .ratio = 2, .compressed_rows = 64, .topk = 64 }, &rng, 32.0, 6, 0, 0, KeyPattern.cancelling); + // Tied scores: the denominator sums equal exponentials; accumulation order picks each probability's side. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "score_tie", .heads = 1, .rows = 1, .position = 300, .ratio = 2, .compressed_rows = 32, .topk = 32 }, &rng, 1.0, 0, 0, 0, KeyPattern.near_tie); + // One dominant key against keys orders of magnitude smaller: denominator and accumulator at the ratio. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "one_dominant", .heads = 1, .rows = 1, .position = 300, .ratio = 2, .compressed_rows = 16, .topk = 16 }, &rng, 1.0, 0, 0, 0, KeyPattern.one_sided); + // Many tiles, so the online correction multiplies the accumulator once per 64 columns. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "many_tiles", .heads = 1, .rows = 1, .position = 511, .ratio = 2, .compressed_rows = 512, .topk = 384 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); + // Cancellation in the value product: alternating-sign keys make the P.V sum small next to its terms. + { + const spec = SparseSpec{ .name = "pv_cancellation", .heads = 1, .rows = 1, .position = 64, .ratio = 0, .compressed_rows = 0, .topk = 0 }; + const q = try gpa.alloc(u16, h.width); + defer gpa.free(q); + for (q) |*cell| cell.* = math.b(rng.scale(1.0)); + const window_rows = try gpa.alloc([]const f32, spec.position + spec.rows); + defer gpa.free(window_rows); + for (window_rows) |*row| { + const cells = try gpa.alloc(f32, h.width); + for (cells, 0..) |*v, d| v.* = if ((d & 1) == 0) rng.scale(64.0) else rng.scale(64.0) * 0.987654321; + row.* = cells; + } + const sinks = try gpa.alloc(f32, 1); + defer gpa.free(sinks); + sinks[0] = 0; + const empty = try gpa.alloc([]const f32, 0); + defer gpa.free(empty); + const indices = try gpa.alloc(i32, 0); + defer gpa.free(indices); + try sparseCase(device, queue, &p, spec, q, window_rows, empty, indices, sinks); + } + // Large cancelling keys beside small ones: score dot products run through near-cancelling terms. + { + const spec = SparseSpec{ .name = "score_cancellation", .heads = 1, .rows = 1, .position = 96, .ratio = 0, .compressed_rows = 0, .topk = 0 }; + const q = try gpa.alloc(u16, h.width); + defer gpa.free(q); + for (q, 0..) |*cell, d| cell.* = if ((d & 1) == 0) math.b(1.0) else math.b(-1.0); + const window_rows = try gpa.alloc([]const f32, spec.position + spec.rows); + defer gpa.free(window_rows); + for (window_rows, 0..) |*row, i| { + const cells = try gpa.alloc(f32, h.width); + const bias: f32 = @as(f32, @floatFromInt(@rem(@as(i32, @intCast(i)), 8))) * 4.0; + for (cells, 0..) |*v, d| v.* = bias + (if ((d & 1) == 0) rng.scale(0.001) else rng.scale(0.001) * 0.99999); + row.* = cells; + } + const sinks = try gpa.alloc(f32, 1); + defer gpa.free(sinks); + sinks[0] = 0; + const empty = try gpa.alloc([]const f32, 0); + defer gpa.free(empty); + const indices = try gpa.alloc(i32, 0); + defer gpa.free(indices); + try sparseCase(device, queue, &p, spec, q, window_rows, empty, indices, sinks); + } +} + +/// One live column pins the maximum, so sink - max is the whole normalization across the FP32 exp limits. +fn sinkCase( + gpa: std.mem.Allocator, + device: mtl.Device, + queue: mtl.Queue, + p: *const csa2.Pipelines, + name: []const u8, + rng: *h.Lcg, + sink: f32, +) !void { + const spec = SparseSpec{ .name = name, .heads = 1, .rows = 1, .position = 0, .ratio = 0, .compressed_rows = 0, .topk = 0 }; + const q = try gpa.alloc(u16, h.width); + defer gpa.free(q); + for (q) |*cell| cell.* = math.b(rng.scale(1.0)); + const window_rows = try gpa.alloc([]const f32, 1); + defer gpa.free(window_rows); + const cells = try gpa.alloc(f32, h.width); + defer gpa.free(cells); + for (cells) |*v| v.* = rng.scale(1.0); + window_rows[0] = cells; + const sinks = try gpa.alloc(f32, 1); + defer gpa.free(sinks); + sinks[0] = sink; + const empty = try gpa.alloc([]const f32, 0); + defer gpa.free(empty); + const indices = try gpa.alloc(i32, 0); + defer gpa.free(indices); + try sparseCase(device, queue, p, spec, q, window_rows, empty, indices, sinks); +} + +test "sink normalization sits at rounding level across the exp overflow boundaries" { + var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_instance.deinit(); + const gpa = arena_instance.allocator(); + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var pipelines = try csa2.Pipelines.load(device); + defer pipelines.deinit(); + var rng = h.Lcg{ .state = 0x51c0de51c0de51c0 }; + + // One case per FP32 exp() boundary: underflow, overflow edge and the balanced middle where the sink decides. + const sweep = [_]struct { sink: f32, name: []const u8 }{ + .{ .sink = -104, .name = "sink_underflow" }, + .{ .sink = -90, .name = "sink_near_underflow" }, + .{ .sink = -60, .name = "sink_tiny" }, + .{ .sink = -12, .name = "sink_small" }, + .{ .sink = -1, .name = "sink_minus_one" }, + .{ .sink = 0, .name = "sink_zero" }, + .{ .sink = 1, .name = "sink_one" }, + .{ .sink = 12, .name = "sink_medium" }, + .{ .sink = 60, .name = "sink_large" }, + .{ .sink = 80, .name = "sink_edge80" }, + .{ .sink = 83, .name = "sink_edge83" }, + .{ .sink = 84, .name = "sink_edge84" }, + .{ .sink = 85, .name = "sink_edge85" }, + .{ .sink = 86, .name = "sink_over_edge" }, + .{ .sink = 90, .name = "sink_overflow" }, + }; + for (sweep) |entry| try sinkCase(gpa, device, queue, &pipelines, entry.name, &rng, entry.sink); + + // Per-head sinks: the kernel indexes the sink array by head, so heads orders apart check the indexing. + { + const spec = SparseSpec{ .name = "sink_per_head", .heads = 4, .rows = 1, .position = 0, .ratio = 0, .compressed_rows = 0, .topk = 0 }; + var head_rng = h.Lcg{ .state = 0xabcdef0123456789 }; + const q = try gpa.alloc(u16, spec.heads * h.width); + defer gpa.free(q); + for (q) |*cell| cell.* = math.b(head_rng.scale(1.0)); + const window_rows = try gpa.alloc([]const f32, 1); + defer gpa.free(window_rows); + const cells = try gpa.alloc(f32, h.width); + defer gpa.free(cells); + for (cells) |*v| v.* = head_rng.scale(1.0); + window_rows[0] = cells; + const sinks = try gpa.alloc(f32, spec.heads); + defer gpa.free(sinks); + for (0..spec.heads) |head| sinks[head] = @as(f32, @floatFromInt(@as(i32, @intCast(head)) * 24 - 36)); + const empty = try gpa.alloc([]const f32, 0); + defer gpa.free(empty); + const indices = try gpa.alloc(i32, 0); + defer gpa.free(indices); + try sparseCase(device, queue, &pipelines, spec, q, window_rows, empty, indices, sinks); + } +} + +// Tile denominators on flat, ramped and descending scores: the ramp drives the correction through FP32 underflow. +test "tile denominators sit at rounding level for every tile shape" { + var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_instance.deinit(); + const gpa = arena_instance.allocator(); + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var pipelines = try csa2.Pipelines.load(device); + defer pipelines.deinit(); + var rng = h.Lcg{ .state = 0x1a2b3c4d5e6f7081 }; + + // Ten tiles: 639 exponentials and 639 products; identical keys tie every score and cancel the sum. + try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = "tiles_equal_scores", .heads = 1, .rows = 1, .position = 1022, .ratio = 2, .compressed_rows = 512, .topk = 512 }, &rng, 1.0, 0, 0, 0, KeyPattern.cancelling); + // Ascending scores by tile: the correction falls one power of two per step until FP32 flushes it. + for ([_]i32{ -2, 0, 1, 2 }) |step| { + var name_buf: [24]u8 = undefined; + const label = std.fmt.bufPrint(&name_buf, "tiles_ramp{d}", .{step}) catch unreachable; + try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = label, .heads = 1, .rows = 1, .position = 1000, .ratio = 2, .compressed_rows = 512, .topk = 512 }, &rng, 1.0, step, 0, 0, KeyPattern.near_tie); + } + // Descending scores: the running maximum never moves, so every tile lands at full weight. + try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = "tiles_descending", .heads = 1, .rows = 1, .position = 1000, .ratio = 2, .compressed_rows = 512, .topk = 512 }, &rng, 1.0, 0, 1, 0, KeyPattern.near_tie); + // One tile only, the boundary case of the online loop: the correction is applied zero times. + try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = "tiles_single", .heads = 1, .rows = 1, .position = 20, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); +} + +// Output mix: window and compressed rows on different scale grids, and mirror rows whose shared shape cancels. +test "the output mix sits at rounding level when window and compressed rows cancel" { + var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_instance.deinit(); + const gpa = arena_instance.allocator(); + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var pipelines = try csa2.Pipelines.load(device); + defer pipelines.deinit(); + var rng = h.Lcg{ .state = 0x77a1c0de_5b5e1a2b }; + + // Large window rows against tiny compressed rows, then the reverse: one accumulator six orders apart. + try sparseCaseScaled(gpa, device, queue, &pipelines, .{ .name = "mix_window_dominates", .heads = 1, .rows = 1, .position = 100, .ratio = 2, .compressed_rows = 100, .topk = 64 }, &rng, 1.0, 14, -12, 0, 0, KeyPattern.random); + try sparseCaseScaled(gpa, device, queue, &pipelines, .{ .name = "mix_compressed_dominates", .heads = 1, .rows = 1, .position = 100, .ratio = 2, .compressed_rows = 100, .topk = 64 }, &rng, 1.0, -12, 14, 0, 0, KeyPattern.random); + try sparseCaseScaled(gpa, device, queue, &pipelines, .{ .name = "mix_balanced_magnitudes", .heads = 1, .rows = 1, .position = 100, .ratio = 2, .compressed_rows = 100, .topk = 64 }, &rng, 1.0, 6, 6, 0, 0, KeyPattern.cancelling); + + // Amplification probe: 128 window rows of a large alternating shape that cancels from scores and sum alike. + for ([_]i32{ 8, 16, 23 }) |shape_exponent| { + var name_buf: [24]u8 = undefined; + const label = std.fmt.bufPrint(&name_buf, "mirror_shape_e{d}", .{shape_exponent}) catch unreachable; + const spec = SparseSpec{ .name = label, .heads = 1, .rows = 1, .position = 127, .ratio = 0, .compressed_rows = 0, .topk = 0 }; + const q = try gpa.alloc(u16, h.width); + defer gpa.free(q); + // q dot shape == 0: a uniform query against a sign-alternating shape, so the large part drops out. + for (q) |*cell| cell.* = math.b(1.0); + const window_rows = try gpa.alloc([]const f32, 128); + defer gpa.free(window_rows); + const shape = std.math.pow(f32, 2, @floatFromInt(shape_exponent)); + const shared = try gpa.alloc(f32, h.width); + defer gpa.free(shared); + // One 64th of the cancelling shape near the FP8 window floor: partial sums run 2^16 times the final. + for (shared) |*v| v.* = rng.scale(shape * 0x1p-16); + for (window_rows, 0..) |*row, i| { + const cells = try gpa.alloc(f32, h.width); + const side: f32 = if ((i & 1) == 0) 1 else -1; + for (0..h.width) |d| { + const flip: f32 = if ((d & 1) == 0) 1 else -1; + cells[d] = side * flip * shape + shared[d]; + } + row.* = cells; + } + const sinks = try gpa.alloc(f32, 1); + defer gpa.free(sinks); + sinks[0] = 0; + const empty = try gpa.alloc([]const f32, 0); + defer gpa.free(empty); + const indices = try gpa.alloc(i32, 0); + defer gpa.free(indices); + try sparseCase(device, queue, &pipelines, spec, q, window_rows, empty, indices, sinks); + } +} diff --git a/zig/tests/deepseek_v41_reference/operator_hc_amplify_fp64.zig b/zig/tests/deepseek_v41_reference/operator_hc_amplify_fp64.zig new file mode 100644 index 0000000000..78c54e00a7 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_hc_amplify_fp64.zig @@ -0,0 +1,284 @@ +//! mHC FP64 stage propagation and 1-ULP amplification test (DeepSeek-V4.1-Flash inference, MIT). +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const math = @import("math.zig"); +const h = @import("operator_hc_fp64.zig"); +const mhc = tf.deepseek_v41.mhc; +const ops = tf.deepseek_v41.ops; +const Hc = tf.deepseek_v41_weights.Hc; +const Config = tf.deepseek_v41_config.Config; + +// Ideal FP64 stage propagation: statistic, dense, split, collapse and post, pre-rounding outputs kept. +fn stageF64(stream: []const u16, hc: Hc, cfg: Config, branch: []const u16, y_pre: []f64, y_bf: *[h.stream_width]u16, coll_pre: []f64, inv_out: *f64, comb_out: *[16]f64) void { + var proj: [h.mix_count]f64 = undefined; + math.denseOf(f64, hc.projection, stream, &proj); + const inv = math.inverseOf(f64, stream, cfg.text_config.rms_norm_eps); + const mix = math.splitOf(f64, &proj, inv, hc, cfg.text_config); + inv_out.* = inv; + comb_out.* = mix.comb; + for (0..h.width) |d| { + var s: f64 = 0; + for (0..h.copies) |c| s += mix.pre[c] * @as(f64, math.f(stream[c * h.width + d])); + coll_pre[d] = s; + } + for (0..h.width) |d| { + for (0..h.copies) |k| { + var acc: f64 = mix.post[k] * @as(f64, math.f(branch[d])); + for (0..h.copies) |j| acc += mix.comb[j * 4 + k] * @as(f64, math.f(stream[j * h.width + d])); + y_pre[k * h.width + d] = acc; + } + } + var expanded: [h.stream_width]u16 = undefined; + @memcpy(&expanded, stream); + math.expandOf(f64, &expanded, branch, mix); + @memcpy(y_bf, &expanded); +} + +fn perturbOneUlp(x: u16) u16 { + return if ((x & 0x8000) == 0) x + 1 else x - 1; +} + +test "mHC 1-ULP input amplification through the hyperconnection mix" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try mhc.Pipelines.load(device); + defer p.deinit(); + var o = try ops.Pipelines.load(device); + defer o.deinit(); + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const gpa = arena.allocator(); + const cfg = try Config.parse(gpa, h.released_config); + + const fixture = try std.json.parseFromSlice(h.Fixture, gpa, @embedFile(h.fixture_path), .{ .ignore_unknown_fields = true }); + const eight_rows = try h.decodeU16(gpa, fixture.value.eight_rows_bf16); + const ffn_stream = try h.decodeU16(gpa, fixture.value.ffn_stream_bf16); + const ffn_branch = try h.decodeU16(gpa, fixture.value.ffn_branch_bf16); + const ffn_scale = try h.decodeF32(gpa, fixture.value.ffn_scale_f32); + const ffn_base = try h.decodeF32(gpa, fixture.value.ffn_base_f32); + const weight = try h.syntheticWeight(gpa); + + const scale = try device.buffer(12, mtl.ResourceOptions.shared); + defer scale.deinit(); + h.upload(f32, scale, ffn_scale); + const base = try device.buffer(h.mix_count * 4, mtl.ResourceOptions.shared); + defer base.deinit(); + h.upload(f32, base, ffn_base); + const weight_buffer = try device.buffer(h.mix_count * h.stream_width * 4, mtl.ResourceOptions.shared); + defer weight_buffer.deinit(); + h.upload(f32, weight_buffer, weight); + const hc = Hc{ + .projection = .{ .buffer = weight_buffer, .offset = 0, .bytes = h.mix_count * h.stream_width * 4, .dtype = .f32 }, + .scale = .{ .buffer = scale, .offset = 0, .bytes = 12, .dtype = .f32 }, + .base = .{ .buffer = base, .offset = 0, .bytes = h.mix_count * 4, .dtype = .f32 }, + }; + + const columns = 64; + const column_stride = h.width / columns; + const Row = struct { name: []const u8, stream: []u16 }; + var rows: [9]Row = undefined; + for (0..8) |r| { + rows[r] = .{ .name = "teacher", .stream = try h.fourCopies(gpa, eight_rows[r * h.width ..][0..h.width]) }; + } + rows[8] = .{ .name = "ffn_token1", .stream = try gpa.dupe(u16, ffn_stream) }; + + const y0 = try gpa.alloc(f64, h.stream_width); + const y1 = try gpa.alloc(f64, h.stream_width); + var y0_bf: [h.stream_width]u16 = undefined; + var y1_bf: [h.stream_width]u16 = undefined; + const coll0 = try gpa.alloc(f64, h.width); + const coll1 = try gpa.alloc(f64, h.width); + var inv0: f64 = undefined; + var inv1: f64 = undefined; + var comb0: [16]f64 = undefined; + var comb1: [16]f64 = undefined; + + for (rows, 0..) |row, row_index| { + stageF64(row.stream, hc, cfg, ffn_branch, y0, &y0_bf, coll0, &inv0, &comb0); + var sum_y2: f64 = 0; + for (y0) |v| sum_y2 += v * v; + var sum_x2: f64 = 0; + for (row.stream) |v| sum_x2 += blk: { + const f: f64 = @floatCast(math.f(v)); + break :blk f * f; + }; + var sum_c2: f64 = 0; + for (coll0) |v| sum_c2 += v * v; + var sum_b2: f64 = 0; + for (y0_bf) |v| { + const f: f64 = @floatCast(math.f(v)); + sum_b2 += f * f; + } + + var amp_sum: f64 = 0; + var amp_max: f64 = 0; + var amp_n: usize = 0; + var amp_c_sum: f64 = 0; + var amp_c_max: f64 = 0; + var amp_b_sum: f64 = 0; + var amp_b_max: f64 = 0; + var changed_any: usize = 0; + var changed_cells: usize = 0; + var inv_rel_sum: f64 = 0; + var comb_sq: f64 = 0; + var comb_n: usize = 0; + + for (0..h.copies) |j| { + for (0..columns) |k| { + const col = k * column_stride; + const idx = j * h.width + col; + const orig = row.stream[idx]; + if (math.f(orig) == 0) continue; + const perturbed = perturbOneUlp(orig); + const dx = @as(f64, @floatCast(math.f(perturbed))) - @as(f64, @floatCast(math.f(orig))); + row.stream[idx] = perturbed; + stageF64(row.stream, hc, cfg, ffn_branch, y1, &y1_bf, coll1, &inv1, &comb1); + row.stream[idx] = orig; + + var dy2: f64 = 0; + for (y0, y1) |a, b| { + const d = a - b; + dy2 += d * d; + } + const amp = @sqrt(dy2 / sum_y2) * @sqrt(sum_x2) / @abs(dx); + amp_sum += amp; + if (amp > amp_max) amp_max = amp; + amp_n += 1; + + var dc2: f64 = 0; + for (coll0, coll1) |a, b| { + const d = a - b; + dc2 += d * d; + } + const amp_c = @sqrt(dc2 / sum_c2) * @sqrt(sum_x2) / @abs(dx); + amp_c_sum += amp_c; + if (amp_c > amp_c_max) amp_c_max = amp_c; + + var changed: usize = 0; + var dyb2: f64 = 0; + for (y0_bf, y1_bf) |a, b| { + if (a != b) changed += 1; + const av: f64 = @floatCast(math.f(a)); + const bv: f64 = @floatCast(math.f(b)); + const d = av - bv; + dyb2 += d * d; + } + const amp_b = @sqrt(dyb2 / sum_b2) * @sqrt(sum_x2) / @abs(dx); + amp_b_sum += amp_b; + if (amp_b > amp_b_max) amp_b_max = amp_b; + if (changed > 0) { + changed_any += 1; + changed_cells += changed; + } + inv_rel_sum += @abs(inv1 - inv0) / inv0; + var comb_d2: f64 = 0; + var comb_m2: f64 = 0; + for (comb0, comb1) |a, b| { + const d = a - b; + comb_d2 += d * d; + comb_m2 += b * b; + } + comb_sq += @sqrt(comb_d2 / comb_m2); + comb_n += 1; + } + } + std.debug.print("HC64 {{\"amplification\":{{\"row\":{d},\"row_name\":\"{s}\",\"perturbations\":{d},\"amp_mean\":{d},\"amp_max\":{d},\"amp_collapse_mean\":{d},\"amp_collapse_max\":{d},\"amp_bf16_mean\":{d},\"amp_bf16_max\":{d},\"bf16_changed_fraction\":{d},\"mean_changed_cells\":{d},\"inv_rel_mean\":{d},\"comb_nrms_mean\":{d}}}}}\n", .{ row_index, row.name, amp_n, amp_sum / @as(f64, @floatFromInt(amp_n)), amp_max, amp_c_sum / @as(f64, @floatFromInt(amp_n)), amp_c_max, amp_b_sum / @as(f64, @floatFromInt(amp_n)), amp_b_max, changed_any, @as(f64, @floatFromInt(changed_cells)) / @as(f64, @floatFromInt(amp_n)), inv_rel_sum / @as(f64, @floatFromInt(amp_n)), comb_sq / @as(f64, @floatFromInt(comb_n)) }); + // The mix attenuates a single-ULP input in FP64 (gain < 1) and stays near one BF16-visible ULP. + try std.testing.expect(amp_sum / @as(f64, @floatFromInt(amp_n)) < 1.0); + try std.testing.expect(amp_max < 1.0); + try std.testing.expect(amp_c_max < 2.5); + try std.testing.expect(amp_b_sum / @as(f64, @floatFromInt(amp_n)) < 3.0); + try std.testing.expect(inv_rel_sum / @as(f64, @floatFromInt(amp_n)) < 1e-5); + } + + // Metal confirmation on the first teacher row: the base row plus 16 stream-cell perturbations through the kernels. + { + const metal_perturbations = 16; + const batch = metal_perturbations + 1; + var batch_stream = try gpa.alloc(u16, batch * h.stream_width); + var cells: [metal_perturbations]usize = undefined; + var dxs: [metal_perturbations]f64 = undefined; + var filled: usize = 0; + @memcpy(batch_stream[0..h.stream_width], rows[0].stream); + outer: for (0..h.copies) |j| { + for (0..columns) |k| { + const idx = j * h.width + k * column_stride; + const orig = rows[0].stream[idx]; + if (math.f(orig) == 0) continue; + cells[filled] = idx; + dxs[filled] = @as(f64, @floatCast(math.f(perturbOneUlp(orig)))) - @as(f64, @floatCast(math.f(orig))); + filled += 1; + if (filled == metal_perturbations) break :outer; + } + } + for (0..filled) |i| { + @memcpy(batch_stream[(i + 1) * h.stream_width ..][0..h.stream_width], rows[0].stream); + batch_stream[(i + 1) * h.stream_width + cells[i]] = perturbOneUlp(rows[0].stream[cells[i]]); + } + const stream_buf = try device.buffer(batch * h.stream_width * 2, mtl.ResourceOptions.shared); + defer stream_buf.deinit(); + h.upload(u16, stream_buf, batch_stream); + const inv_buf = try device.buffer(batch * 4, mtl.ResourceOptions.shared); + defer inv_buf.deinit(); + const mixes_buf = try device.buffer(batch * h.mix_count * 4, mtl.ResourceOptions.shared); + defer mixes_buf.deinit(); + const pre_buf = try device.buffer(batch * 4 * 4, mtl.ResourceOptions.shared); + defer pre_buf.deinit(); + const post_buf = try device.buffer(batch * 4 * 4, mtl.ResourceOptions.shared); + defer post_buf.deinit(); + const comb_buf = try device.buffer(batch * 16 * 4, mtl.ResourceOptions.shared); + defer comb_buf.deinit(); + const collapsed_buf = try device.buffer(batch * h.width * 2, mtl.ResourceOptions.shared); + defer collapsed_buf.deinit(); + const branch_buf = try device.buffer(batch * h.width * 2, mtl.ResourceOptions.shared); + defer branch_buf.deinit(); + for (0..batch) |r| @memcpy(branch_buf.slice(u16, batch * h.width)[r * h.width ..][0..h.width], ffn_branch); + const y_buf = try device.buffer(batch * h.stream_width * 2, mtl.ResourceOptions.shared); + defer y_buf.deinit(); + try h.runStage(queue, &p, &o, @intCast(batch), cfg, h.view(stream_buf), h.view(weight_buffer), h.view(inv_buf), h.view(mixes_buf), h.view(scale), h.view(base), h.view(pre_buf), h.view(post_buf), h.view(comb_buf), h.view(collapsed_buf), h.view(branch_buf), h.view(y_buf)); + + const y_metal = y_buf.slice(u16, batch * h.stream_width); + var nan_count: usize = 0; + var first_nan: usize = 0; + for (y_metal[0..h.stream_width], 0..) |v, i| { + const f = math.f(v); + if (std.math.isNan(f)) { + nan_count += 1; + if (nan_count == 1) first_nan = i; + } + } + var sum_x2_row0: f64 = 0; + for (rows[0].stream) |v| { + const f: f64 = @floatCast(math.f(v)); + sum_x2_row0 += f * f; + } + var sum_base2: f64 = 0; + for (0..h.stream_width) |i| { + const f: f64 = @floatCast(math.f(y_metal[i])); + sum_base2 += f * f; + } + var amp_sum: f64 = 0; + var changed_total: usize = 0; + for (0..filled) |i| { + var dy2: f64 = 0; + var changed: usize = 0; + for (0..h.stream_width) |c| { + const a: f64 = @floatCast(math.f(y_metal[(i + 1) * h.stream_width + c])); + const b: f64 = @floatCast(math.f(y_metal[c])); + const d = a - b; + if (d != 0) changed += 1; + dy2 += d * d; + } + changed_total += changed; + amp_sum += @sqrt(dy2 / sum_base2) * @sqrt(sum_x2_row0) / @abs(dxs[i]); + } + std.debug.print("HC64 {{\"amplification\":{{\"row\":0,\"metal\":true,\"perturbations\":{d},\"amp_mean\":{d},\"mean_changed_cells\":{d}}}}}\n", .{ filled, amp_sum / @as(f64, @floatFromInt(filled)), @as(f64, @floatFromInt(changed_total)) / @as(f64, @floatFromInt(filled)) }); + const metal_amp = amp_sum / @as(f64, @floatFromInt(filled)); + try std.testing.expect(metal_amp > 0.5 and metal_amp < 3.0); + } +} diff --git a/zig/tests/deepseek_v41_reference/operator_hc_fp64.zig b/zig/tests/deepseek_v41_reference/operator_hc_fp64.zig index c41381c32e..952f497e61 100644 --- a/zig/tests/deepseek_v41_reference/operator_hc_fp64.zig +++ b/zig/tests/deepseek_v41_reference/operator_hc_fp64.zig @@ -1,6 +1,4 @@ -//! FP64 ideal-reference isolation for the mHC stage: rounding level, Metal deltas and 1-ULP -//! input amplification. Receipt lines print with an "HC64" prefix. No tolerance is relaxed and -//! no source coefficient is injected: retained rows and real checkpoint scale/base are inputs. +//! Shared mHC FP64 ideal-reference harness; receipts print "HC64" (DeepSeek-V4.1-Flash inference, MIT). const std = @import("std"); const mtl = @import("metal"); const tf = @import("tensorfold"); @@ -8,17 +6,17 @@ const math = @import("math.zig"); const mhc = tf.deepseek_v41.mhc; const ops = tf.deepseek_v41.ops; -const View = tf.deepseek_v41.attention.View; -const Tensor = tf.checkpoint_metal.Tensor; -const Hc = tf.deepseek_v41_weights.Hc; -const Config = tf.deepseek_v41_config.Config; +pub const View = tf.deepseek_v41.attention.View; +pub const Tensor = tf.checkpoint_metal.Tensor; +pub const Hc = tf.deepseek_v41_weights.Hc; +pub const Config = tf.deepseek_v41_config.Config; -const width: usize = 5120; -const copies: usize = 4; -const stream_width: usize = width * copies; -const mix_count: usize = 24; +pub const width: usize = 5120; +pub const copies: usize = 4; +pub const stream_width: usize = width * copies; +pub const mix_count: usize = 24; -const released_config = +pub const released_config = \\{"model_type":"deepseek_v41","quantization_config":{"quant_method":"exl3","version":"1"}, \\"text_config":{"vocab_size":129280,"hidden_size":5120,"moe_intermediate_size":2304, \\"num_hidden_layers":40,"num_attention_heads":64,"num_key_value_heads":1,"head_dim":512, @@ -39,17 +37,17 @@ const released_config = ; /// The 16 retained HC unary arguments (hc-exp-variants-input.json), including 1.4393715858459473. -const retained_exp_argument_bits = [_]u32{ +pub const retained_exp_argument_bits = [_]u32{ 3217026610, 1081519143, 1049174916, 1078264130, 1073892416, 1078760804, 1083574034, 1069038932, 1043563522, 1083490981, 1087210444, 1094911558, 3195364771, 1082252091, 1096635047, 1091866698, }; /// The documented controlling exponent input: Metal FP32 0x4086fa39, ideal 0x4086fa38 (1 ULP). -const controlling_exponent_input: f32 = 1.4393715858459473; +pub const controlling_exponent_input: f32 = 1.4393715858459473; -const fixture_path = "../../src/families/deepseek_v41/fixtures/hc_fp64_rows.json"; +pub const fixture_path = "../../src/families/deepseek_v41/fixtures/hc_fp64_rows.json"; -const Fixture = struct { +pub const Fixture = struct { eight_rows_bf16: []const u8, ffn_stream_bf16: []const u8, ffn_branch_bf16: []const u8, @@ -66,7 +64,7 @@ fn decodeBase64(gpa: std.mem.Allocator, text: []const u8) ![]u8 { return out; } -fn decodeU16(gpa: std.mem.Allocator, text: []const u8) ![]u16 { +pub fn decodeU16(gpa: std.mem.Allocator, text: []const u8) ![]u16 { const bytes = try decodeBase64(gpa, text); defer gpa.free(bytes); const out = try gpa.alloc(u16, bytes.len / 2); @@ -74,7 +72,7 @@ fn decodeU16(gpa: std.mem.Allocator, text: []const u8) ![]u16 { return out; } -fn decodeF32(gpa: std.mem.Allocator, text: []const u8) ![]f32 { +pub fn decodeF32(gpa: std.mem.Allocator, text: []const u8) ![]f32 { const bytes = try decodeBase64(gpa, text); defer gpa.free(bytes); const out = try gpa.alloc(f32, bytes.len / 4); @@ -82,14 +80,14 @@ fn decodeF32(gpa: std.mem.Allocator, text: []const u8) ![]f32 { return out; } -const Lcg = struct { +pub const Lcg = struct { state: u64 = 0x853c49e6748fea9b, - fn next(s: *Lcg) u64 { + pub fn next(s: *Lcg) u64 { s.state = s.state *% 6364136223846793005 +% 1442695040888963407; return s.state >> 11; } /// Uniform [-1, 1) with 24-bit granularity, exactly representable in FP32. - fn uniform(s: *Lcg) f32 { + pub fn uniform(s: *Lcg) f32 { const v: f64 = @as(f64, @floatFromInt(s.next() & 0xffffff)) / 16777216.0; return @floatCast(v * 2 - 1); } @@ -113,7 +111,7 @@ fn ulp32At(v: f64) f64 { return @as(f64, up) - @as(f64, a); } -const Delta = struct { +pub const Delta = struct { nrms: f64 = 0, max_abs: f64 = 0, max_ulp: f64 = 0, @@ -121,7 +119,7 @@ const Delta = struct { count: usize = 0, }; -fn deltaF32VsF64(actual: []const f32, ref: []const f64) Delta { +pub fn deltaF32VsF64(actual: []const f32, ref: []const f64) Delta { var d: Delta = .{ .count = actual.len }; var squared: f64 = 0; var magnitude: f64 = 0; @@ -202,32 +200,32 @@ fn referenceRow(comptime T: type, stream: []const u16, hc: Hc, cfg: Config, bran return out; } -const Buffers = struct { +pub const Buffers = struct { device: mtl.Device, items: [24]mtl.Buffer = undefined, count: usize = 0, - fn get(b: *Buffers, comptime T: type, n: usize) !mtl.Buffer { + pub fn get(b: *Buffers, comptime T: type, n: usize) !mtl.Buffer { const x = try b.device.buffer(try std.math.mul(usize, @sizeOf(T), n), mtl.ResourceOptions.shared); b.items[b.count] = x; b.count += 1; @memset(x.slice(T, n), 0); return x; } - fn deinit(b: *Buffers) void { + pub fn deinit(b: *Buffers) void { for (b.items[0..b.count]) |x| x.deinit(); } }; -fn upload(comptime T: type, buffer: mtl.Buffer, data: []const T) void { +pub fn upload(comptime T: type, buffer: mtl.Buffer, data: []const T) void { @memcpy(buffer.slice(T, data.len), data); } -fn view(buffer: mtl.Buffer) View { +pub fn view(buffer: mtl.Buffer) View { return .{ .buffer = buffer }; } -fn runStage( +pub fn runStage( queue: mtl.Queue, p: *const mhc.Pipelines, o: *const ops.Pipelines, @@ -262,7 +260,7 @@ fn runStage( if (cb.failure() != null) return error.MhcFp64GpuFailure; } -const Case = struct { +pub const Case = struct { name: []const u8, rows: usize, stream: []u16, @@ -272,14 +270,14 @@ const Case = struct { base: []f32, }; -const StageMetric = struct { +pub const StageMetric = struct { name: []const u8, metal: Delta = .{}, f32ref: Delta = .{}, bf16: ?BfDelta = null, }; -fn printMetric(case_name: []const u8, m: StageMetric) void { +pub fn printMetric(case_name: []const u8, m: StageMetric) void { if (m.bf16) |bf| { std.debug.print("HC64 {{\"case\":\"{s}\",\"stage\":\"{s}\",\"elements\":{d},\"nrms_metal_vs_ideal\":{d},\"bf16_unequal_metal_vs_ideal\":{d},\"bf16_unequal_f32ref_vs_ideal\":{d},\"worst_gap\":{d},\"worst_index\":{d}}}\n", .{ case_name, m.name, bf.count, bf.nrms, m.bf16.?.unequal_ideal, m.bf16.?.unequal_f32_ref, m.bf16.?.worst_gap, m.bf16.?.worst }); } else { @@ -287,8 +285,7 @@ fn printMetric(case_name: []const u8, m: StageMetric) void { } } -/// One full HC stage over `rows` streams: statistic, dense, split, collapse and post-expansion, -/// each side (Metal, FP32 source-contract reference, FP64 ideal reference) on identical inputs. +/// One full HC stage over rows streams on identical inputs: Metal, the FP32 source contract and the FP64 ideal. fn runCase( gpa: std.mem.Allocator, device: mtl.Device, @@ -415,8 +412,7 @@ fn runCase( }; for (metrics) |m| printMetric(case.name, m); - // Rounding-level gate: Metal must sit at the FP32 source-contract reference's own distance - // from the FP64 ideal (measured bounds; a real operator defect breaks these by orders). + // Rounding-level gate: Metal must sit at the FP32 source contract's own distance from the FP64 ideal. for (metrics, 0..) |m, i| { if (i < 5) { try std.testing.expect(m.metal.nrms < 2e-7); @@ -436,14 +432,14 @@ fn runCase( try std.testing.expect(metrics[6].bf16.?.unequal_f32_ref * 4096 <= metrics[6].bf16.?.count + 4096); } -fn syntheticWeight(gpa: std.mem.Allocator) ![]f32 { +pub fn syntheticWeight(gpa: std.mem.Allocator) ![]f32 { var lcg = Lcg{}; const w = try gpa.alloc(f32, mix_count * stream_width); for (w) |*v| v.* = lcg.uniform() * 0.03; return w; } -fn fourCopies(gpa: std.mem.Allocator, row: []const u16) ![]u16 { +pub fn fourCopies(gpa: std.mem.Allocator, row: []const u16) ![]u16 { var stream = try gpa.alloc(u16, stream_width); for (0..copies) |c| @memcpy(stream[c * width ..][0..width], row); return stream; @@ -475,8 +471,7 @@ test "mHC fp64 rounding level and Metal deltas on retained streams" { const weight = try syntheticWeight(gpa); - // Token 671 embedding expanded to four copies; real FFN scale/base; synthetic branch-free case - // uses the retained branch row. All three sides consume identical inputs. + // Token 671 embedding in four copies against real FFN scale/base; the branch row feeds a branch-free case. { const stream = try fourCopies(gpa, token671); const branch = try gpa.alloc(u16, width); @@ -518,578 +513,3 @@ test "mHC fp64 rounding level and Metal deltas on retained streams" { try runCase(gpa, device, queue, &p, &o, cfg, .{ .name = "wide_dynamic_range", .rows = 1, .stream = stream, .weight = weight, .branch = branch, .scale = ffn_scale, .base = ffn_base }); } } - -// The comptime parameterization must keep the FP32 source contract bitwise; this compares -// math.mixesOf(f32) against the pre-parameterization algorithm verbatim on a retained row. -test "mHC fp32 reference path stays bit-identical under the comptime parameterization" { - const device = mtl.Device.init() catch return error.SkipZigTest; - defer device.deinit(); - const pool = mtl.objc.Pool.push(); - defer pool.pop(); - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const gpa = arena.allocator(); - const cfg = try Config.parse(gpa, released_config); - const fixture = try std.json.parseFromSlice(Fixture, gpa, @embedFile(fixture_path), .{ .ignore_unknown_fields = true }); - const eight_rows = try decodeU16(gpa, fixture.value.eight_rows_bf16); - const ffn_scale = try decodeF32(gpa, fixture.value.ffn_scale_f32); - const ffn_base = try decodeF32(gpa, fixture.value.ffn_base_f32); - const scale = try device.buffer(12, mtl.ResourceOptions.shared); - defer scale.deinit(); - upload(f32, scale, ffn_scale); - const base = try device.buffer(mix_count * 4, mtl.ResourceOptions.shared); - defer base.deinit(); - upload(f32, base, ffn_base); - var lcg = Lcg{}; - const weight = try device.buffer(mix_count * stream_width * 4, mtl.ResourceOptions.shared); - defer weight.deinit(); - const weight_host = try gpa.alloc(f32, mix_count * stream_width); - for (weight_host) |*v| v.* = lcg.uniform() * 0.03; - upload(f32, weight, weight_host); - const hc = Hc{ - .projection = .{ .buffer = weight, .offset = 0, .bytes = mix_count * stream_width * 4, .dtype = .f32 }, - .scale = .{ .buffer = scale, .offset = 0, .bytes = 12, .dtype = .f32 }, - .base = .{ .buffer = base, .offset = 0, .bytes = mix_count * 4, .dtype = .f32 }, - }; - for (0..8) |r| { - const row = eight_rows[r * width ..][0..width]; - var stream: [stream_width]u16 = undefined; - for (0..copies) |c| @memcpy(stream[c * width ..][0..width], row); - const parameterized = math.mixesOf(f32, &stream, hc, cfg.text_config); - const legacy = mixesLegacy(&stream, hc, cfg); - try std.testing.expect(std.meta.eql(parameterized, legacy)); - } -} - -fn mixesLegacy(x: []const u16, hc: Hc, c: Config) math.Mix { - @setFloatMode(.strict); - var projected: [24]f32 = undefined; - math.denseOf(f32, hc.projection, x, &projected); - var squares: f64 = 0; - for (x) |v| squares += @as(f64, math.f(v)) * math.f(v); - const mean = math.meanFromWide(squares, x.len); - const r = math.idealRsqrt(mean + c.text_config.rms_norm_eps); - var m: math.Mix = undefined; - for (0..4) |i| { - m.pre[i] = math.sigmoid(@mulAdd(f32, projected[i] * r, math.value(hc.scale, 0), math.value(hc.base, i))) + c.text_config.hc_eps; - m.post[i] = 2 * math.sigmoid(@mulAdd(f32, projected[i + 4] * r, math.value(hc.scale, 1), math.value(hc.base, i + 4))); - } - for (0..4) |i| { - var high: f32 = -std.math.inf(f32); - for (0..4) |j| { - const at = i * 4 + j; - m.comb[at] = @mulAdd(f32, projected[at + 8] * r, math.value(hc.scale, 2), math.value(hc.base, at + 8)); - high = @max(high, m.comb[at]); - } - for (0..4) |j| { - m.comb[i * 4 + j] = @exp(m.comb[i * 4 + j] - high); - } - const denominator = (m.comb[i * 4] + m.comb[i * 4 + 2]) + - (m.comb[i * 4 + 1] + m.comb[i * 4 + 3]); - for (0..4) |j| m.comb[i * 4 + j] = m.comb[i * 4 + j] / denominator + c.text_config.hc_eps; - } - for (0..c.text_config.hc_sinkhorn_iters) |iteration| { - if (iteration != 0) for (0..4) |i| { - const denominator = ((m.comb[i * 4] + m.comb[i * 4 + 2]) + - (m.comb[i * 4 + 1] + m.comb[i * 4 + 3])) + c.text_config.hc_eps; - for (0..4) |j| m.comb[i * 4 + j] /= denominator; - }; - for (0..4) |j| { - const denominator = ((m.comb[j] + m.comb[8 + j]) + - (m.comb[4 + j] + m.comb[12 + j])) + c.text_config.hc_eps; - for (0..4) |i| m.comb[i * 4 + j] /= denominator; - } - } - return m; -} - -// Direct split-stage isolation: identical projected mixes and inverse on both sides, so every -// delta is a unary (sigmoid/exp) or reduction boundary, not a projection difference. -test "mHC split unary boundaries against the FP64 ideal on retained arguments" { - const device = mtl.Device.init() catch return error.SkipZigTest; - defer device.deinit(); - const pool = mtl.objc.Pool.push(); - defer pool.pop(); - const queue = try device.queue(); - defer queue.deinit(); - var p = try mhc.Pipelines.load(device); - defer p.deinit(); - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const gpa = arena.allocator(); - const cfg = try Config.parse(gpa, released_config); - - const rows = 7 + 65; // retained arguments plus every integer exponent input -32..32 - var b = Buffers{ .device = device }; - defer b.deinit(); - const mixes_host = try gpa.alloc(f32, rows * mix_count); - @memset(mixes_host, 0); - // Rows 0..3: the 16 retained unary arguments rotated through pre, post and comb positions. - for (0..4) |i| { - for (0..4) |j| { - mixes_host[i * mix_count + j] = @bitCast(retained_exp_argument_bits[(4 * i + j) % 16]); - mixes_host[i * mix_count + 4 + j] = @bitCast(retained_exp_argument_bits[(4 * i + j) % 16]); - } - for (0..16) |q| mixes_host[i * mix_count + 8 + q] = @bitCast(retained_exp_argument_bits[(i + q) % 16]); - } - // Row 4: the documented controlling exponent input as a post argument (exp(+1.439...) path). - mixes_host[4 * mix_count + 5] = -controlling_exponent_input; - // Row 5: comb affine arguments spanning the exponent range. - mixes_host[5 * mix_count + 8] = -32; - mixes_host[5 * mix_count + 9] = -16; - mixes_host[5 * mix_count + 10] = 16; - mixes_host[5 * mix_count + 11] = 32; - // Row 6: the same span through the sigmoid pre/post arguments. - const span = [_]f32{ -32, -16, 16, 32 }; - for (span, 0..) |v, j| { - mixes_host[6 * mix_count + j] = v; - mixes_host[6 * mix_count + 4 + j] = v; - } - - // Rows 7..69: one integer exponent input per row at a pre, post and comb position each. - for (0..65) |i| { - const r = 7 + i; - const v: f32 = @floatFromInt(@as(i32, @intCast(i)) - 32); - mixes_host[r * mix_count] = v; - mixes_host[r * mix_count + 4] = v; - mixes_host[r * mix_count + 8] = v; - } - - const mixes = try b.get(f32, rows * mix_count); - upload(f32, mixes, mixes_host); - const inv = try b.get(f32, rows); - @memset(inv.slice(f32, rows), 1.0); - const scale = try b.get(f32, 3); - @memset(scale.slice(f32, 3), 1.0); - const base = try b.get(f32, mix_count); - const pre = try b.get(f32, rows * 4); - const post = try b.get(f32, rows * 4); - const comb = try b.get(f32, rows * 16); - - const cb = queue.commandBuffer(); - const e = cb.compute(.serial); - mhc.encodeSplit(e, &p, view(mixes), view(inv), view(scale), view(base), view(pre), view(post), view(comb), .{ .rows = rows, .eps = cfg.text_config.hc_eps }); - e.end(); - cb.commit(); - cb.wait(); - if (cb.failure() != null) return error.MhcFp64GpuFailure; - - const scale_tensor: Tensor = .{ .buffer = scale, .offset = 0, .bytes = 12, .dtype = .f32 }; - const base_tensor: Tensor = .{ .buffer = base, .offset = 0, .bytes = mix_count * 4, .dtype = .f32 }; - const hc = Hc{ .projection = base_tensor, .scale = scale_tensor, .base = base_tensor }; - - var max_ulp_metal: f64 = 0; - var max_ulp_f32: f64 = 0; - var worst_name: []const u8 = ""; - const stages = [_]struct { name: []const u8, offset: usize, len: usize }{ - .{ .name = "pre", .offset = 0, .len = 4 }, - .{ .name = "post", .offset = 4, .len = 4 }, - .{ .name = "comb", .offset = 8, .len = 16 }, - }; - for (stages) |stage| { - for (0..rows) |r| { - const m: []const f32 = mixes_host[r * mix_count ..][0..mix_count]; - var m64: [mix_count]f64 = undefined; - for (m, 0..) |v, i| m64[i] = v; - const mix64 = math.splitOf(f64, &m64, 1.0, hc, cfg.text_config); - const mix32 = math.splitOf(f32, m, 1.0, hc, cfg.text_config); - const src64: []const f64 = if (std.mem.eql(u8, stage.name, "pre")) &mix64.pre else if (std.mem.eql(u8, stage.name, "post")) &mix64.post else &mix64.comb; - const src32: []const f32 = if (std.mem.eql(u8, stage.name, "pre")) &mix32.pre else if (std.mem.eql(u8, stage.name, "post")) &mix32.post else &mix32.comb; - const metal: []const f32 = if (std.mem.eql(u8, stage.name, "pre")) pre.slice(f32, rows * 4)[r * 4 ..][0..4] else if (std.mem.eql(u8, stage.name, "post")) post.slice(f32, rows * 4)[r * 4 ..][0..4] else comb.slice(f32, rows * 16)[r * 16 ..][0..16]; - const d = deltaF32VsF64(metal[0..stage.len], src64[0..stage.len]); - const d32 = deltaF32VsF64(src32[0..stage.len], src64[0..stage.len]); - if (d.max_ulp > max_ulp_metal) { - max_ulp_metal = d.max_ulp; - worst_name = stage.name; - } - if (d32.max_ulp > max_ulp_f32) max_ulp_f32 = d32.max_ulp; - std.debug.print("HC64 {{\"case\":\"split_args\",\"stage\":\"{s}\",\"row\":{d},\"nrms_metal_vs_f64\":{d},\"max_ulp_metal_vs_f64\":{d:.4},\"max_ulp_f32_vs_f64\":{d:.4}}}\n", .{ stage.name, r, d.nrms, d.max_ulp, d32.max_ulp }); - } - } - std.debug.print("HC64 {{\"case\":\"split_args\",\"summary\":true,\"max_ulp_metal_vs_f64\":{d:.4},\"worst_stage\":\"{s}\",\"max_ulp_f32_vs_f64\":{d:.4}}}\n", .{ max_ulp_metal, worst_name, max_ulp_f32 }); - // The kernel may exceed the FP32 reference by at most the documented Metal exp slack (1 ULP). - // Absolute ULPs inflate on near-cancelled Sinkhorn cells; NRMS stays below 1e-7 everywhere. - try std.testing.expect(max_ulp_metal <= max_ulp_f32 + 2.0); - try std.testing.expect(max_ulp_metal < 48.0); -} - -// The retained cancellation fixtures: the kernel must keep the source FP32 accumulation -// contract exactly; the FP64 ideal column documents the by-design half-ULP divergence. -test "mHC collapse and post keep source FP32 accumulation under cancellation" { - const device = mtl.Device.init() catch return error.SkipZigTest; - defer device.deinit(); - const pool = mtl.objc.Pool.push(); - defer pool.pop(); - const queue = try device.queue(); - defer queue.deinit(); - var p = try mhc.Pipelines.load(device); - defer p.deinit(); - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const gpa = arena.allocator(); - const cfg = try Config.parse(gpa, released_config); - - var b = Buffers{ .device = device }; - defer b.deinit(); - // Collapse fixture: stream {3,3,0,0} (width 1), previous_pre {-1, 1+2^-23, 0, 0}. - const stream_a = try b.get(u16, 4); - const values_a = [_]u16{ math.b(3), math.b(3), math.b(0), math.b(0) }; - upload(u16, stream_a, &values_a); - const previous_a = try b.get(f32, 4); - const coeff = 1 + 0x1p-23; - const previous_values = [_]f32{ -1, coeff, 0, 0 }; - upload(f32, previous_a, &previous_values); - const collapsed = try b.get(u16, 1); - - // Post fixture: stream {1,1,1,1}, branch {1}, post {-1,0,0,0}, - // comb column 0 = {1, 2^-24, 2^-24, 0} (two half-ULP terms after 1.0). - const stream_b = try b.get(u16, 4); - const values_b = [_]u16{ math.b(1), math.b(1), math.b(1), math.b(1) }; - upload(u16, stream_b, &values_b); - const branch_b = try b.get(u16, 1); - const branch_values = [_]u16{math.b(1)}; - upload(u16, branch_b, &branch_values); - const post_b = try b.get(f32, 4); - const post_values = [_]f32{ -1, 0, 0, 0 }; - upload(f32, post_b, &post_values); - const comb_b = try b.get(f32, 16); - comb_b.slice(f32, 16)[0] = 1; - comb_b.slice(f32, 16)[4] = 0x1p-24; - comb_b.slice(f32, 16)[8] = 0x1p-24; - const y_b = try b.get(u16, 4); - - // Branch-product fixture: stream {3,0,0,0}, post {1+2^-23,...}, comb[0] = -1, branch {3}. - const stream_c = try b.get(u16, 4); - const values_c = [_]u16{ math.b(3), math.b(0), math.b(0), math.b(0) }; - upload(u16, stream_c, &values_c); - const branch_c = try b.get(u16, 1); - const branch_values_c = [_]u16{math.b(3)}; - upload(u16, branch_c, &branch_values_c); - const post_c = try b.get(f32, 4); - const post_values_c = [_]f32{ coeff, 0, 0, 0 }; - upload(f32, post_c, &post_values_c); - const comb_c = try b.get(f32, 16); - comb_c.slice(f32, 16)[0] = -1; - const y_c = try b.get(u16, 4); - - const cb = queue.commandBuffer(); - const e = cb.compute(.serial); - mhc.encodeCollapse(e, &p, view(stream_a), view(previous_a), view(collapsed), .{ .rows = 1, .width = 1, .stream_stride = 4, .y_stride = 1 }); - mhc.encodePost(e, &p, view(branch_b), view(stream_b), view(post_b), view(comb_b), view(y_b), .{ .rows = 1, .width = 1, .branch_stride = 1, .residual_stride = 4, .y_stride = 4 }); - mhc.encodePost(e, &p, view(branch_c), view(stream_c), view(post_c), view(comb_c), view(y_c), .{ .rows = 1, .width = 1, .branch_stride = 1, .residual_stride = 4, .y_stride = 4 }); - e.end(); - cb.commit(); - cb.wait(); - if (cb.failure() != null) return error.MhcFp64GpuFailure; - - // Source-contract expectations (math.zig retained boundaries). - try std.testing.expectEqual(@as(u16, 0x3500), collapsed.slice(u16, 1)[0]); - try std.testing.expectEqual(@as(u16, 0), y_b.slice(u16, 4)[0]); - try std.testing.expectEqual(@as(u16, 0x3500), y_c.slice(u16, 4)[0]); - - // Reference columns for the same fixtures. - var collapsed32: [1]u16 = undefined; - var collapsed64: [1]u16 = undefined; - math.collapseOf(f32, &values_a, .{ -1, coeff, 0, 0 }, &collapsed32); - math.collapseOf(f64, &values_a, .{ -1, @as(f64, coeff), 0, 0 }, &collapsed64); - std.debug.print("HC64 {{\"case\":\"cancellation\",\"fixture\":\"collapse_half_ulps\",\"metal\":\"0x{x}\",\"f32_ref\":\"0x{x}\",\"f64_ideal\":\"0x{x}\"}}\n", .{ collapsed.slice(u16, 1)[0], collapsed32[0], collapsed64[0] }); - try std.testing.expectEqual(collapsed32[0], collapsed.slice(u16, 1)[0]); - - const mix_b = math.MixOf(f32){ .pre = .{ 1, 0, 0, 0 }, .post = .{ -1, 0, 0, 0 }, .comb = .{ 1, 0, 0, 0, 0x1p-24, 0, 0, 0, 0x1p-24, 0, 0, 0, 0, 0, 0, 0 } }; - var y32: [4]u16 = undefined; - var h32 = values_b; - math.expandOf(f32, &h32, &branch_values, mix_b); - @memcpy(&y32, &h32); - const mix_b64 = math.MixOf(f64){ .pre = .{ 1, 0, 0, 0 }, .post = .{ -1, 0, 0, 0 }, .comb = .{ 1, 0, 0, 0, 0x1p-24, 0, 0, 0, 0x1p-24, 0, 0, 0, 0, 0, 0, 0 } }; - var h64 = values_b; - math.expandOf(f64, &h64, &branch_values, mix_b64); - std.debug.print("HC64 {{\"case\":\"cancellation\",\"fixture\":\"post_half_ulps\",\"metal\":\"0x{x}\",\"f32_ref\":\"0x{x}\",\"f64_ideal\":\"0x{x}\"}}\n", .{ y_b.slice(u16, 4)[0], y32[0], h64[0] }); - try std.testing.expectEqual(y32[0], y_b.slice(u16, 4)[0]); - - const mix_c = math.MixOf(f32){ .pre = .{ 1, 0, 0, 0 }, .post = .{ coeff, 0, 0, 0 }, .comb = .{ -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; - var hc32 = values_c; - math.expandOf(f32, &hc32, &branch_values_c, mix_c); - const mix_c64 = math.MixOf(f64){ .pre = .{ 1, 0, 0, 0 }, .post = .{ @as(f64, coeff), 0, 0, 0 }, .comb = .{ -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; - var hc64 = values_c; - math.expandOf(f64, &hc64, &branch_values_c, mix_c64); - std.debug.print("HC64 {{\"case\":\"cancellation\",\"fixture\":\"branch_product\",\"metal\":\"0x{x}\",\"f32_ref\":\"0x{x}\",\"f64_ideal\":\"0x{x}\"}}\n", .{ y_c.slice(u16, 4)[0], hc32[0], hc64[0] }); - try std.testing.expectEqual(hc32[0], y_c.slice(u16, 4)[0]); - _ = cfg; -} - -// Ideal FP64 stage propagation used by the amplification measurement: statistic, dense, split, -// collapse and post-expansion, with the pre-rounding FP64 outputs kept for analog-gain figures. -fn stageF64(stream: []const u16, hc: Hc, cfg: Config, branch: []const u16, y_pre: []f64, y_bf: *[stream_width]u16, coll_pre: []f64, inv_out: *f64, comb_out: *[16]f64) void { - var proj: [mix_count]f64 = undefined; - math.denseOf(f64, hc.projection, stream, &proj); - const inv = math.inverseOf(f64, stream, cfg.text_config.rms_norm_eps); - const mix = math.splitOf(f64, &proj, inv, hc, cfg.text_config); - inv_out.* = inv; - comb_out.* = mix.comb; - for (0..width) |d| { - var s: f64 = 0; - for (0..copies) |c| s += mix.pre[c] * @as(f64, math.f(stream[c * width + d])); - coll_pre[d] = s; - } - for (0..width) |d| { - for (0..copies) |k| { - var acc: f64 = mix.post[k] * @as(f64, math.f(branch[d])); - for (0..copies) |j| acc += mix.comb[j * 4 + k] * @as(f64, math.f(stream[j * width + d])); - y_pre[k * width + d] = acc; - } - } - var h: [stream_width]u16 = undefined; - @memcpy(&h, stream); - math.expandOf(f64, &h, branch, mix); - @memcpy(y_bf, &h); -} - -fn perturbOneUlp(x: u16) u16 { - return if ((x & 0x8000) == 0) x + 1 else x - 1; -} - -test "mHC 1-ULP input amplification through the hyperconnection mix" { - const device = mtl.Device.init() catch return error.SkipZigTest; - defer device.deinit(); - const pool = mtl.objc.Pool.push(); - defer pool.pop(); - const queue = try device.queue(); - defer queue.deinit(); - var p = try mhc.Pipelines.load(device); - defer p.deinit(); - var o = try ops.Pipelines.load(device); - defer o.deinit(); - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const gpa = arena.allocator(); - const cfg = try Config.parse(gpa, released_config); - - const fixture = try std.json.parseFromSlice(Fixture, gpa, @embedFile(fixture_path), .{ .ignore_unknown_fields = true }); - const eight_rows = try decodeU16(gpa, fixture.value.eight_rows_bf16); - const ffn_stream = try decodeU16(gpa, fixture.value.ffn_stream_bf16); - const ffn_branch = try decodeU16(gpa, fixture.value.ffn_branch_bf16); - const ffn_scale = try decodeF32(gpa, fixture.value.ffn_scale_f32); - const ffn_base = try decodeF32(gpa, fixture.value.ffn_base_f32); - const weight = try syntheticWeight(gpa); - - const scale = try device.buffer(12, mtl.ResourceOptions.shared); - defer scale.deinit(); - upload(f32, scale, ffn_scale); - const base = try device.buffer(mix_count * 4, mtl.ResourceOptions.shared); - defer base.deinit(); - upload(f32, base, ffn_base); - const weight_buffer = try device.buffer(mix_count * stream_width * 4, mtl.ResourceOptions.shared); - defer weight_buffer.deinit(); - upload(f32, weight_buffer, weight); - const hc = Hc{ - .projection = .{ .buffer = weight_buffer, .offset = 0, .bytes = mix_count * stream_width * 4, .dtype = .f32 }, - .scale = .{ .buffer = scale, .offset = 0, .bytes = 12, .dtype = .f32 }, - .base = .{ .buffer = base, .offset = 0, .bytes = mix_count * 4, .dtype = .f32 }, - }; - - const columns = 64; - const column_stride = width / columns; - const Row = struct { name: []const u8, stream: []u16 }; - var rows: [9]Row = undefined; - for (0..8) |r| { - rows[r] = .{ .name = "teacher", .stream = try fourCopies(gpa, eight_rows[r * width ..][0..width]) }; - } - rows[8] = .{ .name = "ffn_token1", .stream = try gpa.dupe(u16, ffn_stream) }; - - const y0 = try gpa.alloc(f64, stream_width); - const y1 = try gpa.alloc(f64, stream_width); - var y0_bf: [stream_width]u16 = undefined; - var y1_bf: [stream_width]u16 = undefined; - const coll0 = try gpa.alloc(f64, width); - const coll1 = try gpa.alloc(f64, width); - var inv0: f64 = undefined; - var inv1: f64 = undefined; - var comb0: [16]f64 = undefined; - var comb1: [16]f64 = undefined; - - for (rows, 0..) |row, row_index| { - stageF64(row.stream, hc, cfg, ffn_branch, y0, &y0_bf, coll0, &inv0, &comb0); - var sum_y2: f64 = 0; - for (y0) |v| sum_y2 += v * v; - var sum_x2: f64 = 0; - for (row.stream) |v| sum_x2 += blk: { - const f: f64 = @floatCast(math.f(v)); - break :blk f * f; - }; - var sum_c2: f64 = 0; - for (coll0) |v| sum_c2 += v * v; - var sum_b2: f64 = 0; - for (y0_bf) |v| { - const f: f64 = @floatCast(math.f(v)); - sum_b2 += f * f; - } - - var amp_sum: f64 = 0; - var amp_max: f64 = 0; - var amp_n: usize = 0; - var amp_c_sum: f64 = 0; - var amp_c_max: f64 = 0; - var amp_b_sum: f64 = 0; - var amp_b_max: f64 = 0; - var changed_any: usize = 0; - var changed_cells: usize = 0; - var inv_rel_sum: f64 = 0; - var comb_sq: f64 = 0; - var comb_n: usize = 0; - - for (0..copies) |j| { - for (0..columns) |k| { - const col = k * column_stride; - const idx = j * width + col; - const orig = row.stream[idx]; - if (math.f(orig) == 0) continue; - const perturbed = perturbOneUlp(orig); - const dx = @as(f64, @floatCast(math.f(perturbed))) - @as(f64, @floatCast(math.f(orig))); - row.stream[idx] = perturbed; - stageF64(row.stream, hc, cfg, ffn_branch, y1, &y1_bf, coll1, &inv1, &comb1); - row.stream[idx] = orig; - - var dy2: f64 = 0; - for (y0, y1) |a, b| { - const d = a - b; - dy2 += d * d; - } - const amp = @sqrt(dy2 / sum_y2) * @sqrt(sum_x2) / @abs(dx); - amp_sum += amp; - if (amp > amp_max) amp_max = amp; - amp_n += 1; - - var dc2: f64 = 0; - for (coll0, coll1) |a, b| { - const d = a - b; - dc2 += d * d; - } - const amp_c = @sqrt(dc2 / sum_c2) * @sqrt(sum_x2) / @abs(dx); - amp_c_sum += amp_c; - if (amp_c > amp_c_max) amp_c_max = amp_c; - - var changed: usize = 0; - var dyb2: f64 = 0; - for (y0_bf, y1_bf) |a, b| { - if (a != b) changed += 1; - const av: f64 = @floatCast(math.f(a)); - const bv: f64 = @floatCast(math.f(b)); - const d = av - bv; - dyb2 += d * d; - } - const amp_b = @sqrt(dyb2 / sum_b2) * @sqrt(sum_x2) / @abs(dx); - amp_b_sum += amp_b; - if (amp_b > amp_b_max) amp_b_max = amp_b; - if (changed > 0) { - changed_any += 1; - changed_cells += changed; - } - inv_rel_sum += @abs(inv1 - inv0) / inv0; - var comb_d2: f64 = 0; - var comb_m2: f64 = 0; - for (comb0, comb1) |a, b| { - const d = a - b; - comb_d2 += d * d; - comb_m2 += b * b; - } - comb_sq += @sqrt(comb_d2 / comb_m2); - comb_n += 1; - } - } - std.debug.print("HC64 {{\"amplification\":{{\"row\":{d},\"row_name\":\"{s}\",\"perturbations\":{d},\"amp_mean\":{d},\"amp_max\":{d},\"amp_collapse_mean\":{d},\"amp_collapse_max\":{d},\"amp_bf16_mean\":{d},\"amp_bf16_max\":{d},\"bf16_changed_fraction\":{d},\"mean_changed_cells\":{d},\"inv_rel_mean\":{d},\"comb_nrms_mean\":{d}}}}}\n", .{ row_index, row.name, amp_n, amp_sum / @as(f64, @floatFromInt(amp_n)), amp_max, amp_c_sum / @as(f64, @floatFromInt(amp_n)), amp_c_max, amp_b_sum / @as(f64, @floatFromInt(amp_n)), amp_b_max, changed_any, @as(f64, @floatFromInt(changed_cells)) / @as(f64, @floatFromInt(amp_n)), inv_rel_sum / @as(f64, @floatFromInt(amp_n)), comb_sq / @as(f64, @floatFromInt(comb_n)) }); - // The mix attenuates a single-ULP input in FP64 (analog gain < 1) and stays near one - // BF16-visible ULP of growth; the collapse gain is pinned near one half. - try std.testing.expect(amp_sum / @as(f64, @floatFromInt(amp_n)) < 1.0); - try std.testing.expect(amp_max < 1.0); - try std.testing.expect(amp_c_max < 2.5); - try std.testing.expect(amp_b_sum / @as(f64, @floatFromInt(amp_n)) < 3.0); - try std.testing.expect(inv_rel_sum / @as(f64, @floatFromInt(amp_n)) < 1e-5); - } - - // Metal confirmation on the first teacher row: batch the base row plus 16 stream-cell - // perturbations through the real kernels and compare against the FP64 analog gain. - { - const metal_perturbations = 16; - const batch = metal_perturbations + 1; - var batch_stream = try gpa.alloc(u16, batch * stream_width); - var cells: [metal_perturbations]usize = undefined; - var dxs: [metal_perturbations]f64 = undefined; - var filled: usize = 0; - @memcpy(batch_stream[0..stream_width], rows[0].stream); - outer: for (0..copies) |j| { - for (0..columns) |k| { - const idx = j * width + k * column_stride; - const orig = rows[0].stream[idx]; - if (math.f(orig) == 0) continue; - cells[filled] = idx; - dxs[filled] = @as(f64, @floatCast(math.f(perturbOneUlp(orig)))) - @as(f64, @floatCast(math.f(orig))); - filled += 1; - if (filled == metal_perturbations) break :outer; - } - } - for (0..filled) |i| { - @memcpy(batch_stream[(i + 1) * stream_width ..][0..stream_width], rows[0].stream); - batch_stream[(i + 1) * stream_width + cells[i]] = perturbOneUlp(rows[0].stream[cells[i]]); - } - const stream_buf = try device.buffer(batch * stream_width * 2, mtl.ResourceOptions.shared); - defer stream_buf.deinit(); - upload(u16, stream_buf, batch_stream); - const inv_buf = try device.buffer(batch * 4, mtl.ResourceOptions.shared); - defer inv_buf.deinit(); - const mixes_buf = try device.buffer(batch * mix_count * 4, mtl.ResourceOptions.shared); - defer mixes_buf.deinit(); - const pre_buf = try device.buffer(batch * 4 * 4, mtl.ResourceOptions.shared); - defer pre_buf.deinit(); - const post_buf = try device.buffer(batch * 4 * 4, mtl.ResourceOptions.shared); - defer post_buf.deinit(); - const comb_buf = try device.buffer(batch * 16 * 4, mtl.ResourceOptions.shared); - defer comb_buf.deinit(); - const collapsed_buf = try device.buffer(batch * width * 2, mtl.ResourceOptions.shared); - defer collapsed_buf.deinit(); - const branch_buf = try device.buffer(batch * width * 2, mtl.ResourceOptions.shared); - defer branch_buf.deinit(); - for (0..batch) |r| @memcpy(branch_buf.slice(u16, batch * width)[r * width ..][0..width], ffn_branch); - const y_buf = try device.buffer(batch * stream_width * 2, mtl.ResourceOptions.shared); - defer y_buf.deinit(); - try runStage(queue, &p, &o, @intCast(batch), cfg, view(stream_buf), view(weight_buffer), view(inv_buf), view(mixes_buf), view(scale), view(base), view(pre_buf), view(post_buf), view(comb_buf), view(collapsed_buf), view(branch_buf), view(y_buf)); - - const y_metal = y_buf.slice(u16, batch * stream_width); - var nan_count: usize = 0; - var first_nan: usize = 0; - for (y_metal[0..stream_width], 0..) |v, i| { - const f = math.f(v); - if (std.math.isNan(f)) { - nan_count += 1; - if (nan_count == 1) first_nan = i; - } - } - var sum_x2_row0: f64 = 0; - for (rows[0].stream) |v| { - const f: f64 = @floatCast(math.f(v)); - sum_x2_row0 += f * f; - } - var sum_base2: f64 = 0; - for (0..stream_width) |i| { - const f: f64 = @floatCast(math.f(y_metal[i])); - sum_base2 += f * f; - } - var amp_sum: f64 = 0; - var changed_total: usize = 0; - for (0..filled) |i| { - var dy2: f64 = 0; - var changed: usize = 0; - for (0..stream_width) |c| { - const a: f64 = @floatCast(math.f(y_metal[(i + 1) * stream_width + c])); - const b: f64 = @floatCast(math.f(y_metal[c])); - const d = a - b; - if (d != 0) changed += 1; - dy2 += d * d; - } - changed_total += changed; - amp_sum += @sqrt(dy2 / sum_base2) * @sqrt(sum_x2_row0) / @abs(dxs[i]); - } - std.debug.print("HC64 {{\"amplification\":{{\"row\":0,\"metal\":true,\"perturbations\":{d},\"amp_mean\":{d},\"mean_changed_cells\":{d}}}}}\n", .{ filled, amp_sum / @as(f64, @floatFromInt(filled)), @as(f64, @floatFromInt(changed_total)) / @as(f64, @floatFromInt(filled)) }); - const metal_amp = amp_sum / @as(f64, @floatFromInt(filled)); - try std.testing.expect(metal_amp > 0.5 and metal_amp < 3.0); - } -} diff --git a/zig/tests/deepseek_v41_reference/operator_hc_split_fp64.zig b/zig/tests/deepseek_v41_reference/operator_hc_split_fp64.zig new file mode 100644 index 0000000000..6739259e8b --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_hc_split_fp64.zig @@ -0,0 +1,303 @@ +//! mHC split and collapse FP64 ideal-reference tests (DeepSeek-V4.1-Flash inference, MIT). +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const math = @import("math.zig"); +const h = @import("operator_hc_fp64.zig"); +const mhc = tf.deepseek_v41.mhc; +const Tensor = tf.checkpoint_metal.Tensor; +const Hc = tf.deepseek_v41_weights.Hc; +const Config = tf.deepseek_v41_config.Config; + +// The comptime parameterization must keep the FP32 source contract bitwise against the old algorithm. +test "mHC fp32 reference path stays bit-identical under the comptime parameterization" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const gpa = arena.allocator(); + const cfg = try Config.parse(gpa, h.released_config); + const fixture = try std.json.parseFromSlice(h.Fixture, gpa, @embedFile(h.fixture_path), .{ .ignore_unknown_fields = true }); + const eight_rows = try h.decodeU16(gpa, fixture.value.eight_rows_bf16); + const ffn_scale = try h.decodeF32(gpa, fixture.value.ffn_scale_f32); + const ffn_base = try h.decodeF32(gpa, fixture.value.ffn_base_f32); + const scale = try device.buffer(12, mtl.ResourceOptions.shared); + defer scale.deinit(); + h.upload(f32, scale, ffn_scale); + const base = try device.buffer(h.mix_count * 4, mtl.ResourceOptions.shared); + defer base.deinit(); + h.upload(f32, base, ffn_base); + var lcg = h.Lcg{}; + const weight = try device.buffer(h.mix_count * h.stream_width * 4, mtl.ResourceOptions.shared); + defer weight.deinit(); + const weight_host = try gpa.alloc(f32, h.mix_count * h.stream_width); + for (weight_host) |*v| v.* = lcg.uniform() * 0.03; + h.upload(f32, weight, weight_host); + const hc = Hc{ + .projection = .{ .buffer = weight, .offset = 0, .bytes = h.mix_count * h.stream_width * 4, .dtype = .f32 }, + .scale = .{ .buffer = scale, .offset = 0, .bytes = 12, .dtype = .f32 }, + .base = .{ .buffer = base, .offset = 0, .bytes = h.mix_count * 4, .dtype = .f32 }, + }; + for (0..8) |r| { + const row = eight_rows[r * h.width ..][0..h.width]; + var stream: [h.stream_width]u16 = undefined; + for (0..h.copies) |c| @memcpy(stream[c * h.width ..][0..h.width], row); + const parameterized = math.mixesOf(f32, &stream, hc, cfg.text_config); + const legacy = mixesLegacy(&stream, hc, cfg); + try std.testing.expect(std.meta.eql(parameterized, legacy)); + } +} + +fn mixesLegacy(x: []const u16, hc: Hc, c: Config) math.Mix { + @setFloatMode(.strict); + var projected: [24]f32 = undefined; + math.denseOf(f32, hc.projection, x, &projected); + var squares: f64 = 0; + for (x) |v| squares += @as(f64, math.f(v)) * math.f(v); + const mean = math.meanFromWide(squares, x.len); + const r = math.idealRsqrt(mean + c.text_config.rms_norm_eps); + var m: math.Mix = undefined; + for (0..4) |i| { + m.pre[i] = math.sigmoid(@mulAdd(f32, projected[i] * r, math.value(hc.scale, 0), math.value(hc.base, i))) + c.text_config.hc_eps; + m.post[i] = 2 * math.sigmoid(@mulAdd(f32, projected[i + 4] * r, math.value(hc.scale, 1), math.value(hc.base, i + 4))); + } + for (0..4) |i| { + var high: f32 = -std.math.inf(f32); + for (0..4) |j| { + const at = i * 4 + j; + m.comb[at] = @mulAdd(f32, projected[at + 8] * r, math.value(hc.scale, 2), math.value(hc.base, at + 8)); + high = @max(high, m.comb[at]); + } + for (0..4) |j| { + m.comb[i * 4 + j] = @exp(m.comb[i * 4 + j] - high); + } + const denominator = (m.comb[i * 4] + m.comb[i * 4 + 2]) + + (m.comb[i * 4 + 1] + m.comb[i * 4 + 3]); + for (0..4) |j| m.comb[i * 4 + j] = m.comb[i * 4 + j] / denominator + c.text_config.hc_eps; + } + for (0..c.text_config.hc_sinkhorn_iters) |iteration| { + if (iteration != 0) for (0..4) |i| { + const denominator = ((m.comb[i * 4] + m.comb[i * 4 + 2]) + + (m.comb[i * 4 + 1] + m.comb[i * 4 + 3])) + c.text_config.hc_eps; + for (0..4) |j| m.comb[i * 4 + j] /= denominator; + }; + for (0..4) |j| { + const denominator = ((m.comb[j] + m.comb[8 + j]) + + (m.comb[4 + j] + m.comb[12 + j])) + c.text_config.hc_eps; + for (0..4) |i| m.comb[i * 4 + j] /= denominator; + } + } + return m; +} + +// Split-stage isolation: identical projected mixes and inverse, so only unary or reduction boundaries differ. +test "mHC split unary boundaries against the FP64 ideal on retained arguments" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try mhc.Pipelines.load(device); + defer p.deinit(); + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const gpa = arena.allocator(); + const cfg = try Config.parse(gpa, h.released_config); + + const rows = 7 + 65; // retained arguments plus every integer exponent input -32..32 + var b = h.Buffers{ .device = device }; + defer b.deinit(); + const mixes_host = try gpa.alloc(f32, rows * h.mix_count); + @memset(mixes_host, 0); + // Rows 0..3: the 16 retained unary arguments rotated through pre, post and comb positions. + for (0..4) |i| { + for (0..4) |j| { + mixes_host[i * h.mix_count + j] = @bitCast(h.retained_exp_argument_bits[(4 * i + j) % 16]); + mixes_host[i * h.mix_count + 4 + j] = @bitCast(h.retained_exp_argument_bits[(4 * i + j) % 16]); + } + for (0..16) |q| mixes_host[i * h.mix_count + 8 + q] = @bitCast(h.retained_exp_argument_bits[(i + q) % 16]); + } + // Row 4: the documented controlling exponent input as a post argument (exp(+1.439...) path). + mixes_host[4 * h.mix_count + 5] = -h.controlling_exponent_input; + // Row 5: comb affine arguments spanning the exponent range. + mixes_host[5 * h.mix_count + 8] = -32; + mixes_host[5 * h.mix_count + 9] = -16; + mixes_host[5 * h.mix_count + 10] = 16; + mixes_host[5 * h.mix_count + 11] = 32; + // Row 6: the same span through the sigmoid pre/post arguments. + const span = [_]f32{ -32, -16, 16, 32 }; + for (span, 0..) |v, j| { + mixes_host[6 * h.mix_count + j] = v; + mixes_host[6 * h.mix_count + 4 + j] = v; + } + + // Rows 7..69: one integer exponent input per row at a pre, post and comb position each. + for (0..65) |i| { + const r = 7 + i; + const v: f32 = @floatFromInt(@as(i32, @intCast(i)) - 32); + mixes_host[r * h.mix_count] = v; + mixes_host[r * h.mix_count + 4] = v; + mixes_host[r * h.mix_count + 8] = v; + } + + const mixes = try b.get(f32, rows * h.mix_count); + h.upload(f32, mixes, mixes_host); + const inv = try b.get(f32, rows); + @memset(inv.slice(f32, rows), 1.0); + const scale = try b.get(f32, 3); + @memset(scale.slice(f32, 3), 1.0); + const base = try b.get(f32, h.mix_count); + const pre = try b.get(f32, rows * 4); + const post = try b.get(f32, rows * 4); + const comb = try b.get(f32, rows * 16); + + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + mhc.encodeSplit(e, &p, h.view(mixes), h.view(inv), h.view(scale), h.view(base), h.view(pre), h.view(post), h.view(comb), .{ .rows = rows, .eps = cfg.text_config.hc_eps }); + e.end(); + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.MhcFp64GpuFailure; + + const scale_tensor: Tensor = .{ .buffer = scale, .offset = 0, .bytes = 12, .dtype = .f32 }; + const base_tensor: Tensor = .{ .buffer = base, .offset = 0, .bytes = h.mix_count * 4, .dtype = .f32 }; + const hc = Hc{ .projection = base_tensor, .scale = scale_tensor, .base = base_tensor }; + + var max_ulp_metal: f64 = 0; + var max_ulp_f32: f64 = 0; + var worst_name: []const u8 = ""; + const stages = [_]struct { name: []const u8, offset: usize, len: usize }{ + .{ .name = "pre", .offset = 0, .len = 4 }, + .{ .name = "post", .offset = 4, .len = 4 }, + .{ .name = "comb", .offset = 8, .len = 16 }, + }; + for (stages) |stage| { + for (0..rows) |r| { + const m: []const f32 = mixes_host[r * h.mix_count ..][0..h.mix_count]; + var m64: [h.mix_count]f64 = undefined; + for (m, 0..) |v, i| m64[i] = v; + const mix64 = math.splitOf(f64, &m64, 1.0, hc, cfg.text_config); + const mix32 = math.splitOf(f32, m, 1.0, hc, cfg.text_config); + const src64: []const f64 = if (std.mem.eql(u8, stage.name, "pre")) &mix64.pre else if (std.mem.eql(u8, stage.name, "post")) &mix64.post else &mix64.comb; + const src32: []const f32 = if (std.mem.eql(u8, stage.name, "pre")) &mix32.pre else if (std.mem.eql(u8, stage.name, "post")) &mix32.post else &mix32.comb; + const metal: []const f32 = if (std.mem.eql(u8, stage.name, "pre")) pre.slice(f32, rows * 4)[r * 4 ..][0..4] else if (std.mem.eql(u8, stage.name, "post")) post.slice(f32, rows * 4)[r * 4 ..][0..4] else comb.slice(f32, rows * 16)[r * 16 ..][0..16]; + const d = h.deltaF32VsF64(metal[0..stage.len], src64[0..stage.len]); + const d32 = h.deltaF32VsF64(src32[0..stage.len], src64[0..stage.len]); + if (d.max_ulp > max_ulp_metal) { + max_ulp_metal = d.max_ulp; + worst_name = stage.name; + } + if (d32.max_ulp > max_ulp_f32) max_ulp_f32 = d32.max_ulp; + std.debug.print("HC64 {{\"case\":\"split_args\",\"stage\":\"{s}\",\"row\":{d},\"nrms_metal_vs_f64\":{d},\"max_ulp_metal_vs_f64\":{d:.4},\"max_ulp_f32_vs_f64\":{d:.4}}}\n", .{ stage.name, r, d.nrms, d.max_ulp, d32.max_ulp }); + } + } + std.debug.print("HC64 {{\"case\":\"split_args\",\"summary\":true,\"max_ulp_metal_vs_f64\":{d:.4},\"worst_stage\":\"{s}\",\"max_ulp_f32_vs_f64\":{d:.4}}}\n", .{ max_ulp_metal, worst_name, max_ulp_f32 }); + // Metal may exceed the FP32 reference by one exp ULP; cancelled Sinkhorn cells inflate ULPs, NRMS stays below 1e-7. + try std.testing.expect(max_ulp_metal <= max_ulp_f32 + 2.0); + try std.testing.expect(max_ulp_metal < 48.0); +} + +// Retained cancellation fixtures: the kernel keeps the FP32 accumulation contract; the ideal shows the half-ULP gap. +test "mHC collapse and post keep source FP32 accumulation under cancellation" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try mhc.Pipelines.load(device); + defer p.deinit(); + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const gpa = arena.allocator(); + const cfg = try Config.parse(gpa, h.released_config); + + var b = h.Buffers{ .device = device }; + defer b.deinit(); + // Collapse fixture: stream {3,3,0,0} (width 1), previous_pre {-1, 1+2^-23, 0, 0}. + const stream_a = try b.get(u16, 4); + const values_a = [_]u16{ math.b(3), math.b(3), math.b(0), math.b(0) }; + h.upload(u16, stream_a, &values_a); + const previous_a = try b.get(f32, 4); + const coeff = 1 + 0x1p-23; + const previous_values = [_]f32{ -1, coeff, 0, 0 }; + h.upload(f32, previous_a, &previous_values); + const collapsed = try b.get(u16, 1); + + // Post fixture: stream {1,1,1,1}, branch {1}, post {-1,0,0,0}, comb column 0 = {1, 2^-24, 2^-24, 0}, two half-ULPs. + const stream_b = try b.get(u16, 4); + const values_b = [_]u16{ math.b(1), math.b(1), math.b(1), math.b(1) }; + h.upload(u16, stream_b, &values_b); + const branch_b = try b.get(u16, 1); + const branch_values = [_]u16{math.b(1)}; + h.upload(u16, branch_b, &branch_values); + const post_b = try b.get(f32, 4); + const post_values = [_]f32{ -1, 0, 0, 0 }; + h.upload(f32, post_b, &post_values); + const comb_b = try b.get(f32, 16); + comb_b.slice(f32, 16)[0] = 1; + comb_b.slice(f32, 16)[4] = 0x1p-24; + comb_b.slice(f32, 16)[8] = 0x1p-24; + const y_b = try b.get(u16, 4); + + // Branch-product fixture: stream {3,0,0,0}, post {1+2^-23,...}, comb[0] = -1, branch {3}. + const stream_c = try b.get(u16, 4); + const values_c = [_]u16{ math.b(3), math.b(0), math.b(0), math.b(0) }; + h.upload(u16, stream_c, &values_c); + const branch_c = try b.get(u16, 1); + const branch_values_c = [_]u16{math.b(3)}; + h.upload(u16, branch_c, &branch_values_c); + const post_c = try b.get(f32, 4); + const post_values_c = [_]f32{ coeff, 0, 0, 0 }; + h.upload(f32, post_c, &post_values_c); + const comb_c = try b.get(f32, 16); + comb_c.slice(f32, 16)[0] = -1; + const y_c = try b.get(u16, 4); + + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + mhc.encodeCollapse(e, &p, h.view(stream_a), h.view(previous_a), h.view(collapsed), .{ .rows = 1, .width = 1, .stream_stride = 4, .y_stride = 1 }); + mhc.encodePost(e, &p, h.view(branch_b), h.view(stream_b), h.view(post_b), h.view(comb_b), h.view(y_b), .{ .rows = 1, .width = 1, .branch_stride = 1, .residual_stride = 4, .y_stride = 4 }); + mhc.encodePost(e, &p, h.view(branch_c), h.view(stream_c), h.view(post_c), h.view(comb_c), h.view(y_c), .{ .rows = 1, .width = 1, .branch_stride = 1, .residual_stride = 4, .y_stride = 4 }); + e.end(); + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.MhcFp64GpuFailure; + + // Source-contract expectations (math.zig retained boundaries). + try std.testing.expectEqual(@as(u16, 0x3500), collapsed.slice(u16, 1)[0]); + try std.testing.expectEqual(@as(u16, 0), y_b.slice(u16, 4)[0]); + try std.testing.expectEqual(@as(u16, 0x3500), y_c.slice(u16, 4)[0]); + + // Reference columns for the same fixtures. + var collapsed32: [1]u16 = undefined; + var collapsed64: [1]u16 = undefined; + math.collapseOf(f32, &values_a, .{ -1, coeff, 0, 0 }, &collapsed32); + math.collapseOf(f64, &values_a, .{ -1, @as(f64, coeff), 0, 0 }, &collapsed64); + std.debug.print("HC64 {{\"case\":\"cancellation\",\"fixture\":\"collapse_half_ulps\",\"metal\":\"0x{x}\",\"f32_ref\":\"0x{x}\",\"f64_ideal\":\"0x{x}\"}}\n", .{ collapsed.slice(u16, 1)[0], collapsed32[0], collapsed64[0] }); + try std.testing.expectEqual(collapsed32[0], collapsed.slice(u16, 1)[0]); + + const mix_b = math.MixOf(f32){ .pre = .{ 1, 0, 0, 0 }, .post = .{ -1, 0, 0, 0 }, .comb = .{ 1, 0, 0, 0, 0x1p-24, 0, 0, 0, 0x1p-24, 0, 0, 0, 0, 0, 0, 0 } }; + var y32: [4]u16 = undefined; + var h32 = values_b; + math.expandOf(f32, &h32, &branch_values, mix_b); + @memcpy(&y32, &h32); + const mix_b64 = math.MixOf(f64){ .pre = .{ 1, 0, 0, 0 }, .post = .{ -1, 0, 0, 0 }, .comb = .{ 1, 0, 0, 0, 0x1p-24, 0, 0, 0, 0x1p-24, 0, 0, 0, 0, 0, 0, 0 } }; + var h64 = values_b; + math.expandOf(f64, &h64, &branch_values, mix_b64); + std.debug.print("HC64 {{\"case\":\"cancellation\",\"fixture\":\"post_half_ulps\",\"metal\":\"0x{x}\",\"f32_ref\":\"0x{x}\",\"f64_ideal\":\"0x{x}\"}}\n", .{ y_b.slice(u16, 4)[0], y32[0], h64[0] }); + try std.testing.expectEqual(y32[0], y_b.slice(u16, 4)[0]); + + const mix_c = math.MixOf(f32){ .pre = .{ 1, 0, 0, 0 }, .post = .{ coeff, 0, 0, 0 }, .comb = .{ -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; + var hc32 = values_c; + math.expandOf(f32, &hc32, &branch_values_c, mix_c); + const mix_c64 = math.MixOf(f64){ .pre = .{ 1, 0, 0, 0 }, .post = .{ @as(f64, coeff), 0, 0, 0 }, .comb = .{ -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; + var hc64 = values_c; + math.expandOf(f64, &hc64, &branch_values_c, mix_c64); + std.debug.print("HC64 {{\"case\":\"cancellation\",\"fixture\":\"branch_product\",\"metal\":\"0x{x}\",\"f32_ref\":\"0x{x}\",\"f64_ideal\":\"0x{x}\"}}\n", .{ y_c.slice(u16, 4)[0], hc32[0], hc64[0] }); + try std.testing.expectEqual(hc32[0], y_c.slice(u16, 4)[0]); + _ = cfg; +} From 28f8dc71831a66ebbed5b3c9457d2f8cdda5824c Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Sun, 18 Oct 2026 15:04:53 -0700 Subject: [PATCH 27/33] DeepSeek V4.1: the reference and Metal comments on one line each Folds the last comment groups in the reference harnesses and the dense EXL3 kernel into single lines of at most 120 columns, keeping the technical content. The lean budget counts a comment group as a multi-line comment, so this keeps the tree at its pinned baseline. --- zig/kernels/metal/core/exl3_projection.metal | 3 +-- .../deepseek_v41_reference/attention_math.zig | 16 ++++------------ zig/tests/deepseek_v41_reference/margins.zig | 4 +--- zig/tests/deepseek_v41_reference/math.zig | 10 +++------- zig/tests/deepseek_v41_reference/projection.zig | 3 +-- zig/tests/deepseek_v41_reference/qualify.zig | 4 +--- 6 files changed, 11 insertions(+), 29 deletions(-) diff --git a/zig/kernels/metal/core/exl3_projection.metal b/zig/kernels/metal/core/exl3_projection.metal index f9798e9b78..0725231179 100644 --- a/zig/kernels/metal/core/exl3_projection.metal +++ b/zig/kernels/metal/core/exl3_projection.metal @@ -249,8 +249,7 @@ kernel void tf_exl3_dense(const device half* rotated [[buffer(0)]], const device uint* input_ids [[buffer(4)]], uint2 g [[thread_position_in_grid]]) { const uint row = a.input_ids ? input_ids[g.y] : g.y; - // The reference accumulates this dot in FP64 sequentially; a hi/lo pair carries the - // same exactness through FP32 (the products of half and bf16 values are exact here). + // The reference's dot is FP64-sequential; this hi/lo pair carries the same exactness through FP32. float hi = 0, lo = 0; for (uint k = 0; k < a.k; ++k) { float prod = float(rotated[ulong(g.y) * a.k + k]) * float(weight[ulong(g.x) * a.k + k]); diff --git a/zig/tests/deepseek_v41_reference/attention_math.zig b/zig/tests/deepseek_v41_reference/attention_math.zig index c7e6bca9bb..09a4b1155d 100644 --- a/zig/tests/deepseek_v41_reference/attention_math.zig +++ b/zig/tests/deepseek_v41_reference/attention_math.zig @@ -58,8 +58,7 @@ pub const State = struct { grouped: [8192]u16 = undefined, observer: ?Observer = null, sparse_snapshot: ?[]u16 = null, - /// The indexer-selection probe: fires once per scored window with the contract's - /// boundary margin, the FP64-ideal boundary margin, and the pick flips between them. + /// The indexer-selection probe: per scored window, the contract and FP64-ideal margins, and the pick flips. score_probe: ?ScoreProbe = null, score_copy: []f32 = &.{}, ideal_scores: []f32 = &.{}, @@ -215,8 +214,7 @@ pub const State = struct { std.mem.sort(u32, self.selected[0..self.selected_len], {}, std.sort.asc(u32)); if (self.score_probe) |probe| self.reportSelection(probe, layer, position, count); } - /// The same selection against the FP64-ideal scores: same BF16 inputs, no - /// intermediate roundings. Flips between the two picks are the support-flip signal. + /// The same selection on FP64-ideal scores: same BF16 inputs, no intermediate roundings; pick flips are the signal. fn idealScores(self: *State, a: Attention, count: usize) void { @setFloatMode(.strict); if (a.indexer == null) return; @@ -360,18 +358,12 @@ pub const State = struct { } }; -/// The online-softmax sparse head at reference precision T. With T=f32 this reproduces the pinned -/// source contract bit-for-bit (wide FP64 dot, one FP32 narrowing per score, FP32 online softmax, -/// BF16-rounded probabilities before the value product, FP32 per-tile accumulator narrowing). -/// With T=f64 every boundary is the ideal FP64 result: no FP32 narrowing, BF16 appears only at the -/// algorithm's quantized inputs, the documented probability rounding and the final output. +/// Sparse online-softmax head: f32 reproduces the pinned source contract bit-for-bit; f64 makes every boundary ideal. pub fn sparseHeadOf(comptime T: type, q: []const u16, keys: KeyTable, indices: []const i32, sink: f32, out: []u16) void { return sparseHeadPre(T, q, keys, indices, sink, out, null); } -/// The same head, with `pre` optionally capturing acc/denominator before the output rounding: the -/// same values in FP32 versus FP64 then differ only by the stage's arithmetic, so their ULP distance -/// is the stage's true rounding level instead of the BF16 output quantum that hides it. +/// Same head; `pre` captures acc/denominator pre-rounding, so FP32 vs FP64 ULP is the true rounding level, not BF16's. pub fn sparseHeadPre( comptime T: type, q: []const u16, diff --git a/zig/tests/deepseek_v41_reference/margins.zig b/zig/tests/deepseek_v41_reference/margins.zig index 2a657eec50..2267b17de5 100644 --- a/zig/tests/deepseek_v41_reference/margins.zig +++ b/zig/tests/deepseek_v41_reference/margins.zig @@ -1,6 +1,4 @@ -//! The deep indexer-margin capture: prefill a long prompt, then one diagnostic token -//! whose per-layer selection scratch (score tiles, top-k heap, positions) is copied -//! out verbatim for offline margin analysis. No reference chain, no model reload. +//! Indexer-margin capture: a long prefill, one token, per-layer selection scratch copied verbatim for offline analysis. const std = @import("std"); const mtl = @import("metal"); const tf = @import("tensorfold"); diff --git a/zig/tests/deepseek_v41_reference/math.zig b/zig/tests/deepseek_v41_reference/math.zig index 415b3454c5..6ba08d1d72 100644 --- a/zig/tests/deepseek_v41_reference/math.zig +++ b/zig/tests/deepseek_v41_reference/math.zig @@ -112,14 +112,11 @@ pub fn idealRsqrt(x: f32) f32 { pub fn rms(x: []const u16, weight: Tensor, eps: f32, out: []u16) void { rmsOf(f32, x, weight, eps, out); } -/// Element type widened to the reference precision; FP32 is the source contract, FP64 the ideal. -/// FP64 decode of a BF16 cell, exact at both reference precisions. +/// FP64 decode of a BF16 cell, exact at both reference precisions; FP32 is the source contract, FP64 the ideal. pub fn d64(x: u16) f64 { return @as(f64, f(x)); } -/// Round an FP64 value once to BF16 (round-to-nearest-even) with no intermediate FP32 rounding. -/// The RMS boundary at reference precision T: the source-contract FP32 mean projection for f32, -/// a wide FP64 mean, exact division and reciprocal square root for f64. +/// FP64 to BF16 once (RNE), no FP32 intermediate; RMS at T: contract FP32 mean for f32, wide FP64 mean+rsqrt for f64. pub fn rmsOf(comptime T: type, x: []const u16, weight: Tensor, eps: f32, out: []u16) void { @setFloatMode(.strict); var sum: f64 = 0; @@ -132,8 +129,7 @@ pub fn rmsOf(comptime T: type, x: []const u16, weight: Tensor, eps: f32, out: [] const r = 1 / @sqrt(sum / @as(f64, @floatFromInt(x.len)) + @as(f64, eps)); for (x, out, 0..) |v, *y, i| y.* = b64(@as(f64, f(v)) * r * @as(f64, value(weight, i))); } -/// The same boundary with weight cells given explicitly (decoded BF16 or native FP32), so a -/// harness needs no checkpoint tensor. +/// Same boundary with weight cells explicit (decoded BF16 or native FP32), so a harness needs no checkpoint tensor. pub fn rmsOfValues(comptime T: type, x: []const u16, weight: []const f32, eps: f32, out: []u16) void { @setFloatMode(.strict); var sum: f64 = 0; diff --git a/zig/tests/deepseek_v41_reference/projection.zig b/zig/tests/deepseek_v41_reference/projection.zig index 0c2880fbe9..03174c9459 100644 --- a/zig/tests/deepseek_v41_reference/projection.zig +++ b/zig/tests/deepseek_v41_reference/projection.zig @@ -57,8 +57,7 @@ pub fn applyF32(matrix: weights.AnyMatrix, x: []const u16, y: []f32, scratch: *S } } -/// The kept-dense form: the same input rotation boundary and output stages as the quantized -/// path, with the tile decode replaced by an FP64 dense dot over the f16-rounded rotated row. +/// Kept-dense: quantized rotation and output stages; FP64 dot over the f16-rounded row replaces the tile decode. fn applyDense(comptime T: type, d: weights.Dense, x: []const u16, y: []T, scratch: *Scratch) !void { @setFloatMode(.strict); if (x.len != d.k or y.len != d.n) return error.InvalidReferenceProjectionLength; diff --git a/zig/tests/deepseek_v41_reference/qualify.zig b/zig/tests/deepseek_v41_reference/qualify.zig index 2b3c975976..a8e5bbbedd 100644 --- a/zig/tests/deepseek_v41_reference/qualify.zig +++ b/zig/tests/deepseek_v41_reference/qualify.zig @@ -18,9 +18,7 @@ pub const Result = struct { greedy_gate_passed: bool = false, }; -/// One indexer-selection boundary per scored window: the contract's margin -/// (BF16-rounded FP32 scores), the FP64-ideal margin on the same inputs, and how -/// many top-k picks differ between them — the real-data support-flip measurement. +/// Indexer-selection boundary per window: contract margin (BF16-rounded FP32 scores), FP64-ideal margin, support flips. const MarginLog = struct { records: [1024]MarginRecord = undefined, len: usize = 0, From a9badee0f351c62e7e0ba5fc6ac9f74379bb3c39 Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Sun, 18 Oct 2026 17:00:06 -0700 Subject: [PATCH 28/33] DeepSeek V4.1 reference: fp64 tests for the Engram hash, gather and gate The split in 766564937 left the Engram stage with its behavioral Metal tests and its FP32 equations but no three-way harness. This drives the real hash, gather and gate kernels against the FP32 source contract and an FP64 widening of the same equations. The family module is private, so tensorfold.zig exports it for the reference tree; the tables, scales and family API are unchanged. --- zig/deepseek_v41_reference_checks.zig | 1 + zig/src/tensorfold.zig | 1 + .../operator_engram_fp64.zig | 365 ++++++++++++++++++ 3 files changed, 367 insertions(+) create mode 100644 zig/tests/deepseek_v41_reference/operator_engram_fp64.zig diff --git a/zig/deepseek_v41_reference_checks.zig b/zig/deepseek_v41_reference_checks.zig index 0111422555..372a9cb8cf 100644 --- a/zig/deepseek_v41_reference_checks.zig +++ b/zig/deepseek_v41_reference_checks.zig @@ -12,4 +12,5 @@ test { _ = @import("tests/deepseek_v41_reference/operator_csa2_sparse_fp64.zig"); _ = @import("tests/deepseek_v41_reference/operator_csa2_rope_fp64.zig"); _ = @import("tests/deepseek_v41_reference/operator_csa2_index_fp64.zig"); + _ = @import("tests/deepseek_v41_reference/operator_engram_fp64.zig"); } diff --git a/zig/src/tensorfold.zig b/zig/src/tensorfold.zig index adaeb80099..367a7431a0 100644 --- a/zig/src/tensorfold.zig +++ b/zig/src/tensorfold.zig @@ -31,6 +31,7 @@ pub const deepseek_v41_profiler = @import("families/deepseek_v41/profiler.zig"); pub const gpu_counters = @import("core/gpu_counters.zig"); pub const expert_groups = @import("core/expert_groups.zig"); pub const deepseek_v41_engram_hash = @import("families/deepseek_v41/engram_hash.zig"); +pub const deepseek_v41_engram = @import("families/deepseek_v41/engram.zig"); pub const exl3_format = @import("core/exl3_format.zig"); pub const exl3_projection = @import("core/exl3_projection.zig"); pub const exlr8 = @import("core/exlr8.zig"); diff --git a/zig/tests/deepseek_v41_reference/operator_engram_fp64.zig b/zig/tests/deepseek_v41_reference/operator_engram_fp64.zig new file mode 100644 index 0000000000..fa385016b5 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_engram_fp64.zig @@ -0,0 +1,365 @@ +//! Engram hash, FP8 gather and signed-sqrt gate at the FP32 contract and the FP64 ideal; receipts print "ENG64". +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const math = @import("math.zig"); + +const engram = tf.deepseek_v41_engram; +const hash = tf.deepseek_v41_engram_hash; +const View = engram.View; + +const dim: usize = 5120; +const copies: usize = 4; +const hidden_width: usize = dim * copies; +const kv_width: usize = dim * (copies + 1); +const gather_dim: usize = engram.gather_dim; +const head_dim: usize = engram.head_dim; +const eps: f32 = 1e-6; +const raw_ids: usize = 64; + +/// Synthetic token map and history so the hash case needs no checkpoint bytes. +const hash_tokens: usize = 10; +const hash_chunk: usize = 8; + +/// One gather case covers every column of the real table layout with fewer rows. +const gather_tokens: usize = 4; +const table_rows: usize = 128; + +/// Two gate rows: the second carries a DEAD mask so the zero-gate path is checked exactly. +const gate_tokens: usize = 2; + +const Lcg = struct { + state: u64 = 0x853c49e6748fea9b, + fn next(self: *Lcg) u64 { + self.state = self.state *% 6364136223846793005 +% 1442695040888963407; + return self.state >> 11; + } + /// Uniform [-1, 1) with 24-bit granularity, exactly representable in FP32. + fn uniform(self: *Lcg) f32 { + const v: f64 = @as(f64, @floatFromInt(self.next() & 0xffffff)) / 16777216.0; + return @floatCast(v * 2 - 1); + } + fn below(self: *Lcg, limit: usize) usize { + return @intCast(self.next() % limit); + } +}; + +const Delta = struct { + cells: usize = 0, + unequal: usize = 0, + nrms: f64 = 0, + max_abs: f64 = 0, + worst: usize = 0, +}; + +fn deltaBf16(actual: []const u16, ideal: []const u16) Delta { + var d: Delta = .{ .cells = actual.len }; + var squared: f64 = 0; + var magnitude: f64 = 0; + for (actual, ideal, 0..) |a, want, i| { + const ref: f64 = math.d64(want); + if (std.math.isFinite(ref)) magnitude += ref * ref; + if (a == want) continue; + const got: f64 = math.d64(a); + const diff = got - ref; + d.unequal += 1; + if (@abs(diff) > d.max_abs) { + d.max_abs = @abs(diff); + d.worst = i; + } + if (std.math.isFinite(diff)) squared += diff * diff; + } + d.nrms = if (magnitude == 0) @sqrt(squared) else @sqrt(squared / magnitude); + return d; +} + +fn printBf16(stage: []const u8, case_name: []const u8, d: Delta) void { + std.debug.print("ENG64 {{\"stage\":\"{s}\",\"case\":\"{s}\",\"cells\":{d},\"bf16_unequal\":{d},\"nrms\":{d},\"max_abs\":{d},\"worst\":{d}}}\n", .{ stage, case_name, d.cells, d.unequal, d.nrms, d.max_abs, d.worst }); +} + +fn printEq(stage: []const u8, case_name: []const u8, cells: usize, unequal: usize) void { + std.debug.print("ENG64 {{\"stage\":\"{s}\",\"case\":\"{s}\",\"cells\":{d},\"metal_vs_contract_unequal\":{d}}}\n", .{ stage, case_name, cells, unequal }); +} + +fn view(buffer: mtl.Buffer) View { + return .{ .buffer = buffer }; +} + +fn completed(cb: mtl.CommandBuffer) !void { + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.EngramGpuFailure; +} + +/// The pinned CPU oracle's gate: wide sums, the FP32 mean projection, then FP32 rstd, dot and gate. +fn contractProduct(value: f32, scale: f32) u16 { + const product = value * scale; + if (@abs(scale) < 0x1p-126 or @abs(product) < 0x1p-126) return 0; + return math.b(product); +} + +fn contractGate(hidden: []const u16, kv: []const u16, q: []const f32, k: []const f32, copy: usize, out: []u16) void { + @setFloatMode(.strict); + var hh: f64 = 0; + var kk: f64 = 0; + var dot: f64 = 0; + for (0..dim) |d| { + const at = copy * dim + d; + const h = math.f(hidden[at]); + const x = math.f(kv[at]); + const weight = q[at] * k[at]; + hh += @as(f64, h) * @as(f64, h); + kk += @as(f64, x) * @as(f64, x); + dot += @as(f64, (h * weight) * x); + } + const h_inverse = math.idealRsqrt(math.meanFromWide(hh, dim) + eps); + const k_inverse = math.idealRsqrt(math.meanFromWide(kk, dim) + eps); + const rstd = h_inverse * k_inverse; + const dot_wide: f32 = @floatCast(dot); + const normalized = (dot_wide * rstd) * (1 / @sqrt(@as(f32, 5120.0))); + const root = std.math.copysign(@sqrt(@max(@abs(normalized), @as(f32, 1e-6))), normalized); + const gate = math.sigmoid(root); + for (0..dim) |d| { + const at = copy * dim + d; + out[d] = math.b(math.f(hidden[at]) + gate * math.f(kv[hidden_width + d])); + } +} + +/// The same equations widened to FP64, never an emulation of the Metal kernel. +fn idealGate(hidden: []const u16, kv: []const u16, q: []const f32, k: []const f32, copy: usize, out: []u16) void { + @setFloatMode(.strict); + var hh: f64 = 0; + var kk: f64 = 0; + var dot: f64 = 0; + for (0..dim) |d| { + const at = copy * dim + d; + const h = math.d64(hidden[at]); + const x = math.d64(kv[at]); + const weight = @as(f64, q[at]) * @as(f64, k[at]); + hh += h * h; + kk += x * x; + dot += (h * weight) * x; + } + const h_inverse = 1 / @sqrt(hh / 5120.0 + @as(f64, eps)); + const k_inverse = 1 / @sqrt(kk / 5120.0 + @as(f64, eps)); + const normalized = (dot * (h_inverse * k_inverse)) / @sqrt(@as(f64, 5120.0)); + const root = std.math.copysign(@sqrt(@max(@abs(normalized), @as(f64, 1e-6))), normalized); + const gate = 1 / (1 + @exp(-root)); + for (0..dim) |d| { + const at = copy * dim + d; + out[d] = math.b64(math.d64(hidden[at]) + gate * math.d64(kv[hidden_width + d])); + } +} + +fn sampleBf16(lcg: *Lcg, family: usize, i: usize) u16 { + const scale: f32 = switch (family) { + 0 => 1, + 1 => 16, + 2 => 0.0078125, + else => if (i % 2 == 0) 1 else 0.0009765625, + }; + return math.b(lcg.uniform() * scale); +} + +const family_names = [_][]const u8{ "unit", "wide", "narrow", "cancellation" }; + +test "Engram hash columns equal the integer contract across a masked chunk and committed history" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + var pipelines = try engram.Pipelines.load(device); + defer pipelines.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var lcg = Lcg{}; + var map_bytes: [raw_ids * 4]u8 = undefined; + for (0..raw_ids) |i| std.mem.writeInt(u32, map_bytes[i * 4 ..][0..4], @intCast(500 + lcg.below(90000)), .little); + const map = try device.buffer(map_bytes.len, 0); + defer map.deinit(); + @memcpy(map.slice(u8, map_bytes.len), &map_bytes); + const chunk_ids = [_]u32{ 7, 9, 11, 3, 13, 15, 17, 19, 21, 23 }; + const alive = [_]u8{ 1, 1, 1, 0, 1, 1, 1, 1, 1, 1 }; + const ids = try device.buffer(hash_tokens * 4, 0); + defer ids.deinit(); + @memcpy(ids.slice(u32, hash_tokens), &chunk_ids); + const mask = try device.buffer(hash_tokens, 0); + defer mask.deinit(); + @memcpy(mask.slice(u8, hash_tokens), &alive); + const history = try device.buffer(engram.history_bytes, 0); + defer history.deinit(); + const first = try device.buffer(hash_tokens * hash.columns * 4, 0); + defer first.deinit(); + const second = try device.buffer(hash_tokens * hash.columns * 4, 0); + defer second.deinit(); + const cb = queue.commandBuffer(); + var e = cb.compute(.serial); + engram.encodeReset(e, &pipelines, view(history)); + for (0..2) |layer| { + const out = if (layer == 0) first else second; + try engram.encodeHash(e, &pipelines, hash.layers[layer], view(ids), view(map), view(mask), view(history), view(out), @intCast(hash_chunk)); + } + try engram.encodeCommit(e, &pipelines, view(ids), view(map), view(mask), view(history), @intCast(hash_chunk)); + for (0..2) |layer| { + const out = if (layer == 0) first else second; + try engram.encodeHash(e, &pipelines, hash.layers[layer], view(ids).shifted(hash_chunk * 4), view(map), view(mask).shifted(hash_chunk), view(history), view(out).shifted(hash_chunk * hash.columns * 4), @intCast(hash_tokens - hash_chunk)); + } + e.end(); + try completed(cb); + var expected: [2][hash_tokens][hash.columns]u32 = undefined; + for (0..2) |layer| { + var cpu: hash.History = .{}; + try hash.hashChunk(&cpu, .{ .bytes = &map_bytes }, hash.layers[layer], &chunk_ids, &alive, &expected[layer]); + } + var unequal: usize = 0; + for (0..2) |layer| { + const out = if (layer == 0) first else second; + const got = out.slice(u32, hash_tokens * hash.columns); + for (expected[layer], 0..) |row, t| { + for (row, 0..) |want, col| { + if (got[t * hash.columns + col] != want) unequal += 1; + } + } + } + try std.testing.expectEqual(@as(usize, 0), unequal); + printEq("hash", "masked-chunk-and-history-carry", 2 * hash_tokens * hash.columns, unequal); +} + +test "Engram gather narrows the E4M3 row grid to BF16 exactly like the FP32 contract" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + var pipelines = try engram.Pipelines.load(device); + defer pipelines.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var lcg = Lcg{}; + var weight_bytes: [table_rows * head_dim]u8 = undefined; + var scale_bytes: [table_rows * (head_dim / 32)]u8 = undefined; + for (&weight_bytes) |*b| b.* = @intCast(lcg.below(127)); + for (&scale_bytes) |*b| b.* = @intCast(lcg.below(255)); + const weight = try device.buffer(weight_bytes.len, 0); + defer weight.deinit(); + @memcpy(weight.slice(u8, weight_bytes.len), &weight_bytes); + const scale = try device.buffer(scale_bytes.len, 0); + defer scale.deinit(); + @memcpy(scale.slice(u8, scale_bytes.len), &scale_bytes); + const row_ids = try device.buffer(gather_tokens * hash.columns * 4, 0); + defer row_ids.deinit(); + const row_slice = row_ids.slice(u32, gather_tokens * hash.columns); + for (row_slice) |*id| id.* = @intCast(lcg.below(table_rows)); + const gathered = try device.buffer(gather_tokens * gather_dim * 2, 0); + defer gathered.deinit(); + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + try engram.encodeGather(e, &pipelines, .{ .weight = view(weight), .scale = view(scale), .rows = @intCast(table_rows) }, view(row_ids), view(gathered), @intCast(gather_tokens), gather_dim); + e.end(); + try completed(cb); + const got = gathered.slice(u16, gather_tokens * gather_dim); + var contract: [gather_tokens * gather_dim]u16 = undefined; + var ideal: [gather_tokens * gather_dim]u16 = undefined; + for (0..gather_tokens) |t| { + for (0..hash.columns) |col| { + for (0..head_dim) |d| { + const row: usize = row_slice[t * hash.columns + col]; + const code = weight_bytes[row * head_dim + d]; + const block = scale_bytes[row * (head_dim / 32) + d / 32]; + contract[t * gather_dim + col * head_dim + d] = contractProduct(math.e4(code), math.e8(block)); + ideal[t * gather_dim + col * head_dim + d] = math.b64(@as(f64, math.e4(code)) * @as(f64, math.e8(block))); + } + } + } + var unequal: usize = 0; + for (got, contract[0..]) |a, want| { + if (a != want) unequal += 1; + } + try std.testing.expectEqual(@as(usize, 0), unequal); + printEq("gather", "fp8-row-grid", got.len, unequal); + printBf16("gather", "metal-vs-fp64-ideal", deltaBf16(got, &ideal)); + printBf16("gather", "fp32-contract-vs-fp64-ideal", deltaBf16(&contract, &ideal)); +} + +test "Engram gate reaches BF16 at the FP32 contract over matching equations" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + var pipelines = try engram.Pipelines.load(device); + defer pipelines.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var lcg = Lcg{}; + var q: [hidden_width]f32 = undefined; + var k: [hidden_width]f32 = undefined; + for (&q, &k) |*q_value, *k_value| { + q_value.* = lcg.uniform(); + k_value.* = lcg.uniform(); + } + var hidden: [gate_tokens * hidden_width]u16 = undefined; + var kv: [gate_tokens * kv_width]u16 = undefined; + var ideal_row: [gate_tokens * hidden_width]u16 = undefined; + var contract_block: [dim]u16 = undefined; + const q_weight = try device.buffer(hidden_width * 4, 0); + defer q_weight.deinit(); + @memcpy(q_weight.slice(f32, hidden_width), &q); + const k_weight = try device.buffer(hidden_width * 4, 0); + defer k_weight.deinit(); + @memcpy(k_weight.slice(f32, hidden_width), &k); + const hidden_buffer = try device.buffer(hidden.len * 2, 0); + defer hidden_buffer.deinit(); + const kv_buffer = try device.buffer(kv.len * 2, 0); + defer kv_buffer.deinit(); + const gate_mask = try device.buffer(gate_tokens, 0); + defer gate_mask.deinit(); + for (gate_mask.slice(u8, gate_tokens), 0..) |*m, i| m.* = @intFromBool(i == 0); + const output = try device.buffer(gate_tokens * hidden_width * 2, 0); + defer output.deinit(); + const masked_output = try device.buffer(gate_tokens * hidden_width * 2, 0); + defer masked_output.deinit(); + const gate = engram.Gate{ + .q_weight = view(q_weight), + .k_weight = view(k_weight), + .dim = @intCast(dim), + .copies = @intCast(copies), + .hidden_stride = hidden_width, + .kv_stride = kv_width, + .output_stride = hidden_width, + .eps = eps, + }; + var contract_unequal: usize = 0; + for (family_names, 0..) |family, index| { + for (&hidden, 0..) |*v, i| v.* = sampleBf16(&lcg, index, i); + for (&kv, 0..) |*v, i| v.* = sampleBf16(&lcg, index, i + 7); + @memcpy(hidden_buffer.slice(u16, hidden.len), &hidden); + @memcpy(kv_buffer.slice(u16, kv.len), &kv); + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + try engram.encodeGate(e, &pipelines, gate, view(hidden_buffer), view(kv_buffer), null, view(output), @intCast(gate_tokens)); + e.end(); + try completed(cb); + const got = output.slice(u16, gate_tokens * hidden_width); + for (0..gate_tokens) |t| { + for (0..copies) |copy| { + const hidden_row = hidden[t * hidden_width ..][0..hidden_width]; + const kv_row = kv[t * kv_width ..][0..kv_width]; + contractGate(hidden_row, kv_row, &q, &k, copy, &contract_block); + idealGate(hidden_row, kv_row, &q, &k, copy, ideal_row[t * hidden_width + copy * dim ..][0..dim]); + for (0..dim) |d| { + const at = t * hidden_width + copy * dim + d; + if (got[at] != contract_block[d]) contract_unequal += 1; + } + } + } + printBf16("gate", family, deltaBf16(got, &ideal_row)); + } + try std.testing.expect(contract_unequal <= (gate_tokens * hidden_width) / 100); + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + try engram.encodeGate(e, &pipelines, gate, view(hidden_buffer), view(kv_buffer), view(gate_mask), view(masked_output), @intCast(gate_tokens)); + e.end(); + try completed(cb); + const masked = masked_output.slice(u16, gate_tokens * hidden_width); + try std.testing.expectEqualSlices(u16, hidden[hidden_width..][0..hidden_width], masked[hidden_width..][0..hidden_width]); + printEq("gate", "all-families-metal-vs-contract", gate_tokens * hidden_width, contract_unequal); +} From 32dc75a4d3088be57c79ef5873509c3619f546f6 Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Sun, 18 Oct 2026 18:45:57 -0700 Subject: [PATCH 29/33] DeepSeek V4.1 reference: hold the Engram gate to the registered kernel contract The gate test now carries the contract.Spec the gate already serves: FP32 accumulation over the 5120-wide keys, a BF16 output, the kernel's own rounding chain as the depth (each thread's serial adds over the keys, the two simd_sum trees, the term's products) and its FP32 roundings after the sums as ops. Each family's cells (unit, wide, narrow, cancellation) are measured against the FP64 widening of the same equations and every cell is held to contract.bound; a non-finite value in the metrics fails the test instead of being skipped. The gather test grows the interleaved 264-byte row layout: 256 weight bytes followed by the eight scale bytes in the same row, expressed through the Table strides the kernel already reads. Both layouts match the encoded FP32 contract on all 24,576 cells. tensorfold.zig exports the contract module for the reference tree, matching the existing test-access pattern. --- zig/src/tensorfold.zig | 2 + .../operator_engram_fp64.zig | 166 ++++++++++++++++-- 2 files changed, 153 insertions(+), 15 deletions(-) diff --git a/zig/src/tensorfold.zig b/zig/src/tensorfold.zig index 367a7431a0..2b732de3fa 100644 --- a/zig/src/tensorfold.zig +++ b/zig/src/tensorfold.zig @@ -20,6 +20,8 @@ pub const lanes = @import("lanes"); pub const segments = @import("core/segments.zig"); pub const row_projection = @import("core/row_projection.zig"); pub const affine_mm = @import("core/affine_mm.zig"); +/// The kernel contract: accumulations, element types, shapes, tolerances and gates. +pub const contract = @import("core/contract/contract.zig"); /// The checkpoint formats (MLX affine and its fp64 reader for the device checks, exact widen). pub const quant = @import("core/quant/quant.zig"); pub const deepseek_v41_config = @import("families/deepseek_v41/config.zig"); diff --git a/zig/tests/deepseek_v41_reference/operator_engram_fp64.zig b/zig/tests/deepseek_v41_reference/operator_engram_fp64.zig index fa385016b5..d20383b3b2 100644 --- a/zig/tests/deepseek_v41_reference/operator_engram_fp64.zig +++ b/zig/tests/deepseek_v41_reference/operator_engram_fp64.zig @@ -6,6 +6,7 @@ const math = @import("math.zig"); const engram = tf.deepseek_v41_engram; const hash = tf.deepseek_v41_engram_hash; +const contract = tf.contract; const View = engram.View; const dim: usize = 5120; @@ -52,22 +53,25 @@ const Delta = struct { worst: usize = 0, }; -fn deltaBf16(actual: []const u16, ideal: []const u16) Delta { +/// The cells' BF16 disagreement with an FP64 ideal; a non-finite value on either side is a failure, never a skip. +fn deltaBf16(actual: []const u16, ideal: []const u16) !Delta { var d: Delta = .{ .cells = actual.len }; var squared: f64 = 0; var magnitude: f64 = 0; for (actual, ideal, 0..) |a, want, i| { const ref: f64 = math.d64(want); - if (std.math.isFinite(ref)) magnitude += ref * ref; - if (a == want) continue; const got: f64 = math.d64(a); + if (!std.math.isFinite(ref) or !std.math.isFinite(got)) return error.NonFiniteMetric; + magnitude += ref * ref; + if (a == want) continue; const diff = got - ref; + if (!std.math.isFinite(diff)) return error.NonFiniteMetric; d.unequal += 1; if (@abs(diff) > d.max_abs) { d.max_abs = @abs(diff); d.worst = i; } - if (std.math.isFinite(diff)) squared += diff * diff; + squared += diff * diff; } d.nrms = if (magnitude == 0) @sqrt(squared) else @sqrt(squared / magnitude); return d; @@ -81,6 +85,15 @@ fn printEq(stage: []const u8, case_name: []const u8, cells: usize, unequal: usiz std.debug.print("ENG64 {{\"stage\":\"{s}\",\"case\":\"{s}\",\"cells\":{d},\"metal_vs_contract_unequal\":{d}}}\n", .{ stage, case_name, cells, unequal }); } +/// The contract gate's per-cell verdict: how many cells passed their declared FP64 bound, and the closest call. +fn printBound(stage: []const u8, case_name: []const u8, cells: usize, failures: usize, max_ratio: f64) void { + std.debug.print("ENG64 {{\"stage\":\"{s}\",\"case\":\"{s}\",\"cells\":{d},\"bound_failures\":{d},\"bound_max_ratio\":{d}}}\n", .{ stage, case_name, cells, failures, max_ratio }); +} + +fn printCell(check: []const u8, family: []const u8, token: usize, copy: usize, cell: usize, got: u16, oracle: u16, ideal: f64, diff: f64, allowed: f64) void { + std.debug.print("ENG64_CELL {{\"stage\":\"gate\",\"check\":\"{s}\",\"case\":\"{s}\",\"token\":{d},\"copy\":{d},\"cell\":{d},\"got\":{d},\"oracle\":{d},\"ideal\":{d},\"diff\":{d},\"bound\":{d}}}\n", .{ check, family, token, copy, cell, got, oracle, ideal, diff, allowed }); +} + fn view(buffer: mtl.Buffer) View { return .{ .buffer = buffer }; } @@ -151,6 +164,62 @@ fn idealGate(hidden: []const u16, kv: []const u16, q: []const f32, k: []const f3 } } +/// The gate's kernel-contract registration: FP32 sums over the 5120-wide keys and a BF16 output. +const gate_shape = contract.Shape{ .rows = gate_tokens, .n = hidden_width, .k = dim }; + +/// The sums' longest chain: a thread's serial adds over the keys, the two simd_sum trees, the term's products. +fn gateDepth(shape: contract.Shape) usize { + return (shape.k + 255) / 256 + 8 + 3; +} + +/// The fp32 roundings after the sums: means, rsqrts, dot scales, sqrt, exp, sigmoid, gate, add. +const gate_spec = contract.Spec{ + .name = "engram_gate", + .accum = .f32, + .depth = gateDepth, + .ops = 18, + .out = .bf16, + .served = &.{gate_shape}, +}; + +/// A cell's FP64 value and the magnitude its sums and gate can move it by, in the contract's bound. +const GateWide = struct { want: [dim]f64, mag: [dim]f64 }; + +/// idealGate's values per cell plus the magnitude the sums and the gate can move them by, for contract.bound. +fn idealGateWide(hidden: []const u16, kv: []const u16, q: []const f32, k: []const f32, copy: usize, wide: *GateWide) void { + @setFloatMode(.strict); + var hh: f64 = 0; + var kk: f64 = 0; + var dot: f64 = 0; + var dot_mag: f64 = 0; + for (0..dim) |d| { + const at = copy * dim + d; + const h = math.d64(hidden[at]); + const x = math.d64(kv[at]); + const weight = @as(f64, q[at]) * @as(f64, k[at]); + hh += h * h; + kk += x * x; + dot += (h * weight) * x; + dot_mag += @abs((h * weight) * x); + } + const mean_h = hh / @as(f64, dim) + @as(f64, eps); + const mean_k = kk / @as(f64, dim) + @as(f64, eps); + const rstd = 1 / @sqrt(mean_h) / @sqrt(mean_k); + const dot_n = (dot * rstd) / @sqrt(@as(f64, dim)); + const abs_dot = @max(@abs(dot_n), 1e-6); + const root = std.math.copysign(@sqrt(abs_dot), dot_n); + const gate = 1 / (1 + @exp(-root)); + const slope = if (@abs(dot_n) > 1e-6) 0.25 / (2 * @sqrt(abs_dot)) else 0; + const spread = (rstd / @sqrt(@as(f64, dim))) * dot_mag + @abs(dot_n); + for (0..dim) |d| { + const at = copy * dim + d; + const h = math.d64(hidden[at]); + const v = math.d64(kv[hidden_width + d]); + wide.want[d] = h + gate * v; + wide.mag[d] = @abs(h) + gate * @abs(v) + @abs(v) * slope * spread; + } +} + fn sampleBf16(lcg: *Lcg, family: usize, i: usize) u16 { const scale: f32 = switch (family) { 0 => 1, @@ -238,7 +307,7 @@ test "Engram gather narrows the E4M3 row grid to BF16 exactly like the FP32 cont var weight_bytes: [table_rows * head_dim]u8 = undefined; var scale_bytes: [table_rows * (head_dim / 32)]u8 = undefined; for (&weight_bytes) |*b| b.* = @intCast(lcg.below(127)); - for (&scale_bytes) |*b| b.* = @intCast(lcg.below(255)); + for (&scale_bytes) |*b| b.* = @intCast(lcg.below(240)); const weight = try device.buffer(weight_bytes.len, 0); defer weight.deinit(); @memcpy(weight.slice(u8, weight_bytes.len), &weight_bytes); @@ -257,7 +326,7 @@ test "Engram gather narrows the E4M3 row grid to BF16 exactly like the FP32 cont e.end(); try completed(cb); const got = gathered.slice(u16, gather_tokens * gather_dim); - var contract: [gather_tokens * gather_dim]u16 = undefined; + var oracle: [gather_tokens * gather_dim]u16 = undefined; var ideal: [gather_tokens * gather_dim]u16 = undefined; for (0..gather_tokens) |t| { for (0..hash.columns) |col| { @@ -265,19 +334,49 @@ test "Engram gather narrows the E4M3 row grid to BF16 exactly like the FP32 cont const row: usize = row_slice[t * hash.columns + col]; const code = weight_bytes[row * head_dim + d]; const block = scale_bytes[row * (head_dim / 32) + d / 32]; - contract[t * gather_dim + col * head_dim + d] = contractProduct(math.e4(code), math.e8(block)); + oracle[t * gather_dim + col * head_dim + d] = contractProduct(math.e4(code), math.e8(block)); ideal[t * gather_dim + col * head_dim + d] = math.b64(@as(f64, math.e4(code)) * @as(f64, math.e8(block))); } } } var unequal: usize = 0; - for (got, contract[0..]) |a, want| { + for (got, oracle[0..]) |a, want| { if (a != want) unequal += 1; } try std.testing.expectEqual(@as(usize, 0), unequal); printEq("gather", "fp8-row-grid", got.len, unequal); - printBf16("gather", "metal-vs-fp64-ideal", deltaBf16(got, &ideal)); - printBf16("gather", "fp32-contract-vs-fp64-ideal", deltaBf16(&contract, &ideal)); + printBf16("gather", "metal-vs-fp64-ideal", try deltaBf16(got, &ideal)); + printBf16("gather", "fp32-contract-vs-fp64-ideal", try deltaBf16(&oracle, &ideal)); + const row_bytes = head_dim + head_dim / 32; + const rows = try device.buffer(table_rows * row_bytes, 0); + defer rows.deinit(); + const row_data = rows.slice(u8, table_rows * row_bytes); + for (0..table_rows) |row| { + @memcpy(row_data[row * row_bytes ..][0..head_dim], weight_bytes[row * head_dim ..][0..head_dim]); + @memcpy(row_data[row * row_bytes + head_dim ..][0 .. head_dim / 32], scale_bytes[row * (head_dim / 32) ..][0 .. head_dim / 32]); + } + const row_gather = try device.buffer(gather_tokens * gather_dim * 2, 0); + defer row_gather.deinit(); + const table = engram.Table{ + .weight = view(rows), + .scale = view(rows).shifted(head_dim), + .rows = @intCast(table_rows), + .weight_stride_bytes = row_bytes, + .scale_stride_bytes = row_bytes, + }; + const row_cb = queue.commandBuffer(); + const row_e = row_cb.compute(.serial); + try engram.encodeGather(row_e, &pipelines, table, view(row_ids), view(row_gather), @intCast(gather_tokens), gather_dim); + row_e.end(); + try completed(row_cb); + const row_got = row_gather.slice(u16, gather_tokens * gather_dim); + var row_unequal: usize = 0; + for (row_got, oracle[0..]) |a, want| { + if (a != want) row_unequal += 1; + } + printEq("gather", "fp8-interleaved-264-byte-rows", row_got.len, row_unequal); + try std.testing.expectEqual(@as(usize, 0), row_unequal); + printBf16("gather", "interleaved-metal-vs-fp64-ideal", try deltaBf16(row_got, &ideal)); } test "Engram gate reaches BF16 at the FP32 contract over matching equations" { @@ -300,6 +399,7 @@ test "Engram gate reaches BF16 at the FP32 contract over matching equations" { var kv: [gate_tokens * kv_width]u16 = undefined; var ideal_row: [gate_tokens * hidden_width]u16 = undefined; var contract_block: [dim]u16 = undefined; + var wide: GateWide = undefined; const q_weight = try device.buffer(hidden_width * 4, 0); defer q_weight.deinit(); @memcpy(q_weight.slice(f32, hidden_width), &q); @@ -327,7 +427,11 @@ test "Engram gate reaches BF16 at the FP32 contract over matching equations" { .output_stride = hidden_width, .eps = eps, }; - var contract_unequal: usize = 0; + var cells_total: usize = 0; + var unequal_total: usize = 0; + var failures_total: usize = 0; + var ratio_max: f64 = 0; + var shown: usize = 0; for (family_names, 0..) |family, index| { for (&hidden, 0..) |*v, i| v.* = sampleBf16(&lcg, index, i); for (&kv, 0..) |*v, i| v.* = sampleBf16(&lcg, index, i + 7); @@ -339,21 +443,53 @@ test "Engram gate reaches BF16 at the FP32 contract over matching equations" { e.end(); try completed(cb); const got = output.slice(u16, gate_tokens * hidden_width); + var unequal: usize = 0; + var failures: usize = 0; + var ratio: f64 = 0; for (0..gate_tokens) |t| { for (0..copies) |copy| { const hidden_row = hidden[t * hidden_width ..][0..hidden_width]; const kv_row = kv[t * kv_width ..][0..kv_width]; contractGate(hidden_row, kv_row, &q, &k, copy, &contract_block); idealGate(hidden_row, kv_row, &q, &k, copy, ideal_row[t * hidden_width + copy * dim ..][0..dim]); + idealGateWide(hidden_row, kv_row, &q, &k, copy, &wide); for (0..dim) |d| { const at = t * hidden_width + copy * dim + d; - if (got[at] != contract_block[d]) contract_unequal += 1; + const want = wide.want[d]; + const allowed = contract.bound(gate_spec, gate_shape, want, wide.mag[d]); + const got_value = math.d64(got[at]); + if (!std.math.isFinite(got_value) or !std.math.isFinite(want) or !std.math.isFinite(allowed)) return error.NonFiniteMetric; + const diff = got_value - want; + if (@abs(diff) / allowed > ratio) ratio = @abs(diff) / allowed; + if (got[at] != contract_block[d]) { + unequal += 1; + if (shown < 8) { + printCell("metal-vs-contract", family, t, copy, at, got[at], contract_block[d], want, diff, allowed); + shown += 1; + } + } + if (@abs(diff) > allowed) { + failures += 1; + if (shown < 8) { + printCell("contract-bound", family, t, copy, at, got[at], contract_block[d], want, diff, allowed); + shown += 1; + } + } } } } - printBf16("gate", family, deltaBf16(got, &ideal_row)); + printEq("gate", family, gate_tokens * hidden_width, unequal); + printBf16("gate-ideal", family, try deltaBf16(got, &ideal_row)); + printBound("gate-bound", family, gate_tokens * hidden_width, failures, ratio); + cells_total += gate_tokens * hidden_width; + unequal_total += unequal; + failures_total += failures; + if (ratio > ratio_max) ratio_max = ratio; + try std.testing.expectEqual(@as(usize, 0), failures); } - try std.testing.expect(contract_unequal <= (gate_tokens * hidden_width) / 100); + printEq("gate", "all-families-metal-vs-contract", cells_total, unequal_total); + printBound("gate-bound", "all-families", cells_total, failures_total, ratio_max); + try std.testing.expectEqual(@as(usize, 0), failures_total); const cb = queue.commandBuffer(); const e = cb.compute(.serial); try engram.encodeGate(e, &pipelines, gate, view(hidden_buffer), view(kv_buffer), view(gate_mask), view(masked_output), @intCast(gate_tokens)); @@ -361,5 +497,5 @@ test "Engram gate reaches BF16 at the FP32 contract over matching equations" { try completed(cb); const masked = masked_output.slice(u16, gate_tokens * hidden_width); try std.testing.expectEqualSlices(u16, hidden[hidden_width..][0..hidden_width], masked[hidden_width..][0..hidden_width]); - printEq("gate", "all-families-metal-vs-contract", gate_tokens * hidden_width, contract_unequal); + printEq("gate", "dead-mask-zero-gate", hidden_width, 0); } From 46a2fb354f446184e257917e92a99d383eff4867 Mon Sep 17 00:00:00 2001 From: akol1 Date: Tue, 20 Oct 2026 17:46:42 -0700 Subject: [PATCH 30/33] DeepSeek V4.1 reference: the index tie fixture broadcasts row 0 into every key --- zig/tests/deepseek_v41_reference/operator_csa2_index_fp64.zig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zig/tests/deepseek_v41_reference/operator_csa2_index_fp64.zig b/zig/tests/deepseek_v41_reference/operator_csa2_index_fp64.zig index 410a0115fc..3fa6e1b299 100644 --- a/zig/tests/deepseek_v41_reference/operator_csa2_index_fp64.zig +++ b/zig/tests/deepseek_v41_reference/operator_csa2_index_fp64.zig @@ -237,7 +237,8 @@ test "indexer scoring and selection sit at rounding level against the FP64 ideal } // Identical keys: every score is the same value, so the whole selection is tie-driven. if (std.mem.eql(u8, spec.name, "index_ties")) { - for (cells[128..], cells[0..128]) |*dst, src| dst.* = src; + // Broadcast row 0's 128 cells into every later row, so each key's slice is a copy of the first key's. + for (cells[128..], 0..) |*dst, i| dst.* = cells[i % 128]; } // A score cliff just inside the selection boundary: half the keys sit on the boundary. if (std.mem.eql(u8, spec.name, "index_boundary")) { From 2485c137ae5f323f51f1a8ace84ca952d110e06e Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:09:26 -0700 Subject: [PATCH 31/33] test(contract): DeepSeek V4.1 sparse attention through the Metal kernel contract dsv41_sparse_attention is compiled and launched on the served path but had no contract registration, so every gate on it was unchecked. This adds its probe, its launch record, and the source resolution gate 5 needs. Gates 1-5 pass; the receipt's SUMMARY line is in the pull request. The probe covers one chunk of rows with no ring and no compressed keys, so every key a row sees is a row of the chunk buffer. Two properties of the real call are modeled rather than assumed: - the key buffer is the packed FP4 cache, not linear bf16: a value is an e4 code and each 32 values share a power-of-two block scale, decoded to bf16. A NaN cannot be stored in it, so gate 1 plants non-finite inputs in the query, which is plain bf16, and the stated rule says what then happens. - the denominator sums unrounded probabilities while only the product rounds. Gate 4's stress inputs make the width-long dot cancel, where a narrow accumulator loses the result: fp32 is exact, fp16 misses by inf, bf16 by 1x. The M1 and M2 thread limits (768 both) come from --offline. The contract also stops travelling through the tensorfold facade here: it is its own module everywhere else on main, and re-exporting it made the module reachable two ways, which Zig rejects as "file exists in modules 'core' and 'contract'". The engram reference now imports it directly, as the other probes do. --- zig/build/deepseek_v41.zig | 1 + zig/src/core/contract/registry.zig | 1 + zig/src/tensorfold.zig | 2 - zig/tests/contract/metal_csa2.zig | 320 ++++++++++++++++++ zig/tests/contract/metal_limits.zig | 3 + zig/tests/contract/metal_main.zig | 7 + .../operator_engram_fp64.zig | 2 +- 7 files changed, 333 insertions(+), 3 deletions(-) create mode 100644 zig/tests/contract/metal_csa2.zig diff --git a/zig/build/deepseek_v41.zig b/zig/build/deepseek_v41.zig index 2e419e9755..27b571586a 100644 --- a/zig/build/deepseek_v41.zig +++ b/zig/build/deepseek_v41.zig @@ -50,6 +50,7 @@ pub fn checks( .imports = &.{ .{ .name = "tensorfold", .module = engine }, .{ .name = "metal", .module = metal }, + .{ .name = "contract", .module = @import("kernel_contract.zig").module(b, target, optimize) }, }, }), .filters = &.{"tests.deepseek_v41_reference."}, diff --git a/zig/src/core/contract/registry.zig b/zig/src/core/contract/registry.zig index faa99bb0da..308efc6102 100644 --- a/zig/src/core/contract/registry.zig +++ b/zig/src/core/contract/registry.zig @@ -518,6 +518,7 @@ pub const metal = [_]M{ .{ .name = "drama3 codec c_snake", .threads = 256, .tg_memory = 0, .reserved = 256 }, .{ .name = "drama3 codec c_dwconv7", .threads = 256, .tg_memory = 0, .reserved = 256 }, .{ .name = "drama3 codec c_copy", .threads = 256, .tg_memory = 0, .reserved = 256 }, + .{ .name = "deepseek v4.1 sparse attention dsv41_sparse_attention", .threads = 32, .tg_memory = 256, .m1 = 768, .m2 = 768 }, }; /// Gate 5's planted Metal controls: a launch past M1/M2's compiled limit (#708's class), 36 KiB of threadgroup memory. pub const metal_controls = [_]M{ diff --git a/zig/src/tensorfold.zig b/zig/src/tensorfold.zig index 2b732de3fa..367a7431a0 100644 --- a/zig/src/tensorfold.zig +++ b/zig/src/tensorfold.zig @@ -20,8 +20,6 @@ pub const lanes = @import("lanes"); pub const segments = @import("core/segments.zig"); pub const row_projection = @import("core/row_projection.zig"); pub const affine_mm = @import("core/affine_mm.zig"); -/// The kernel contract: accumulations, element types, shapes, tolerances and gates. -pub const contract = @import("core/contract/contract.zig"); /// The checkpoint formats (MLX affine and its fp64 reader for the device checks, exact widen). pub const quant = @import("core/quant/quant.zig"); pub const deepseek_v41_config = @import("families/deepseek_v41/config.zig"); diff --git a/zig/tests/contract/metal_csa2.zig b/zig/tests/contract/metal_csa2.zig new file mode 100644 index 0000000000..409cfc3dfa --- /dev/null +++ b/zig/tests/contract/metal_csa2.zig @@ -0,0 +1,320 @@ +//! DeepSeek V4.1's chunk-only sparse attention: no ring and no compressed keys, every key is a chunk row. +const std = @import("std"); +const mtl = @import("metal"); +const contract = @import("contract"); +const sources = @import("kernel_sources"); +const rig_ = @import("rig.zig"); + +const c = contract.spec; +const g = contract.gates; + +pub const heads = 4; +pub const width = 64; +pub const rows = 4; +pub const window = 128; +/// The chunk's rows at the prepared call: the gates shrink a served shape's rows, never its n or k. +pub var live: usize = rows; +const chunk_stride = width + width / 32; +const q_stride = heads * width; +const scale: f32 = 0.04419417382415922; + +/// One query element a lane, then the compensated lane reduction and the online denominator. +fn onlineDepth(s: c.Shape) usize { + return (s.k + 31) / 32 + 6; +} + +const why = "a bf16 output of a causal softmax over at most 512 keys: each probability is rounded to bf16 before its product, whose error stays under an output ulp a hundred times over"; + +pub fn spec() c.Spec { + const served = &[_]c.Shape{.{ .rows = rows, .n = heads, .k = rows }}; + return .{ + .name = "deepseek_v41 sparse attention dsv41_sparse_attention", + .accum = .f32, + .depth = onlineDepth, + .ops = 512, + .out = .bf16, + .tiles = .{ .{}, .{}, .{} }, + .served = served, + .nan = .{ .stated = "a NaN query makes every column of its head NaN; the packed key cache cannot hold one" }, + .sums = .{ .f16 = 1, .bf16 = 1, .why = why }, + }; +} + +const Params = extern struct { + rows: u32, + heads: u32, + width: u32, + q_stride: u32, + position: u32, + chunk_rows: u32, + window: u32, + ring_stride: u32, + chunk_stride: u32, + main_stride: u32, + main_block: u32, + ratio: u32, + compressed_rows: u32, + topk: u32, + index_stride: u32, + scale: f32, +}; + +fn params() Params { + return .{ + .rows = @intCast(live), + .heads = heads, + .width = width, + .q_stride = q_stride, + .position = 0, + .chunk_rows = @intCast(live), + .window = window, + .ring_stride = chunk_stride, + .chunk_stride = chunk_stride, + .main_stride = width / 2 + width / 16, + .main_block = 16, + .ratio = 0, + .compressed_rows = 0, + .topk = 0, + .index_stride = 512, + .scale = scale, + }; +} + +pub const Probe = struct { + a: std.mem.Allocator, + rig: rig_.Rig, + pipe: mtl.Pipeline, + q: rig_.Ref = undefined, + chunk: rig_.Ref = undefined, + sink: rig_.Ref = undefined, + out: rig_.Ref = undefined, + n: usize = 0, + + pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue) !Probe { + // kv.metal owns the exact packed-cache decoders this source is compiled after. + const source = try std.mem.concat(a, u8, &.{ sources.deepseek_v41_kv, sources.deepseek_v41_csa2 }); + defer a.free(source); + return .{ + .a = a, + .rig = .{ .a = a, .device = device, .queue = queue }, + .pipe = try rig_.pipeline(device, source, "dsv41_sparse_attention"), + }; + } + + pub fn deinit(self: *Probe) void { + self.rig.deinit(); + self.pipe.deinit(); + } + + pub fn kernel(self: *Probe) g.Kernel { + return .{ .spec = spec(), .ctx = self, .vt = &vt }; + } + + const vt = g.hooks(Probe); + + pub fn prepare(self: *Probe, shape: c.Shape, fill: g.Fill, seed: u64) !bool { + if (shape.n != heads or shape.rows == 0 or shape.rows > window or shape.k < 1 or shape.k > shape.rows) return false; + live = shape.rows; + self.rig.reset(); + var prng = std.Random.DefaultPrng.init(seed); + const rng = prng.random(); + const q = try self.a.alloc(u16, live * q_stride); + defer self.a.free(q); + // The chunk is the engine's packed FP4 cache: one byte a value, then a shared power-of-two scale a block. + const cache = try self.a.alloc(u8, live * chunk_stride); + defer self.a.free(cache); + const keys = try self.a.alloc(f32, live * width); + defer self.a.free(keys); + for (q) |*v| v.* = c.toBf16(rng.float(f32) * 2 - 1); + for (keys) |*v| v.* = rng.float(f32) * 2 - 1; + if (fill == .nonfinite) { + // The keys cannot hold a NaN: a value is an e4 code and a block scale is a power of two. + q[0] = c.toBf16(std.math.nan(f32)); + q[q_stride + 1] = c.toBf16(std.math.inf(f32)); + q[2 * q_stride + 2] = c.toBf16(-std.math.inf(f32)); + } + if (fill == .stress) { + // Large integer products that cancel down the column, so a narrow accumulator loses the result. + for (q, 0..) |*v, i| v.* = c.toBf16(if (i % 2 == 0) @as(f32, 3.0e4) else @as(f32, 1.0)); + for (keys, 0..) |*v, i| v.* = if (i % 4 < 2) 448 else -448; + } + for (0..live) |row| packRow(cache[row * chunk_stride ..][0..chunk_stride], keys[row * width ..][0..width]); + var sink: [heads]f32 = undefined; + for (&sink) |*s| s.* = rng.float(f32) - 0.5; + self.q = try self.rig.input(std.mem.sliceAsBytes(q)); + self.chunk = try self.rig.input(std.mem.sliceAsBytes(cache)); + self.sink = try self.rig.input(std.mem.sliceAsBytes(&sink)); + self.out = try self.rig.output(live * q_stride * 2); + self.n = live * heads * width; + return true; + } + + pub fn run(self: *Probe) !void { + const p = self.rig.begin(); + const e = p.e; + e.setPipeline(self.pipe); + // Buffers 1, 3 and 4 are the ring, compressed keys and index list: unused here, bound to a valid view. + e.setBuffer(self.q.buf, self.q.off, 0); + e.setBuffer(self.q.buf, self.q.off, 1); + e.setBuffer(self.chunk.buf, self.chunk.off, 2); + e.setBuffer(self.q.buf, self.q.off, 3); + e.setBuffer(self.q.buf, self.q.off, 4); + e.setBuffer(self.sink.buf, self.sink.off, 5); + e.setBuffer(self.out.buf, self.out.off, 6); + e.setValue(params(), 7); + e.dispatchGroups(mtl.Size.of(heads, @intCast(live), 1), mtl.Size.of(32, 1, 1)); + try self.rig.finish(p); + } + + /// Gate 1's own check: the kernel is trusted for its NaN rule, the planted inputs must both classes show. + pub fn check(self: *Probe, at: g.Gate) !?[]const u8 { + if (at != .nonfinite) return null; + const want = try self.a.alloc(f64, self.n); + defer self.a.free(want); + const mag = try self.a.alloc(f64, self.n); + defer self.a.free(mag); + try self.reference(want, mag); + const out = try self.rig.outputBytes(); + var bad: usize = 0; + var first: usize = self.n; + for (want, 0..) |w, i| { + if (c.classOf(c.read(.bf16, out, i)) == c.classOf(w)) continue; + bad += 1; + if (first == self.n) first = i; + } + if (bad == 0) return null; + return try std.fmt.allocPrint(self.a, "{d} of {d} outputs differ in class at the prepared call; first at {d}: {t} where the reference gives {t}", .{ bad, self.n, first, c.classOf(c.read(.bf16, out, first)), c.classOf(want[first]) }); + } + + pub fn count(self: *Probe) usize { + return self.n; + } + + /// The same products and bf16-rounded probabilities, with the per-column accumulation in `acc` and `order`. + pub fn simulate(self: *Probe, acc: c.Accum, order: g.Order, out: []f64) !void { + const q: [*]const u16 = @ptrCast(@alignCast(self.rig.bytes(self.q, live * q_stride * 2))); + const k: [*]const u8 = @ptrCast(@alignCast(self.rig.bytes(self.chunk, live * chunk_stride))); + const sink: [*]const f32 = @ptrCast(@alignCast(self.rig.bytes(self.sink, heads * 4))); + const scores = try self.a.alloc(f64, window); + defer self.a.free(scores); + const terms = try self.a.alloc(f64, window + 1); + defer self.a.free(terms); + for (0..live) |row| { + const scanned: usize = if (row == 0) 1 else window; + for (0..heads) |head| { + var mx: f64 = -std.math.inf(f64); + for (0..scanned) |j| { + if (j > row) { + scores[j] = -std.math.inf(f64); + continue; + } + // The width-long reduction is where a narrow accumulator loses the cancellations. + for (0..width) |d| terms[d] = bf(q[row * q_stride + head * width + d]) * unpack(k + j * chunk_stride, d); + scores[j] = contract.rows.sumIn(acc, order, terms[0..width]) * @as(f64, scale); + mx = @max(mx, scores[j]); + } + // The denominator is the kernel's own sum, so it accumulates in `acc` and `order` too. + for (scores[0..scanned], 0..) |s, j| terms[j] = @exp(s - mx); + terms[scanned] = @exp(@as(f64, sink[head]) - mx); + const denom = contract.rows.sumIn(acc, order, terms[0 .. scanned + 1]); + for (0..width) |d| { + const products = try self.a.alloc(f64, scanned); + defer self.a.free(products); + for (scores[0..scanned], 0..) |s, j| products[j] = roundBf(@exp(s - mx)) * unpack(k + j * chunk_stride, d); + out[row * q_stride + head * width + d] = contract.rows.sumIn(acc, order, products[0..scanned]) / denom; + } + } + } + } + + /// The same bf16 products and bf16-rounded probabilities, with every sum in FP64. + pub fn reference(self: *Probe, want: []f64, mag: []f64) !void { + const q: [*]const u16 = @ptrCast(@alignCast(self.rig.bytes(self.q, live * q_stride * 2))); + const k: [*]const u8 = @ptrCast(@alignCast(self.rig.bytes(self.chunk, live * chunk_stride))); + const sink: [*]const f32 = @ptrCast(@alignCast(self.rig.bytes(self.sink, heads * 4))); + const scores = try self.a.alloc(f64, window); + defer self.a.free(scores); + for (0..live) |row| { + // The kernel always scans a whole window; slots past the query position are skipped (a null key). + const scanned: usize = if (row == 0) 1 else window; + for (0..heads) |head| { + var mx: f64 = -std.math.inf(f64); + var columns: usize = 0; + for (0..scanned) |j| { + // position 0: slot 0 is chunk row 0, and no slot past the query is visible. + if (j > row) { + scores[j] = -std.math.inf(f64); + continue; + } + var dot: f64 = 0; + for (0..width) |d| { + dot += bf(q[row * q_stride + head * width + d]) * unpack(k + j * chunk_stride, d); + } + scores[j] = dot * @as(f64, scale); + // Metal's max drops a NaN operand, and every term feeding the dot is a rounded bf16. + if (std.math.isNan(scores[j])) scores[j] = -std.math.inf(f64); + mx = @max(mx, scores[j]); + columns += 1; + } + // The denominator adds the unrounded probabilities; only the product rounds. + var sum: f64 = 0; + for (scores[0..scanned]) |s| sum += @exp(s - mx); + const denom = sum + @exp(@as(f64, sink[head]) - mx); + for (0..width) |d| { + var acc: f64 = 0; + for (scores[0..scanned], 0..) |s, j| { + const probability = roundBf(@exp(s - mx)); + acc += probability * unpack(k + j * chunk_stride, d); + } + const i = row * q_stride + head * width + d; + want[i] = acc / denom; + mag[i] = @floatFromInt(columns); + } + } + } + } +}; + +/// A bf16 stored value as its exact float64. +fn bf(v: u16) f64 { + return @floatCast(@as(f32, @bitCast(@as(u32, v) << 16))); +} + +/// The kernel's `dsv41_e4`: a sign, a 4-bit exponent and a 3-bit mantissa. +fn e4(code: u8) f32 { + const e = (code >> 3) & 15; + const m = code & 7; + const v: f32 = if (e == 0) @as(f32, @floatFromInt(m)) * 0x1p-9 else std.math.ldexp(1.0 + @as(f32, @floatFromInt(m)) / 8.0, @as(i32, e) - 7); + return if (code & 128 != 0) -v else v; +} + +/// The kernel's `dsv41_e4code` for a value already reduced to one block's scale. +fn e4code(x: f32) u8 { + const a = @min(@abs(x), 448.0); + if (a == 0) return if (std.math.signbit(x)) 128 else 0; + const e = std.math.clamp(@as(i32, @intCast((@as(u32, @bitCast(a)) >> 23) & 255)) - 127, -6, 7); + const q: i32 = @intFromFloat(@round(std.math.ldexp(a, 3 - e))); + const bits: u32 = if (e == -6 and q < 8) @intCast(q) else @intCast((e + 7) * 8 + q - 8); + return @intCast(@min(bits, 126) | (if (std.math.signbit(x)) @as(u32, 128) else 0)); +} + +/// The kernel's `dsv41_unpack` for kind 0: an e4 value times a power-of-two block scale, then to bf16. +fn unpack(row: [*]const u8, d: usize) f64 { + const s: f32 = @bitCast(@as(u32, row[width + d / 32]) << 23); + return roundBf(@as(f64, e4(row[d])) * @as(f64, s)); +} + +/// Pack one chunk row the way `dsv41_pack` does, at a block scale of one so every code is exact. +fn packRow(out: []u8, x: []const f32) void { + for (x, 0..) |v, d| { + out[width + d / 32] = 127; + out[d] = e4code(v); + } +} + +/// The kernel's `dsv41_round`: to the nearest bf16, even on a tie. +fn roundBf(x: f64) f64 { + const f: f32 = @floatCast(x); + const u: u32 = @bitCast(f); + return @floatCast(@as(f32, @bitCast((u + 0x7fff + ((u >> 16) & 1)) & 0xffff_0000))); +} diff --git a/zig/tests/contract/metal_limits.zig b/zig/tests/contract/metal_limits.zig index 821404d2f0..7982bf1358 100644 --- a/zig/tests/contract/metal_limits.zig +++ b/zig/tests/contract/metal_limits.zig @@ -50,6 +50,9 @@ fn launchOf(a: std.mem.Allocator, device: mtl.Device, d: limits.MetalLaunch) !La frags = true; } else if (std.mem.startsWith(u8, d.name, "h3 ")) { text = @import("h3_kernels").source; + } else if (std.mem.startsWith(u8, d.name, "deepseek v4.1 sparse attention ")) { + // kv.metal owns the packed-cache decoders this source is compiled after. + text = try std.mem.concat(a, u8, &.{ sources.deepseek_v41_kv, sources.deepseek_v41_csa2 }); } else if (std.mem.startsWith(u8, d.name, "glm gather ")) { text = tf.glm.wide.gather_source; } else if (std.mem.startsWith(u8, d.name, "glm ")) { diff --git a/zig/tests/contract/metal_main.zig b/zig/tests/contract/metal_main.zig index 4725b1eae8..7a1e26d099 100644 --- a/zig/tests/contract/metal_main.zig +++ b/zig/tests/contract/metal_main.zig @@ -12,6 +12,7 @@ const h3_k = @import("metal_h3.zig"); const h3_topk = @import("metal_h3_topk.zig"); const drama3_k = @import("metal_drama3.zig"); const glm_eh_k = @import("metal_glm_eh.zig"); +const csa2 = @import("metal_csa2.zig"); const flash_pool = @import("metal_flashnext_pool.zig"); const flash = @import("metal_flashnext.zig"); @@ -233,6 +234,12 @@ pub fn main(init: std.process.Init) !u8 { ntiles += 1; }; for (tiles[0..ntiles]) |*x| try kernels.append(a, x.kernel()); + var csa2_p: ?csa2.Probe = null; + defer if (csa2_p) |*p| p.deinit(); + if (wanted(o, csa2.spec().name)) { + csa2_p = try csa2.Probe.init(a, device, queue); + try kernels.append(a, csa2_p.?.kernel()); + } var t: g.Tally = .{}; const t0 = std.Io.Clock.awake.now(init.io); if (o.gates) { diff --git a/zig/tests/deepseek_v41_reference/operator_engram_fp64.zig b/zig/tests/deepseek_v41_reference/operator_engram_fp64.zig index d20383b3b2..d1020e7834 100644 --- a/zig/tests/deepseek_v41_reference/operator_engram_fp64.zig +++ b/zig/tests/deepseek_v41_reference/operator_engram_fp64.zig @@ -6,7 +6,7 @@ const math = @import("math.zig"); const engram = tf.deepseek_v41_engram; const hash = tf.deepseek_v41_engram_hash; -const contract = tf.contract; +const contract = @import("contract").spec; const View = engram.View; const dim: usize = 5120; From 14db39001a763d6951ea78e339f067fd7f906e97 Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:03:24 -0700 Subject: [PATCH 32/33] Pass the span metadata the counter recorder now requires Recorder.beginSpan gained a metadata argument, but metal_counter_consumer still called it with two, so the tf-metal-counters build stopped compiling: zig build test does not build that consumer, which is why it went unseen. The loop that walks the span classes records its index as metadata, the same way the profiler records its stage. The three lifecycle tests pass zero. Verified with zig build tf-metal-counters and zig build test-metal-counters-host (six of six). --- zig/src/metal_counter_consumer.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zig/src/metal_counter_consumer.zig b/zig/src/metal_counter_consumer.zig index 3ea67c6991..e098e75b16 100644 --- a/zig/src/metal_counter_consumer.zig +++ b/zig/src/metal_counter_consumer.zig @@ -81,7 +81,7 @@ pub const Consumer = struct { const scales = [_]f32{ 2, 1, 0.5 }; const biases = [_]f32{ 1, -3, 2 }; for (classes, 0..) |kind, i| { - try recorder.beginSpan(&encoder, kind); + try recorder.beginSpan(&encoder, kind, @intCast(i)); const input = if (i % 2 == 0) self.a else self.b; const output = if (i % 2 == 0) self.b else self.a; encoder.setPipeline(self.pipeline); @@ -207,11 +207,11 @@ test "bounded recorder rejects nesting, overflow, open spans and premature reuse try std.testing.expectError(error.CounterEncoderStillOpen, recorder.discardAfterCompletion()); try std.testing.expectError(error.CounterRecordingInFlight, recorder.beginCommand(cb)); try std.testing.expectError(error.NoActiveCounterSpan, recorder.endSpan(&encoder)); - try recorder.beginSpan(&encoder, .dense); - try std.testing.expectError(error.NestedCounterSpan, recorder.beginSpan(&encoder, .copy)); + try recorder.beginSpan(&encoder, .dense, 0); + try std.testing.expectError(error.NestedCounterSpan, recorder.beginSpan(&encoder, .copy, 0)); try std.testing.expectError(error.UnclosedCounterSpan, recorder.finishEncoding(&encoder)); try recorder.endSpan(&encoder); - try std.testing.expectError(error.CounterSpanCapacityExceeded, recorder.beginSpan(&encoder, .copy)); + try std.testing.expectError(error.CounterSpanCapacityExceeded, recorder.beginSpan(&encoder, .copy, 0)); try recorder.finishEncoding(&encoder); try std.testing.expectError(error.CounterCommandNotCompleted, recorder.readAfterCompletion()); try recorder.cancelEncoding(&encoder); From e740bf0dc2efacdd54629d3c9a1fcfa565d927e4 Mon Sep 17 00:00:00 2001 From: akol1 <15747200+akol1@users.noreply.github.com> Date: Wed, 21 Oct 2026 14:47:39 -0700 Subject: [PATCH 33/33] Rebase the precision line onto 1.1.2 Keeps this line's own load.zig, whose Plan API is what the weight staging checks consume, and restores kv_state.zig, whose Metal geometry constants 1.1.2's logical-transaction API does not carry. Follows 1.1.2's one-name text config by reading eps, hc_eps and hc_sinkhorn_iters directly, and re-registers the V4.1 check steps beside the other families'. --- zig/build/metal_targets.zig | 9 + zig/src/families/deepseek_v41/attention.zig | 18 +- zig/src/families/deepseek_v41/kv_state.zig | 154 ++++++--- zig/src/families/deepseek_v41/load.zig | 320 +++++++++++++----- zig/src/families/deepseek_v41/model.zig | 16 +- .../deepseek_v41_reference/attention_math.zig | 10 +- .../deepseek_v41_reference/engram_math.zig | 6 +- zig/tests/deepseek_v41_reference/math.zig | 4 +- .../operator_attention.zig | 6 +- .../deepseek_v41_reference/operator_hc.zig | 10 +- .../operator_hc_amplify_fp64.zig | 4 +- .../operator_hc_fp64.zig | 8 +- .../operator_hc_split_fp64.zig | 20 +- .../operator_history.zig | 6 +- .../deepseek_v41_reference/operator_moe.zig | 8 +- zig/tests/deepseek_v41_reference/qualify.zig | 2 +- .../deepseek_v41_reference/reference.zig | 14 +- zig/tests/deepseek_v41_reference/trace.zig | 2 +- 18 files changed, 418 insertions(+), 199 deletions(-) diff --git a/zig/build/metal_targets.zig b/zig/build/metal_targets.zig index 655eeb540e..ebb300be1a 100644 --- a/zig/build/metal_targets.zig +++ b/zig/build/metal_targets.zig @@ -155,6 +155,10 @@ pub fn metalTargets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: s const kernels_mod = embedded(b, lib, "tensorfold.metallib"); counters_build.targets(b, target, optimize, metal, mods.sources, test_step); + // DeepSeek V4.1: the operator qualification steps registered beside the other families'. + const dsv41_build = @import("deepseek_v41.zig"); + test_step.dependOn(dsv41_build.weights(b, target, optimize, metal)); + // The engines registry: the families this platform's binaries serve; the CLI reads it for models/info/pull. const api = b.createModule(.{ .root_source_file = b.path("zig/src/core/engine_api.zig"), .target = target, .optimize = optimize, .link_libc = true, .imports = &.{.{ .name = "lanes", .module = mods.lanes }} }); const engines = b.createModule(.{ @@ -247,6 +251,11 @@ pub fn metalTargets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: s b.getInstallStep().dependOn(&b.addInstallFile(packed_lib, "lib/nemotron_packed.metallib").step); sources.addImport("nemotron_packed_metallib", embedded(b, packed_lib, "nemotron_packed.metallib")); const engine = mods.engine; + { + const dsv41 = dsv41_build.checks(b, target, optimize, metal, sources, engine); + test_step.dependOn(dsv41.device); + test_step.dependOn(dsv41.reference); + } // FlashNext's CPU contracts, with the family module's imports so the replay graph type-checks const flash_host = b.addTest(.{ .root_module = b.createModule(.{ .root_source_file = b.path("zig/flashnext_host.zig"), diff --git a/zig/src/families/deepseek_v41/attention.zig b/zig/src/families/deepseek_v41/attention.zig index a1f76f6886..930b4b5637 100644 --- a/zig/src/families/deepseek_v41/attention.zig +++ b/zig/src/families/deepseek_v41/attention.zig @@ -43,7 +43,7 @@ pub const Pipelines = struct { rotary: rope.Pipelines, sparse: csa2.Pipelines, - pub fn load(device: mtl.Device, capacity: u32, c: config.Text) !Pipelines { + pub fn load(device: mtl.Device, capacity: u32, c: config.Config) !Pipelines { var plain = try ops.Pipelines.load(device); errdefer plain.deinit(); var cache = try kv.Pipelines.load(device); @@ -55,7 +55,7 @@ pub const Pipelines = struct { const sparse = try csa2.Pipelines.load(device); return .{ .plain = plain, .cache = cache, .select = select, .rotary = rotary, .sparse = sparse }; } - pub fn residentBytes(capacity: u32, c: config.Text) usize { + pub fn residentBytes(capacity: u32, c: config.Config) usize { return rope.Pipelines.residentBytes(capacity, c.qk_rope_head_dim); } pub fn registerResidency(p: *const Pipelines, set: mtl.ResidencySet) void { @@ -137,7 +137,7 @@ fn norm(p: *const Pipelines, e: mtl.ComputeEncoder, x: View, weight: checkpoint. ops.encodeRms(e, &p.plain, x, tensor(weight), x, .{ .rows = rows, .width = width, .x_stride = width, .y_stride = width, .eps = eps, .weight_f32 = weight.dtype == .f32 }); e.barrier(); } -fn rotate(p: *const Pipelines, e: mtl.ComputeEncoder, x: View, rows: u32, heads: u32, width: u32, position: u32, ratio: u32, step: u32, inverse: bool, c: config.Text) !void { +fn rotate(p: *const Pipelines, e: mtl.ComputeEncoder, x: View, rows: u32, heads: u32, width: u32, position: u32, ratio: u32, step: u32, inverse: bool, c: config.Config) !void { try p.rotary.encode(e, x, .{ .rows = rows, .heads = heads, .width = width, .rotary = c.qk_rope_head_dim, .stride = heads * width, .position = position, .step = step, .inverse = @intFromBool(inverse), .spectrum = if (ratio == 0) .normal else .compressed }); } fn precedingIndex(layer: u32) u32 { @@ -152,12 +152,12 @@ fn precedingIndex(layer: u32) u32 { /// BF16 x/y [rows,5120]; layers 0..39 share the chunk-start cursor, appended by the caller after all layers. // No extra RMS after wq_b or index-query Hadamard outside EXL3. -pub fn encode(p: *const Pipelines, e: mtl.ComputeEncoder, w: *const weights.Attention, state: *State, work: *Workspace, x: View, y: View, ep: *const exl3.Pipelines, scratch: exl3.View, layer: u32, rows: u32, c: config.Text) !void { +pub fn encode(p: *const Pipelines, e: mtl.ComputeEncoder, w: *const weights.Attention, state: *State, work: *Workspace, x: View, y: View, ep: *const exl3.Pipelines, scratch: exl3.View, layer: u32, rows: u32, c: config.Config) !void { return encodeDiagnostic(p, e, w, state, work, x, y, ep, scratch, layer, rows, c, false, null); } /// Same encoder and arithmetic as encode, with optional copies at operator boundaries. -pub fn encodeDiagnostic(p: *const Pipelines, e: mtl.ComputeEncoder, w: *const weights.Attention, state: *State, work: *Workspace, x: View, y: View, ep: *const exl3.Pipelines, scratch: exl3.View, layer: u32, rows: u32, c: config.Text, comptime diagnostic: bool, observer: ?DiagnosticObserver) !void { +pub fn encodeDiagnostic(p: *const Pipelines, e: mtl.ComputeEncoder, w: *const weights.Attention, state: *State, work: *Workspace, x: View, y: View, ep: *const exl3.Pipelines, scratch: exl3.View, layer: u32, rows: u32, c: config.Config, comptime diagnostic: bool, observer: ?DiagnosticObserver) !void { if (layer >= 40 or rows == 0 or rows > work.max_rows or state.position > state.plan.capacity or rows > state.plan.capacity - state.position) return error.InvalidAttentionChunk; @@ -173,7 +173,7 @@ pub fn encodeDiagnostic(p: *const Pipelines, e: mtl.ComputeEncoder, w: *const we e.barrier(); try projection(ep, e, .{ .trellis = w.wq_a }, x, qr, scratch, rows, c.hidden_size, c.q_lora_rank, .bf16); if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .qr_projection, qr); - norm(p, e, qr, w.q_norm, rows, c.q_lora_rank, c.rms_norm_eps); + norm(p, e, qr, w.q_norm, rows, c.q_lora_rank, c.eps); if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .qr_norm, qr); try projection(ep, e, .{ .trellis = w.wq_b }, qr, q, scratch, rows, c.q_lora_rank, q_width, .bf16); if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .q_projection, q); @@ -181,7 +181,7 @@ pub fn encodeDiagnostic(p: *const Pipelines, e: mtl.ComputeEncoder, w: *const we if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .q, q); try projection(ep, e, .{ .trellis = w.wkv }, x, raw, scratch, rows, c.hidden_size, c.head_dim, .bf16); if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .kv_projection, raw); - norm(p, e, raw, w.kv_norm, rows, c.head_dim, c.rms_norm_eps); + norm(p, e, raw, w.kv_norm, rows, c.head_dim, c.eps); if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .kv_norm, raw); try rotate(p, e, raw, rows, 1, c.head_dim, position, ratio, 1, false, c); if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .kv_rope, raw); @@ -206,14 +206,14 @@ pub fn encodeDiagnostic(p: *const Pipelines, e: mtl.ComputeEncoder, w: *const we const gate = compressor.wgate orelse return error.MissingAttentionCompressorGate; try projection(ep, e, gate, x, gates, scratch, rows, c.hidden_size, c.head_dim, .f32); } - try p.cache.encodePool(e, values, gates, source.partial, tensor(compressor.norm), latent, .{ .rows = rows, .ratio = ratio, .position = position, .stride = c.head_dim, .input_f32 = @intFromBool(ratio > 1), .eps = c.rms_norm_eps }); + try p.cache.encodePool(e, values, gates, source.partial, tensor(compressor.norm), latent, .{ .rows = rows, .ratio = ratio, .position = position, .stride = c.head_dim, .input_f32 = @intFromBool(ratio > 1), .eps = c.eps }); if (complete != 0) { const selector = if (w.indexer) |*value| value else return error.MissingAttentionIndexer; const wk = selector.wk orelse return error.MissingAttentionIndexKeyProjection; const k_norm = selector.k_norm orelse return error.MissingAttentionIndexKeyNorm; const key = work.view(.index_k); try projection(ep, e, wk, latent, key, scratch, complete, c.head_dim, c.index_head_dim, .bf16); - norm(p, e, key, k_norm, complete, c.index_head_dim, c.rms_norm_eps); + norm(p, e, key, k_norm, complete, c.index_head_dim, c.eps); try rotate(p, e, key, complete, 1, c.index_head_dim, position / ratio * ratio, ratio, ratio, false, c); try p.cache.encodePack(e, key, source.index, .{ .rows = complete, .width = c.index_head_dim, .stride = c.index_head_dim, .packed_stride = kv.cache.index_stride, .block = 32, .kind = .index_fp4, .first = position / ratio }); } diff --git a/zig/src/families/deepseek_v41/kv_state.zig b/zig/src/families/deepseek_v41/kv_state.zig index 708257cf8d..f2b70175a7 100644 --- a/zig/src/families/deepseek_v41/kv_state.zig +++ b/zig/src/families/deepseek_v41/kv_state.zig @@ -1,61 +1,111 @@ -// Shared logical KV transactions. Backends supply copies and explicit physical formats. +//! Shared cache ownership and exact transaction snapshots; appended rows are cursor-delimited. const std = @import("std"); -pub const Encoding = enum { bf16, f32, cuda_fp8_584, cuda_index_fp8_132, metal_fp4_288, metal_index_fp4_68 }; -pub const Region = struct { - live: u64, - backup: u64, - row_bytes: usize, - rows: u32, - kind: enum { ring, carry }, - encoding: Encoding = .bf16, - pub fn bytes(r: Region) usize { - return r.row_bytes * r.rows; +const mtl = @import("metal"); +const View = @import("view.zig").View; +pub const owners = [_]u32{ 2, 8, 14, 20 }; +pub const ratios = [_]u32{ 2, 2, 2, 1 }; +pub const index_sources = [_]u32{ 2, 8, 14, 20, 24, 28, 32, 36 }; +pub const main_stride = 288; +pub const index_stride = 68; +pub const ring_stride = 528; +pub const window = 128; +pub const max_context = 1048576; + +pub fn ownerFor(layer: u32, ratio: u32) !usize { + if (ratio == 0) return error.NoCompressedCache; + if (layer >= 40) return error.InvalidCacheLayer; + if (layer < 2 or ratio != (if (layer < 20) @as(u32, 2) else 1)) return error.InvalidCacheOwner; + var found: ?usize = null; + for (owners, 0..) |owner, i| { + if (owner <= layer and ratios[i] == ratio) found = i; } -}; -pub const Copy = struct { - ctx: *anyopaque, - run: *const fn (*anyopaque, u64, u64, usize) anyerror!void, - pub fn copy(c: Copy, dst: u64, src: u64, bytes: usize) !void { - try c.run(c.ctx, dst, src, bytes); + return found orelse error.InvalidCacheOwner; +} +pub fn isIndexSource(layer: u32) bool { + return std.mem.indexOfScalar(u32, &index_sources, layer) != null; +} +pub const Source = struct { owner: u32, ratio: u32, rows: u32, main: View, index: View, partial: View }; +pub const Plan = struct { + capacity: u32, + layers: u32, + pub fn validate(p: Plan) !void { + if (p.capacity == 0 or p.capacity > max_context or p.layers != 40) + return error.InvalidCacheCapacity; + } + pub fn packedBytes(p: Plan) usize { + var bytes: usize = 0; + for (ratios) |r| bytes += @as(usize, p.capacity / r) * (main_stride + index_stride); + return bytes; + } + pub fn mutableBytes(p: Plan) usize { + return @as(usize, p.layers) * window * ring_stride + 7 * 2 * 512 * 4; + } + pub fn residentBytes(p: Plan) usize { + return p.packedBytes() + 2 * p.mutableBytes(); } }; pub const State = struct { - position: u64 = 0, - pending: ?u32 = null, - // A failed copy leaves the transaction active. The owner must retry or retain its buffers. - pub fn begin(s: *State, regions: []const Region, c: Copy, n: u32) !void { - if (s.pending != null) return error.CacheTransactionActive; - if (n == 0 or n > 64 or s.position > std.math.maxInt(u64) - @as(u64, n)) return error.BadWindow; - for (regions) |r| { - _ = std.math.mul(usize, r.row_bytes, r.rows) catch return error.BadRegion; - if (r.row_bytes == 0 or r.rows == 0 or (r.kind == .ring and n > r.rows)) return error.BadRegion; - } - for (regions) |r| try c.copy(r.backup, r.live, r.bytes()); - s.pending = n; - } - // Carries are selected from the accepted projection before keep. A full rollback restores them here. - pub fn keep(s: *State, regions: []const Region, c: Copy, count: u32) !void { - const n = s.pending orelse return error.NoCacheTransaction; - if (count > n) return error.BadWindow; - for (regions) |r| { - if (count == 0) { - try c.copy(r.live, r.backup, r.bytes()); - } else if (r.kind == .ring) { - // Only rejected positions were overwritten. Accepted rows stay in their physical ring slots. - for (count..n) |i| { - const off = ((s.position + i) % r.rows) * r.row_bytes; - try c.copy(r.live + off, r.backup + off, r.row_bytes); - } - } + plan: Plan, + storage: mtl.Buffer, + live: mtl.Buffer, + backup: mtl.Buffer, + position: u32 = 0, + saved_position: ?u32 = null, + + pub fn init(device: mtl.Device, plan: Plan) !State { + try plan.validate(); + const storage = try device.buffer(plan.packedBytes(), mtl.ResourceOptions.shared); + errdefer storage.deinit(); + const live = try device.buffer(plan.mutableBytes(), mtl.ResourceOptions.shared); + errdefer live.deinit(); + const backup = try device.buffer(plan.mutableBytes(), mtl.ResourceOptions.shared); + @memset(live.contents()[0..plan.mutableBytes()], 0); + return .{ .plan = plan, .storage = storage, .live = live, .backup = backup }; + } + pub fn deinit(s: *State) void { + s.storage.deinit(); + s.live.deinit(); + s.backup.deinit(); + } + pub fn residentBytes(s: *const State) usize { + return s.plan.residentBytes(); + } + pub fn registerResidency(s: *const State, set: mtl.ResidencySet) void { + set.add(s.storage); + set.add(s.live); + set.add(s.backup); + } + pub fn ring(s: *const State, layer: u32) !View { + if (layer >= s.plan.layers) return error.InvalidCacheLayer; + return .{ .buffer = s.live, .offset = @as(usize, layer) * window * ring_stride }; + } + pub fn source(s: *const State, layer: u32, ratio: u32) !Source { + if (layer >= s.plan.layers) return error.InvalidCacheLayer; + if (s.position > s.plan.capacity) return error.CacheCapacityExceeded; + const id = try ownerFor(layer, ratio); + var packed_off: usize = 0; + var partial_off: usize = @as(usize, s.plan.layers) * window * ring_stride; + for (ratios[0..id]) |r| { + packed_off += @as(usize, s.plan.capacity / r) * (main_stride + index_stride); + partial_off += 2 * r * 512 * 4; } - s.position += count; - s.pending = null; + return .{ .owner = owners[id], .ratio = ratio, .rows = s.position / ratio, .main = .{ .buffer = s.storage, .offset = packed_off }, .index = .{ .buffer = s.storage, .offset = packed_off + @as(usize, s.plan.capacity / ratio) * main_stride }, .partial = .{ .buffer = s.live, .offset = partial_off } }; + } + pub fn append(s: *State, rows: u32) !void { + if (s.position > s.plan.capacity or rows == 0 or rows > s.plan.capacity - s.position) + return error.CacheCapacityExceeded; + s.position += rows; } - pub fn rollback(s: *State, regions: []const Region, c: Copy) !void { - try s.keep(regions, c, 0); + pub fn begin(s: *State) !void { + if (s.saved_position != null) return error.CacheTransactionActive; + s.saved_position = s.position; + } + pub fn commit(s: *State) !void { + if (s.saved_position == null) return error.NoCacheTransaction; + s.saved_position = null; + } + pub fn rollback(s: *State) !void { + s.position = s.saved_position orelse return error.NoCacheTransaction; + s.saved_position = null; } }; - -test { - _ = @import("kv_state_test.zig"); -} diff --git a/zig/src/families/deepseek_v41/load.zig b/zig/src/families/deepseek_v41/load.zig index 6d9476e97b..422af3555c 100644 --- a/zig/src/families/deepseek_v41/load.zig +++ b/zig/src/families/deepseek_v41/load.zig @@ -1,89 +1,249 @@ -//! Shared checkpoint admission and host images. Backend resources belong in adapters. -pub const Pack = @import("pack.zig").Pack; -pub const Info = @import("pack.zig").Info; -pub const plan = @import("plan.zig"); -pub const stage = @import("stage.zig"); -pub const named = @import("named.zig"); -pub const prepared = @import("prepared.zig"); +//! Header-first resident loading: no mapped weights, deferred disk lookup or duplicate Engram projections. const std = @import("std"); -/// Strict production checkpoint admission. Pack itself also supports partial header inspection. -pub fn open(a: std.mem.Allocator, io: std.Io, dir: []const u8) !Pack { - var p = try Pack.open(a, io, dir); - errdefer p.deinit(); - if (p.missing != 0) return error.CheckpointIndexMismatch; - var owners = p.owner.iterator(); - while (owners.next()) |e| { - const info = p.get(e.key_ptr.*) orelse return error.CheckpointIndexMismatch; - if (!std.mem.eql(u8, p.files.items[info.file], e.value_ptr.*)) return error.CheckpointIndexMismatch; +const mtl = @import("metal"); +const st = @import("../../core/safetensors.zig"); +const cp = @import("../../core/checkpoint_metal.zig"); +const paced = @import("../../core/paced_read.zig"); + +pub const Reference = struct { shard: usize, entry: st.Entry }; +pub const Shard = struct { + path: [:0]const u8, + file_bytes: usize, + data_start: usize, + begin: usize, + end: usize, + names: st.Header, +}; + +pub const Plan = struct { + arena: std.heap.ArenaAllocator, + shards: std.ArrayList(Shard) = .empty, + tensors: std.StringHashMapUnmanaged(Reference) = .empty, + bytes: usize = 0, + + pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, table_dir: []const u8) !Plan { + var self = Plan{ .arena = std.heap.ArenaAllocator.init(gpa) }; + errdefer self.deinit(); + const a = self.arena.allocator(); + const index_path = try std.fs.path.join(a, &.{ dir, "model.safetensors.index.json" }); + const index_bytes = try std.Io.Dir.cwd().readFileAlloc(io, index_path, a, .limited(64 << 20)); + const index = try std.json.parseFromSliceLeaky(struct { weight_map: std.json.Value }, a, index_bytes, .{ + .ignore_unknown_fields = true, + }); + if (index.weight_map != .object or index.weight_map.object.count() == 0) return error.BadCheckpointIndex; + var files: std.StringHashMapUnmanaged(void) = .empty; + var entries = index.weight_map.object.iterator(); + while (entries.next()) |item| { + if (item.value_ptr.* != .string) return error.BadCheckpointIndex; + const file = item.value_ptr.string; + if (!safeBasename(file)) return error.BadCheckpointIndex; + try files.put(a, file, {}); + } + const ordered = try a.alloc([]const u8, files.count()); + var file_it = files.keyIterator(); + var n: usize = 0; + while (file_it.next()) |file| : (n += 1) ordered[n] = file.*; + std.mem.sort([]const u8, ordered, {}, less); + for (ordered) |file| try self.add(try std.fs.path.join(a, &.{ dir, file }), null); + if (self.tensors.count() != index.weight_map.object.count()) return error.CheckpointIndexMismatch; + entries = index.weight_map.object.iterator(); + while (entries.next()) |item| { + const ref = self.tensors.get(item.key_ptr.*) orelse return error.CheckpointIndexMismatch; + if (!std.mem.eql(u8, std.fs.path.basename(self.shards.items[ref.shard].path), item.value_ptr.string)) + return error.CheckpointIndexMismatch; + } + for ([_]struct { layer: u32, file: []const u8, rows: usize }{ + .{ .layer = 1, .file = "model-00047-of-00048.safetensors", .rows = 384006168 }, + .{ .layer = 14, .file = "model-00048-of-00048.safetensors", .rows = 384016682 }, + }) |table| { + const prefix = try std.fmt.allocPrint(a, "layers.{d}.engram.embed.", .{table.layer}); + try self.add(try std.fs.path.join(a, &.{ table_dir, table.file }), prefix); + const weight = try std.fmt.allocPrint(a, "{s}weight", .{prefix}); + const scale = try std.fmt.allocPrint(a, "{s}scale", .{prefix}); + try self.require(weight, .f8_e4m3, &.{ table.rows, 256 }); + try self.require(scale, .f8_e8m0, &.{ table.rows, 8 }); + if (self.shards.items[self.shards.items.len - 1].names.count() != 2) return error.UnexpectedEngramTensor; + } + return self; + } + + pub fn deinit(self: *Plan) void { + self.arena.deinit(); + self.* = undefined; + } + + pub fn get(self: *const Plan, name: []const u8) !st.Entry { + return (self.tensors.get(name) orelse return error.MissingTensor).entry; + } + + pub fn has(self: *const Plan, name: []const u8) bool { + return self.tensors.contains(name); + } + + pub fn require(self: *const Plan, name: []const u8, dtype: st.DType, shape: []const usize) !void { + const e = try self.get(name); + if (e.dtype != dtype or e.rank != shape.len or !std.mem.eql(usize, e.shape[0..e.rank], shape)) { + std.log.err("unsupported tensor {s}: dtype {s}, shape {any}", .{ name, @tagName(e.dtype), e.shape[0..e.rank] }); + return error.UnsupportedTensor; + } } - return p; + + fn add(self: *Plan, path: []const u8, prefix: ?[]const u8) !void { + const a = self.arena.allocator(); + const own_path = try a.dupeSentinel(u8, path, 0); + const fd = std.c.open(own_path, .{ .ACCMODE = .RDONLY }); + if (fd < 0) return error.OpenFailed; + defer _ = std.c.close(fd); + const length = std.c.lseek(fd, 0, std.c.SEEK.END); + if (length < 8) return error.BadSafetensors; + const file_bytes: usize = @intCast(length); + var head: [8]u8 = undefined; + try readAll(fd, &head, 0); + const header_bytes = std.mem.readInt(u64, &head, .little); + if (header_bytes > @min(file_bytes - 8, 64 << 20)) return error.BadSafetensors; + const json = try a.alloc(u8, @intCast(header_bytes)); + try readAll(fd, json, 8); + const data_start: usize = 8 + @as(usize, @intCast(header_bytes)); + const names = try st.parseHeaderPrefix(a, json, file_bytes - data_start, prefix); + if (names.count() == 0) return error.MissingTensor; + var begin: usize = if (prefix == null) 0 else file_bytes - data_start; + var end: usize = if (prefix == null) file_bytes - data_start else 0; + for (names.keys(), names.values()) |name, entry| { + begin = @min(begin, entry.begin); + end = @max(end, entry.end); + const slot = try self.tensors.getOrPut(a, name); + if (slot.found_existing) return error.DuplicateTensor; + slot.value_ptr.* = .{ .shard = self.shards.items.len, .entry = entry }; + } + if (prefix != null) { + var selected_bytes: usize = 0; + for (names.values()) |entry| selected_bytes = try std.math.add(usize, selected_bytes, entry.end - entry.begin); + if (selected_bytes != end - begin) return error.NoncontiguousEngramTables; + } + self.bytes = try std.math.add(usize, self.bytes, end - begin); + try self.shards.append(a, .{ .path = own_path, .file_bytes = file_bytes, .data_start = data_start, .begin = begin, .end = end, .names = names }); + } + + /// Only called after architecture validation and the complete memory admission calculation. + pub fn materialize(self: *const Plan, gpa: std.mem.Allocator, device: mtl.Device, residency: mtl.ResidencySet, queue: mtl.Queue) !cp.Checkpoint { + var out = cp.Checkpoint.init(gpa); + errdefer out.deinit(); + const stage = try paced.stage(gpa, 32 << 20); + defer gpa.free(stage); + var warm = mtl.keepalive.Target{ .queue = queue, .sets = @as(*const [1]mtl.ResidencySet, &residency) }; + var last_tick: f64 = -std.math.inf(f64); + var short: paced.Memory = undefined; + for (self.shards.items, 0..) |shard, i| { + const fd = std.c.open(shard.path, .{ .ACCMODE = .RDONLY }); + if (fd < 0) return error.OpenFailed; + defer _ = std.c.close(fd); + if (std.c.fcntl(fd, 48, @as(c_int, 1)) != 0) return error.UncachedReadUnavailable; + if (std.c.lseek(fd, 0, std.c.SEEK.END) != shard.file_bytes) return error.CheckpointChanged; + const bytes = shard.end - shard.begin; + const buffer = try device.buffer(@max(bytes, 16), mtl.ResourceOptions.shared | mtl.ResourceOptions.untracked); + var owned = false; + errdefer if (!owned) buffer.deinit(); + try out.own(buffer, bytes); + owned = true; + residency.add(buffer); + residency.commit(); + residency.requestResidency(); + var copied: usize = 0; + while (copied < bytes) { + const count = @min(bytes - copied, stage.len - paced.alignment); + paced.read(fd, buffer.contents()[copied..][0..count], shard.data_start + shard.begin + copied, stage, &short) catch |err| { + if (err == error.MemoryShort) { + var msg: [512]u8 = undefined; + std.log.err("deepseek-v41: {s}", .{paced.refusal(&msg, short, paced.floor())}); + } + return err; + }; + copied += count; + const now = mtl.clock.seconds(); + if (now - last_tick >= 0.4) { + mtl.keepalive.Target.tick(&warm); + last_tick = now; + } + } + for (shard.names.keys(), shard.names.values()) |name, e| { + try out.set(name, .{ .buffer = buffer, .offset = e.begin - shard.begin, .bytes = e.end - e.begin, .dtype = e.dtype, .shape = e.shape, .rank = e.rank }); + } + std.log.info("resident checkpoint shard {d}/{d}: {d} bytes, residency set {d} bytes", .{ i + 1, self.shards.items.len, bytes, residency.allocatedSize() }); + } + residency.commit(); + residency.requestResidency(); + return out; + } +}; + +fn readAll(fd: std.c.fd_t, dest: []u8, at: usize) !void { + var done: usize = 0; + while (done < dest.len) { + const n = std.c.pread(fd, dest.ptr + done, dest.len - done, @intCast(at + done)); + if (n <= 0) return error.ShortRead; + done += @intCast(n); + } +} + +fn safeBasename(s: []const u8) bool { + return s.len > 0 and !std.mem.eql(u8, s, ".") and !std.mem.eql(u8, s, "..") and + std.mem.indexOfAny(u8, s, "/\\") == null and std.mem.indexOfScalar(u8, s, 0) == null and + std.mem.endsWith(u8, s, ".safetensors"); } -pub const Projection = Pack.Projection; -pub const projection = @import("projection.zig").read; -test "shared projection requires dense rotation scales and refuses half-bit groups" { - var p = Pack.init(std.testing.allocator); - defer p.deinit(); - try p.addHeader("dense", "{\"p.weight\":{\"dtype\":\"BF16\",\"shape\":[128,128],\"data_offsets\":[0,32768]}}", 32768); - try std.testing.expectError(error.MissingTensor, projection(&p, "p")); - try std.testing.expectError(error.BadTrellis, @import("core").exl3_layout.Group.fromShape(&.{ 8, 8, 8 })); +fn less(_: void, a: []const u8, b: []const u8) bool { + return std.mem.lessThan(u8, a, b); } -test "shared projection admits packed sign owners and gives dense siblings precedence" { + +test "selected table plan excludes projections and preserves tensor byte offsets" { const a = std.testing.allocator; - const image = try @import("pack.zig").image(a, &.{ - .{ .name = "p.trellis", .dtype = "I16", .shape = &.{ 8, 8, 32 } }, - .{ .name = "p.su", .dtype = "I16", .shape = &.{8} }, - .{ .name = "p.sv", .dtype = "I16", .shape = &.{8} }, - }); - defer a.free(image); - var p = Pack.init(a); - defer p.deinit(); - try p.addHeaderDump("packed", image); - var packed_plan = try @import("model.zig").Plan.init(a, &p, &.{"p"}); - defer packed_plan.deinit(); - const packed_storage = packed_plan.projections[0].storage; - try std.testing.expect(packed_storage == .exl3); - try std.testing.expectEqual(p.get("p.su").?, packed_storage.exl3.scales.input); - try std.testing.expectEqual(p.get("p.sv").?, packed_storage.exl3.scales.output); - try std.testing.expectError(error.UnsupportedPackedScales, packed_storage.cudaGroup()); - const dense = try @import("pack.zig").image(a, &.{ - .{ .name = "p.weight", .dtype = "BF16", .shape = &.{ 128, 256 } }, - .{ .name = "p.suh", .dtype = "F16", .shape = &.{256} }, - .{ .name = "p.svh", .dtype = "F16", .shape = &.{128} }, - }); - defer a.free(dense); - var both = Pack.init(a); - defer both.deinit(); - try both.addHeaderDump("dense", dense); - const trellis = try @import("pack.zig").image(a, &.{.{ .name = "p.trellis", .dtype = "I16", .shape = &.{ 16, 8, 32 } }}); - defer a.free(trellis); - try both.addHeaderDump("trellis", trellis); - var dense_plan = try @import("model.zig").Plan.init(a, &both, &.{"p"}); - defer dense_plan.deinit(); - const dense_storage = dense_plan.projections[0].storage; - try std.testing.expect(dense_storage == .dense); - try std.testing.expectEqual(@as(usize, 256), dense_storage.dense.k); - try std.testing.expectEqual(@as(usize, 128), dense_storage.dense.n); - try std.testing.expectEqual(both.get("p.weight").?, dense_storage.dense.weight); - try std.testing.expectEqual(both.get("p.suh").?, dense_storage.dense.scales.input); - try std.testing.expectEqual(both.get("p.svh").?, dense_storage.dense.scales.output); - try std.testing.expectError(error.UnsupportedDenseProjection, dense_storage.cudaGroup()); + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const header = + \\{"padding":{"dtype":"U8","shape":[3],"data_offsets":[0,3]}, + \\ "engram.embed.weight":{"dtype":"U8","shape":[4],"data_offsets":[3,7]}, + \\ "engram.embed.scale":{"dtype":"U8","shape":[2],"data_offsets":[7,9]}, + \\ "engram.wkv":{"dtype":"U8","shape":[1],"data_offsets":[9,10]}} + ; + var data: [8 + header.len + 10]u8 = undefined; + std.mem.writeInt(u64, data[0..8], header.len, .little); + @memcpy(data[8..][0..header.len], header); + @memset(data[8 + header.len ..], 0); + try tmp.dir.writeFile(io, .{ .sub_path = "table.safetensors", .data = &data }); + const path = try std.fmt.allocPrint(a, ".zig-cache/tmp/{s}/table.safetensors", .{tmp.sub_path}); + defer a.free(path); + var plan = Plan{ .arena = std.heap.ArenaAllocator.init(a) }; + defer plan.deinit(); + try plan.add(path, "engram.embed."); + try std.testing.expectEqual(@as(usize, 6), plan.bytes); + try std.testing.expectEqual(@as(usize, 3), plan.shards.items[0].begin); + try std.testing.expectEqual(@as(usize, 9), plan.shards.items[0].end); + try std.testing.expectEqual(@as(usize, 7), (try plan.get("engram.embed.scale")).begin); + try std.testing.expectError(error.MissingTensor, plan.get("engram.wkv")); + try std.testing.expectError(error.DuplicateTensor, plan.add(path, "engram.embed.")); } -test "odd half-widths require a mul1 marker and scale pairs cannot be ambiguous" { + +test "selected table payload must be contiguous and indexed files cannot escape checkpoint directory" { const a = std.testing.allocator; - const data = try @import("pack.zig").image(a, &.{ - .{ .name = "p.trellis", .dtype = "I16", .shape = &.{ 8, 8, 24 } }, - .{ .name = "p.suh", .dtype = "F16", .shape = &.{128} }, - .{ .name = "p.svh", .dtype = "F16", .shape = &.{128} }, - }); - defer a.free(data); - var p = Pack.init(a); - defer p.deinit(); - try p.addHeaderDump("odd", data); - try std.testing.expectError(error.BadMarker, p.codebook(std.testing.io, "p")); - try std.testing.expectError(error.BadMarker, projection(&p, "p")); - const signs = try @import("pack.zig").image(a, &.{.{ .name = "p.su", .dtype = "I16", .shape = &.{8} }}); - defer a.free(signs); - try p.addHeaderDump("signs", signs); - try std.testing.expectError(error.AmbiguousScales, projection(&p, "p")); + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const header = + \\{"engram.embed.weight":{"dtype":"U8","shape":[4],"data_offsets":[0,4]}, + \\ "projection":{"dtype":"U8","shape":[3],"data_offsets":[4,7]}, + \\ "engram.embed.scale":{"dtype":"U8","shape":[2],"data_offsets":[7,9]}} + ; + var data: [8 + header.len + 9]u8 = undefined; + std.mem.writeInt(u64, data[0..8], header.len, .little); + @memcpy(data[8..][0..header.len], header); + @memset(data[8 + header.len ..], 0); + try tmp.dir.writeFile(io, .{ .sub_path = "gap.safetensors", .data = &data }); + const path = try std.fmt.allocPrint(a, ".zig-cache/tmp/{s}/gap.safetensors", .{tmp.sub_path}); + defer a.free(path); + var plan = Plan{ .arena = std.heap.ArenaAllocator.init(a) }; + defer plan.deinit(); + try std.testing.expectError(error.NoncontiguousEngramTables, plan.add(path, "engram.embed.")); + for ([_][]const u8{ "../x.safetensors", "/x.safetensors", "a\\x.safetensors", ".", "..", "x.safetensors\x00", "x.json", "" }) |name| try std.testing.expect(!safeBasename(name)); + try std.testing.expect(safeBasename("model-00001-of-00039.safetensors")); } diff --git a/zig/src/families/deepseek_v41/model.zig b/zig/src/families/deepseek_v41/model.zig index 443b3b2747..f1187afa00 100644 --- a/zig/src/families/deepseek_v41/model.zig +++ b/zig/src/families/deepseek_v41/model.zig @@ -39,7 +39,7 @@ const Pipelines = struct { attention: attention.Pipelines, groups: expert_groups.Pipelines, - fn init(device: mtl.Device, capacity: u32, c: config.Text) !Pipelines { + fn init(device: mtl.Device, capacity: u32, c: config.Config) !Pipelines { var ep = try exl3.Pipelines.load(device); errdefer ep.deinit(); var op = try ops.Pipelines.load(device); @@ -109,7 +109,7 @@ pub const Model = struct { const workspace_bytes = try std.math.add(usize, wp.bytes, attention.Workspace.bytes(o.chunk_rows)); const cache_bytes = try std.math.add(usize, kp.residentBytes(), weights.sizeofDescriptors()); const execution_bytes = try std.math.add(usize, workspace_bytes, cache_bytes); - const rotary_bytes = attention.Pipelines.residentBytes(capacity, c.text_config); + const rotary_bytes = attention.Pipelines.residentBytes(capacity, c); const needed = try std.math.add(usize, plan.bytes, try std.math.add(usize, execution_bytes, rotary_bytes)); const device = try mtl.Device.init(); errdefer device.deinit(); @@ -122,7 +122,7 @@ pub const Model = struct { const max_buffer = mtl.objc.msg(u64, device.id, "maxBufferLength", .{}); for (plan.shards.items) |shard| if (shard.end - shard.begin > max_buffer) return error.ShardExceedsMetalBufferLimit; std.log.info("full resident admission: weights={d}, total={d}, context={d}, headroom={d}", .{ plan.bytes, needed, capacity, available - needed }); - var pipelines = try Pipelines.init(device, capacity, c.text_config); + var pipelines = try Pipelines.init(device, capacity, c); errdefer pipelines.deinit(); const queue = try device.queue(); errdefer queue.deinit(); @@ -233,7 +233,7 @@ pub const Model = struct { try engram.encodeGate(enc, &p.engram, .{ .q_weight = tensor(g.q_weight), .k_weight = tensor(g.k_weight), .dim = 5120, .copies = 4, .weight_dtype = switch (g.q_weight.dtype) { .bf16 => .bf16, else => .f32, - }, .hidden_stride = 20480, .kv_stride = 25600, .output_stride = 20480, .eps = m.cfg.text_config.rms_norm_eps }, ws.view(.stream), ws.view(.engram_kv), null, ws.view(.stream), rows); + }, .hidden_stride = 20480, .kv_stride = 25600, .output_stride = 20480, .eps = m.cfg.eps }, ws.view(.stream), ws.view(.engram_kv), null, ws.view(.stream), rows); } try m.profileSpan(&enc, .mhc); m.mixes(enc, layer.hc_attn, ws.view(.attention_pre), rows); @@ -241,7 +241,7 @@ pub const Model = struct { enc.barrier(); m.normalize(enc, ws.view(.collapsed), layer.attn_norm, ws.view(.normalized), rows); try m.profileSpan(&enc, .attention); - try attention.encode(&p.attention, enc, &layer.attn, &m.cache, &m.attention_workspace, ws.view(.normalized), ws.view(.branch), &p.projection, ws.projection(), @intCast(index), rows, m.cfg.text_config); + try attention.encode(&p.attention, enc, &layer.attn, &m.cache, &m.attention_workspace, ws.view(.normalized), ws.view(.branch), &p.projection, ws.projection(), @intCast(index), rows, m.cfg); enc.barrier(); try m.profileSpan(&enc, .mhc); m.post(enc, rows); @@ -291,16 +291,16 @@ pub const Model = struct { } fn normalize(m: *Model, enc: mtl.ComputeEncoder, x: View, w: checkpoint.Tensor, y: View, rows: u32) void { - ops.encodeRms(enc, &m.pipelines.ops, x, tensor(w), y, .{ .rows = rows, .width = 5120, .x_stride = 5120, .y_stride = 5120, .eps = m.cfg.text_config.rms_norm_eps }); + ops.encodeRms(enc, &m.pipelines.ops, x, tensor(w), y, .{ .rows = rows, .width = 5120, .x_stride = 5120, .y_stride = 5120, .eps = m.cfg.eps }); enc.barrier(); } fn mixes(m: *Model, enc: mtl.ComputeEncoder, w: weights.Hc, pre: View, rows: u32) void { const ws = &m.workspace; - mhc.encodeRms(enc, &m.pipelines.mhc, ws.view(.stream), ws.view(.inv), .{ .rows = rows, .width = 5120, .stream_stride = 20480, .eps = m.cfg.text_config.rms_norm_eps }); + mhc.encodeRms(enc, &m.pipelines.mhc, ws.view(.stream), ws.view(.inv), .{ .rows = rows, .width = 5120, .stream_stride = 20480, .eps = m.cfg.eps }); ops.encodeDense(enc, &m.pipelines.ops, ws.view(.stream), tensor(w.projection), ws.view(.mixes), .{ .rows = rows, .k = 20480, .n = 24, .x_stride = 20480, .w_stride = 20480, .y_stride = 24, .w_dtype = .f32, .y_dtype = .f32 }); enc.barrier(); - mhc.encodeSplit(enc, &m.pipelines.mhc, ws.view(.mixes), ws.view(.inv), tensor(w.scale), tensor(w.base), pre, ws.view(.post), ws.view(.comb), .{ .rows = rows, .eps = m.cfg.text_config.hc_eps }); + mhc.encodeSplit(enc, &m.pipelines.mhc, ws.view(.mixes), ws.view(.inv), tensor(w.scale), tensor(w.base), pre, ws.view(.post), ws.view(.comb), .{ .rows = rows, .eps = m.cfg.hc_eps }); enc.barrier(); } diff --git a/zig/tests/deepseek_v41_reference/attention_math.zig b/zig/tests/deepseek_v41_reference/attention_math.zig index 09a4b1155d..35317f6a10 100644 --- a/zig/tests/deepseek_v41_reference/attention_math.zig +++ b/zig/tests/deepseek_v41_reference/attention_math.zig @@ -3,7 +3,7 @@ const std = @import("std"); const tf = @import("tensorfold"); const m = @import("math.zig"); const projection = @import("projection.zig"); -const Config = tf.deepseek_v41_config.Text; +const Config = tf.deepseek_v41_config.Config; const Attention = tf.deepseek_v41_weights.Attention; pub const max_capacity = 4096; @@ -136,10 +136,10 @@ pub const State = struct { if ((position + 1) % ratio != 0) return false; poolOf(f32, .{ &self.partial_kv[owner][0], &self.partial_kv[owner][1] }, .{ &self.partial_gate[owner][0], &self.partial_gate[owner][1] }, &self.latent); } - m.rms(&self.latent, compressor.norm, c.rms_norm_eps, &self.latent); + m.rms(&self.latent, compressor.norm, c.eps, &self.latent); const indexer = a.indexer.?; try projection.apply(indexer.wk.?, &self.latent, &self.index_key, scratch); - m.rms(&self.index_key, indexer.k_norm.?, c.rms_norm_eps, &self.index_key); + m.rms(&self.index_key, indexer.k_norm.?, c.eps, &self.index_key); const group_position = position + 1 - ratio; m.rope(&self.index_key, group_position, true, false, c); m.quantize(&self.index_key, .index); @@ -293,7 +293,7 @@ pub const State = struct { const compressed = c.compress_ratios[layer] != 0; try projection.apply(.{ .trellis = a.wq_a }, x, &self.qr, scratch); self.observe(.qr_projection, &self.qr); - m.rms(&self.qr, a.q_norm, c.rms_norm_eps, &self.qr); + m.rms(&self.qr, a.q_norm, c.eps, &self.qr); self.observe(.qr_norm, &self.qr); try projection.apply(.{ .trellis = a.wq_b }, &self.qr, &self.q, scratch); self.observe(.q_projection, &self.q); @@ -301,7 +301,7 @@ pub const State = struct { self.observe(.q, &self.q); try projection.apply(.{ .trellis = a.wkv }, x, &self.kv, scratch); self.observe(.kv_projection, &self.kv); - m.rms(&self.kv, a.kv_norm, c.rms_norm_eps, &self.kv); + m.rms(&self.kv, a.kv_norm, c.eps, &self.kv); self.observe(.kv_norm, &self.kv); m.rope(&self.kv, position, compressed, false, c); self.observe(.kv_rope, &self.kv); diff --git a/zig/tests/deepseek_v41_reference/engram_math.zig b/zig/tests/deepseek_v41_reference/engram_math.zig index 88772898de..9b7c275900 100644 --- a/zig/tests/deepseek_v41_reference/engram_math.zig +++ b/zig/tests/deepseek_v41_reference/engram_math.zig @@ -4,7 +4,7 @@ const tf = @import("tensorfold"); const m = @import("math.zig"); const projection = @import("projection.zig"); const Engram = tf.deepseek_v41_weights.Engram; -const Config = tf.deepseek_v41_config.Text; +const Config = tf.deepseek_v41_config.Config; const multipliers = [2][4]u64{ .{ 76632096046245, 4839876093313, 35959672319349, 73987337458391 }, .{ 67716810739261, 51510806800915, 30921347202721, 82619226485591 }, @@ -91,8 +91,8 @@ pub const State = struct { } const h_mean = m.meanFromWide(hh, 5120); const k_mean = m.meanFromWide(kk, 5120); - const h_inverse = m.idealRsqrt(h_mean + c.rms_norm_eps); - const k_inverse = m.idealRsqrt(k_mean + c.rms_norm_eps); + const h_inverse = m.idealRsqrt(h_mean + c.eps); + const k_inverse = m.idealRsqrt(k_mean + c.eps); const rstd: f32 = h_inverse * k_inverse; const dot_f32: f32 = @floatCast(dot); const normalized: f32 = (dot_f32 * rstd) * (1 / @sqrt(@as(f32, 5120))); diff --git a/zig/tests/deepseek_v41_reference/math.zig b/zig/tests/deepseek_v41_reference/math.zig index 6ba08d1d72..6106888f27 100644 --- a/zig/tests/deepseek_v41_reference/math.zig +++ b/zig/tests/deepseek_v41_reference/math.zig @@ -3,7 +3,7 @@ const std = @import("std"); const tf = @import("tensorfold"); const Tensor = tf.checkpoint_metal.Tensor; const Hc = tf.deepseek_v41_weights.Hc; -const Config = tf.deepseek_v41_config.Text; +const Config = tf.deepseek_v41_config.Config; pub fn f(x: u16) f32 { return @bitCast(@as(u32, x) << 16); @@ -212,7 +212,7 @@ pub fn mixes(x: []const u16, hc: Hc, c: Config) Mix { pub fn mixesOf(comptime T: type, x: []const u16, hc: Hc, c: Config) MixOf(T) { var projected: [24]T = undefined; denseOf(T, hc.projection, x, &projected); - const r = inverseOf(T, x, c.rms_norm_eps); + const r = inverseOf(T, x, c.eps); return splitOf(T, &projected, r, hc, c); } /// The affine, softmax and Sinkhorn coefficient stage at reference precision T. diff --git a/zig/tests/deepseek_v41_reference/operator_attention.zig b/zig/tests/deepseek_v41_reference/operator_attention.zig index 73e94641c3..4a8cb51556 100644 --- a/zig/tests/deepseek_v41_reference/operator_attention.zig +++ b/zig/tests/deepseek_v41_reference/operator_attention.zig @@ -20,7 +20,7 @@ const hyperconnections = @import("operator_hc.zig").hyperconnections; pub fn runAttention(gpa: std.mem.Allocator, io: std.Io, model: *Model, snapshot: []const u8, dir: []const u8, layer: usize) !void { const blob = try std.Io.Dir.cwd().readFileAlloc(io, snapshot, gpa, .limited(8 << 20)); defer gpa.free(blob); - const cfg = model.cfg.text_config; + const cfg = model.cfg; if (layer >= model.weights.layers.len) return error.InvalidAttentionLayer; const snapshot_offset = layer * 2 * 61488; if (blob.len < snapshot_offset + 40960 + 10240) return error.InvalidAttentionSnapshot; @@ -103,9 +103,9 @@ pub fn runAttention(gpa: std.mem.Allocator, io: std.Io, model: *Model, snapshot: try attentionProjection(gpa, model, weights.wo_b, capture.view(.grouped), "wo_b", &scratch, expected, &report, &count); var same_input: [32768]u16 = undefined; - math.rms(capture.row(.qr_projection), weights.q_norm, cfg.rms_norm_eps, same_input[0..1280]); + math.rms(capture.row(.qr_projection), weights.q_norm, cfg.eps, same_input[0..1280]); try attentionMetric(u16, gpa, &report, &count, "qr_norm", "same_native_input_reference_rms", capture.row(.qr_norm), same_input[0..1280]); - math.rms(capture.row(.kv_projection), weights.kv_norm, cfg.rms_norm_eps, same_input[0..512]); + math.rms(capture.row(.kv_projection), weights.kv_norm, cfg.eps, same_input[0..512]); try attentionMetric(u16, gpa, &report, &count, "kv_norm", "same_native_input_reference_rms", capture.row(.kv_norm), same_input[0..512]); @memcpy(same_input[0..512], capture.row(.kv_rope)); math.quantize(same_input[0..512], .window); diff --git a/zig/tests/deepseek_v41_reference/operator_hc.zig b/zig/tests/deepseek_v41_reference/operator_hc.zig index 4aec23362a..bd3cc0425a 100644 --- a/zig/tests/deepseek_v41_reference/operator_hc.zig +++ b/zig/tests/deepseek_v41_reference/operator_hc.zig @@ -12,7 +12,7 @@ const attentionMetric = diagnostic.attentionMetric; pub fn hyperconnections(gpa: std.mem.Allocator, model: *Model, stream_bytes: []const u8, branch: attention.View, report: *std.ArrayList(u8), count: *usize, layer: usize) !void { const ws = &model.workspace; const hc = model.weights.layers[layer].hc_ffn; - const cfg = model.cfg.text_config; + const cfg = model.cfg; const stream = ws.view(.stream); @memcpy(stream.buffer.contents()[stream.offset..][0..40960], stream_bytes[0..40960]); var input: [20480]u16 = undefined; @@ -23,11 +23,11 @@ pub fn hyperconnections(gpa: std.mem.Allocator, model: *Model, stream_bytes: []c var squares: f64 = 0; for (input) |v| squares += @as(f64, math.f(v)) * math.f(v); const mean = math.meanFromWide(squares, input.len); - const expected_inv = [_]f32{math.idealRsqrt(mean + cfg.rms_norm_eps)}; + const expected_inv = [_]f32{math.idealRsqrt(mean + cfg.eps)}; const cb = model.queue.commandBufferUnretained(); const enc = cb.compute(.serial); for (model.checkpoint.shards.items) |shard| enc.useResource(shard.buffer, mtl.ResourceUsage.read); - tf.deepseek_v41.mhc.encodeRms(enc, &model.pipelines.mhc, stream, ws.view(.inv), .{ .rows = 1, .width = 5120, .stream_stride = 20480, .eps = cfg.rms_norm_eps }); + tf.deepseek_v41.mhc.encodeRms(enc, &model.pipelines.mhc, stream, ws.view(.inv), .{ .rows = 1, .width = 5120, .stream_stride = 20480, .eps = cfg.eps }); tf.deepseek_v41.ops.encodeDense(enc, &model.pipelines.ops, stream, .{ .buffer = hc.projection.buffer, .offset = hc.projection.offset }, ws.view(.mixes), .{ .rows = 1, .k = 20480, .n = 24, .x_stride = 20480, .w_stride = 20480, .y_stride = 24, .w_dtype = .f32, .y_dtype = .f32 }); enc.barrier(); tf.deepseek_v41.mhc.encodeSplit(enc, &model.pipelines.mhc, ws.view(.mixes), ws.view(.inv), .{ .buffer = hc.scale.buffer, .offset = hc.scale.offset }, .{ .buffer = hc.base.buffer, .offset = hc.base.offset }, ws.view(.attention_pre), ws.view(.post), ws.view(.comb), .{ .rows = 1, .eps = cfg.hc_eps }); @@ -62,7 +62,7 @@ pub fn hyperconnections(gpa: std.mem.Allocator, model: *Model, stream_bytes: []c next_enc.useResource(norm.buffer, mtl.ResourceUsage.read); tf.deepseek_v41.mhc.encodeCollapse(next_enc, &model.pipelines.mhc, stream, ws.view(.attention_pre), ws.view(.collapsed), .{ .rows = 1, .width = 5120, .stream_stride = 20480, .y_stride = 5120 }); next_enc.barrier(); - tf.deepseek_v41.ops.encodeRms(next_enc, &model.pipelines.ops, ws.view(.collapsed), .{ .buffer = norm.buffer, .offset = norm.offset }, ws.view(.branch), .{ .rows = 1, .width = 5120, .x_stride = 5120, .y_stride = 5120, .eps = cfg.rms_norm_eps }); + tf.deepseek_v41.ops.encodeRms(next_enc, &model.pipelines.ops, ws.view(.collapsed), .{ .buffer = norm.buffer, .offset = norm.offset }, ws.view(.branch), .{ .rows = 1, .width = 5120, .x_stride = 5120, .y_stride = 5120, .eps = cfg.eps }); next_enc.end(); next_cb.commit(); next_cb.wait(); @@ -71,7 +71,7 @@ pub fn hyperconnections(gpa: std.mem.Allocator, model: *Model, stream_bytes: []c try attentionMetric(u16, gpa, report, count, "next_collapse", "same_retained_ffn_stream_and_native_pre", diagnosticRow(u16, ws.view(.collapsed), 5120), &collapsed); math.collapse(&input, expected_mix.pre, &collapsed); try attentionMetric(u16, gpa, report, count, "next_collapse", "same_retained_ffn_stream_reference_pre", diagnosticRow(u16, ws.view(.collapsed), 5120), &collapsed); - math.rms(&collapsed, norm, cfg.rms_norm_eps, &collapsed); + math.rms(&collapsed, norm, cfg.eps, &collapsed); try attentionMetric(u16, gpa, report, count, "next_normalized", "same_retained_ffn_stream_reference_pre_and_norm", diagnosticRow(u16, ws.view(.branch), 5120), &collapsed); } } diff --git a/zig/tests/deepseek_v41_reference/operator_hc_amplify_fp64.zig b/zig/tests/deepseek_v41_reference/operator_hc_amplify_fp64.zig index 78c54e00a7..02b5ebfa0d 100644 --- a/zig/tests/deepseek_v41_reference/operator_hc_amplify_fp64.zig +++ b/zig/tests/deepseek_v41_reference/operator_hc_amplify_fp64.zig @@ -13,8 +13,8 @@ const Config = tf.deepseek_v41_config.Config; fn stageF64(stream: []const u16, hc: Hc, cfg: Config, branch: []const u16, y_pre: []f64, y_bf: *[h.stream_width]u16, coll_pre: []f64, inv_out: *f64, comb_out: *[16]f64) void { var proj: [h.mix_count]f64 = undefined; math.denseOf(f64, hc.projection, stream, &proj); - const inv = math.inverseOf(f64, stream, cfg.text_config.rms_norm_eps); - const mix = math.splitOf(f64, &proj, inv, hc, cfg.text_config); + const inv = math.inverseOf(f64, stream, cfg.eps); + const mix = math.splitOf(f64, &proj, inv, hc, cfg); inv_out.* = inv; comb_out.* = mix.comb; for (0..h.width) |d| { diff --git a/zig/tests/deepseek_v41_reference/operator_hc_fp64.zig b/zig/tests/deepseek_v41_reference/operator_hc_fp64.zig index 952f497e61..3ab5051dec 100644 --- a/zig/tests/deepseek_v41_reference/operator_hc_fp64.zig +++ b/zig/tests/deepseek_v41_reference/operator_hc_fp64.zig @@ -186,9 +186,9 @@ fn StageRefOf(comptime T: type) type { fn referenceRow(comptime T: type, stream: []const u16, hc: Hc, cfg: Config, branch: []const u16, collapsed: *[width]u16, y: *[stream_width]u16) StageRefOf(T) { var out: StageRefOf(T) = undefined; - out.inv = math.inverseOf(T, stream, cfg.text_config.rms_norm_eps); + out.inv = math.inverseOf(T, stream, cfg.eps); math.denseOf(T, hc.projection, stream, &out.proj); - const mix = math.splitOf(T, &out.proj, out.inv, hc, cfg.text_config); + const mix = math.splitOf(T, &out.proj, out.inv, hc, cfg); out.pre = mix.pre; out.post = mix.post; out.comb = mix.comb; @@ -246,10 +246,10 @@ pub fn runStage( ) !void { const cb = queue.commandBuffer(); const e = cb.compute(.serial); - mhc.encodeRms(e, p, stream, inv, .{ .rows = rows, .width = width, .stream_stride = stream_width, .eps = cfg.text_config.rms_norm_eps }); + mhc.encodeRms(e, p, stream, inv, .{ .rows = rows, .width = width, .stream_stride = stream_width, .eps = cfg.eps }); ops.encodeDense(e, o, stream, weight, mixes, .{ .rows = rows, .k = stream_width, .n = mix_count, .x_stride = stream_width, .w_stride = stream_width, .y_stride = mix_count, .w_dtype = .f32, .y_dtype = .f32 }); e.barrier(); - mhc.encodeSplit(e, p, mixes, inv, scale, base, pre, post, comb, .{ .rows = rows, .eps = cfg.text_config.hc_eps }); + mhc.encodeSplit(e, p, mixes, inv, scale, base, pre, post, comb, .{ .rows = rows, .eps = cfg.hc_eps }); e.barrier(); mhc.encodeCollapse(e, p, stream, pre, collapsed, .{ .rows = rows, .width = width, .stream_stride = stream_width, .y_stride = width }); e.barrier(); diff --git a/zig/tests/deepseek_v41_reference/operator_hc_split_fp64.zig b/zig/tests/deepseek_v41_reference/operator_hc_split_fp64.zig index 6739259e8b..016d1cea4f 100644 --- a/zig/tests/deepseek_v41_reference/operator_hc_split_fp64.zig +++ b/zig/tests/deepseek_v41_reference/operator_hc_split_fp64.zig @@ -44,7 +44,7 @@ test "mHC fp32 reference path stays bit-identical under the comptime parameteriz const row = eight_rows[r * h.width ..][0..h.width]; var stream: [h.stream_width]u16 = undefined; for (0..h.copies) |c| @memcpy(stream[c * h.width ..][0..h.width], row); - const parameterized = math.mixesOf(f32, &stream, hc, cfg.text_config); + const parameterized = math.mixesOf(f32, &stream, hc, cfg); const legacy = mixesLegacy(&stream, hc, cfg); try std.testing.expect(std.meta.eql(parameterized, legacy)); } @@ -57,10 +57,10 @@ fn mixesLegacy(x: []const u16, hc: Hc, c: Config) math.Mix { var squares: f64 = 0; for (x) |v| squares += @as(f64, math.f(v)) * math.f(v); const mean = math.meanFromWide(squares, x.len); - const r = math.idealRsqrt(mean + c.text_config.rms_norm_eps); + const r = math.idealRsqrt(mean + c.eps); var m: math.Mix = undefined; for (0..4) |i| { - m.pre[i] = math.sigmoid(@mulAdd(f32, projected[i] * r, math.value(hc.scale, 0), math.value(hc.base, i))) + c.text_config.hc_eps; + m.pre[i] = math.sigmoid(@mulAdd(f32, projected[i] * r, math.value(hc.scale, 0), math.value(hc.base, i))) + c.hc_eps; m.post[i] = 2 * math.sigmoid(@mulAdd(f32, projected[i + 4] * r, math.value(hc.scale, 1), math.value(hc.base, i + 4))); } for (0..4) |i| { @@ -75,17 +75,17 @@ fn mixesLegacy(x: []const u16, hc: Hc, c: Config) math.Mix { } const denominator = (m.comb[i * 4] + m.comb[i * 4 + 2]) + (m.comb[i * 4 + 1] + m.comb[i * 4 + 3]); - for (0..4) |j| m.comb[i * 4 + j] = m.comb[i * 4 + j] / denominator + c.text_config.hc_eps; + for (0..4) |j| m.comb[i * 4 + j] = m.comb[i * 4 + j] / denominator + c.hc_eps; } - for (0..c.text_config.hc_sinkhorn_iters) |iteration| { + for (0..c.hc_sinkhorn_iters) |iteration| { if (iteration != 0) for (0..4) |i| { const denominator = ((m.comb[i * 4] + m.comb[i * 4 + 2]) + - (m.comb[i * 4 + 1] + m.comb[i * 4 + 3])) + c.text_config.hc_eps; + (m.comb[i * 4 + 1] + m.comb[i * 4 + 3])) + c.hc_eps; for (0..4) |j| m.comb[i * 4 + j] /= denominator; }; for (0..4) |j| { const denominator = ((m.comb[j] + m.comb[8 + j]) + - (m.comb[4 + j] + m.comb[12 + j])) + c.text_config.hc_eps; + (m.comb[4 + j] + m.comb[12 + j])) + c.hc_eps; for (0..4) |i| m.comb[i * 4 + j] /= denominator; } } @@ -156,7 +156,7 @@ test "mHC split unary boundaries against the FP64 ideal on retained arguments" { const cb = queue.commandBuffer(); const e = cb.compute(.serial); - mhc.encodeSplit(e, &p, h.view(mixes), h.view(inv), h.view(scale), h.view(base), h.view(pre), h.view(post), h.view(comb), .{ .rows = rows, .eps = cfg.text_config.hc_eps }); + mhc.encodeSplit(e, &p, h.view(mixes), h.view(inv), h.view(scale), h.view(base), h.view(pre), h.view(post), h.view(comb), .{ .rows = rows, .eps = cfg.hc_eps }); e.end(); cb.commit(); cb.wait(); @@ -179,8 +179,8 @@ test "mHC split unary boundaries against the FP64 ideal on retained arguments" { const m: []const f32 = mixes_host[r * h.mix_count ..][0..h.mix_count]; var m64: [h.mix_count]f64 = undefined; for (m, 0..) |v, i| m64[i] = v; - const mix64 = math.splitOf(f64, &m64, 1.0, hc, cfg.text_config); - const mix32 = math.splitOf(f32, m, 1.0, hc, cfg.text_config); + const mix64 = math.splitOf(f64, &m64, 1.0, hc, cfg); + const mix32 = math.splitOf(f32, m, 1.0, hc, cfg); const src64: []const f64 = if (std.mem.eql(u8, stage.name, "pre")) &mix64.pre else if (std.mem.eql(u8, stage.name, "post")) &mix64.post else &mix64.comb; const src32: []const f32 = if (std.mem.eql(u8, stage.name, "pre")) &mix32.pre else if (std.mem.eql(u8, stage.name, "post")) &mix32.post else &mix32.comb; const metal: []const f32 = if (std.mem.eql(u8, stage.name, "pre")) pre.slice(f32, rows * 4)[r * 4 ..][0..4] else if (std.mem.eql(u8, stage.name, "post")) post.slice(f32, rows * 4)[r * 4 ..][0..4] else comb.slice(f32, rows * 16)[r * 16 ..][0..16]; diff --git a/zig/tests/deepseek_v41_reference/operator_history.zig b/zig/tests/deepseek_v41_reference/operator_history.zig index 499a1cd463..5490c1ed88 100644 --- a/zig/tests/deepseek_v41_reference/operator_history.zig +++ b/zig/tests/deepseek_v41_reference/operator_history.zig @@ -41,7 +41,7 @@ pub fn runAttentionHistory(gpa: std.mem.Allocator, io: std.Io, model: *Model, tr if (token >= attention_math.max_capacity) return error.DiagnosticHistoryTooLarge; const prefix_elements = try std.math.mul(usize, token + 1, (layer + 1) * 5120); if (prefix_elements > (256 << 20) / @sizeOf(u16)) return error.DiagnosticHistoryTooLarge; - const cfg = model.cfg.text_config; + const cfg = model.cfg; const history_count = token + 1; const ratio = cfg.compress_ratios[layer]; const compressed_rows = if (ratio == 0) 0 else history_count / ratio; @@ -236,9 +236,9 @@ pub fn runAttentionHistory(gpa: std.mem.Allocator, io: std.Io, model: *Model, tr native_input_state.sparseHead(capture.row(.q)[head * 512 ..][0..512], native_window, helper_indices, sinks[head], same_input[head * 512 ..][0..512]); try attentionMetric(u16, gpa, &report, &count, "sparse_o", "same_native_query_window_compressed_cache_selection_reference_sparse_diagnostic", capture.row(.sparse_o), &same_input); try attentionWrite(gpa, io, dir, "attention-sparse-same-native-input-reference.bf16", std.mem.sliceAsBytes(&same_input)); - math.rms(capture.row(.qr_projection), weights.q_norm, cfg.rms_norm_eps, same_input[0..1280]); + math.rms(capture.row(.qr_projection), weights.q_norm, cfg.eps, same_input[0..1280]); try attentionMetric(u16, gpa, &report, &count, "qr_norm", "same_native_input_reference_rms_diagnostic", capture.row(.qr_norm), same_input[0..1280]); - math.rms(capture.row(.kv_projection), weights.kv_norm, cfg.rms_norm_eps, same_input[0..512]); + math.rms(capture.row(.kv_projection), weights.kv_norm, cfg.eps, same_input[0..512]); try attentionMetric(u16, gpa, &report, &count, "kv_norm", "same_native_input_reference_rms_diagnostic", capture.row(.kv_norm), same_input[0..512]); @memcpy(&same_input, capture.row(.q_projection)); for (0..64) |head| math.rope(same_input[head * 512 ..][0..512], token, ratio != 0, false, cfg); diff --git a/zig/tests/deepseek_v41_reference/operator_moe.zig b/zig/tests/deepseek_v41_reference/operator_moe.zig index 558234569f..53048e6644 100644 --- a/zig/tests/deepseek_v41_reference/operator_moe.zig +++ b/zig/tests/deepseek_v41_reference/operator_moe.zig @@ -137,7 +137,7 @@ fn moeReferenceRows(model: *Model, layer: usize, input: []const u16, ids: *const try projection.apply(.{ .trellis = matrices[0] }, input, &rows.gate[slot], scratch); try projection.apply(.{ .trellis = matrices[1] }, input, &rows.up[slot], scratch); } - moe_reference.moeActivation(&rows.gate[slot], &rows.up[slot], if (slot == 6) 1 else routing[slot], model.cfg.text_config, &rows.activation[slot]); + moe_reference.moeActivation(&rows.gate[slot], &rows.up[slot], if (slot == 6) 1 else routing[slot], model.cfg, &rows.activation[slot]); // A shared or identical-weight expert result can be reused only when the BF16 input is identical. if (reuse != null and std.mem.eql(u16, &rows.activation[slot], &previous.?.activation[reuse.?])) { rows.down[slot] = previous.?.down[reuse.?]; @@ -290,17 +290,17 @@ pub fn runMoe(gpa: std.mem.Allocator, io: std.Io, model: *Model, snapshot: []con @memcpy(&native_ids, diagnosticRow(u32, model.workspace.view(.route_ids), 6)); @memcpy(&native_weights, diagnosticRow(f32, model.workspace.view(.route_weights), 6)); @memcpy(&native_logits_routing.logits, diagnosticRow(f32, model.workspace.view(.router_logits), 384)); - moe_reference.moeSelect(model.weights.layers[layer].moe, model.cfg.text_config, &native_logits_routing.logits, &native_logits_routing.scores, &native_logits_routing.ids, &native_logits_routing.weights); + moe_reference.moeSelect(model.weights.layers[layer].moe, model.cfg, &native_logits_routing.logits, &native_logits_routing.scores, &native_logits_routing.ids, &native_logits_routing.weights); var source_routing: moe_reference.MoeRouting = .{}; math.dense(model.weights.layers[layer].moe.gate, input_row, &source_routing.logits); - moe_reference.moeSelect(model.weights.layers[layer].moe, model.cfg.text_config, &source_routing.logits, &source_routing.scores, &source_routing.ids, &source_routing.weights); + moe_reference.moeSelect(model.weights.layers[layer].moe, model.cfg, &source_routing.logits, &source_routing.scores, &source_routing.ids, &source_routing.weights); try moeReferenceRows(model, layer, input_row, &source_routing.ids, &source_routing.weights, source, &scratch, null, null); try moeReferenceRows(model, layer, input_row, &native_ids, &native_weights, native_weights_reference, &scratch, source, &source_routing.ids); try moeReferenceRows(model, layer, input_row, &retained_ids, &retained_weights, retained_reference, &scratch, native_weights_reference, &native_ids); for (0..7) |slot| { native_inputs_reference.gate[slot] = native.gate[slot]; native_inputs_reference.up[slot] = native.up[slot]; - moe_reference.moeActivation(&native.gate[slot], &native.up[slot], if (slot == 6) 1 else native_weights[slot], model.cfg.text_config, &native_inputs_reference.activation[slot]); + moe_reference.moeActivation(&native.gate[slot], &native.up[slot], if (slot == 6) 1 else native_weights[slot], model.cfg, &native_inputs_reference.activation[slot]); // Deliberately use the captured native activation, not the preceding reference activation. if (std.mem.eql(u16, &native.activation[slot], &native_weights_reference.activation[slot])) { native_inputs_reference.down[slot] = native_weights_reference.down[slot]; diff --git a/zig/tests/deepseek_v41_reference/qualify.zig b/zig/tests/deepseek_v41_reference/qualify.zig index a8e5bbbedd..2a3ce5384f 100644 --- a/zig/tests/deepseek_v41_reference/qualify.zig +++ b/zig/tests/deepseek_v41_reference/qualify.zig @@ -93,7 +93,7 @@ fn maximum(row: []const f32) !u32 { pub fn run(gpa: std.mem.Allocator, io: std.Io, model: *Model, ids: []const u32, token_map: []const u8, dir: []const u8) !Result { if (ids.len == 0 or ids.len > 16) return error.InvalidOracleWindow; - var reference = try Reference.init(gpa, model.cfg.text_config, &model.weights, &model.checkpoint, token_map, @intCast(ids.len)); + var reference = try Reference.init(gpa, model.cfg, &model.weights, &model.checkpoint, token_map, @intCast(ids.len)); defer reference.deinit(); var margins = MarginLog{}; reference.attention_state.score_probe = .{ .ctx = &margins, .record = &MarginLog.capture }; diff --git a/zig/tests/deepseek_v41_reference/reference.zig b/zig/tests/deepseek_v41_reference/reference.zig index 98069068f7..168e1be583 100644 --- a/zig/tests/deepseek_v41_reference/reference.zig +++ b/zig/tests/deepseek_v41_reference/reference.zig @@ -21,7 +21,7 @@ pub const MoeRouting = struct { weights: [6]f32 = undefined, }; -pub fn moeSelect(moe_weights: weights.Moe, c: config.Text, logits: *const [384]f32, scores: *[384]f32, ids: *[6]u32, routing_weights: *[6]f32) void { +pub fn moeSelect(moe_weights: weights.Moe, c: config.Config, logits: *const [384]f32, scores: *[384]f32, ids: *[6]u32, routing_weights: *[6]f32) void { @setFloatMode(.strict); for (logits, scores) |z, *score| { const softplus = if (z > 20) z else std.math.log1p(@exp(z)); @@ -48,7 +48,7 @@ pub fn moeSelect(moe_weights: weights.Moe, c: config.Text, logits: *const [384]f for (routing_weights) |*v| v.* = (v.* / denominator) * c.routed_scaling_factor; } -pub fn moeActivation(gate: []const u16, up: []const u16, route: f32, c: config.Text, activation: []u16) void { +pub fn moeActivation(gate: []const u16, up: []const u16, route: f32, c: config.Config, activation: []u16) void { @setFloatMode(.strict); for (activation, gate, up) |*v, g_value, u_value| { const g = @min(math.f(g_value), c.swiglu_limit); @@ -59,7 +59,7 @@ pub fn moeActivation(gate: []const u16, up: []const u16, route: f32, c: config.T pub const Reference = struct { gpa: std.mem.Allocator, - c: config.Text, + c: config.Config, w: *const weights.Model, checkpoint: *const cp.Checkpoint, capacity: u32, @@ -87,7 +87,7 @@ pub const Reference = struct { }; /// Borrowed model/checkpoint/config/map must outlive this instance; capacity is the teacher sequence length. - pub fn init(gpa: std.mem.Allocator, c: config.Text, w: *const weights.Model, checkpoint: *const cp.Checkpoint, token_map: []const u8, capacity: u32) !Reference { + pub fn init(gpa: std.mem.Allocator, c: config.Config, w: *const weights.Model, checkpoint: *const cp.Checkpoint, token_map: []const u8, capacity: u32) !Reference { if (capacity == 0 or capacity > c.max_position_embeddings or capacity > attention.max_capacity) return error.InvalidReferenceCapacity; if (c.hidden_size != 5120 or c.num_hidden_layers != 40 or c.vocab_size != 129280 or @@ -120,20 +120,20 @@ pub const Reference = struct { if (layer.engram) |e| try self.engram_state.apply(e, if (layer_id == 1) 0 else 1, compressed, &self.h, self.c, &self.projection_scratch); const attn_mix = math.mixes(&self.h, layer.hc_attn, self.c); math.collapse(&self.h, pre, &self.input); - math.rms(&self.input, layer.attn_norm, self.c.rms_norm_eps, &self.input); + math.rms(&self.input, layer.attn_norm, self.c.eps, &self.input); try self.attention_state.apply(layer.attn, layer_id, &self.input, self.position, self.c, &self.output, &self.projection_scratch); math.expand(&self.h, &self.output, attn_mix); if (self.observer) |observer| observer.read(observer.ctx, layer_id, .attention, self); const ffn_mix = math.mixes(&self.h, layer.hc_ffn, self.c); math.collapse(&self.h, attn_mix.pre, &self.input); - math.rms(&self.input, layer.ffn_norm, self.c.rms_norm_eps, &self.input); + math.rms(&self.input, layer.ffn_norm, self.c.eps, &self.input); try self.moe(layer.moe, layer_id); math.expand(&self.h, &self.output, ffn_mix); if (self.observer) |observer| observer.read(observer.ctx, layer_id, .moe, self); pre = ffn_mix.pre; } math.collapse(&self.h, pre, &self.input); - math.rms(&self.input, self.w.norm, self.c.rms_norm_eps, &self.input); + math.rms(&self.input, self.w.norm, self.c.eps, &self.input); try projection.applyF32(.{ .trellis = self.w.head }, &self.input, self.logits, &self.projection_scratch); for (self.logits) |v| if (!std.math.isFinite(v)) return error.NonFiniteReferenceLogits; self.engram_state.advance(compressed); diff --git a/zig/tests/deepseek_v41_reference/trace.zig b/zig/tests/deepseek_v41_reference/trace.zig index 79cdef0fa4..0d80a1a104 100644 --- a/zig/tests/deepseek_v41_reference/trace.zig +++ b/zig/tests/deepseek_v41_reference/trace.zig @@ -74,7 +74,7 @@ pub fn run(gpa: std.mem.Allocator, io: std.Io, model: *Model, ids: []const u32, model.residency.remove(buffer); model.residency.commit(); } - var reference = try Reference.init(gpa, model.cfg.text_config, &model.weights, &model.checkpoint, token_map, @intCast(ids.len)); + var reference = try Reference.init(gpa, model.cfg, &model.weights, &model.checkpoint, token_map, @intCast(ids.len)); defer reference.deinit(); reference.observer = .{ .ctx = &capture, .read = Capture.reference }; const expected = try gpa.alloc(f32, ids.len * 129280);