diff --git a/.gitignore b/.gitignore index 75ba703b9a..070b8ca186 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ zig-out/ # Versioned GUI build resources, generated app engines stay ignored in app/.gitignore. !app/build/ +# mHC fp64 qualification receipts. +/receipts/ +*.log diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 018947342d..5b34839762 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -217,6 +217,13 @@ TensorFold ships no model weights. The `z-lab/Qwen3.8-27B-DFlash2` model card st The optional `incoai/GLM-5.3-Flash-DFlash2` model card states CC BY-NC-ND 4.0, for non-commercial use without derivatives; only the Python engine reads that drafter. Each checkpoint keeps its own license. +The native `zig/src/families/deepseek_v41/` implementation and `zig/tests/deepseek_v41_reference/` +follow [DeepSeek-V4.1-Flash's official inference](https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/tree/2cba9e42aa026125f3ed06c6d98c1db82f7ca027/inference), +MIT License, Copyright (c) 2023 DeepSeek: the reference's equations, the Engram normalization, +PCG64 multipliers and prime bucket layout, and the pinned chat template come from that revision. +The EXL3 checkpoint reads through the ExLlamaV3 format covered above; no captured CUDA/Triton +kernel assets, full model shards or binary traces are bundled. + ## Structured output: libtfgrammar `lib/libtfgrammar` in the release archives is `zig/src/grammar/tf_grammar.cc` compiled with the C++ sources of diff --git a/docs/deepseek-v41-mhc-fp64.json b/docs/deepseek-v41-mhc-fp64.json new file mode 100644 index 0000000000..833a67ae98 --- /dev/null +++ b/docs/deepseek-v41-mhc-fp64.json @@ -0,0 +1,103 @@ +{ + "scope": "mHC (hyperconnection mix) stage fp64 qualification: rounding level, Metal deltas and 1-ULP input amplification. Independent CPU reference parameterized comptime (math.zig MixOf/denseOf/mixesOf/splitOf/inverseOf/collapseOf/expandOf, f32 source-contract path bit-identical, f64 ideal path); harness zig/tests/deepseek_v41_reference/operator_hc_fp64.zig; fixtures hc_attention_rms.json and hc_fp64_rows.json (retained inputs only).", + "nrms_definition": "sqrt(sum(delta^2)/sum(ref^2)) in f64; ULP = |delta| / fp32 spacing at the fp64 reference value.", + "inputs": [ + "token671_attention (retained token-671 embedding, four copies)", + "token1_ffn (retained token-1 FFN stream and branch, real hc_ffn scale/base)", + "eight_teacher_rows (the eight retained teacher embeddings)", + "wide_dynamic_range (copy magnitudes 2^24..2^-12, rotating signs)", + "split_args (16 retained unary arguments incl. 1.4393715858459473, controlling post argument, integer exp sweep -32..32, plus sigmoid/softmax spans)", + "cancellation fixtures (half-ULP terms after 1.0)" + ], + "rounding_level_fp32_reference_vs_fp64": { + "inverse_statistic_nrms": [ + 2.7e-09, + 2.8e-08 + ], + "projection_dense_nrms": [ + 2.2e-08, + 2.9e-08 + ], + "pre_sigmoid_nrms": [ + 1.1e-08, + 4.2e-08 + ], + "post_nrms": [ + 3.9e-08, + 7.2e-08 + ], + "comb_sinkhorn_nrms": [ + 1.4e-08, + 6.8e-08 + ], + "collapse_bf16_unequal_cells": 0, + "post_expand_bf16_unequal_cells": [ + 0, + 5 + ] + }, + "metal_vs_fp64": { + "inverse_max_ulp": 0.6, + "projection_max_ulp": 0.5, + "pre_max_ulp": 2.75, + "post_max_ulp": 2.29, + "comb_max_ulp_near_cancelled_cells_only": 34.2, + "comb_max_abs": 1.5e-07, + "post_controlling_exponent_row_max_ulp": 1.61, + "exponent_sweep_minus32_to_32_worst_stage_nrms": 9e-08, + "bf16_unequal_cells": { + "token671": 0, + "token1_ffn": "3 of 20480", + "eight_teacher_rows": "8 of 163840", + "wide_dynamic_range": 0 + }, + "verdict": "every sub-stage sits at the fp32 source-contract rounding level; Metal exceeds the fp32 reference by at most ~1 ULP at the documented Metal exp boundary" + }, + "cancellation_boundaries": [ + { + "fixture": "collapse_half_ulps", + "metal": "0x3500", + "f32_ref": "0x3500", + "f64_ideal": "0x34c0" + }, + { + "fixture": "post_half_ulps", + "metal": "0x0", + "f32_ref": "0x0", + "f64_ideal": "0x3400" + }, + { + "fixture": "branch_product", + "metal": "0x3500", + "f32_ref": "0x3500", + "f64_ideal": "0x34c0" + } + ], + "amplification_1ulp_input": { + "method": "perturb one bf16 stream cell (or branch cell) by 1 ULP, propagate the ideal fp64 stage, output NRMS growth = sqrt(sum(dy^2)/sum(y^2)) / (|dx|/sqrt(sum(x^2))); 256 sampled cells per row; 8 teacher positions plus the token-1 FFN row", + "analog_fp64_gain_mean_per_row": [ + 0.182, + 0.76 + ], + "analog_fp64_gain_max_overall": 0.762, + "collapse_gain_mean": 0.5003, + "collapse_gain_max": 1.944, + "bf16_visible_gain_mean_per_row": [ + 0.74, + 2.382 + ], + "bf16_visible_gain_max": 34.5, + "bf16_rows_changed_fraction": "224-250 of 256 perturbations change at least one of 20480 output cells", + "mean_changed_cells": "8.1-13.1 of 20480", + "statistic_relative_shift_mean": 3.6e-07, + "comb_nrms_shift_mean": 3.6e-08, + "metal_confirmation_row0": { + "amp_mean": 1.886, + "fp64_bf16_amp_mean": 1.945, + "mean_changed_cells": 12.06 + } + }, + "gate_reading": "A single-ULP input difference is attenuated by the mix in exact arithmetic (gain < 1) and appears as 1-bf16-ULP flips on about 0.06 percent of output cells; the stage's own rounding-level output NRMS is 0 to 2.5e-5. Hyperconnection mixing therefore cannot turn fp32 ULP noise into the observed 4.096 percent whole-model NRMS; a specific operator-level difference elsewhere (sparse/compressed attention candidates) remains the plausible cause.", + "kernel_changes": "none required; no tolerance relaxed, no source coefficient injected", + "receipts": "gitignored receipts/ (mhc-fp64-run.log, mhc-fp64-parsed.json, build-summary.log); commit contains code, fixture and this summary" +} \ No newline at end of file diff --git a/tools/zig/nfkd_tables.py b/tools/zig/nfkd_tables.py new file mode 100644 index 0000000000..d785458eab --- /dev/null +++ b/tools/zig/nfkd_tables.py @@ -0,0 +1,96 @@ +"""Emit the Engram token-map normalizer's NFKD, mark, lowercase and combining-class tables as Zig source.""" +import argparse +import subprocess +import sys +import tempfile +import unicodedata +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +OUT = ROOT / "zig" / "src" / "core" / "tokenizer" / "nfkd_data.zig" +VERSION = "16.0.0" + + +def rows(values, per_line, fmt): + """Comma-terminated lines of at most `per_line` values, four-space indented.""" + for i in range(0, len(values), per_line): + yield " " + " ".join(fmt(v) + "," for v in values[i:i + per_line]) + + +def render(): + """The generated Zig source before the compiler's formatter aligns the columns.""" + flat, decompositions = [], [] + for cp in range(0x110000): + decomposed = unicodedata.normalize("NFKD", chr(cp)) + if decomposed != chr(cp): + decompositions.append((cp, len(flat), len(decomposed))) + flat.extend(ord(c) for c in decomposed) + marks = [cp for cp in range(0x110000) if unicodedata.category(chr(cp)) in ("Mn", "Mc", "Me")] + lower_flat, lowercases = [], [] + for cp in range(0x110000): + low = chr(cp).lower() + if low != chr(cp): + lowercases.append((cp, len(lower_flat), len(low))) + lower_flat.extend(ord(c) for c in low) + cccs = [] + for cp in range(0x110000): + ccc = unicodedata.combining(chr(cp)) + if not ccc: + continue + if cccs and cccs[-1][2] == ccc and cccs[-1][1] + 1 == cp: + cccs[-1][1] = cp + else: + cccs.append([cp, cp, ccc]) + decomp = lambda e: ".{ .cp = 0x%04X, .start = %d, .len = %d }" % e + ccc = lambda t: ".{ 0x%04X, 0x%04X, 0x%04X }" % tuple(t) + return "\n".join([ + "// Generated by tools/zig/nfkd_tables.py from the Unicode %s character database; do not edit." % VERSION, + 'pub const version = "%s";' % VERSION, + "", + "pub const Decomp = struct { cp: u21, start: u32, len: u8 };", + "", + "// All NFKD sequences concatenated, indexed by Decomp.start/.len.", + "pub const sequences = [_]u21{", *rows(flat, 8, lambda v: "0x%04X" % v), "};", + "", + "pub const decompositions = [_]Decomp{", *rows(decompositions, 1, decomp), "};", + "", + "// General-category Mn/Mc/Me codepoints, sorted; the Rust tokenizers' StripAccents removes all of category M*.", + "pub const marks = [_]u21{", *rows(marks, 8, lambda v: "0x%04X" % v), "};", + "", + "pub const Lower = struct { cp: u21, start: u32, len: u8 };", + "", + "// Full lowercase mappings concatenated (Rust char::to_lowercase semantics).", + "pub const lower_sequences = [_]u21{", *rows(lower_flat, 8, lambda v: "0x%04X" % v), "};", + "", + "pub const lowercases = [_]Lower{", *rows(lowercases, 1, decomp), "};", + "", + "// Canonical combining class ranges [first, last, ccc], unicodedata %s." % VERSION, + "pub const cccs = [_][3]u21{", *rows(cccs, 4, ccc), "};", + ]) + "\n" + + +def formatted(text, zig): + """`text` grown through the Zig compiler's formatter, which is how the committed file is shaped.""" + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "nfkd_data.zig" + path.write_text(text) + subprocess.run([zig, "fmt", str(path)], check=True, capture_output=True) + return path.read_text() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true", help="fail if the committed file differs") + parser.add_argument("--zig", default="zig", help="the Zig compiler whose formatter shapes the output") + args = parser.parse_args() + if unicodedata.unidata_version != VERSION: + sys.exit("Python has Unicode %s; the Engram tables need %s" % (unicodedata.unidata_version, VERSION)) + text = formatted(render(), args.zig) + if args.check: + sys.exit(0 if OUT.read_text() == text else "%s is stale; rerun tools/zig/nfkd_tables.py" % OUT.relative_to(ROOT)) + OUT.write_text(text) + print("Wrote %s (Unicode %s)" % (OUT.relative_to(ROOT), VERSION)) + + +if __name__ == "__main__": + main() diff --git a/zig/build/deepseek_v41.zig b/zig/build/deepseek_v41.zig new file mode 100644 index 0000000000..2e419e9755 --- /dev/null +++ b/zig/build/deepseek_v41.zig @@ -0,0 +1,60 @@ +//! Experimental DS4.1 weight-loading qualification targets. +const std = @import("std"); + +pub fn weights( + b: *std.Build, + target: std.Build.ResolvedTarget, + optimize: std.builtin.OptimizeMode, + metal: *std.Build.Module, +) *std.Build.Step { + const host = b.addTest(.{ .root_module = b.createModule(.{ + .root_source_file = b.path("zig/deepseek_v41_weights_checks.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + .imports = &.{ + .{ .name = "metal", .module = metal }, + }, + }) }); + const step = b.step("test-deepseek-v41-weights", "V4.1 configuration, Engram table plans and resident weight staging"); + step.dependOn(&b.addRunArtifact(host).step); + return step; +} + +pub fn checks( + b: *std.Build, + target: std.Build.ResolvedTarget, + optimize: std.builtin.OptimizeMode, + metal: *std.Build.Module, + sources: *std.Build.Module, + engine: *std.Build.Module, +) struct { device: *std.Build.Step, reference: *std.Build.Step } { + const device = b.addTest(.{ .root_module = b.createModule(.{ + .root_source_file = b.path("zig/deepseek_v41_checks.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + .imports = &.{ + .{ .name = "metal", .module = metal }, + .{ .name = "kernel_sources", .module = sources }, + }, + }) }); + const device_step = b.step("test-deepseek-v41", "Experimental DS4.1 EXL3, Engram, mHC and sparse attention on Metal"); + device_step.dependOn(&b.addRunArtifact(device).step); + const reference = b.addTest(.{ + .root_module = b.createModule(.{ + .root_source_file = b.path("zig/deepseek_v41_reference_checks.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + .imports = &.{ + .{ .name = "tensorfold", .module = engine }, + .{ .name = "metal", .module = metal }, + }, + }), + .filters = &.{"tests.deepseek_v41_reference."}, + }); + const reference_step = b.step("test-deepseek-v41-reference", "Independent CPU teacher-oracle arithmetic boundaries"); + reference_step.dependOn(&b.addRunArtifact(reference).step); + return .{ .device = device_step, .reference = reference_step }; +} diff --git a/zig/deepseek_v41_checks.zig b/zig/deepseek_v41_checks.zig new file mode 100644 index 0000000000..755a35c618 --- /dev/null +++ b/zig/deepseek_v41_checks.zig @@ -0,0 +1,13 @@ +//! Source-built V4.1 operator qualification; real Metal execution, no checkpoint needed. +test { + _ = @import("src/core/exl3_projection_gpu_test.zig"); + _ = @import("src/families/deepseek_v41/engram.zig"); + _ = @import("src/families/deepseek_v41/csa2.zig"); + _ = @import("src/families/deepseek_v41/ops_check.zig"); + _ = @import("src/families/deepseek_v41/load.zig"); + _ = @import("src/families/deepseek_v41/index_precision_test.zig"); + _ = @import("src/families/deepseek_v41/config.zig"); + _ = @import("src/families/deepseek_v41/profiler.zig"); + _ = @import("src/core/gpu_counters.zig"); + _ = @import("src/core/expert_groups.zig"); +} diff --git a/zig/deepseek_v41_reference_checks.zig b/zig/deepseek_v41_reference_checks.zig new file mode 100644 index 0000000000..0111422555 --- /dev/null +++ b/zig/deepseek_v41_reference_checks.zig @@ -0,0 +1,15 @@ +//! Independent CPU arithmetic boundaries used by the resident-checkpoint teacher oracle. +test { + _ = @import("tests/deepseek_v41_reference/projection.zig"); + _ = @import("tests/deepseek_v41_reference/math.zig"); + _ = @import("tests/deepseek_v41_reference/attention_math.zig"); + _ = @import("tests/deepseek_v41_reference/qualify.zig"); + _ = @import("tests/deepseek_v41_reference/operator_hc_fp64.zig"); + _ = @import("tests/deepseek_v41_reference/operator_hc_split_fp64.zig"); + _ = @import("tests/deepseek_v41_reference/operator_hc_amplify_fp64.zig"); + _ = @import("tests/deepseek_v41_reference/operator_csa2_fp64.zig"); + _ = @import("tests/deepseek_v41_reference/operator_csa2_pack_fp64.zig"); + _ = @import("tests/deepseek_v41_reference/operator_csa2_sparse_fp64.zig"); + _ = @import("tests/deepseek_v41_reference/operator_csa2_rope_fp64.zig"); + _ = @import("tests/deepseek_v41_reference/operator_csa2_index_fp64.zig"); +} diff --git a/zig/deepseek_v41_run.zig b/zig/deepseek_v41_run.zig new file mode 100644 index 0000000000..10cae27f87 --- /dev/null +++ b/zig/deepseek_v41_run.zig @@ -0,0 +1,310 @@ +//! Full-checkpoint admission and execution receipts; no synthetic weights or replay substitute. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const ds = tf.deepseek_v41; +const config = tf.deepseek_v41_config; +const load = tf.deepseek_v41_load; +const weights = tf.deepseek_v41_weights; +const process_memory = @import("src/server/process_memory.zig"); +const oracle = @import("tests/deepseek_v41_reference/qualify.zig"); +const margins = @import("tests/deepseek_v41_reference/margins.zig"); +const profile = @import("tests/deepseek_v41_reference/profile.zig"); +const trace = @import("tests/deepseek_v41_reference/trace.zig"); +const operators = @import("tests/deepseek_v41_reference/operators.zig"); +const attention_operators = @import("tests/deepseek_v41_reference/operator_attention.zig"); +const history_operators = @import("tests/deepseek_v41_reference/operator_history.zig"); +const hc_operators = @import("tests/deepseek_v41_reference/operator_hc.zig"); +const moe_operators = @import("tests/deepseek_v41_reference/operator_moe.zig"); + +fn usage(code: u8) noreturn { + std.debug.print("usage: tf-dsv41-run plan MODEL TABLES\n" ++ + " tf-dsv41-run teacher MODEL TABLES IDS_JSON OUT_DIR [CHUNK]\n" ++ + " tf-dsv41-run seams MODEL TABLES IDS_JSON OUT_DIR\n" ++ + " tf-dsv41-run oracle MODEL TABLES IDS_JSON OUT_DIR\n" ++ + " tf-dsv41-run margins MODEL TABLES IDS_JSON OUT_DIR [DEEP]\n" ++ + " tf-dsv41-run profile MODEL TABLES IDS_JSON OUT_DIR\n" ++ + " tf-dsv41-run trace MODEL TABLES IDS_JSON OUT_DIR\n" ++ + " tf-dsv41-run operators MODEL TABLES OUT_DIR\n" ++ + " tf-dsv41-run attention-sameinput MODEL TABLES TRACE_CPU_BIN OUT_DIR [LAYER]\n" ++ + " tf-dsv41-run attention-history-sameinput MODEL TABLES TRACE_DIR OUT_DIR TOKEN LAYER\n" ++ + " tf-dsv41-run hc-sameinput MODEL TABLES TRACE_CPU_BIN OUT_DIR [LAYER]\n" ++ + " tf-dsv41-run moe-sameinput MODEL TABLES TRACE_GPU_BIN OUT_DIR [LAYER]\n" ++ + " tf-dsv41-run generate MODEL TABLES IDS_JSON OUT_DIR [TOKENS]\n" ++ + "IDS_JSON: nonempty JSON array of token IDs below 129280.\n" ++ + "Limits: oracle/trace 16 IDs; profile 256 IDs; teacher/seams 4096 IDs.\n" ++ + "History: TOKEN below 4096; retained normalized prefix at most 256 MiB.\n" ++ + "Local diagnostics do not qualify the model. Oracle retains the independent 1% NRMS gate.\n", .{}); + std.process.exit(code); +} +fn number(s: []const u8) !u32 { + return std.fmt.parseInt(u32, s, 10); +} +fn validArguments(args: []const [:0]const u8) bool { + if (args.len < 2) return false; + const mode = args[1]; + for ([_][]const u8{ "plan", "operators" }, [_]usize{ 4, 5 }) |name, count| + if (std.mem.eql(u8, mode, name)) return args.len == count; + if (std.mem.eql(u8, mode, "attention-history-sameinput")) return args.len == 8; + for ([_][]const u8{ "teacher", "generate", "attention-sameinput", "hc-sameinput", "moe-sameinput", "margins" }) |name| + if (std.mem.eql(u8, mode, name)) return args.len == 6 or args.len == 7; + for ([_][]const u8{ "seams", "oracle", "profile", "trace" }) |name| + if (std.mem.eql(u8, mode, name)) return args.len == 6; + return false; +} +fn argmax(row: []const f32) !u32 { + var best: u32 = 0; + if (row.len == 0) return error.EmptyLogits; + var score = -std.math.inf(f32); + for (row, 0..) |v, i| { + if (!std.math.isFinite(v)) return error.NonfiniteLogits; + if (v > score) { + best = @intCast(i); + score = v; + } + } + return best; +} +fn write(io: std.Io, a: std.mem.Allocator, dir: []const u8, name: []const u8, data: []const u8) !void { + const path = try std.fs.path.join(a, &.{ dir, name }); + defer a.free(path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = data }); +} +const Counters = struct { disk_read: u64, pageins: u64, footprint: u64 }; +/// Darwin's rusage_info_v4 fields the runner reads around execution (page-ins, disk reads, footprint). +const RUsage = extern struct { + uuid: [16]u8, + user_time: u64, + system_time: u64, + pkg_idle_wkups: u64, + interrupt_wkups: u64, + pageins: u64, + wired_size: u64, + resident_size: u64, + phys_footprint: u64, + proc_start_abstime: u64, + proc_exit_abstime: u64, + child_user_time: u64, + child_system_time: u64, + child_pkg_idle_wkups: u64, + child_interrupt_wkups: u64, + child_pageins: u64, + child_elapsed_abstime: u64, + diskio_bytesread: u64, +}; +fn counters() !Counters { + const proc_pid_rusage = struct { + extern "c" fn proc_pid_rusage(pid: c_int, flavor: c_int, buffer: *anyopaque) c_int; + }.proc_pid_rusage; + const getpid = struct { + extern "c" fn getpid() c_int; + }.getpid; + var r: RUsage = undefined; + if (proc_pid_rusage(getpid(), 4, &r) != 0) return error.MemoryCountersUnavailable; + return .{ .disk_read = r.diskio_bytesread, .pageins = r.pageins, .footprint = r.phys_footprint }; +} +fn teacher(m: *ds.Model, ids: []const u32, chunk: u32, out: []f32) !f64 { + if (chunk == 0 or chunk > m.workspace.plan.rows or ids.len > m.cache.plan.capacity) + return error.InvalidTeacherWindow; + if (out.len != ids.len * 129280) return error.InvalidTeacherOutputLength; + try m.reset(); + var elapsed: f64 = 0; + var start: usize = 0; + while (start < ids.len) { + const end = @min(start + chunk, ids.len); + const row = try m.forward(ids[start..end], .all); + @memcpy(out[start * 129280 .. end * 129280], row); + for (row) |value| if (!std.math.isFinite(value)) return error.NonfiniteLogits; + elapsed += m.last_gpu_seconds; + start = end; + } + return elapsed; +} + +pub fn main(init: std.process.Init) !void { + const a = init.gpa; + const io = init.io; + const args = try init.minimal.args.toSlice(init.arena.allocator()); + if (args.len == 2 and (std.mem.eql(u8, args[1], "--help") or std.mem.eql(u8, args[1], "-h"))) usage(0); + if (!validArguments(args)) usage(2); + const arena = init.arena.allocator(); + const mode = args[1]; + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + if (std.mem.eql(u8, mode, "plan")) { + const cfg = try config.Config.load(arena, io, args[2]); + var plan = try load.Plan.open(a, io, args[2], args[3]); + defer plan.deinit(); + try weights.validate(&plan); + const device = try mtl.Device.init(); + defer device.deinit(); + std.debug.print("{{\"mode\":\"plan\",\"weight_bytes\":{d},\"shards\":{d},\"tensors\":{d},\"context\":{d},\"metal_working_set_bytes\":{d}}}\n", .{ plan.bytes, plan.shards.items.len, plan.tensors.count(), cfg.text_config.max_position_embeddings, device.maxWorkingSet() }); + return; + } + const is_operators = std.mem.eql(u8, mode, "operators"); + const is_attention = std.mem.eql(u8, mode, "attention-sameinput"); + const is_attention_history = std.mem.eql(u8, mode, "attention-history-sameinput"); + const is_hc = std.mem.eql(u8, mode, "hc-sameinput"); + const is_moe = std.mem.eql(u8, mode, "moe-sameinput"); + const max_ids: usize = if (std.mem.eql(u8, mode, "margins")) 1048576 else if (std.mem.eql(u8, mode, "oracle") or std.mem.eql(u8, mode, "trace")) 16 else if (std.mem.eql(u8, mode, "profile")) ds.prefill_step else if (std.mem.eql(u8, mode, "teacher") or std.mem.eql(u8, mode, "seams")) 4096 else 1048576; + const input = if (is_operators or is_attention or is_attention_history or is_hc or is_moe) &.{} else try std.Io.Dir.cwd().readFileAlloc(io, args[4], arena, .limited(max_ids * 12 + 2)); + const ids: []const u32 = if (is_operators or is_attention or is_attention_history or is_hc or is_moe) &.{} else try std.json.parseFromSliceLeaky([]const u32, arena, input, .{}); + if (!is_operators and !is_attention and !is_attention_history and !is_hc and !is_moe and (ids.len == 0 or ids.len > 1048576)) return error.InvalidPrompt; + if (ids.len > max_ids) return error.InvalidPrompt; + for (ids) |id| if (id >= 129280) return error.InvalidToken; + const is_oracle = std.mem.eql(u8, mode, "oracle"); + const is_trace = std.mem.eql(u8, mode, "trace"); + const dir = args[if (is_operators) @as(usize, 4) else 5]; + try std.Io.Dir.cwd().createDirPath(io, dir); + const map = try std.fs.path.join(arena, &.{ args[2], "engram-token-map.bin" }); + const map_blob = if (is_oracle or is_trace) + try std.Io.Dir.cwd().readFileAlloc(io, map, arena, .limited(2 << 20)) + else + &.{}; + const model = try a.create(ds.Model); + defer a.destroy(model); + const begin = mtl.clock.seconds(); + model.* = try ds.Model.open(a, io, .{ .dir = args[2], .table_dir = args[3], .token_map = map }); + defer model.deinit(); + const ready = mtl.clock.seconds(); + const before = try counters(); + var execution_after: ?Counters = null; + var execution_error: ?anyerror = null; + execute(a, io, model, args, ids, map_blob, dir, before, &execution_after) catch |err| { + execution_error = err; + }; + const after = execution_after orelse try counters(); + if (after.disk_read < before.disk_read or after.pageins < before.pageins) return error.NonmonotonicCounters; + const resident_gate_passed = after.disk_read == before.disk_read; + const receipt = try std.json.Stringify.valueAlloc(a, .{ + .mode = mode, + .evidence_origin = "current_invocation_not_historical_qualification", + .context_capacity = model.cache.plan.capacity, + .consumed_tokens = model.cache.position, + .weight_bytes = model.checkpoint.residentBytes(), + .resident_allocation_bytes = model.residency.allocatedSize(), + .ready_footprint_bytes = before.footprint, + .final_footprint_bytes = after.footprint, + .load_seconds = ready - begin, + .measurement_scope = if (execution_after != null) "model_execution" else "execution_and_diagnostic_io", + .execution_disk_read_bytes = after.disk_read - before.disk_read, + .execution_pageins = after.pageins - before.pageins, + .resident_gate_passed = resident_gate_passed, + .execution_error = if (execution_error) |err| @as(?[]const u8, @errorName(err)) else null, + }, .{}); + defer a.free(receipt); + std.debug.print("{s}\n", .{receipt}); + try write(io, a, dir, "residency.json", receipt); + if (!resident_gate_passed) std.debug.print("residency gate failed: ResidentExecutionReadFromDisk\n", .{}); + if (execution_error) |err| return err; + if (!resident_gate_passed) return error.ResidentExecutionReadFromDisk; +} + +fn execute(a: std.mem.Allocator, io: std.Io, model: *ds.Model, args: []const [:0]const u8, ids: []const u32, map_blob: []const u8, dir: []const u8, before: Counters, execution_after: *?Counters) !void { + const mode = args[1]; + const is_operators = std.mem.eql(u8, mode, "operators"); + const is_attention = std.mem.eql(u8, mode, "attention-sameinput"); + const is_attention_history = std.mem.eql(u8, mode, "attention-history-sameinput"); + const is_hc = std.mem.eql(u8, mode, "hc-sameinput"); + const is_moe = std.mem.eql(u8, mode, "moe-sameinput"); + const is_oracle = std.mem.eql(u8, mode, "oracle"); + const is_profile = std.mem.eql(u8, mode, "profile"); + const is_trace = std.mem.eql(u8, mode, "trace"); + const is_teacher = std.mem.eql(u8, mode, "teacher"); + const is_seams = std.mem.eql(u8, mode, "seams"); + if (is_operators) { + try operators.run(a, io, model, dir); + } else if (is_attention) { + try attention_operators.runAttention(a, io, model, args[4], dir, if (args.len > 6) try number(args[6]) else 0); + } else if (is_attention_history) { + try history_operators.runAttentionHistory(a, io, model, args[4], dir, try number(args[6]), try number(args[7])); + } else if (is_hc) { + try hc_operators.runHyperconnections(a, io, model, args[4], dir, if (args.len > 6) try number(args[6]) else 0); + } else if (is_moe) { + try moe_operators.runMoe(a, io, model, args[4], dir, if (args.len > 6) try number(args[6]) else 21); + } else if (is_oracle) { + try operators.run(a, io, model, dir); + _ = try oracle.run(a, io, model, ids, map_blob, dir); + } else if (std.mem.eql(u8, mode, "margins")) { + try margins.run(a, io, model, ids, dir, if (args.len > 6) try number(args[6]) else 1536); + } else if (is_profile) { + try profile.run(a, io, model, ids, dir); + } else if (is_trace) { + try trace.run(a, io, model, ids, map_blob, dir); + } else if (is_teacher or is_seams) { + const chunk = if (args.len > 6) try number(args[6]) else model.workspace.plan.rows; + if (chunk == 0 or chunk > model.workspace.plan.rows) return error.InvalidPrefillChunk; + const outputs = try a.alloc(f32, ids.len * 129280); + defer a.free(outputs); + const elapsed = try teacher(model, ids, chunk, outputs); + // Capture execution before writing the receipt, not filesystem metadata reads from the receipt itself. + if (is_seams) { + const alternative = try a.alloc(f32, outputs.len); + defer a.free(alternative); + for ([_]u32{ 1, 2, 3, 7, 31, 127, 128, 129, 255, 256 }) |split| { + if (split > model.workspace.plan.rows) continue; + _ = try teacher(model, ids, split, alternative); + var error_max: f64 = 0; + var disagreements: usize = 0; + var bit_disagreements: usize = 0; + for (outputs, alternative) |x, y| { + error_max = @max(error_max, @abs(@as(f64, x) - y)); + bit_disagreements += @intFromBool(@as(u32, @bitCast(x)) != @as(u32, @bitCast(y))); + } + for (0..ids.len) |i| { + if (try argmax(outputs[i * 129280 ..][0..129280]) != try argmax(alternative[i * 129280 ..][0..129280])) + disagreements += 1; + } + std.debug.print("{{\"mode\":\"seam\",\"chunk\":{d},\"max_logit_error\":{d},\"bit_disagreements\":{d},\"greedy_disagreements\":{d}}}\n", .{ split, error_max, bit_disagreements, disagreements }); + if (disagreements != 0 or bit_disagreements != 0) { + var name: [48]u8 = undefined; + const file = try std.fmt.bufPrint(&name, "seam-{d}-logits.f32", .{split}); + execution_after.* = try counters(); + try write(io, a, dir, file, std.mem.sliceAsBytes(alternative)); + return error.ChunkSeamMismatch; + } + } + } + std.debug.print("{{\"mode\":\"teacher\",\"tokens\":{d},\"gpu_seconds\":{d}}}\n", .{ ids.len, elapsed }); + execution_after.* = try counters(); + try write(io, a, dir, "logits.f32", std.mem.sliceAsBytes(outputs)); + } else if (std.mem.eql(u8, mode, "generate")) { + const n = if (args.len > 6) try number(args[6]) else 64; + if (ids.len > model.cache.plan.capacity or n == 0 or n > model.cache.plan.capacity - ids.len) + return error.ContextFull; + try model.reset(); + const prefill_start = mtl.clock.seconds(); + var row: []const f32 = &.{}; + var start: usize = 0; + var progress_at = prefill_start; + while (start < ids.len) { + const end = @min(start + model.workspace.plan.rows, ids.len); + row = try model.forward(ids[start..end], if (end == ids.len) .last else .none); + start = end; + const now = mtl.clock.seconds(); + if (now - progress_at >= 60 or start == ids.len) { + const memory = try counters(); + std.debug.print("{{\"mode\":\"prefill_progress\",\"completed_tokens\":{d},\"prompt_tokens\":{d},\"elapsed_seconds\":{d},\"footprint_bytes\":{d},\"disk_read_bytes\":{d},\"pageins\":{d}}}\n", .{ start, ids.len, now - prefill_start, memory.footprint, memory.disk_read - before.disk_read, memory.pageins - before.pageins }); + progress_at = now; + } + } + const prefill_end = mtl.clock.seconds(); + const generated = try a.alloc(u32, n); + defer a.free(generated); + var gpu_seconds: f64 = 0; + const decode_start = mtl.clock.seconds(); + for (generated, 0..) |*token, i| { + token.* = try argmax(row); + if (i + 1 < generated.len) { + row = try model.forward(&.{token.*}, .last); + gpu_seconds += model.last_gpu_seconds; + } + } + const decode_end = mtl.clock.seconds(); + execution_after.* = try counters(); + const json = try std.json.Stringify.valueAlloc(a, generated, .{}); + defer a.free(json); + try write(io, a, dir, "tokens.json", json); + std.debug.print("{{\"mode\":\"generate\",\"prompt_tokens\":{d},\"completion_tokens\":{d},\"prefill_seconds\":{d},\"decode_seconds\":{d},\"decode_gpu_seconds\":{d}}}\n", .{ ids.len, n, prefill_end - prefill_start, decode_end - decode_start, gpu_seconds }); + } else usage(2); +} diff --git a/zig/deepseek_v41_weights_checks.zig b/zig/deepseek_v41_weights_checks.zig new file mode 100644 index 0000000000..b3d4fedcac --- /dev/null +++ b/zig/deepseek_v41_weights_checks.zig @@ -0,0 +1,8 @@ +//! Host checks for V4.1 configuration, Engram table plans and resident weight staging; no kernels compiled. +test { + _ = @import("src/families/deepseek_v41/config.zig"); + _ = @import("src/families/deepseek_v41/view.zig"); + _ = @import("src/families/deepseek_v41/engram_hash.zig"); + _ = @import("src/families/deepseek_v41/load.zig"); + _ = @import("src/families/deepseek_v41/weights.zig"); +} diff --git a/zig/kernels/metal/core/exl3_projection.metal b/zig/kernels/metal/core/exl3_projection.metal index 0eca0d80c5..0725231179 100644 --- a/zig/kernels/metal/core/exl3_projection.metal +++ b/zig/kernels/metal/core/exl3_projection.metal @@ -241,6 +241,25 @@ kernel void tf_exl3_state_window_check(const device uint4* inputs [[buffer(0)]], outputs[tid] = tf_exl3_mapped_state(tile, (off << 6) | (1u << 11)); } +// Kept-dense: rotated rows x bf16 weight, f32 in rotate_out sums; sequential k, one thread an output, in fp64 order. +kernel void tf_exl3_dense(const device half* rotated [[buffer(0)]], + const device bfloat* weight [[buffer(1)]], + device float* sums [[buffer(2)]], + constant TfExl3Args& a [[buffer(3)]], + const device uint* input_ids [[buffer(4)]], + uint2 g [[thread_position_in_grid]]) { + const uint row = a.input_ids ? input_ids[g.y] : g.y; + // The reference's dot is FP64-sequential; this hi/lo pair carries the same exactness through FP32. + float hi = 0, lo = 0; + for (uint k = 0; k < a.k; ++k) { + float prod = float(rotated[ulong(g.y) * a.k + k]) * float(weight[ulong(g.x) * a.k + k]); + float t = hi + prod; + lo += (hi - t) + prod; + hi = t; + } + sums[ulong(g.y) * a.n + g.x] = hi + lo; +} + // Four SIMDgroups share decoded weights while each retains its original eight-row compensated dots. kernel void tf_exl3_matmul_prefill(const device half* x [[buffer(0)]], const device uchar* trellis [[buffer(1)]], diff --git a/zig/kernels/metal/core/expert_groups.metal b/zig/kernels/metal/core/expert_groups.metal new file mode 100644 index 0000000000..be543d5780 --- /dev/null +++ b/zig/kernels/metal/core/expert_groups.metal @@ -0,0 +1,36 @@ +// Group routed slots into padded eight-row expert tiles; scatter preserves original token and output IDs. +#include +using namespace metal; +struct TfExpertGroups { uint rows, experts, repeat, padded; }; +#define TF_GROUP_ARGS \ + const device uint* ids [[buffer(0)]], device atomic_uint* counts [[buffer(1)]], \ + device uint* offsets [[buffer(2)]], device uint* ranks [[buffer(3)]], \ + device uint* expert_ids [[buffer(4)]], device uint* token_ids [[buffer(5)]], \ + device uint* slot_ids [[buffer(6)]], constant TfExpertGroups& d [[buffer(7)]], \ + uint i [[thread_position_in_grid]] + +kernel void tf_expert_group_clear(TF_GROUP_ARGS) { + if (i < d.experts) atomic_store_explicit(counts + i, 0u, memory_order_relaxed); + if (i < d.padded) expert_ids[i] = 0xffffffffu; +} +kernel void tf_expert_group_count(TF_GROUP_ARGS) { + if (i >= d.rows || ids[i] >= d.experts) return; + ranks[i] = atomic_fetch_add_explicit(counts + ids[i], 1u, memory_order_relaxed); +} +kernel void tf_expert_group_prefix(TF_GROUP_ARGS) { + if (i != 0) return; + uint offset = 0; + for (uint expert = 0; expert < d.experts; ++expert) { + offsets[expert] = offset; + const uint count = atomic_load_explicit(counts + expert, memory_order_relaxed); + offset += (count + 7u) & ~7u; + } +} +kernel void tf_expert_group_scatter(TF_GROUP_ARGS) { + if (i >= d.rows || ids[i] >= d.experts) return; + const uint destination = offsets[ids[i]] + ranks[i]; + if (destination >= d.padded) return; + expert_ids[destination] = ids[i]; + token_ids[destination] = i / d.repeat; + slot_ids[destination] = i; +} diff --git a/zig/kernels/metal/deepseek_v41/csa2.metal b/zig/kernels/metal/deepseek_v41/csa2.metal new file mode 100644 index 0000000000..9109e9aaff --- /dev/null +++ b/zig/kernels/metal/deepseek_v41/csa2.metal @@ -0,0 +1,116 @@ +// Source is compiled after kv.metal, which provides exact packed-cache decoders. +struct Dsv41Attention { + uint rows, heads, width, q_stride, position, chunk_rows, window, ring_stride; + uint chunk_stride, main_stride, main_block, ratio, compressed_rows, topk, index_stride; + float scale; +}; +// BF16 query/key products are exact in FP32; compensate accumulation and lane reduction. +inline void dsv41_dot_add(thread float &sum, thread float &error, float value) { + float next = sum + value; + float recovered = next - sum; + error += (sum - (next - recovered)) + (value - recovered); + sum = next; +} +inline float dsv41_dot_reduce(float sum, float error) { + for (uint offset = 16; offset != 0; offset >>= 1) { + float other = simd_shuffle_xor(sum, offset); + float other_error = simd_shuffle_xor(error, offset); + dsv41_dot_add(sum, error, other); + dsv41_dot_add(sum, error, other_error); + } + return sum + error; +} +// BF16 probability rounding uses fixed query-based tiles, never a chunk-dependent physical key layout. +inline uint dsv41_window_columns(constant Dsv41Attention &p, uint row) { + return p.position + row == 0 ? 1 : p.window; +} +inline device const uchar *dsv41_attn_key(uint slot, uint row, device const uchar *ring, + device const uchar *chunk, device const uchar *main, device const int *indices, + constant Dsv41Attention &p, thread uint &kind) { + uint query = p.position + row; + uint columns = dsv41_window_columns(p, row); + if (slot < columns) { + uint pos; + if (query == 0) { + pos = 0; + } else if (query + 1 < p.window) { + pos = (query + 1 + slot) % p.window; + } else { + uint begin = query + 1 > p.window ? query + 1 - p.window : 0; + pos = begin + slot; + } + if (pos > query) return nullptr; + kind = 0; + return pos >= p.position ? chunk + (pos - p.position) * p.chunk_stride + : ring + (pos % p.window) * p.ring_stride; + } + if (!p.ratio) return nullptr; + int pos = indices[row * p.index_stride + slot - columns]; + if (pos < 0 || uint(pos) >= min(p.compressed_rows, (query + 1) / p.ratio)) return nullptr; + kind = 1; + return main + uint(pos) * p.main_stride; +} +kernel void dsv41_sparse_attention(device const ushort *q [[buffer(0)]], device const uchar *ring [[buffer(1)]], + device const uchar *chunk [[buffer(2)]], device const uchar *main [[buffer(3)]], + device const int *indices [[buffer(4)]], device const float *sink [[buffer(5)]], + device ushort *out [[buffer(6)]], constant Dsv41Attention &p [[buffer(7)]], + uint2 g [[threadgroup_position_in_grid]], uint lane [[thread_index_in_threadgroup]]) { + uint head = g.x, row = g.y; + if (head >= p.heads || row >= p.rows) return; + threadgroup float scores[64]; + float query[16], acc[16]; + for (uint j = 0; j < p.width / 32; ++j) { + query[j] = dsv41_bf(q[row * p.q_stride + head * p.width + lane + j * 32]); acc[j] = 0; + } + float mx = -1e30f, sum = 0; + uint query_position = p.position + row; + uint compressed_columns = p.ratio ? min(p.topk, min(p.compressed_rows, (query_position + 1) / p.ratio)) : 0; + uint count = dsv41_window_columns(p, row) + compressed_columns; + for (uint base = 0; base < count; base += 64) { + float next_max = mx; + for (uint j = 0; j < 64; ++j) { + uint kind = 0; + device const uchar *key = base + j < count ? dsv41_attn_key(base + j, row, ring, chunk, main, indices, p, kind) : nullptr; + float dot = 0, dot_error = 0; + if (key) for (uint d = 0; d < p.width / 32; ++d) + dsv41_dot_add(dot, dot_error, query[d] * dsv41_unpack(key, lane + d * 32, p.width, kind ? p.main_block : 32, kind)); + dot = key ? dsv41_dot_reduce(dot, dot_error) * p.scale : -INFINITY; + next_max = max(next_max, dot); + if (lane == 0) scores[j] = dot; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + float correction = exp(mx - next_max); + float tile_sum_hi = 0, tile_sum_lo = 0; + float acc_hi[16], acc_lo[16]; + for (uint d = 0; d < p.width / 32; ++d) { + acc_hi[d] = acc[d] * correction; + acc_lo[d] = fma(acc[d], correction, -acc_hi[d]); + } + for (uint j = 0; j < 64; ++j) { + float probability = exp(scores[j] - next_max); + float t = tile_sum_hi + probability; + tile_sum_lo += (tile_sum_hi - t) + probability; + tile_sum_hi = t; + float rounded = dsv41_bf(dsv41_round(probability)); + uint kind = 0; + device const uchar *key = base + j < count ? dsv41_attn_key(base + j, row, ring, chunk, main, indices, p, kind) : nullptr; + if (key) for (uint d = 0; d < p.width / 32; ++d) { + float v = dsv41_unpack(key, lane + d * 32, p.width, kind ? p.main_block : 32, kind); + float prod_hi = rounded * v; + float prod_lo = fma(rounded, v, -prod_hi); + float t2 = acc_hi[d] + prod_hi; + acc_lo[d] += (acc_hi[d] - t2) + prod_hi; + acc_hi[d] = t2; + acc_lo[d] += prod_lo; + } + } + for (uint d = 0; d < p.width / 32; ++d) acc[d] = acc_hi[d] + acc_lo[d]; + sum = sum * correction + (tile_sum_hi + tile_sum_lo); + mx = next_max; + threadgroup_barrier(mem_flags::mem_threadgroup); + } + // The source adds sink mass to the completed online denominator before the FP32 division. + float denom = sum + exp(sink[head] - mx); + for (uint d = 0; d < p.width / 32; ++d) + out[row * p.q_stride + head * p.width + lane + d * 32] = dsv41_round(acc[d] / denom); +} diff --git a/zig/kernels/metal/deepseek_v41/engram.metal b/zig/kernels/metal/deepseek_v41/engram.metal new file mode 100644 index 0000000000..b3d454cf59 --- /dev/null +++ b/zig/kernels/metal/deepseek_v41/engram.metal @@ -0,0 +1,155 @@ +// Math/layout attribution: DeepSeek AI DeepSeek-V4.1-Flash inference/{engram,model}.py (MIT). +#include +using namespace metal; +inline float rd(const device uchar* p, ulong i, uint dtype) { + if (dtype == 1u) return ((const device float*)p)[i]; + if (dtype == 2u) return float(((const device half*)p)[i]); + return as_type(uint(((const device ushort*)p)[i]) << 16); +} + +struct EngramLayer { + uint layer_id, rows; + ulong multipliers[4]; + uint primes[24], offsets[24]; +}; +struct EngramHistory { int tail[3]; uint available; }; +struct EngramHashArgs { uint tokens, pad, masked; }; + +inline int engram_token(const device uint* ids, const device uint* map, + const device uchar* mask, uint pos, uint masked) { + return masked && mask[pos] == 0 ? -1 : int(map[ids[pos]]); +} + +// One lane per token; history is immutable until all layers have hashed this chunk. +kernel void tf_v41_engram_hash(const device uint* ids [[buffer(0)]], + const device uint* map [[buffer(1)]], + const device uchar* mask [[buffer(2)]], + const device EngramHistory& history [[buffer(3)]], + device uint* out [[buffer(4)]], + constant EngramLayer& layer [[buffer(5)]], + constant EngramHashArgs& args [[buffer(6)]], + uint pos [[thread_position_in_grid]]) { + if (pos >= args.tokens) return; + bool blocked = false; + ulong rolling = 0; + for (uint shift = 0; shift < 4; shift++) { + int token; + if (shift <= pos) token = engram_token(ids, map, mask, pos - shift, args.masked); + else { + const uint back = shift - pos - 1; + token = back < history.available ? history.tail[back] : -1; + } + blocked = blocked || token == -1; + const ulong compressed = blocked ? args.pad : uint(token); + rolling ^= compressed * layer.multipliers[shift]; + if (shift != 0) for (uint head = 0; head < 8; head++) { + const uint col = (shift - 1) * 8 + head; + out[size_t(pos) * 24 + col] = uint(rolling % layer.primes[col]) + layer.offsets[col]; + } + } +} + +// Called exactly once after all Engram layers hash a chunk, never once per layer. +kernel void tf_v41_engram_commit(const device uint* ids [[buffer(0)]], + const device uint* map [[buffer(1)]], + const device uchar* mask [[buffer(2)]], + device EngramHistory& history [[buffer(3)]], + constant EngramHashArgs& args [[buffer(4)]]) { + int tail[3]; + for (uint i = 0; i < 3; i++) { + tail[i] = i < args.tokens ? engram_token(ids, map, mask, args.tokens - i - 1, args.masked) + : i - args.tokens < history.available ? history.tail[i - args.tokens] : -1; + } + for (uint i = 0; i < 3; i++) history.tail[i] = tail[i]; + history.available = min(3u, history.available + args.tokens); +} + +kernel void tf_v41_engram_reset(device EngramHistory& history [[buffer(0)]]) { + for (uint i = 0; i < 3; i++) history.tail[i] = -1; + history.available = 0; +} + +inline float engram_e4m3(uchar bits) { + const uint magnitude = bits & 127, exponent = magnitude >> 3, mantissa = magnitude & 7; + float value = magnitude == 127 ? NAN : exponent == 0 ? float(mantissa) * 0x1p-9f + : ldexp(1.0f + float(mantissa) * 0.125f, int(exponent) - 7); + return bits & 128 ? -value : value; +} +inline float engram_ue8m0(uchar bits) { + return bits == 255 ? NAN : as_type(bits == 0 ? 0x00400000u : uint(bits) << 23); +} + +struct EngramGatherArgs { + ulong weight_stride, scale_stride, output_stride; + uint tokens, columns, head_dim, scale_block; +}; + +// Read only the selected E4M3FN rows; one UE8M0 scale per 32 values of each row. +kernel void tf_v41_engram_gather(const device uint* ids [[buffer(0)]], + const device uchar* table [[buffer(1)]], + const device uchar* scales [[buffer(2)]], + device bfloat* out [[buffer(3)]], + constant EngramGatherArgs& args [[buffer(4)]], + uint2 pos [[thread_position_in_grid]]) { + if (pos.y >= args.tokens || pos.x >= args.columns * args.head_dim) return; + const uint col = pos.x / args.head_dim, d = pos.x % args.head_dim; + const size_t row = ids[size_t(pos.y) * args.columns + col]; + const float value = engram_e4m3(table[row * args.weight_stride + d]); + const float scale = engram_ue8m0(scales[row * args.scale_stride + d / args.scale_block]); + out[size_t(pos.y) * args.output_stride + pos.x] = bfloat(value * scale); +} + +struct EngramGateArgs { + ulong hidden_stride, kv_stride, output_stride; + uint dim, copies, masked, weight_dtype; + float eps; +}; + +// Each copy gets its own RMS dot gate; all copies share the value following the flattened keys. +kernel void tf_v41_engram_gate(const device bfloat* hidden [[buffer(0)]], + const device bfloat* kv [[buffer(1)]], + const device uchar* q_weight [[buffer(2)]], + const device uchar* k_weight [[buffer(3)]], + const device uchar* mask [[buffer(4)]], + device bfloat* out [[buffer(5)]], + constant EngramGateArgs& args [[buffer(6)]], + uint2 group [[threadgroup_position_in_grid]], + uint t [[thread_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]], + uint sg [[simdgroup_index_in_threadgroup]]) { + threadgroup float3 sums[8]; + threadgroup float gate; + const size_t hbase = size_t(group.x) * args.hidden_stride + size_t(group.y) * args.dim; + const size_t kbase = size_t(group.x) * args.kv_stride + size_t(group.y) * args.dim; + const size_t wbase = size_t(group.y) * args.dim; + float3 acc = 0; + for (uint d = t; d < args.dim; d += 256) { + const float h = float(hidden[hbase + d]), k = float(kv[kbase + d]); + const float weight = rd(q_weight, wbase + d, args.weight_dtype) * rd(k_weight, wbase + d, args.weight_dtype); + acc += float3(h * h, k * k, (h * weight) * k); + } + acc = float3(simd_sum(acc.x), simd_sum(acc.y), simd_sum(acc.z)); + if (lane == 0) sums[sg] = acc; + threadgroup_barrier(mem_flags::mem_threadgroup); + if (sg == 0) { + acc = lane < 8 ? sums[lane] : float3(0); + acc = float3(simd_sum(acc.x), simd_sum(acc.y), simd_sum(acc.z)); + if (lane == 0) { + float rstd; + { +#pragma clang fp contract(off) + const float factor = 1.0f / float(args.dim); + rstd = metal::precise::rsqrt(acc.x * factor + args.eps) + * metal::precise::rsqrt(acc.y * factor + args.eps); + } + const float dot = (acc.z * rstd) * metal::precise::rsqrt(float(args.dim)); + const float signed_root = copysign(metal::precise::sqrt(max(abs(dot), 1e-6f)), dot); + gate = args.masked && mask[group.x] == 0 ? 0.0f : 1.0f / (1.0f + metal::precise::exp(-signed_root)); + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + const size_t vbase = size_t(group.x) * args.kv_stride + size_t(args.copies) * args.dim; + const size_t obase = size_t(group.x) * args.output_stride + size_t(group.y) * args.dim; + for (uint d = t; d < args.dim; d += 256) + out[obase + d] = bfloat(float(hidden[hbase + d]) + gate * float(kv[vbase + d])); +} diff --git a/zig/kernels/metal/deepseek_v41/index.metal b/zig/kernels/metal/deepseek_v41/index.metal new file mode 100644 index 0000000000..8d62337624 --- /dev/null +++ b/zig/kernels/metal/deepseek_v41/index.metal @@ -0,0 +1,103 @@ +// Source is compiled after kv.metal; no query-by-key-by-head score tensor is materialized. +struct Dsv41Index { + uint rows, heads, width, q_stride, weight_stride, position, ratio, keys, key_stride; + uint tile_start, tile_width, topk, candidate_topk, block_size, use_candidates, make_candidates; + float scale; +}; +struct Dsv41Pick { float score; int position; }; +inline bool dsv41_less(Dsv41Pick a, Dsv41Pick b) { + return a.score < b.score || (a.score == b.score && a.position > b.position); +} +inline void dsv41_heap(device Dsv41Pick *h, uint size, Dsv41Pick v) { + if (!dsv41_less(h[0], v)) return; + uint at = 0; + while (2 * at + 1 < size) { + uint child = 2 * at + 1; + if (child + 1 < size && dsv41_less(h[child + 1], h[child])) ++child; + if (!dsv41_less(h[child], v)) break; + h[at] = h[child]; at = child; + } + h[at] = v; +} +inline int dsv41_index_position(uint local, uint row, device const Dsv41Pick *candidates, + constant Dsv41Index &p) { + uint slot = p.tile_start + local; + if (!p.use_candidates) return int(slot); + uint block = slot / p.block_size; + if (block >= p.candidate_topk) return -1; + int id = candidates[row * p.candidate_topk + block].position; + return id < 0 ? -1 : id * int(p.block_size) + int(slot % p.block_size); +} +kernel void dsv41_index_score(device const ushort *q [[buffer(0)]], device const ushort *weight [[buffer(1)]], + device const uchar *key [[buffer(2)]], device const Dsv41Pick *candidates [[buffer(3)]], + device float *scores [[buffer(4)]], constant Dsv41Index &p [[buffer(5)]], + uint2 g [[threadgroup_position_in_grid]], uint lane [[thread_index_in_threadgroup]]) { + uint local = g.x, row = g.y; + if (local >= p.tile_width || row >= p.rows) return; + int pos = dsv41_index_position(local, row, candidates, p); + uint visible = min(p.keys, (p.position + row + 1) / p.ratio); + if (pos < 0 || uint(pos) >= visible) { + if (lane == 0) scores[row * p.tile_width + local] = -INFINITY; + return; + } + device const uchar *k = key + uint(pos) * p.key_stride; + float key_values[4]; + for (uint d = 0; d < p.width / 32; ++d) key_values[d] = dsv41_unpack(k, lane + 32 * d, p.width, 32, 2); + float total = 0; + for (uint head = 0; head < p.heads; ++head) { + float dot = 0; + for (uint d = 0; d < p.width / 32; ++d) + dot += dsv41_bf(q[row * p.q_stride + head * p.width + lane + 32 * d]) * key_values[d]; + dot = dsv41_bf(dsv41_round(simd_sum(dot))); + const float scaled_weight = dsv41_bf(dsv41_round(dsv41_bf(weight[row * p.weight_stride + head]) * p.scale)); + total += dsv41_bf(dsv41_round(max(dot, 0.0f) * scaled_weight)); + } + if (lane == 0) scores[row * p.tile_width + local] = dsv41_bf(dsv41_round(total)); +} +kernel void dsv41_index_clear(device Dsv41Pick *top [[buffer(0)]], device Dsv41Pick *candidate [[buffer(1)]], + constant Dsv41Index &p [[buffer(2)]], uint2 i [[thread_position_in_grid]]) { + if (i.y >= p.rows) return; + if (i.x < p.topk) top[i.y * p.topk + i.x] = {-INFINITY, -1}; + if (p.make_candidates && i.x < p.candidate_topk) candidate[i.y * p.candidate_topk + i.x] = {-INFINITY, -1}; +} +kernel void dsv41_index_merge(device const float *scores [[buffer(0)]], device Dsv41Pick *top [[buffer(1)]], + device Dsv41Pick *candidate [[buffer(2)]], constant Dsv41Index &p [[buffer(3)]], + uint row [[thread_position_in_grid]]) { + if (row >= p.rows) return; + device Dsv41Pick *heap = top + row * p.topk; + for (uint i = 0; i < p.tile_width; ++i) { + float score = scores[row * p.tile_width + i]; + int pos = dsv41_index_position(i, row, candidate, p); + if (isfinite(score)) dsv41_heap(heap, p.topk, {score, pos}); + } + if (!p.make_candidates) return; + uint visible = min(p.keys, (p.position + row + 1) / p.ratio); + device Dsv41Pick *blocks = candidate + row * p.candidate_topk; + for (uint base = 0; base < p.tile_width; base += p.block_size) { + float score = -INFINITY; + for (uint j = 0; j < p.block_size; ++j) score = max(score, scores[row * p.tile_width + base + j]); + uint block = (p.tile_start + base) / p.block_size; + if (visible && block == (visible - 1) / p.block_size) score = INFINITY; + if (score > -INFINITY) dsv41_heap(blocks, p.candidate_topk, {score, int(block)}); + } +} +inline void dsv41_sort_positions(device Dsv41Pick *p, uint size) { + // Selection is score-ordered; attention sees the reference's ascending position order. + for (uint i = 1; i < size; ++i) { + Dsv41Pick value = p[i]; uint j = i; + while (j > 0 && (p[j - 1].position < 0 || + (value.position >= 0 && p[j - 1].position > value.position))) { + if (value.position < 0) break; + p[j] = p[j - 1]; --j; + } + p[j] = value; + } +} +kernel void dsv41_index_finish(device Dsv41Pick *top [[buffer(0)]], device Dsv41Pick *candidate [[buffer(1)]], + device int *positions [[buffer(2)]], constant Dsv41Index &p [[buffer(3)]], uint row [[thread_position_in_grid]]) { + if (row >= p.rows) return; + device Dsv41Pick *heap = top + row * p.topk; + dsv41_sort_positions(heap, p.topk); + for (uint i = 0; i < p.topk; ++i) positions[row * p.topk + i] = heap[i].position; + if (p.make_candidates) dsv41_sort_positions(candidate + row * p.candidate_topk, p.candidate_topk); +} diff --git a/zig/kernels/metal/deepseek_v41/kv.metal b/zig/kernels/metal/deepseek_v41/kv.metal new file mode 100644 index 0000000000..6ac2017efb --- /dev/null +++ b/zig/kernels/metal/deepseek_v41/kv.metal @@ -0,0 +1,141 @@ +// DeepSeek V4.1 cache quantization and pooling, following the official inference kernels. +#include +using namespace metal; +inline float dsv41_bf(ushort x) { return as_type(uint(x) << 16); } +inline ushort dsv41_round(float x) { + uint u = as_type(x); + return ushort((u + 0x7fff + ((u >> 16) & 1)) >> 16); +} +inline float dsv41_e4(uchar c) { + uint e = (c >> 3) & 15, m = c & 7; + float v = e == 0 ? float(m) * 0x1p-9f : ldexp(1.0f + float(m) / 8.0f, int(e) - 7); + return (c & 128) ? -v : v; +} +inline uchar dsv41_e4code(float x) { + float a = min(abs(x), 448.0f); + if (a == 0) return signbit(x) ? 128 : 0; + int e = max(-6, int((as_type(a) >> 23) & 255) - 127); + int q = int(rint(ldexp(a, 3 - e))); + uint c = e == -6 && q < 8 ? uint(q) : uint((e + 7) * 8 + q - 8); + return uchar(min(c, 126u) | (signbit(x) ? 128u : 0u)); +} +inline float dsv41_e2(uchar c) { + const float v[8] = {0, .5f, 1, 1.5f, 2, 3, 4, 6}; + return (c & 8) ? -v[c & 7] : v[c & 7]; +} +inline uchar dsv41_e2code(float x) { + float a = min(abs(x), 6.0f); + uint pick = a <= .25f ? 0 : a < .75f ? 1 : a <= 1.25f ? 2 : a < 1.75f ? 3 + : a <= 2.5f ? 4 : a < 3.5f ? 5 : a <= 5.0f ? 6 : 7; + return uchar(pick | (signbit(x) ? 8 : 0)); +} +inline float dsv41_unpack(device const uchar *p, uint d, uint width, uint block, uint kind) { + if (kind == 0) { + float s = as_type(uint(p[width + d / 32]) << 23); + return dsv41_bf(dsv41_round(dsv41_e4(p[d]) * s)); + } + uchar c = p[d / 2]; + float s = kind == 1 ? dsv41_e4(p[width / 2 + d / block]) + : as_type(uint(p[width / 2 + d / block]) << 23); + return dsv41_bf(dsv41_round(dsv41_e2((d & 1) ? c >> 4 : c & 15) * s)); +} +struct Dsv41Pack { uint rows, width, stride, packed_stride, block, kind, first, ring, inplace; }; +kernel void dsv41_pack(device ushort *x [[buffer(0)]], device uchar *out [[buffer(1)]], + constant Dsv41Pack &p [[buffer(2)]], uint2 g [[threadgroup_position_in_grid]], + uint lane [[thread_index_in_threadgroup]]) { + uint row = g.y, group = g.x, d = group * p.block + lane; + if (row >= p.rows) return; + float v = lane < p.block ? dsv41_bf(x[row * p.stride + d]) : 0; + float a = simd_max(abs(v)); + float scale; uchar sc; + if (p.kind == 1) { + sc = dsv41_e4code(max(a, 6.0f * 0x1p-9f) / 6.0f); scale = dsv41_e4(sc); + } else { + float minimum = p.kind == 0 ? 1e-4f : 6.0f * 0x1p-126f; + float z = max(a, minimum) / (p.kind == 0 ? 448.0f : 6.0f); + uint bits = as_type(z); + int exp = int((bits >> 23) & 255) - 127 + ((bits & 0x7fffff) != 0); + sc = uchar(exp + 127); scale = ldexp(1.0f, exp); + } + uint dst = p.ring ? (p.first + row) % p.ring : p.first + row; + device uchar *packed = out + dst * p.packed_stride; + if (lane == 0) packed[(p.kind == 0 ? p.width : p.width / 2) + group] = sc; + if (lane >= p.block) return; + if (p.kind == 0) { + uchar c = dsv41_e4code(v / scale); packed[d] = c; + if (p.inplace) x[row * p.stride + d] = dsv41_round(dsv41_e4(c) * scale); + } else { + uchar c = dsv41_e2code(v / scale); + uchar other = uchar(simd_shuffle_xor(uint(c), 1)); + if (!(lane & 1)) packed[d / 2] = c | (other << 4); + if (p.inplace) x[row * p.stride + d] = dsv41_round(dsv41_e2(c) * scale); + } +} +struct Dsv41Pool { uint rows, width, ratio, position, stride, input_f32; float eps; }; +kernel void dsv41_pool(device const uchar *values [[buffer(0)]], device const float *gate [[buffer(1)]], + device float *partial [[buffer(2)]], device const ushort *weight [[buffer(3)]], + device ushort *latent [[buffer(4)]], constant Dsv41Pool &p [[buffer(5)]], + uint group [[threadgroup_position_in_grid]], uint lane [[thread_index_in_threadgroup]]) { + // A group reads the prior chunk's tail before the independent tail-update dispatch. + uint start = (p.position / p.ratio + group) * p.ratio; + threadgroup float pooled[512]; + float square = 0; + for (uint d = lane; d < p.width; d += 128) { + float mx = -INFINITY, sum = 0, total = 0; + for (uint j = 0; j < p.ratio; ++j) { + uint pos = start + j; + float score = p.ratio == 1 ? 0 : (pos < p.position ? partial[p.ratio * p.width + (pos % p.ratio) * p.width + d] + : gate[(pos - p.position) * p.stride + d]); + mx = max(mx, score); + } + float slots[2], exponent[2]; + for (uint j = 0; j < p.ratio; ++j) { + uint pos = start + j; + slots[j] = pos < p.position ? partial[(pos % p.ratio) * p.width + d] + : (p.input_f32 ? reinterpret_cast(values)[(pos - p.position) * p.stride + d] + : dsv41_bf(reinterpret_cast(values)[(pos - p.position) * p.stride + d])); + float score = p.ratio == 1 ? 0 : (pos < p.position ? partial[p.ratio * p.width + (pos % p.ratio) * p.width + d] + : gate[(pos - p.position) * p.stride + d]); + exponent[j] = exp(score - mx); sum += exponent[j]; + } + for (uint j = 0; j < p.ratio; ++j) total += slots[j] * (exponent[j] / sum); + float v = dsv41_bf(dsv41_round(total)); pooled[d] = v; square += v * v; + } + threadgroup float sums[4]; + float ss = simd_sum(square); + if (!(lane % 32)) sums[lane / 32] = ss; + threadgroup_barrier(mem_flags::mem_threadgroup); + float inv = rsqrt((sums[0] + sums[1] + sums[2] + sums[3]) / p.width + p.eps); + for (uint d = lane; d < p.width; d += 128) + latent[group * p.width + d] = dsv41_round(pooled[d] * inv * dsv41_bf(weight[d])); +} +kernel void dsv41_pool_tail(device const float *values [[buffer(0)]], device const float *gate [[buffer(1)]], + device float *partial [[buffer(2)]], constant Dsv41Pool &p [[buffer(3)]], + uint d [[thread_position_in_grid]]) { + if (d >= p.width || p.ratio == 1) return; + uint end = p.position + p.rows, tail = end % p.ratio; + for (uint j = 0; j < tail; ++j) { + uint pos = end - tail + j; + if (pos < p.position) continue; + partial[j * p.width + d] = values[(pos - p.position) * p.stride + d]; + partial[p.ratio * p.width + j * p.width + d] = gate[(pos - p.position) * p.stride + d]; + } +} +kernel void dsv41_copy(device const uchar *src [[buffer(0)]], device uchar *dst [[buffer(1)]], + constant uint &bytes [[buffer(2)]], uint i [[thread_position_in_grid]]) { + if (i < bytes) dst[i] = src[i]; +} +kernel void dsv41_publish_window(device const uchar *chunk [[buffer(0)]], device uchar *ring [[buffer(1)]], + constant uint4 &p [[buffer(2)]], uint2 i [[thread_position_in_grid]]) { + uint count = min(p.x, 128u), first = p.x - count; + if (i.x >= p.z || i.y >= count) return; + uint row = first + i.y, slot = (p.y + row) % 128; + ring[slot * p.w + i.x] = chunk[row * p.z + i.x]; +} +kernel void dsv41_unpack_rows(device const uchar *packed [[buffer(0)]], device ushort *out [[buffer(1)]], + constant Dsv41Pack &p [[buffer(2)]], uint2 i [[thread_position_in_grid]]) { + if (i.x >= p.width || i.y >= p.rows) return; + uint row = p.ring ? (p.first + i.y) % p.ring : p.first + i.y; + out[i.y * p.stride + i.x] = dsv41_round(dsv41_unpack(packed + row * p.packed_stride, + i.x, p.width, p.block, p.kind)); +} diff --git a/zig/kernels/metal/deepseek_v41/mhc.metal b/zig/kernels/metal/deepseek_v41/mhc.metal new file mode 100644 index 0000000000..8b6203a4cf --- /dev/null +++ b/zig/kernels/metal/deepseek_v41/mhc.metal @@ -0,0 +1,120 @@ +// Reference: DeepSeek-V4.1-Flash inference/model.py and kernel.py, revision 2cba9e42aa026125f3ed06c6d98c1db82f7ca027. +#include +using namespace metal; +inline float bf(ushort x) { return as_type(uint(x)<<16); } +inline ushort rb(float x) { + uint u=as_type(x); + if ((u&0x7fffffff)>0x7f800000) return ushort((u>>16)|0x40); + return ushort((u+0x7fff+((u>>16)&1))>>16); +} +// Torch CUDA materializes FP32 squares before MeanOps accumulation. +#pragma clang fp contract(off) +struct Rms { uint rows,width,stream_stride; float eps; }; +kernel void dsv41_hc_rms(const device ushort* x [[buffer(0)]], device float* inv [[buffer(1)]], + constant Rms& d [[buffer(2)]], uint row [[threadgroup_position_in_grid]], + uint t [[thread_index_in_threadgroup]], uint lane [[thread_index_in_simdgroup]], + uint sg [[simdgroup_index_in_threadgroup]]) { + // Fixed single-row CUDA tree: four vector accumulators, stride 512, independent of prefill chunks. + threadgroup float sums[512]; + float4 accum=0.0f; + for (uint i=4*t;i<4*d.width;i+=4*512) { + float4 a=float4(bf(x[ulong(row)*d.stream_stride+i]), + bf(x[ulong(row)*d.stream_stride+i+1]), + bf(x[ulong(row)*d.stream_stride+i+2]), + bf(x[ulong(row)*d.stream_stride+i+3])); + float4 square=a*a; + accum+=square; + } + float s=((accum.x+accum.y)+accum.z)+accum.w; + sums[t]=s; + for (uint offset=256;offset>=32;offset>>=1) { + threadgroup_barrier(mem_flags::mem_threadgroup); + if (t0;offset>>=1) s+=simd_shuffle_down(s,offset); + // MeanOps projects with a rounded FP32 factor, not a division of the sum. + float factor=1.0f/float(4*d.width); + if (lane==0) inv[row]=rsqrt(s*factor+d.eps); + } +} +#pragma clang fp contract(on) +struct Split { uint rows,mix_stride,coefficient_stride,comb_stride; float eps; }; +kernel void dsv41_hc_split(const device float* mixes [[buffer(0)]], const device float* inv [[buffer(1)]], + const device float* scale [[buffer(2)]], const device float* base [[buffer(3)]], + device float* pre [[buffer(4)]], device float* post [[buffer(5)]], + device float* comb [[buffer(6)]], constant Split& d [[buffer(7)]], + uint row [[thread_position_in_grid]]) { + if (row>=d.rows) return; + float m[16]; + for (uint j=0;j<4;j++) { + float a=fma(mixes[ulong(row)*d.mix_stride+j]*inv[row],scale[0],base[j]); + float b=fma(mixes[ulong(row)*d.mix_stride+4+j]*inv[row],scale[1],base[4+j]); + pre[ulong(row)*d.coefficient_stride+j]=1.0f/(1.0f+exp(-a))+d.eps; + post[ulong(row)*d.coefficient_stride+j]=2.0f/(1.0f+exp(-b)); + for (uint k=0;k<4;k++) { + uint q=4*j+k; + m[q]=fma(mixes[ulong(row)*d.mix_stride+8+q]*inv[row],scale[2],base[8+q]); + } + } + // Initial row softmax + eps, then a column normalization. This counts as iteration one. + for (uint j=0;j<4;j++) { + float mx=max(max(m[4*j],m[4*j+1]),max(m[4*j+2],m[4*j+3])); + for (uint k=0;k<4;k++) m[4*j+k]=exp(m[4*j+k]-mx); + // Pinned TileLang HC row/column sums use FP32 butterfly pairs. + float s=(m[4*j]+m[4*j+2])+(m[4*j+1]+m[4*j+3]); + for (uint k=0;k<4;k++) m[4*j+k]=m[4*j+k]/s+d.eps; + } + for (uint it=0;it<20;it++) { + if (it>0) { + for (uint j=0;j<4;j++) { + float s=((m[4*j]+m[4*j+2])+(m[4*j+1]+m[4*j+3]))+d.eps; + for (uint k=0;k<4;k++) m[4*j+k]/=s; + } + } + for (uint k=0;k<4;k++) { + float s=((m[k]+m[8+k])+(m[4+k]+m[12+k]))+d.eps; + for (uint j=0;j<4;j++) m[4*j+k]/=s; + } + } + for (uint q=0;q<16;q++) comb[ulong(row)*d.comb_stride+q]=m[q]; +} +// PyTorch materializes these FP32 products before reduction and branch addition. +#pragma clang fp contract(off) +struct Collapse { uint rows,width,stream_stride,y_stride,pre_stride; }; +kernel void dsv41_hc_collapse(const device ushort* x [[buffer(0)]], const device float* previous_pre [[buffer(1)]], + device ushort* y [[buffer(2)]], constant Collapse& d [[buffer(3)]], + uint2 i [[thread_position_in_grid]]) { + if (i.x>=d.width || i.y>=d.rows) return; + float s=0; + for (uint c=0;c<4;c++) s+=previous_pre[ulong(i.y)*d.pre_stride+c]*bf(x[ulong(i.y)*d.stream_stride+ulong(c)*d.width+i.x]); + y[ulong(i.y)*d.y_stride+i.x]=rb(s); +} +struct Post { uint rows,width,branch_stride,residual_stride,y_stride,coefficient_stride,comb_stride; }; +kernel void dsv41_hc_post(const device ushort* branch [[buffer(0)]], const device ushort* residual [[buffer(1)]], + const device float* post [[buffer(2)]], const device float* comb [[buffer(3)]], + device ushort* y [[buffer(4)]], constant Post& d [[buffer(5)]], + uint2 i [[thread_position_in_grid]]) { + if (i.x>=d.width || i.y>=d.rows) return; + float r[4]; + for (uint j=0;j<4;j++) r[j]=bf(residual[ulong(i.y)*d.residual_stride+ulong(j)*d.width+i.x]); + float b=bf(branch[ulong(i.y)*d.branch_stride+i.x]); + for (uint k=0;k<4;k++) { + float s=0; + for (uint j=0;j<4;j++) s+=comb[ulong(i.y)*d.comb_stride+4*j+k]*r[j]; + y[ulong(i.y)*d.y_stride+ulong(k)*d.width+i.x]=rb(post[ulong(i.y)*d.coefficient_stride+k]*b+s); + } +} +#pragma clang fp contract(on) +struct Expand { uint rows,width,x_stride,stream_stride,pre_stride; }; +kernel void dsv41_hc_expand(const device ushort* x [[buffer(0)]], device ushort* stream [[buffer(1)]], + device float* pre [[buffer(2)]], constant Expand& d [[buffer(3)]], + uint2 i [[thread_position_in_grid]]) { + if (i.x>=d.width || i.y>=d.rows) return; + for (uint c=0;c<4;c++) stream[ulong(i.y)*d.stream_stride+ulong(c)*d.width+i.x]=x[ulong(i.y)*d.x_stride+i.x]; + if (i.x==0) for (uint c=0;c<4;c++) pre[ulong(i.y)*d.pre_stride+c]=c==0?1.0f:0.0f; +} diff --git a/zig/kernels/metal/deepseek_v41/ops.metal b/zig/kernels/metal/deepseek_v41/ops.metal new file mode 100644 index 0000000000..d831901dd5 --- /dev/null +++ b/zig/kernels/metal/deepseek_v41/ops.metal @@ -0,0 +1,124 @@ +// Math: deepseek-ai/DeepSeek-V4.1-Flash inference/model.py, revision 2cba9e42aa026125f3ed06c6d98c1db82f7ca027. +#include +using namespace metal; +inline float bf(ushort x) { return as_type(uint(x) << 16); } +inline ushort rb(float x) { + uint u = as_type(x); + if ((u & 0x7fffffff) > 0x7f800000) return ushort((u >> 16) | 0x40); + return ushort((u + 0x7fff + ((u >> 16) & 1)) >> 16); +} +inline float rd(const device uchar* p, ulong i, uint dtype) { + if (dtype==1) return ((const device float*)p)[i]; + if (dtype==2) return float(((const device half*)p)[i]); + return bf(((const device ushort*)p)[i]); +} +struct Rms { uint rows, width, x_stride, y_stride; float eps; uint weight_f32; }; +#pragma clang fp contract(off) +kernel void dsv41_rms(const device ushort* x [[buffer(0)]], const device uchar* w [[buffer(1)]], + device ushort* y [[buffer(2)]], constant Rms& d [[buffer(3)]], + uint row [[threadgroup_position_in_grid]], uint t [[thread_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]], uint sg [[simdgroup_index_in_threadgroup]]) { + threadgroup float sums[8]; + float s = 0; + for (uint i = t; i < d.width; i += 256) { float a = bf(x[ulong(row)*d.x_stride+i]); s += a*a; } + s = simd_sum(s); + if (lane == 0) sums[sg] = s; + threadgroup_barrier(mem_flags::mem_threadgroup); + if (sg == 0) { + float total = simd_sum(lane < 8 ? sums[lane] : 0.0f); + const float factor = 1.0f / float(d.width); + if (lane == 0) sums[0] = rsqrt(total * factor + d.eps); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint i = t; i < d.width; i += 256) + y[ulong(row)*d.y_stride+i] = rb((bf(x[ulong(row)*d.x_stride+i])*sums[0])*rd(w,i,d.weight_f32)); +} +#pragma clang fp contract(on) +// Each SIMD group emits four columns; compensated dots do not depend on token count. + +// Precision: 0=BF16, 1=FP32, 2=FP16; FP16 weights never round through BF16. +inline void dense_add(thread float& high, thread float& low, float value) { + float next = high + value; + float borrowed = next - high; + low += (high - (next - borrowed)) + (value - borrowed); + high = next; +} +struct Dense { uint rows, k, n, groups, x_stride, w_stride, y_stride, x_dtype, w_dtype, y_dtype; }; +kernel void dsv41_dense(const device uchar* x [[buffer(0)]], const device uchar* w [[buffer(1)]], + device uchar* y [[buffer(2)]], constant Dense& d [[buffer(3)]], + uint3 g [[threadgroup_position_in_grid]], uint lane [[thread_index_in_simdgroup]], + uint sg [[simdgroup_index_in_threadgroup]]) { + uint col = g.x*16 + sg*4; + float a[4] = {0,0,0,0}, residual[4] = {0,0,0,0}; + for (uint k = lane; k < d.k; k += 32) { + float v = rd(x,ulong(g.y)*d.x_stride+ulong(g.z)*d.k+k,d.x_dtype); + for (uint j=0;j<4;j++) if (col+j>= 1) { + float other = simd_shuffle_xor(a[j], offset); + float other_residual = simd_shuffle_xor(residual[j], offset); + dense_add(a[j], residual[j], other); + residual[j] += other_residual; + } + float v = a[j] + residual[j]; + if (lane==0 && col+j=d.width || i.y>=d.rows) return; + ushort v=table[ulong(ids[i.y])*d.table_stride+i.x]; + for (uint c=0;c=d.width || i.y>=d.rows) return; + ulong k=ulong(i.y)*d.input_stride+i.x; + float a=min(rd(gate,k,d.input_f32),10.0f), b=clamp(rd(up,k,d.input_f32),-10.0f,10.0f); + float v=(a/(1.0f+exp(-a)))*b; + if (d.weighted) v=weights[i.y]*v; + y[ulong(i.y)*d.output_stride+i.x]=rb(v); +} +struct Combine { uint rows,width,routed_row_stride,slot_stride,shared_stride,y_stride,input_f32,id_stride; }; +kernel void dsv41_combine(const device uchar* routed [[buffer(0)]], const device uchar* shared [[buffer(1)]], + const device uint* ids [[buffer(2)]], device ushort* y [[buffer(3)]], + constant Combine& d [[buffer(4)]], + uint2 i [[thread_position_in_grid]], uint t [[thread_index_in_threadgroup]]) { + threadgroup uint slots[6]; + if (t==0) { + for (uint a=0;a<6;a++) slots[a]=a; + for (uint a=1;a<6;a++) { + uint slot=slots[a], j=a; + while (j>0 && ids[ulong(i.y)*d.id_stride+slots[j-1]]>ids[ulong(i.y)*d.id_stride+slot]) { + slots[j]=slots[j-1]; j--; + } + slots[j]=slot; + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (i.x>=d.width || i.y>=d.rows) return; + for (uint slot=0;slot<6;slot++) if (ids[ulong(i.y)*d.id_stride+slot]>=384) { + y[ulong(i.y)*d.y_stride+i.x]=rb(NAN); + return; + } + float s=0; + for (uint rank=0;rank<6;rank++) + s+=rd(routed,ulong(i.y)*d.routed_row_stride+ulong(slots[rank])*d.slot_stride+i.x,d.input_f32); + s+=rd(shared,ulong(i.y)*d.shared_stride+i.x,d.input_f32); + y[ulong(i.y)*d.y_stride+i.x]=rb(s); +} diff --git a/zig/kernels/metal/deepseek_v41/rope.metal b/zig/kernels/metal/deepseek_v41/rope.metal new file mode 100644 index 0000000000..09c4d1cb2e --- /dev/null +++ b/zig/kernels/metal/deepseek_v41/rope.metal @@ -0,0 +1,17 @@ +// Adjacent-pair RoPE and YaRN match DeepSeek V4.1's in-place BF16 rotation. +#include +using namespace metal; +struct Dsv41Rope { uint rows, heads, width, rotary, stride, position, step, inverse, spectrum; }; +kernel void dsv41_rope(device ushort *x [[buffer(0)]], constant Dsv41Rope &p [[buffer(1)]], + device const float2 *factors [[buffer(2)]], + uint3 i [[thread_position_in_grid]]) { + if (i.x >= p.rotary / 2 || i.y >= p.heads || i.z >= p.rows) return; + // The encoder binds the selected spectrum; rows are absolute source positions. + float2 rotation = factors[(ulong(p.position) + ulong(i.z) * p.step) * (p.rotary / 2) + i.x]; + float c = rotation.x, s = p.inverse ? -rotation.y : rotation.y; + uint at = i.z * p.stride + i.y * p.width + p.width - p.rotary + 2 * i.x; + float a = as_type(uint(x[at]) << 16), b = as_type(uint(x[at + 1]) << 16); + uint u = as_type(a * c - b * s), v = as_type(a * s + b * c); + x[at] = ushort((u + 0x7fff + ((u >> 16) & 1)) >> 16); + x[at + 1] = ushort((v + 0x7fff + ((v >> 16) & 1)) >> 16); +} diff --git a/zig/kernels/metal/deepseek_v41/route.metal b/zig/kernels/metal/deepseek_v41/route.metal new file mode 100644 index 0000000000..51f5454dd3 --- /dev/null +++ b/zig/kernels/metal/deepseek_v41/route.metal @@ -0,0 +1,53 @@ +// Reference: DeepSeek-V4.1-Flash inference/model.py Gate, revision 2cba9e42aa026125f3ed06c6d98c1db82f7ca027. +#include +using namespace metal; +inline float lp(float x) { float u=1.0f+x; return u==1.0f?x:x*(log(u)/(u-1.0f)); } +inline float score(float x) { return sqrt(x>20.0f?x:lp(exp(x))); } +struct Shape { uint rows,logit_stride,output_stride; float temperature; uint bias_dtype; }; +kernel void dsv41_route(const device float* logits [[buffer(0)]], const device uchar* bias [[buffer(1)]], + device uint* ids [[buffer(2)]], device float* weights [[buffer(3)]], + constant Shape& d [[buffer(4)]], uint row [[threadgroup_position_in_grid]], + uint t [[thread_index_in_threadgroup]], uint lane [[thread_index_in_simdgroup]], + uint sg [[simdgroup_index_in_threadgroup]]) { + threadgroup float scores[384], select[384], best[4], picked[6]; + threadgroup uint best_id[4], winner; + for (uint e=t;e<384;e+=128) { + float b; + if (d.bias_dtype==1) b=((const device float*)bias)[e]; + else if (d.bias_dtype==2) b=float(((const device half*)bias)[e]); + else b=as_type(uint(((const device ushort*)bias)[e])<<16); + scores[e]=score(logits[ulong(row)*d.logit_stride+e]/d.temperature); + select[e]=scores[e]+b; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + // Every expert participates in every rank selection; ties use the lower expert id deterministically. + for (uint slot=0;slot<6;slot++) { + float v=-INFINITY; + uint id=0xffffffffu; + for (uint e=t;e<384;e+=128) if (select[e]>v || (select[e]==v && e std.math.maxInt(u32)) return error.InvalidExl3Experts; diff --git a/zig/src/core/expert_groups.zig b/zig/src/core/expert_groups.zig new file mode 100644 index 0000000000..5b672ab6fd --- /dev/null +++ b/zig/src/core/expert_groups.zig @@ -0,0 +1,101 @@ +//! Borrowed bounded GPU route grouping; padded eight-row tiles preserve each original output slot. +const std = @import("std"); +const mtl = @import("metal"); +const sources = @import("kernel_sources"); +const exl3 = @import("exl3_projection.zig"); +pub const Field = enum { counts, offsets, ranks, expert_ids, token_ids, slot_ids }; +const field_count = std.enums.values(Field).len; + +pub const Plan = struct { + rows: u32, + experts: u32, + padded: u32, + offsets: [field_count]usize, + bytes: usize, + + pub fn init(rows: usize, experts: usize) !Plan { + if (rows == 0 or experts == 0 or rows > std.math.maxInt(u32) or experts > std.math.maxInt(u32)) + return error.InvalidExpertGrouping; + const active = @min(rows, experts); + const tiles = try std.math.add(usize, active, (rows - active) / 8); + const padded = try std.math.mul(usize, tiles, 8); + if (padded > std.math.maxInt(u32)) return error.ExpertGroupingTooLarge; + const sizes = [_]usize{ experts * 4, experts * 4, rows * 4, padded * 4, padded * 4, padded * 4 }; + var plan = Plan{ .rows = @intCast(rows), .experts = @intCast(experts), .padded = @intCast(padded), .offsets = undefined, .bytes = 0 }; + for (sizes, 0..) |size, i| { + plan.offsets[i] = std.mem.alignForward(usize, plan.bytes, 16); + plan.bytes = try std.math.add(usize, plan.offsets[i], size); + } + plan.bytes = std.mem.alignForward(usize, plan.bytes, 16); + return plan; + } +}; + +pub const Scratch = struct { + buffer: mtl.Buffer, + offset: usize = 0, + plan: Plan, + + pub fn view(self: Scratch, field: Field) exl3.View { + return .{ .buffer = self.buffer, .offset = self.offset + self.plan.offsets[@backingInt(field)] }; + } +}; + +pub const Pipelines = struct { + stages: [4]mtl.Pipeline, + + pub fn load(device: mtl.Device) !Pipelines { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const lib = try mtl.Library.fromSource(device, sources.core_expert_groups, mtl.CompileOptions.mlx()); + defer lib.deinit(); + var stages: [4]mtl.Pipeline = undefined; + var loaded: usize = 0; + errdefer for (stages[0..loaded]) |pipeline| pipeline.deinit(); + inline for (.{ "tf_expert_group_clear", "tf_expert_group_count", "tf_expert_group_prefix", "tf_expert_group_scatter" }, 0..) |name, i| { + stages[i] = try mtl.Pipeline.init(device, lib, name, false); + loaded += 1; + } + return .{ .stages = stages }; + } + pub fn deinit(self: Pipelines) void { + for (self.stages) |pipeline| pipeline.deinit(); + } +}; +const Args = extern struct { rows: u32, experts: u32, repeat: u32, padded: u32 }; + +pub fn encode(enc: mtl.ComputeEncoder, pipelines: *const Pipelines, ids: exl3.View, scratch: Scratch, repeat: u32) !void { + // Plan must come from Plan.init; route IDs remain GPU-resident and are bounds-checked by the kernels. + if (repeat == 0 or scratch.plan.rows == 0 or scratch.plan.experts == 0 or + scratch.plan.rows % repeat != 0) return error.InvalidExpertGrouping; + if (ids.offset % 4 != 0 or scratch.offset % 4 != 0 or + ids.offset > ids.buffer.length() or @as(usize, scratch.plan.rows) * 4 > ids.buffer.length() - ids.offset or + scratch.offset > scratch.buffer.length() or scratch.plan.bytes > scratch.buffer.length() - scratch.offset) + return error.InvalidExpertGroupingStorage; + const args = Args{ .rows = scratch.plan.rows, .experts = scratch.plan.experts, .repeat = repeat, .padded = scratch.plan.padded }; + for (pipelines.stages, 0..) |pipeline, phase| { + enc.setPipeline(pipeline); + enc.setBuffer(ids.buffer, ids.offset, 0); + inline for (std.enums.values(Field), 1..) |field, i| { + const view = scratch.view(field); + enc.setBuffer(view.buffer, view.offset, i); + } + enc.setValue(args, 7); + enc.dispatchThreads(.{ .width = switch (phase) { + 0 => @max(args.experts, args.padded), + 2 => 1, + else => args.rows, + } }, .{ .width = 256 }); + enc.barrier(); + } +} + +test "expert grouping bounds padding without context-sized allocation" { + const small = try Plan.init(6, 384); + try std.testing.expectEqual(@as(u32, 48), small.padded); + const full = try Plan.init(1536, 384); + try std.testing.expectEqual(@as(u32, 4224), full.padded); + try std.testing.expect(full.bytes < 64 * 1024); + try std.testing.expectError(error.InvalidExpertGrouping, Plan.init(0, 384)); + try std.testing.expectError(error.ExpertGroupingTooLarge, Plan.init(std.math.maxInt(u32), 384)); +} diff --git a/zig/src/core/gpu_counters.zig b/zig/src/core/gpu_counters.zig index 2e0b216165..72edee1343 100644 --- a/zig/src/core/gpu_counters.zig +++ b/zig/src/core/gpu_counters.zig @@ -1,4 +1,4 @@ -//! Same-command-buffer GPU timestamps, bounded resources and no per-span CPU waits. +//! Bounded same-command GPU timestamps; disabled init allocates nothing, no per-span CPU waits or CPU timing fallback. const std = @import("std"); const mtl = @import("metal"); const counters = mtl.counters; @@ -9,6 +9,8 @@ pub const max_span_capacity: usize = 65536; pub const Options = struct { enabled: bool = false, max_spans: usize = 512 }; pub const Span = struct { kind: Class = .dense, + /// Caller-owned diagnostic label; does not affect sampling or generic class aggregation. + metadata: u32 = 0, begin_timestamp: u64 = 0, end_timestamp: u64 = 0, gpu_seconds: f64 = 0, @@ -16,7 +18,7 @@ pub const Span = struct { pub const Aggregate = struct { calls: u64 = 0, gpu_seconds: f64 = 0 }; const State = enum { idle, recording, resolved, read, broken }; -/// Two reference samples convert GPU ticks to nanoseconds; never use CPU span time. +/// Apple's paired reference samples convert GPU ticks to CPU nanoseconds, not elapsed operator time. pub const Calibration = struct { before: counters.ClockSample, after: counters.ClockSample, @@ -29,7 +31,8 @@ pub const Calibration = struct { } pub fn duration(self: Calibration, begin: u64, end: u64) !f64 { - if (begin == 0 or end == 0 or begin == counters.error_value or end == counters.error_value) return error.InvalidCounterSample; + if (begin == 0 or end == 0 or begin == counters.error_value or end == counters.error_value) + return error.InvalidCounterSample; if (end < begin) return error.ReversedCounterSample; if (begin < self.before.gpu or end > self.after.gpu) return error.CounterOutsideCalibration; const ticks: f64 = @floatFromInt(end - begin); @@ -79,7 +82,7 @@ pub const Recorder = struct { calibration: ?Calibration = null, totals: [class_count]Aggregate = @splat(.{}), - /// Init-only bounded storage; disabled init allocates nothing. + /// Init-only storage, descriptors and fence; 1..65536 spans, at most 16 CPU-resolve bytes per span. pub fn init(allocator: std.mem.Allocator, device: mtl.Device, options: Options) !Recorder { var self = Recorder{ .allocator = allocator }; if (!options.enabled) return self; @@ -120,8 +123,8 @@ pub const Recorder = struct { self.state = .recording; } - /// Non-nested spans; stage mode rotates fenced encoders, requiring resource and pipeline rebinding. - pub fn beginSpan(self: *Recorder, encoder: *mtl.ComputeEncoder, kind: Class) !void { + /// No nesting; stage sampling rotates fenced serial encoders: callers must rebind and redeclare resources. + pub fn beginSpan(self: *Recorder, encoder: *mtl.ComputeEncoder, kind: Class, metadata: u32) !void { if (!self.enabled()) return; if (self.state != .recording or !self.encoder_open) return error.CounterNotRecording; if (self.active_span != null) return error.NestedCounterSpan; @@ -141,7 +144,7 @@ pub const Recorder = struct { self.encoder_open = true; resources.fence.waitCompute(encoder.*); } - self.spans[index] = .{ .kind = kind }; + self.spans[index] = .{ .kind = kind, .metadata = metadata }; self.span_count += 1; self.active_span = index; } @@ -172,7 +175,7 @@ pub const Recorder = struct { self.active_span = null; } - /// Ends the final encoder before the caller's existing commit/wait; resolution requires completion. + /// End the final encoder before existing commit/wait; resolve counters only after command completion. pub fn finishEncoding(self: *Recorder, encoder: *mtl.ComputeEncoder) !void { if (!self.enabled()) { encoder.end(); @@ -186,7 +189,7 @@ pub const Recorder = struct { self.state = .resolved; } - /// Cancels unsubmitted work, closing any surviving encoder, including after encoder creation failure. + /// Close and discard failed, unsubmitted encoding without committing; safe after encoder creation failure. pub fn cancelEncoding(self: *Recorder, encoder: *mtl.ComputeEncoder) !void { if (self.command_buffer) |cb| { if (cb.status() != .not_enqueued) return error.CounterCommandAlreadySubmitted; @@ -198,7 +201,7 @@ pub const Recorder = struct { try self.discardAfterCompletion(); } - /// No wait or fallback; results are borrowed until beginCommand, discardAfterCompletion or deinit. + /// No wait or fallback; results are borrowed until reuse/discard/deinit, errors and invalid calibration propagate. pub fn readAfterCompletion(self: *Recorder) ![]const Span { if (!self.enabled()) return &.{}; if (self.state == .read) return self.spans[0..self.span_count]; @@ -244,7 +247,7 @@ pub const Recorder = struct { return self.totals; } - /// Discards closed, unsubmitted or completed recordings; in-flight resources are never reused. + /// Discard completed/unsubmitted recordings after closing encoders; in-flight reuse is rejected, never waited. pub fn discardAfterCompletion(self: *Recorder) !void { if (self.encoder_open) return error.CounterEncoderStillOpen; if (self.command_buffer) |cb| { @@ -288,14 +291,14 @@ test "disabled recorder allocates nothing and never touches a device or encoder var recorder = try Recorder.init(std.testing.failing_allocator, undefined, .{ .enabled = false }); defer recorder.deinit() catch unreachable; var encoder: mtl.ComputeEncoder = undefined; - try recorder.beginSpan(&encoder, .attention); + try recorder.beginSpan(&encoder, .attention, 0); try recorder.endSpan(&encoder); try std.testing.expect(!recorder.enabled()); try std.testing.expectEqual(@as(usize, 0), (try recorder.readAfterCompletion()).len); for (recorder.aggregates()) |aggregate| try std.testing.expectEqual(@as(u64, 0), aggregate.calls); } -test "calibration rejects missing samples, both reversed clocks and both range boundaries" { +test "calibration rejects missing samples and both clock and range boundaries" { const c = Calibration{ .before = .{ .cpu = 1000, .gpu = 100 }, .after = .{ .cpu = 5000, .gpu = 300 } }; try std.testing.expectError(error.InvalidCounterSample, c.duration(0, 200)); try std.testing.expectError(error.InvalidCounterSample, c.duration(100, counters.error_value)); @@ -308,12 +311,45 @@ test "calibration rejects missing samples, both reversed clocks and both range b } test "enabled capacity validation occurs before any device access" { - try std.testing.expectError(error.InvalidSpanCapacity, Recorder.init(std.testing.failing_allocator, undefined, .{ - .enabled = true, - .max_spans = 0, - })); - try std.testing.expectError(error.InvalidSpanCapacity, Recorder.init(std.testing.failing_allocator, undefined, .{ - .enabled = true, - .max_spans = max_span_capacity + 1, - })); + try std.testing.expectError(error.InvalidSpanCapacity, Recorder.init(std.testing.failing_allocator, undefined, .{ .enabled = true, .max_spans = 0 })); + try std.testing.expectError(error.InvalidSpanCapacity, Recorder.init(std.testing.failing_allocator, undefined, .{ .enabled = true, .max_spans = max_span_capacity + 1 })); +} + +test "counter ownership rejects open and in-flight reuse without a CPU span wait" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + if (counters.query(device).mode() == .unsupported) return error.SkipZigTest; + const queue = try device.queue(); + defer queue.deinit(); + const gate = try device.sharedEvent(); + defer gate.deinit(); + var recorder = try Recorder.init(std.testing.allocator, device, .{ .enabled = true, .max_spans = 1 }); + defer recorder.deinit() catch unreachable; + const cb = queue.commandBuffer(); + cb.waitFor(gate, 1); + var encoder = cb.compute(.serial); + try recorder.beginCommand(cb); + var submitted = false; + defer { + if (submitted) { + gate.set(1); + cb.wait(); + recorder.discardAfterCompletion() catch unreachable; + } else recorder.cancelEncoding(&encoder) catch unreachable; + } + try std.testing.expectError(error.CounterEncoderStillOpen, recorder.deinit()); + try recorder.beginSpan(&encoder, .dense, 42); + try std.testing.expectError(error.NestedCounterSpan, recorder.beginSpan(&encoder, .copy, 43)); + try std.testing.expectError(error.UnclosedCounterSpan, recorder.finishEncoding(&encoder)); + try recorder.endSpan(&encoder); + try std.testing.expectError(error.CounterSpanCapacityExceeded, recorder.beginSpan(&encoder, .copy, 43)); + try recorder.finishEncoding(&encoder); + cb.commit(); + submitted = true; + try std.testing.expectError(error.CounterCommandNotCompleted, recorder.readAfterCompletion()); + try std.testing.expectError(error.CounterRecordingInFlight, recorder.beginCommand(cb)); + try std.testing.expectError(error.CounterCommandNotCompleted, recorder.discardAfterCompletion()); + try std.testing.expectError(error.CounterCommandNotCompleted, recorder.deinit()); } diff --git a/zig/src/core/tokenizer/nfkd_data.zig b/zig/src/core/tokenizer/nfkd_data.zig index 1bc3595eb2..38864b3c76 100644 --- a/zig/src/core/tokenizer/nfkd_data.zig +++ b/zig/src/core/tokenizer/nfkd_data.zig @@ -1,10 +1,9 @@ -//! NFKD data for the Engram token-map normalizer chain, generated from -//! unicodedata 16.0.0 (the version unicode_data.zig pins). Fork tool: -//! .agents/tools/gen_nfkd_tables.py regenerates this file. +// Generated by tools/zig/nfkd_tables.py from the Unicode 16.0.0 character database; do not edit. +pub const version = "16.0.0"; pub const Decomp = struct { cp: u21, start: u32, len: u8 }; -/// All NFKD sequences concatenated, indexed by Decomp.start/.len. +// All NFKD sequences concatenated, indexed by Decomp.start/.len. pub const sequences = [_]u21{ 0x0020, 0x0020, 0x0308, 0x0061, 0x0020, 0x0304, 0x0032, 0x0033, 0x0020, 0x0301, 0x03BC, 0x0020, 0x0327, 0x0031, 0x006F, 0x0031, @@ -22385,8 +22384,7 @@ pub const decompositions = [_]Decomp{ .{ .cp = 0x2FA1D, .start = 42308, .len = 1 }, }; -/// General-category Mn/Mc/Me codepoints, sorted; StripAccents removes these -/// (the Rust tokenizers' StripAccents filters all of category M*). +// General-category Mn/Mc/Me codepoints, sorted; the Rust tokenizers' StripAccents removes all of category M*. pub const marks = [_]u21{ 0x0300, 0x0301, 0x0302, 0x0303, 0x0304, 0x0305, 0x0306, 0x0307, 0x0308, 0x0309, 0x030A, 0x030B, 0x030C, 0x030D, 0x030E, 0x030F, @@ -22705,7 +22703,7 @@ pub const marks = [_]u21{ pub const Lower = struct { cp: u21, start: u32, len: u8 }; -/// Full lowercase mappings concatenated (Rust char::to_lowercase semantics). +// Full lowercase mappings concatenated (Rust char::to_lowercase semantics). pub const lower_sequences = [_]u21{ 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F, 0x0070, @@ -24355,7 +24353,7 @@ pub const lowercases = [_]Lower{ .{ .cp = 0x1E921, .start = 1460, .len = 1 }, }; -/// Canonical combining class ranges [first, last, ccc], unicodedata 16.0.0. +// Canonical combining class ranges [first, last, ccc], unicodedata 16.0.0. pub const cccs = [_][3]u21{ .{ 0x0300, 0x0314, 0x00E6 }, .{ 0x0315, 0x0315, 0x00E8 }, .{ 0x0316, 0x0319, 0x00DC }, .{ 0x031A, 0x031A, 0x00E8 }, .{ 0x031B, 0x031B, 0x00D8 }, .{ 0x031C, 0x0320, 0x00DC }, .{ 0x0321, 0x0322, 0x00CA }, .{ 0x0323, 0x0326, 0x00DC }, diff --git a/zig/src/families/deepseek_v41/attention.zig b/zig/src/families/deepseek_v41/attention.zig new file mode 100644 index 0000000000..a1f76f6886 --- /dev/null +++ b/zig/src/families/deepseek_v41/attention.zig @@ -0,0 +1,271 @@ +//! Pinned DeepSeek V4.1 attention, resident EXL3 projections and causal packed-cache chunk execution. +const mtl = @import("metal"); +const exl3 = @import("../../core/exl3_projection.zig"); +const checkpoint = @import("../../core/checkpoint_metal.zig"); +const weights = @import("weights.zig"); +const config = @import("config.zig"); +const ops = @import("ops.zig"); +const csa2 = @import("csa2.zig"); +const kv = @import("kv.zig"); +const index = @import("csa2_index.zig"); +const rope = @import("rope.zig"); +pub const View = @import("view.zig").View; +pub const State = kv.cache.State; + +/// Opt-in bounded intermediate capture; ordinary execution does not install an observer. +pub const DiagnosticStage = enum { + qr_projection, + qr_norm, + q_projection, + q, + kv_projection, + kv_norm, + kv_rope, + kv_quantized, + sparse_o, + o, + grouped, + wo_b, +}; +pub const DiagnosticObserver = struct { + ctx: *anyopaque, + capture: *const fn (*anyopaque, mtl.ComputeEncoder, DiagnosticStage, View) anyerror!void, + + fn observe(comptime enabled: bool, self: ?DiagnosticObserver, e: mtl.ComputeEncoder, stage: DiagnosticStage, source: View) !void { + if (enabled) if (self) |observer| try observer.capture(observer.ctx, e, stage, source); + } +}; + +pub const Pipelines = struct { + plain: ops.Pipelines, + cache: kv.Pipelines, + select: index.Pipelines, + rotary: rope.Pipelines, + sparse: csa2.Pipelines, + + pub fn load(device: mtl.Device, capacity: u32, c: config.Text) !Pipelines { + var plain = try ops.Pipelines.load(device); + errdefer plain.deinit(); + var cache = try kv.Pipelines.load(device); + errdefer cache.deinit(); + var select = try index.Pipelines.load(device); + errdefer select.deinit(); + var rotary = try rope.Pipelines.load(device, capacity, c.qk_rope_head_dim, .{ .theta = c.rope_theta, .original = 0 }, .{ .theta = c.compress_rope_theta, .factor = c.rope_scaling.factor, .original = c.rope_scaling.original_max_position_embeddings, .beta_fast = c.rope_scaling.beta_fast, .beta_slow = c.rope_scaling.beta_slow }); + errdefer rotary.deinit(); + const sparse = try csa2.Pipelines.load(device); + return .{ .plain = plain, .cache = cache, .select = select, .rotary = rotary, .sparse = sparse }; + } + pub fn residentBytes(capacity: u32, c: config.Text) usize { + return rope.Pipelines.residentBytes(capacity, c.qk_rope_head_dim); + } + pub fn registerResidency(p: *const Pipelines, set: mtl.ResidencySet) void { + p.rotary.registerResidency(set); + } + pub fn deinit(p: *Pipelines) void { + p.plain.deinit(); + p.cache.deinit(); + p.select.deinit(); + p.rotary.deinit(); + p.sparse.deinit(); + } +}; + +// Row-sized regions never contain dequantized weights or a query-by-context tensor. +const regions = [_]usize{ 1280 * 2, 32768 * 2, 512 * 2, 512 * 4, 512 * 4, 512 * 2, 128 * 2, 4096 * 2, 32 * 2, 8192 * 2 }; +const Region = enum(usize) { qr, query, raw_kv, values, gates, latent, index_k, index_q, index_weights, grouped }; + +pub const Workspace = struct { + buffer: mtl.Buffer, + selection: index.Scratch, + max_rows: u32, + selection_layer: ?u32 = null, + selection_position: u32 = 0, + selection_rows: u32 = 0, + candidate_position: u32 = 0, + candidate_rows: u32 = 0, + + pub fn bytes(max_rows: u32) usize { + var per_row: usize = 0; + for (regions) |size| per_row += size; + return @as(usize, max_rows) * per_row + index.Scratch.bytes(max_rows); + } + pub fn init(device: mtl.Device, max_rows: u32) !Workspace { + if (max_rows == 0 or max_rows > 4096) return error.InvalidAttentionWorkspace; + const buffer = try device.buffer(bytes(max_rows) - index.Scratch.bytes(max_rows), mtl.ResourceOptions.shared | mtl.ResourceOptions.untracked); + errdefer buffer.deinit(); + const selection = try index.Scratch.init(device, max_rows); + return .{ .buffer = buffer, .selection = selection, .max_rows = max_rows }; + } + pub fn deinit(w: *Workspace) void { + w.buffer.deinit(); + w.selection.deinit(); + } + pub fn residentBytes(w: *const Workspace) usize { + return bytes(w.max_rows); + } + pub fn registerResidency(w: *const Workspace, set: mtl.ResidencySet) void { + set.add(w.buffer); + set.add(w.selection.buffer); + } + fn view(w: *const Workspace, region: Region) View { + var offset: usize = 0; + for (regions[0..@backingInt(region)]) |size| offset += size * w.max_rows; + return .{ .buffer = w.buffer, .offset = offset }; + } +}; + +fn tensor(t: checkpoint.Tensor) View { + return .{ .buffer = t.buffer, .offset = t.offset }; +} +fn core(v: View) exl3.View { + return .{ .buffer = v.buffer, .offset = v.offset }; +} +fn projection(p: *const exl3.Pipelines, e: mtl.ComputeEncoder, m: weights.AnyMatrix, x: View, y: View, scratch: exl3.View, rows: u32, xs: u32, ys: u32, output: exl3.Output) !void { + switch (m) { + .trellis => |t| try exl3.encode(p, e, t, core(x), core(y), scratch, .{ .count = rows, .x_stride = xs, .y_stride = ys, .output = output }), + // A kept-dense repack: the same rotate stages, the middle a plain dense product. + .dense => |d| try exl3.encodeDense(p, e, .{ + .weight = core(tensor(d.weight)), + .suh = core(tensor(d.suh)), + .svh = core(tensor(d.svh)), + .k = d.k, + .n = d.n, + }, core(x), core(y), scratch, .{ .count = rows, .x_stride = xs, .y_stride = ys, .output = output }), + } +} +fn norm(p: *const Pipelines, e: mtl.ComputeEncoder, x: View, weight: checkpoint.Tensor, rows: u32, width: u32, eps: f32) void { + ops.encodeRms(e, &p.plain, x, tensor(weight), x, .{ .rows = rows, .width = width, .x_stride = width, .y_stride = width, .eps = eps, .weight_f32 = weight.dtype == .f32 }); + e.barrier(); +} +fn rotate(p: *const Pipelines, e: mtl.ComputeEncoder, x: View, rows: u32, heads: u32, width: u32, position: u32, ratio: u32, step: u32, inverse: bool, c: config.Text) !void { + try p.rotary.encode(e, x, .{ .rows = rows, .heads = heads, .width = width, .rotary = c.qk_rope_head_dim, .stride = heads * width, .position = position, .step = step, .inverse = @intFromBool(inverse), .spectrum = if (ratio == 0) .normal else .compressed }); +} +fn precedingIndex(layer: u32) u32 { + var source: u32 = 2; + for (kv.cache.index_sources) |id| { + if (id > layer) break; + source = id; + } + return source; +} + +/// BF16 x/y [rows,5120]; layers 0..39 share the chunk-start cursor, appended by the caller after all layers. + +// No extra RMS after wq_b or index-query Hadamard outside EXL3. +pub fn encode(p: *const Pipelines, e: mtl.ComputeEncoder, w: *const weights.Attention, state: *State, work: *Workspace, x: View, y: View, ep: *const exl3.Pipelines, scratch: exl3.View, layer: u32, rows: u32, c: config.Text) !void { + return encodeDiagnostic(p, e, w, state, work, x, y, ep, scratch, layer, rows, c, false, null); +} + +/// Same encoder and arithmetic as encode, with optional copies at operator boundaries. +pub fn encodeDiagnostic(p: *const Pipelines, e: mtl.ComputeEncoder, w: *const weights.Attention, state: *State, work: *Workspace, x: View, y: View, ep: *const exl3.Pipelines, scratch: exl3.View, layer: u32, rows: u32, c: config.Text, comptime diagnostic: bool, observer: ?DiagnosticObserver) !void { + if (layer >= 40 or rows == 0 or rows > work.max_rows or + state.position > state.plan.capacity or rows > state.plan.capacity - state.position) + return error.InvalidAttentionChunk; + const position = state.position; + const ratio = c.compress_ratios[layer]; + const end = position + rows; + const q_width = c.num_attention_heads * c.head_dim; + const qr = work.view(.qr); + const q = work.view(.query); + const raw = work.view(.raw_kv); + const ring = try state.ring(layer); + const chunk = work.selection.windowChunk(); + e.barrier(); + try projection(ep, e, .{ .trellis = w.wq_a }, x, qr, scratch, rows, c.hidden_size, c.q_lora_rank, .bf16); + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .qr_projection, qr); + norm(p, e, qr, w.q_norm, rows, c.q_lora_rank, c.rms_norm_eps); + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .qr_norm, qr); + try projection(ep, e, .{ .trellis = w.wq_b }, qr, q, scratch, rows, c.q_lora_rank, q_width, .bf16); + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .q_projection, q); + try rotate(p, e, q, rows, c.num_attention_heads, c.head_dim, position, ratio, 1, false, c); + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .q, q); + try projection(ep, e, .{ .trellis = w.wkv }, x, raw, scratch, rows, c.hidden_size, c.head_dim, .bf16); + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .kv_projection, raw); + norm(p, e, raw, w.kv_norm, rows, c.head_dim, c.rms_norm_eps); + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .kv_norm, raw); + try rotate(p, e, raw, rows, 1, c.head_dim, position, ratio, 1, false, c); + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .kv_rope, raw); + try p.cache.encodePack(e, raw, chunk, .{ .rows = rows, .width = c.head_dim, .stride = c.head_dim, .packed_stride = kv.cache.ring_stride, .block = 32, .kind = .window_fp8 }); + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .kv_quantized, chunk); + + var main = chunk; + var compressed_rows: u32 = 0; + if (ratio != 0) { + const source = try state.source(layer, ratio); + main = source.main; + compressed_rows = end / ratio; + const complete = compressed_rows - position / ratio; + const latent = work.view(.latent); + if (source.owner == layer) { + const compressor = if (w.compressor) |*value| value else return error.MissingAttentionCompressor; + const output: exl3.Output = if (ratio == 1) .bf16 else .f32; + const values = work.view(.values); + const gates = work.view(.gates); + try projection(ep, e, compressor.wkv, x, values, scratch, rows, c.hidden_size, c.head_dim, output); + if (ratio > 1) { + const gate = compressor.wgate orelse return error.MissingAttentionCompressorGate; + try projection(ep, e, gate, x, gates, scratch, rows, c.hidden_size, c.head_dim, .f32); + } + try p.cache.encodePool(e, values, gates, source.partial, tensor(compressor.norm), latent, .{ .rows = rows, .ratio = ratio, .position = position, .stride = c.head_dim, .input_f32 = @intFromBool(ratio > 1), .eps = c.rms_norm_eps }); + if (complete != 0) { + const selector = if (w.indexer) |*value| value else return error.MissingAttentionIndexer; + const wk = selector.wk orelse return error.MissingAttentionIndexKeyProjection; + const k_norm = selector.k_norm orelse return error.MissingAttentionIndexKeyNorm; + const key = work.view(.index_k); + try projection(ep, e, wk, latent, key, scratch, complete, c.head_dim, c.index_head_dim, .bf16); + norm(p, e, key, k_norm, complete, c.index_head_dim, c.rms_norm_eps); + try rotate(p, e, key, complete, 1, c.index_head_dim, position / ratio * ratio, ratio, ratio, false, c); + try p.cache.encodePack(e, key, source.index, .{ .rows = complete, .width = c.index_head_dim, .stride = c.index_head_dim, .packed_stride = kv.cache.index_stride, .block = 32, .kind = .index_fp4, .first = position / ratio }); + } + } + if (kv.cache.isIndexSource(layer)) { + const selector = if (w.indexer) |*value| value else return error.MissingAttentionIndexer; + const iq = work.view(.index_q); + const iw = work.view(.index_weights); + const iq_width = c.index_n_heads * c.index_head_dim; + if (compressed_rows != 0) { + try projection(ep, e, selector.wq_b, qr, iq, scratch, rows, c.q_lora_rank, iq_width, .bf16); + try rotate(p, e, iq, rows, c.index_n_heads, c.index_head_dim, position, ratio, 1, false, c); + try p.cache.encodePack(e, iq, work.selection.queryPacked(), .{ .rows = rows * c.index_n_heads, .width = c.index_head_dim, .stride = c.index_head_dim, .packed_stride = kv.cache.index_stride, .block = 32, .kind = .index_fp4, .inplace = 1 }); + ops.encodeDense(e, &p.plain, x, tensor(selector.weights_proj), iw, .{ .rows = rows, .k = c.hidden_size, .n = c.index_n_heads, .x_stride = c.hidden_size, .w_stride = c.hidden_size, .y_stride = c.index_n_heads, .w_dtype = switch (selector.weights_proj.dtype) { + .bf16 => .bf16, + else => .f16, + } }); + e.barrier(); + } + const make_candidates = layer == c.candidate_source_layer_id; + const use_candidates = layer > c.candidate_source_layer_id; + if (use_candidates and (work.candidate_position != position or work.candidate_rows != rows)) + return error.MissingAttentionCandidates; + try p.select.encode(e, iq, iw, source.index, work.selection, .{ .rows = rows, .q_stride = iq_width, .weight_stride = c.index_n_heads, .position = position, .ratio = ratio, .keys = compressed_rows, .make_candidates = @intFromBool(make_candidates), .use_candidates = @intFromBool(use_candidates) }); + work.selection_layer = layer; + work.selection_position = position; + work.selection_rows = rows; + if (make_candidates) { + work.candidate_position = position; + work.candidate_rows = rows; + } + } else if (work.selection_layer != precedingIndex(layer) or + work.selection_position != position or work.selection_rows != rows) return error.MissingAttentionSelection; + // Index K consumes the normalized latent before this in-place RoPE mutates it. + if (source.owner == layer and complete != 0) { + try rotate(p, e, latent, complete, 1, c.head_dim, position / ratio * ratio, ratio, ratio, false, c); + try p.cache.encodePack(e, latent, source.main, .{ .rows = complete, .width = c.head_dim, .stride = c.head_dim, .packed_stride = kv.cache.main_stride, .block = 16, .kind = .main_fp4, .first = position / ratio }); + } + } + try p.sparse.encode(e, q, ring, chunk, main, work.selection.positions(), tensor(w.sink), q, .{ .rows = rows, .q_stride = q_width, .position = position, .chunk_rows = rows, .ratio = ratio, .compressed_rows = compressed_rows, .topk = @min(c.index_topk, compressed_rows) }); + e.barrier(); + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .sparse_o, q); + try p.cache.publishWindow(e, chunk, ring, rows, position); + try rotate(p, e, q, rows, c.num_attention_heads, c.head_dim, position, ratio, 1, true, c); + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .o, q); + const grouped = work.view(.grouped); + const group_input = q_width / c.o_groups; + const group_output = c.o_groups * c.o_lora_rank; + for (w.wo_a, 0..) |matrix, group| { + try projection(ep, e, .{ .trellis = matrix }, q.shifted(group * group_input * 2), grouped.shifted(group * c.o_lora_rank * 2), scratch, rows, q_width, group_output, .bf16); + } + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .grouped, grouped); + try projection(ep, e, .{ .trellis = w.wo_b }, grouped, y, scratch, rows, group_output, c.hidden_size, .bf16); + if (diagnostic) try DiagnosticObserver.observe(true, observer, e, .wo_b, y); +} diff --git a/zig/src/families/deepseek_v41/csa2.zig b/zig/src/families/deepseek_v41/csa2.zig new file mode 100644 index 0000000000..395de9107a --- /dev/null +++ b/zig/src/families/deepseek_v41/csa2.zig @@ -0,0 +1,77 @@ +//! Exact causal SWA plus shared compressed KV; fixed-buffer chunked prefill and decode use one path. +const std = @import("std"); +const mtl = @import("metal"); +const sources = @import("kernel_sources"); +const View = @import("view.zig").View; +pub const index = @import("csa2_index.zig"); +pub const kv = @import("kv.zig"); +pub const rope = @import("rope.zig"); +pub const Params = extern struct { + rows: u32, + heads: u32 = 64, + width: u32 = 512, + q_stride: u32, + position: u32, + chunk_rows: u32, + window: u32 = 128, + ring_stride: u32 = 528, + chunk_stride: u32 = 528, + main_stride: u32 = 288, + main_block: u32 = 16, + ratio: u32, + compressed_rows: u32, + topk: u32 = 512, + index_stride: u32 = 512, + scale: f32 = 0.04419417382415922, + pub fn validate(p: Params) !void { + if (p.rows == 0 or p.heads == 0 or p.width == 0 or p.width > 512 or p.width % 32 != 0 or + @as(u64, p.q_stride) < @as(u64, p.heads) * p.width or + p.chunk_rows < p.rows or p.window != 128 or + p.ring_stride < p.width + p.width / 32 or p.chunk_stride < p.width + p.width / 32 or + p.main_stride < p.width / 2 + p.width / 16 or p.main_block != 16 or + (p.ratio != 0 and p.ratio != 1 and p.ratio != 2) or p.topk > 512 or p.index_stride < p.topk or + @as(u64, p.position) + p.rows > kv.cache.max_context or + p.compressed_rows > kv.cache.max_context or !std.math.isFinite(p.scale) or p.scale <= 0 or + @as(u64, p.rows -| 1) * p.q_stride + @as(u64, p.heads) * p.width > std.math.maxInt(u32) or + @as(u64, p.rows) * p.index_stride > std.math.maxInt(u32) or + @as(u64, p.rows) * p.chunk_stride > std.math.maxInt(u32) or + @as(u64, p.window) * p.ring_stride > std.math.maxInt(u32) or + @as(u64, p.compressed_rows) * p.main_stride > std.math.maxInt(u32)) + return error.InvalidSparseAttentionShape; + } +}; +pub const Pipelines = struct { + attention: mtl.Pipeline, + pub fn load(device: mtl.Device) !Pipelines { + var options = mtl.CompileOptions.mlx(); + options.math_functions = .precise; + const lib = try mtl.Library.fromSource(device, sources.deepseek_v41_kv ++ sources.deepseek_v41_csa2, options); + defer lib.deinit(); + return .{ .attention = try mtl.Pipeline.init(device, lib, "dsv41_sparse_attention", false) }; + } + pub fn deinit(p: *Pipelines) void { + p.attention.deinit(); + } + /// BF16 Q/output [rows,heads,width], FP32 sinks, I32 indices; publish this chunk to the ring only after dispatch. + pub fn encode(p: *const Pipelines, enc: mtl.ComputeEncoder, q: View, ring: View, chunk: View, compressed: View, positions: View, sink: View, out: View, args: Params) !void { + try args.validate(); + try q.requireRows(args.rows, args.q_stride, @as(usize, args.heads) * args.width, 2); + try out.requireRows(args.rows, args.q_stride, @as(usize, args.heads) * args.width, 2); + try sink.require(@as(usize, args.heads) * 4, 4); + if (args.position != 0) try ring.requireRows(@min(args.position, args.window), args.ring_stride, args.width + args.width / 32, 1); + try chunk.requireRows(args.rows, args.chunk_stride, args.width + args.width / 32, 1); + if (args.ratio != 0 and args.compressed_rows != 0 and args.topk != 0) { + try compressed.requireRows(args.compressed_rows, args.main_stride, args.width / 2 + args.width / 16, 1); + try positions.requireRows(args.rows, args.index_stride, args.topk, 4); + } + enc.setPipeline(p.attention); + for ([_]View{ q, ring, chunk, compressed, positions, sink, out }, 0..) |v, i| + enc.setBuffer(v.buffer, v.offset, i); + enc.setValue(args, 7); + enc.dispatchGroups(mtl.Size.of(args.heads, args.rows, 1), mtl.Size.of(32, 1, 1)); + } +}; + +test { + _ = @import("kv_test.zig"); +} diff --git a/zig/src/families/deepseek_v41/csa2_index.zig b/zig/src/families/deepseek_v41/csa2_index.zig new file mode 100644 index 0000000000..2ac845ef07 --- /dev/null +++ b/zig/src/families/deepseek_v41/csa2_index.zig @@ -0,0 +1,139 @@ +//! Tiled GPU scores and bounded online heaps; the memory footprint does not grow with context length. +const std = @import("std"); +const max_context = @import("kv_state.zig").max_context; +const mtl = @import("metal"); +const sources = @import("kernel_sources"); +const View = @import("view.zig").View; +pub const score_tile_width = 4096; +pub const Params = extern struct { + rows: u32, + heads: u32 = 32, + width: u32 = 128, + q_stride: u32, + weight_stride: u32, + position: u32, + ratio: u32, + keys: u32, + key_stride: u32 = 68, + tile_start: u32 = 0, + tile_width: u32 = score_tile_width, + topk: u32 = 512, + candidate_topk: u32 = 2048, + block_size: u32 = 8, + use_candidates: u32 = 0, + make_candidates: u32 = 0, + scale: f32 = 0.015625, + pub fn validate(p: Params) !void { + if (p.rows == 0 or p.heads == 0 or p.heads > 32 or p.width == 0 or p.width > 128 or p.width % 32 != 0 or + p.q_stride < p.heads * p.width or p.weight_stride < p.heads or + (p.ratio != 1 and p.ratio != 2) or p.key_stride < p.width / 2 + p.width / 32 or + p.topk != 512 or p.candidate_topk != 2048 or p.block_size != 8 or + p.tile_width != score_tile_width or p.use_candidates > 1 or p.make_candidates > 1 or + (p.use_candidates != 0 and p.make_candidates != 0) or + @as(u64, p.position) + p.rows > max_context or p.keys > max_context or + !std.math.isFinite(p.scale) or p.scale <= 0 or + @as(u64, p.rows) * p.q_stride > std.math.maxInt(u32) or + @as(u64, p.rows) * p.weight_stride > std.math.maxInt(u32) or + @as(u64, p.keys) * p.key_stride > std.math.maxInt(u32)) + return error.InvalidIndexerShape; + } +}; +pub const Scratch = struct { + buffer: mtl.Buffer, + rows: u32, + pub fn bytes(rows: u32) usize { + return @as(usize, rows) * (score_tile_width * 4 + 512 * 8 + 2048 * 8 + 512 * 4 + 528 + 32 * 68); + } + pub fn init(device: mtl.Device, rows: u32) !Scratch { + if (rows == 0 or rows > 4096) return error.InvalidIndexerScratch; + return .{ .buffer = try device.buffer(bytes(rows), mtl.ResourceOptions.shared), .rows = rows }; + } + pub fn deinit(s: *Scratch) void { + s.buffer.deinit(); + } + pub fn scores(s: Scratch) View { + return .{ .buffer = s.buffer }; + } + pub fn heap(s: Scratch) View { + return s.scores().shifted(@as(usize, s.rows) * score_tile_width * 4); + } + pub fn candidates(s: Scratch) View { + return s.heap().shifted(@as(usize, s.rows) * 512 * 8); + } + pub fn positions(s: Scratch) View { + return s.candidates().shifted(@as(usize, s.rows) * 2048 * 8); + } + pub fn windowChunk(s: Scratch) View { + return s.positions().shifted(@as(usize, s.rows) * 512 * 4); + } + pub fn queryPacked(s: Scratch) View { + return s.windowChunk().shifted(@as(usize, s.rows) * 528); + } +}; +pub const Pipelines = struct { + score: mtl.Pipeline, + clear: mtl.Pipeline, + merge: mtl.Pipeline, + finish: mtl.Pipeline, + pub fn load(device: mtl.Device) !Pipelines { + var options = mtl.CompileOptions.mlx(); + options.math_functions = .precise; + const lib = try mtl.Library.fromSource(device, sources.deepseek_v41_kv ++ sources.deepseek_v41_index, options); + defer lib.deinit(); + var p: Pipelines = undefined; + p.score = try mtl.Pipeline.init(device, lib, "dsv41_index_score", false); + errdefer p.score.deinit(); + p.clear = try mtl.Pipeline.init(device, lib, "dsv41_index_clear", false); + errdefer p.clear.deinit(); + p.merge = try mtl.Pipeline.init(device, lib, "dsv41_index_merge", false); + errdefer p.merge.deinit(); + p.finish = try mtl.Pipeline.init(device, lib, "dsv41_index_finish", false); + return p; + } + pub fn deinit(p: *Pipelines) void { + p.score.deinit(); + p.clear.deinit(); + p.merge.deinit(); + p.finish.deinit(); + } + /// Q has already undergone BF16 RoPE and in-place FP4/E8M0 quantization; weights are BF16 projection output. + pub fn encode(p: *const Pipelines, enc: mtl.ComputeEncoder, q: View, weights: View, keys: View, scratch: Scratch, args: Params) !void { + try args.validate(); + if (args.rows > scratch.rows) return error.IndexerScratchExceeded; + try (View{ .buffer = scratch.buffer }).require(Scratch.bytes(scratch.rows), 4); + if (args.keys != 0) { + try q.requireRows(args.rows, args.q_stride, @as(usize, args.heads) * args.width, 2); + try weights.requireRows(args.rows, args.weight_stride, args.heads, 2); + try keys.requireRows(args.keys, args.key_stride, args.width / 2 + args.width / 32, 1); + } + enc.setPipeline(p.clear); + for ([_]View{ scratch.heap(), scratch.candidates() }, 0..) |v, i| enc.setBuffer(v.buffer, v.offset, i); + enc.setValue(args, 2); + enc.dispatchThreads(mtl.Size.of(@max(args.topk, args.candidate_topk), args.rows, 1), mtl.Size.of(128, 1, 1)); + enc.barrier(); + const count = if (args.use_candidates != 0) args.candidate_topk * args.block_size else args.keys; + var at: u32 = 0; + while (at < count) : (at += score_tile_width) { + var tile = args; + tile.tile_start = at; + enc.setPipeline(p.score); + for ([_]View{ q, weights, keys, scratch.candidates(), scratch.scores() }, 0..) |v, i| + enc.setBuffer(v.buffer, v.offset, i); + enc.setValue(tile, 5); + enc.dispatchGroups(mtl.Size.of(score_tile_width, args.rows, 1), mtl.Size.of(32, 1, 1)); + enc.barrier(); + enc.setPipeline(p.merge); + for ([_]View{ scratch.scores(), scratch.heap(), scratch.candidates() }, 0..) |v, i| + enc.setBuffer(v.buffer, v.offset, i); + enc.setValue(tile, 3); + enc.dispatchThreads(mtl.Size.of(args.rows, 1, 1), mtl.Size.of(1, 1, 1)); + enc.barrier(); + } + enc.setPipeline(p.finish); + for ([_]View{ scratch.heap(), scratch.candidates(), scratch.positions() }, 0..) |v, i| + enc.setBuffer(v.buffer, v.offset, i); + enc.setValue(args, 3); + enc.dispatchThreads(mtl.Size.of(args.rows, 1, 1), mtl.Size.of(1, 1, 1)); + enc.barrier(); + } +}; diff --git a/zig/src/families/deepseek_v41/csa2_test.zig b/zig/src/families/deepseek_v41/csa2_test.zig new file mode 100644 index 0000000000..0ac81cacca --- /dev/null +++ b/zig/src/families/deepseek_v41/csa2_test.zig @@ -0,0 +1,309 @@ +const std = @import("std"); +const mtl = @import("metal"); +const csa2 = @import("csa2.zig"); +const index = @import("csa2_index.zig"); +const rope = @import("rope.zig"); +const View = @import("view.zig").View; +fn bf(x: f32) u16 { + const bits: u32 = @bitCast(x); + return @intCast((bits +% 0x7fff +% ((bits >> 16) & 1)) >> 16); +} +fn view(b: mtl.Buffer) View { + return .{ .buffer = b }; +} +fn submit(cb: mtl.CommandBuffer, enc: mtl.ComputeEncoder) !void { + enc.end(); + cb.commit(); + cb.wait(); + try std.testing.expect(cb.failure() == null); +} + +test "sparse attention preserves a causal window across ring wrap and masks future compressed keys" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try csa2.Pipelines.load(device); + defer p.deinit(); + const q = try device.buffer(3 * 512 * 2, mtl.ResourceOptions.shared); + defer q.deinit(); + const ring = try device.buffer(128 * 528, mtl.ResourceOptions.shared); + defer ring.deinit(); + const chunk = try device.buffer(3 * 528, mtl.ResourceOptions.shared); + defer chunk.deinit(); + const main = try device.buffer(201 * 288, mtl.ResourceOptions.shared); + defer main.deinit(); + const positions = try device.buffer(6 * 4, mtl.ResourceOptions.shared); + defer positions.deinit(); + const sink = try device.buffer(4, mtl.ResourceOptions.shared); + defer sink.deinit(); + const out = try device.buffer(3 * 512 * 2, mtl.ResourceOptions.shared); + defer out.deinit(); + @memset(q.contents()[0 .. 3 * 512 * 2], 0); + sink.slice(f32, 1)[0] = 0; + for (0..128) |r| { + @memset(ring.contents()[r * 528 .. r * 528 + 512], 0x38); + @memset(ring.contents()[r * 528 + 512 .. (r + 1) * 528], 127); + } + for (0..3) |r| { + @memset(chunk.contents()[r * 528 .. r * 528 + 512], 0x40); + @memset(chunk.contents()[r * 528 + 512 .. (r + 1) * 528], 127); + positions.slice(i32, 6)[2 * r] = 0; + positions.slice(i32, 6)[2 * r + 1] = 200; + } + @memset(main.contents()[0..256], 0x66); + @memset(main.contents()[256..288], 0x38); + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encode(enc, view(q), view(ring), view(chunk), view(main), view(positions), view(sink), view(out), .{ .rows = 3, .heads = 1, .q_stride = 512, .position = 129, .chunk_rows = 3, .ratio = 2, .compressed_rows = 201, .topk = 2, .index_stride = 2 }); + try submit(cb, enc); + for (0..3) |r| { + const expected = bf((133 + @as(f32, @floatFromInt(r))) / 130); + for (out.slice(u16, 3 * 512)[r * 512 .. (r + 1) * 512]) |v| try std.testing.expectEqual(expected, v); + } +} + +test "sparse QK cancellation retains a small score before sink normalization" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try csa2.Pipelines.load(device); + defer p.deinit(); + const q = try device.buffer(512 * 2, mtl.ResourceOptions.shared); + defer q.deinit(); + const chunk = try device.buffer(528, mtl.ResourceOptions.shared); + defer chunk.deinit(); + const sink = try device.buffer(4, mtl.ResourceOptions.shared); + defer sink.deinit(); + const out = try device.buffer(512 * 2, mtl.ResourceOptions.shared); + defer out.deinit(); + @memset(q.slice(u16, 512), 0); + q.slice(u16, 512)[0] = bf(0x1p24); + q.slice(u16, 512)[1] = bf(-0x1p24); + q.slice(u16, 512)[32] = bf(0x1p-6); + @memset(chunk.contents()[0..512], 0x38); + @memset(chunk.contents()[512..528], 127); + sink.slice(f32, 1)[0] = 0; + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + // No prior window or compressed keys: their unused bindings share the immutable chunk. + try p.encode(enc, view(q), view(chunk), view(chunk), view(chunk), view(chunk), view(sink), view(out), .{ .rows = 1, .heads = 1, .q_stride = 512, .position = 0, .chunk_rows = 1, .ratio = 0, .compressed_rows = 0, .scale = 1 }); + try submit(cb, enc); + // Exact dot = 2^-6; 1 / (1 + exp(-2^-6)) rounds to this BF16 value. + for (out.slice(u16, 512)) |v| try std.testing.expectEqual(bf(0.50390625), v); +} + +test "the first query retains BF16 probability arithmetic regardless of future prefill rows" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try csa2.Pipelines.load(device); + defer p.deinit(); + const q = try device.buffer(32 * 2, mtl.ResourceOptions.shared); + defer q.deinit(); + const ring = try device.buffer(128 * 33, mtl.ResourceOptions.shared); + defer ring.deinit(); + const chunk = try device.buffer(128 * 33, mtl.ResourceOptions.shared); + defer chunk.deinit(); + const main = try device.buffer(18, mtl.ResourceOptions.shared); + defer main.deinit(); + const positions = try device.buffer(4, mtl.ResourceOptions.shared); + defer positions.deinit(); + const sink = try device.buffer(4, mtl.ResourceOptions.shared); + defer sink.deinit(); + const out = try device.buffer(32 * 2, mtl.ResourceOptions.shared); + defer out.deinit(); + @memset(q.slice(u16, 32), 0); + q.slice(u16, 32)[0] = bf(1); + @memset(chunk.contents()[0..32], 0x38); + chunk.contents()[32] = 127; + @memset(main.contents()[0..16], 0x44); + @memset(main.contents()[16..18], 0x38); + positions.slice(i32, 1)[0] = 0; + sink.slice(f32, 1)[0] = 0.005; + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encode(enc, view(q), view(ring), view(chunk), view(main), view(positions), view(sink), view(out), .{ .rows = 1, .heads = 1, .width = 32, .q_stride = 32, .position = 0, .chunk_rows = 1, .ring_stride = 33, .chunk_stride = 33, .main_stride = 18, .ratio = 1, .compressed_rows = 1, .topk = 1, .index_stride = 1, .scale = 1 }); + try submit(cb, enc); + // BF16(exp(-1))*1 + 1*2; padded windows would move key 2 into a later online-softmax tile. + for (out.slice(u16, 32)) |v| try std.testing.expectEqual(bf(1.5703125), v); + const cb2 = queue.commandBuffer(); + const enc2 = cb2.compute(.serial); + try p.encode(enc2, view(q), view(ring), view(chunk), view(main), view(positions), view(sink), view(out), .{ .rows = 1, .heads = 1, .width = 32, .q_stride = 32, .position = 0, .chunk_rows = 128, .ring_stride = 33, .chunk_stride = 33, .main_stride = 18, .ratio = 1, .compressed_rows = 1, .topk = 1, .index_stride = 1, .scale = 1 }); + try submit(cb2, enc2); + for (out.slice(u16, 32)) |v| try std.testing.expectEqual(bf(1.5703125), v); +} + +test "hierarchical candidate selection pins the newest partial block and subsequent reindex uses those blocks" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try index.Pipelines.load(device); + defer p.deinit(); + var scratch = try index.Scratch.init(device, 1); + defer scratch.deinit(); + const q = try device.buffer(32 * 2, mtl.ResourceOptions.shared); + defer q.deinit(); + const weights = try device.buffer(2, mtl.ResourceOptions.shared); + defer weights.deinit(); + const keys_count = 2048 * 8 + 3; + const keys = try device.buffer(keys_count * 17, mtl.ResourceOptions.shared); + defer keys.deinit(); + @memset(q.slice(u16, 32), bf(1)); + weights.slice(u16, 1)[0] = bf(1); + for (0..keys_count) |r| { + @memset(keys.contents()[r * 17 .. r * 17 + 16], 0x22); + keys.contents()[r * 17 + 16] = if (r >= 2048 * 8) 126 else 127; + } + var args = index.Params{ .rows = 1, .heads = 1, .width = 32, .q_stride = 32, .weight_stride = 1, .position = keys_count - 1, .ratio = 1, .keys = keys_count, .key_stride = 17, .make_candidates = 1, .scale = 1 }; + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encode(enc, view(q), view(weights), view(keys), scratch, args); + try submit(cb, enc); + const Pick = extern struct { score: f32, position: i32 }; + const candidates = @as([*]Pick, @ptrCast(@alignCast(scratch.candidates().buffer.contents() + scratch.candidates().offset)))[0..2048]; + try std.testing.expectEqual(@as(i32, 2048), candidates[2047].position); + for (candidates[0..2047], 0..) |pick, i| try std.testing.expectEqual(@as(i32, @intCast(i)), pick.position); + const cb2 = queue.commandBuffer(); + const enc2 = cb2.compute(.serial); + args.make_candidates = 0; + args.use_candidates = 1; + try p.encode(enc2, view(q), view(weights), view(keys), scratch, args); + try submit(cb2, enc2); + const selected = @as([*]i32, @ptrCast(@alignCast(scratch.positions().buffer.contents() + scratch.positions().offset)))[0..512]; + for (selected, 0..) |pos, i| try std.testing.expectEqual(@as(i32, @intCast(i)), pos); +} + +test "RoPE rotates adjacent tail pairs with BF16 assignment and compressed positions use group starts" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try rope.Pipelines.load(device, 3, 32, .{ .theta = 10000 }, .{ .theta = 160000 }); + defer p.deinit(); + const x = try device.buffer(2 * 64 * 2, mtl.ResourceOptions.shared); + defer x.deinit(); + @memset(x.slice(u16, 128), bf(1)); + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try std.testing.expectError(error.InvalidRotaryShape, p.encode(enc, view(x), .{ .rows = 2, .heads = 1, .width = 64, .rotary = 32, .stride = 64, .position = 1, .step = 2 })); + try std.testing.expectError(error.InvalidRotaryShape, p.encode(enc, view(x), .{ .rows = 1, .heads = 1, .width = 64, .rotary = 16, .stride = 64, .position = 0 })); + try std.testing.expectError(error.InvalidRotaryShape, p.encode(enc, view(x), .{ .rows = 2, .heads = 1, .width = 64, .rotary = 32, .stride = 64, .position = std.math.maxInt(u32), .step = std.math.maxInt(u32) })); + try p.encode(enc, view(x), .{ .rows = 2, .heads = 1, .width = 64, .rotary = 32, .stride = 64, .position = 0, .step = 2 }); + try submit(cb, enc); + for (x.slice(u16, 128)[0..96]) |v| try std.testing.expectEqual(bf(1), v); + for (0..16) |pair| { + const power: f32 = @floatCast(std.math.pow(f64, 10000, @as(f32, @floatFromInt(pair * 2)) / 32)); + const frequency: f32 = 1 / power; + const angle: f32 = 2 * frequency; + const c: f32 = @floatCast(@cos(@as(f64, angle))); + const s: f32 = @floatCast(@sin(@as(f64, angle))); + const a = bf(c - s); + const b = bf(s + c); + try std.testing.expectEqual(a, x.slice(u16, 128)[96 + pair * 2]); + try std.testing.expectEqual(b, x.slice(u16, 128)[97 + pair * 2]); + } +} + +test "indexer visibility follows completed compressor groups for every query in a chunk" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try index.Pipelines.load(device); + defer p.deinit(); + var scratch = try index.Scratch.init(device, 3); + defer scratch.deinit(); + const q = try device.buffer(3 * 32 * 2, mtl.ResourceOptions.shared); + defer q.deinit(); + const weights = try device.buffer(3 * 2, mtl.ResourceOptions.shared); + defer weights.deinit(); + const keys = try device.buffer(3 * 17, mtl.ResourceOptions.shared); + defer keys.deinit(); + @memset(q.slice(u16, 3 * 32), bf(1)); + @memset(weights.slice(u16, 3), bf(1)); + for (0..3) |r| { + @memset(keys.contents()[r * 17 .. r * 17 + 16], 0x22); + keys.contents()[r * 17 + 16] = 127; + } + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encode(enc, view(q), view(weights), view(keys), scratch, .{ .rows = 3, .heads = 1, .width = 32, .q_stride = 32, .weight_stride = 1, .position = 0, .ratio = 2, .keys = 3, .key_stride = 17, .scale = 1 }); + try submit(cb, enc); + const selected = @as([*]i32, @ptrCast(@alignCast(scratch.positions().buffer.contents() + + scratch.positions().offset)))[0 .. 3 * 512]; + for (selected[0..512]) |pos| try std.testing.expectEqual(@as(i32, -1), pos); + for (1..3) |r| { + try std.testing.expectEqual(@as(i32, 0), selected[r * 512]); + for (selected[r * 512 + 1 .. (r + 1) * 512]) |pos| try std.testing.expectEqual(@as(i32, -1), pos); + } +} + +test "YaRN interpolates its correction band and inverse rotation conjugates the frequency" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try rope.Pipelines.load(device, 65537, 32, .{ .theta = 10000 }, .{ .theta = 160000, .original = 65536 }); + defer p.deinit(); + const x = try device.buffer(64 * 2, mtl.ResourceOptions.shared); + defer x.deinit(); + @memset(x.slice(u16, 64), bf(1)); + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encode(enc, view(x), .{ .rows = 1, .heads = 1, .width = 64, .rotary = 32, .stride = 64, .position = 65536, .inverse = 1, .spectrum = .compressed }); + try submit(cb, enc); + const low: f32 = @floatCast(@max(@floor(32 * @log(@as(f64, 65536) / (32 * 2 * std.math.pi)) / (2 * @log(@as(f64, 160000)))), 0)); + const high: f32 = @floatCast(@min(@ceil(32 * @log(@as(f64, 65536) / (2 * std.math.pi)) / (2 * @log(@as(f64, 160000)))), 31)); + for ([_]usize{ 0, 10, 15 }) |pair| { + const power: f32 = @floatCast(std.math.pow(f64, 160000, @as(f32, @floatFromInt(pair * 2)) / 32)); + const frequency: f32 = 1 / power; + const smooth: f32 = 1 - std.math.clamp((@as(f32, @floatFromInt(pair)) - low) / @max(high - low, 1e-3), 0, 1); + const slow: f32 = frequency / 16 * (1 - smooth); + const fast: f32 = frequency * smooth; + const angle: f32 = 65536 * (slow + fast); + const c: f32 = @floatCast(@cos(@as(f64, angle))); + const s: f32 = -@as(f32, @floatCast(@sin(@as(f64, angle)))); + try std.testing.expectEqual(bf(c - s), x.slice(u16, 64)[32 + pair * 2]); + try std.testing.expectEqual(bf(s + c), x.slice(u16, 64)[33 + pair * 2]); + } +} + +test "RoPE retained source pair rounds the position-five imaginary consumer to pinned CUDA BF16" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try rope.Pipelines.load(device, 6, 64, .{ .theta = 10000 }, .{ .theta = 160000, .original = 65536 }); + defer p.deinit(); + const x = try device.buffer(512 * 2, mtl.ResourceOptions.shared); + defer x.deinit(); + @memset(x.slice(u16, 512), bf(0)); + // Token-5/layer-0 rotary pair 21 crosses a BF16 tie under per-head cos; pinned source factors give .546875. + x.slice(u16, 512)[490] = bf(1.15625); + x.slice(u16, 512)[491] = bf(0.53125); + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encode(enc, view(x), .{ .rows = 1, .heads = 1, .width = 512, .rotary = 64, .stride = 512, .position = 5 }); + try submit(cb, enc); + try std.testing.expectEqual(bf(0.546875), x.slice(u16, 512)[491]); +} diff --git a/zig/src/families/deepseek_v41/engram.zig b/zig/src/families/deepseek_v41/engram.zig new file mode 100644 index 0000000000..867cc78d39 --- /dev/null +++ b/zig/src/families/deepseek_v41/engram.zig @@ -0,0 +1,207 @@ +//! Resident Engram FP8 gathers and signed-sqrt gates; WKV is projected separately by the caller. +const std = @import("std"); +const mtl = @import("metal"); +const sources = @import("kernel_sources"); +pub const View = @import("view.zig").View; +const Dtype = @import("ops.zig").Dtype; +pub const hash = @import("engram_hash.zig"); + +pub const head_dim = 256; +pub const gather_dim = hash.columns * head_dim; +pub const history_bytes = 16; + +/// Borrowed full-resident E4M3FN rows and UE8M0 rowwise scale blocks, never a 32x32 scale matrix. +pub const Table = struct { + weight: View, + scale: View, + rows: u32, + weight_stride_bytes: usize = head_dim, + scale_stride_bytes: usize = head_dim / 32, + scale_block: u32 = 32, + + pub fn validate(self: Table) !void { + if (self.rows == 0 or self.weight_stride_bytes < head_dim or self.scale_stride_bytes < head_dim / 32 or self.scale_block != 32) return error.InvalidEngramTable; + try self.weight.requireRows(self.rows, self.weight_stride_bytes, head_dim, 1); + try self.scale.requireRows(self.rows, self.scale_stride_bytes, head_dim / 32, 1); + } +}; + +/// BF16 hidden [tokens,copies,dim], WKV [tokens,(copies+1)*dim]; norm weights contiguous FP32 [copies,dim]. +pub const Gate = struct { + q_weight: View, + k_weight: View, + /// The released checkpoints store these f32; rank-split repacks store bf16. + weight_dtype: Dtype = .f32, + dim: u32, + copies: u32, + hidden_stride: usize, + kv_stride: usize, + output_stride: usize, + eps: f32, + + pub fn validate(self: Gate) !void { + const width = try std.math.mul(usize, self.dim, self.copies); + const projected_width = try std.math.add(usize, width, self.dim); + if (self.dim == 0 or self.copies == 0 or self.hidden_stride < width or self.output_stride < width or self.kv_stride < projected_width or !std.math.isFinite(self.eps) or self.eps <= 0 or + self.dim > std.math.maxInt(u32) - 255) return error.InvalidEngramGate; + const weight_bytes = try std.math.mul(usize, width, 4); + try self.q_weight.require(weight_bytes, 4); + try self.k_weight.require(weight_bytes, 4); + } +}; + +pub const Pipelines = struct { + hash_ids: mtl.Pipeline, + commit: mtl.Pipeline, + reset: mtl.Pipeline, + gather: mtl.Pipeline, + gate: mtl.Pipeline, + + pub fn load(device: mtl.Device) !Pipelines { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const lib = try mtl.Library.fromSource(device, sources.deepseek_v41_engram, mtl.CompileOptions.mlx()); + defer lib.deinit(); + const h = try mtl.Pipeline.init(device, lib, "tf_v41_engram_hash", false); + errdefer h.deinit(); + const c = try mtl.Pipeline.init(device, lib, "tf_v41_engram_commit", false); + errdefer c.deinit(); + const r = try mtl.Pipeline.init(device, lib, "tf_v41_engram_reset", false); + errdefer r.deinit(); + const g = try mtl.Pipeline.init(device, lib, "tf_v41_engram_gather", false); + errdefer g.deinit(); + const gate = try mtl.Pipeline.init(device, lib, "tf_v41_engram_gate", false); + return .{ .hash_ids = h, .commit = c, .reset = r, .gather = g, .gate = gate }; + } + + pub fn deinit(self: *Pipelines) void { + self.hash_ids.deinit(); + self.commit.deinit(); + self.reset.deinit(); + self.gather.deinit(); + self.gate.deinit(); + } +}; + +const HashArgs = extern struct { tokens: u32, pad: u32, masked: u32 }; +const GatherArgs = extern struct { + weight_stride: u64, + scale_stride: u64, + output_stride: u64, + tokens: u32, + columns: u32, + head_dim: u32, + scale_block: u32, +}; +const GateArgs = extern struct { + hidden_stride: u64, + kv_stride: u64, + output_stride: u64, + dim: u32, + copies: u32, + masked: u32, + weight_dtype: u32, + eps: f32, +}; + +fn bind(e: mtl.ComputeEncoder, view: View, at: usize) void { + e.setBuffer(view.buffer, view.offset, at); +} + +/// Each history View is one sequence's 16-byte GPU record; initialize before its first hash. +pub fn encodeReset(e: mtl.ComputeEncoder, p: *const Pipelines, history: View) void { + e.setPipeline(p.reset); + bind(e, history, 0); + e.dispatchGroups(.{ .width = 1 }, .{ .width = 1 }); + e.barrier(); +} + +/// Snapshot/rollback are GPU copies; caller ends compute encoding before beginning a blit encoder. +pub fn encodeSnapshot(e: mtl.BlitEncoder, history: View, saved: View) void { + e.copy(history.buffer, history.offset, saved.buffer, saved.offset, history_bytes); +} + +pub fn encodeRollback(e: mtl.BlitEncoder, saved: View, history: View) void { + encodeSnapshot(e, saved, history); +} + +/// Raw u32 IDs -> [tokens,24] hashes; map covers valid IDs, optional 0=DEAD mask, history is unchanged. +pub fn encodeHash(e: mtl.ComputeEncoder, p: *const Pipelines, meta: hash.Layer, ids: View, token_map: View, mask: ?View, history: View, out: View, tokens: u32) !void { + if (tokens == 0) return error.EmptyEngramChunk; + if (!std.mem.eql(u8, std.mem.asBytes(&meta), std.mem.asBytes(&hash.layers[0])) and + !std.mem.eql(u8, std.mem.asBytes(&meta), std.mem.asBytes(&hash.layers[1]))) return error.InvalidEngramMetadata; + try ids.requireRows(tokens, 1, 1, 4); + try token_map.require(4, 4); + try history.require(history_bytes, 4); + try out.requireRows(tokens, hash.columns, hash.columns, 4); + if (mask) |m| try m.require(tokens, 1); + e.setPipeline(p.hash_ids); + bind(e, ids, 0); + bind(e, token_map, 1); + bind(e, mask orelse ids, 2); + bind(e, history, 3); + bind(e, out, 4); + e.setValue(meta, 5); + e.setValue(HashArgs{ .tokens = tokens, .pad = hash.pad_id, .masked = @intFromBool(mask != null) }, 6); + e.dispatchThreads(.{ .width = tokens }, .{ .width = 64 }); + e.barrier(); +} + +/// Commit exactly once per chunk after both layers have hashed; remains bounded at three past tokens. +pub fn encodeCommit(e: mtl.ComputeEncoder, p: *const Pipelines, ids: View, token_map: View, mask: ?View, history: View, tokens: u32) !void { + if (tokens == 0) return error.EmptyEngramChunk; + try ids.requireRows(tokens, 1, 1, 4); + try token_map.require(4, 4); + try history.require(history_bytes, 4); + if (mask) |m| try m.require(tokens, 1); + e.barrier(); + e.setPipeline(p.commit); + bind(e, ids, 0); + bind(e, token_map, 1); + bind(e, mask orelse ids, 2); + bind(e, history, 3); + e.setValue(HashArgs{ .tokens = tokens, .pad = hash.pad_id, .masked = @intFromBool(mask != null) }, 4); + e.dispatchGroups(.{ .width = 1 }, .{ .width = 1 }); + e.barrier(); +} + +/// Gather uint32 [tokens,24] row IDs into BF16 [tokens,6144]; IDs must be within the borrowed table. +pub fn encodeGather(e: mtl.ComputeEncoder, p: *const Pipelines, table: Table, ids: View, out: View, tokens: u32, output_stride: usize) !void { + try table.validate(); + if (tokens == 0 or output_stride < gather_dim) return error.InvalidEngramGather; + try ids.requireRows(tokens, hash.columns, hash.columns, 4); + try out.requireRows(tokens, output_stride, gather_dim, 2); + e.setPipeline(p.gather); + bind(e, ids, 0); + bind(e, table.weight, 1); + bind(e, table.scale, 2); + bind(e, out, 3); + e.setValue(GatherArgs{ .weight_stride = table.weight_stride_bytes, .scale_stride = table.scale_stride_bytes, .output_stride = output_stride, .tokens = tokens, .columns = hash.columns, .head_dim = head_dim, .scale_block = table.scale_block }, 4); + e.dispatchThreads(.{ .width = gather_dim, .height = tokens }, .{ .width = 256 }); + e.barrier(); +} + +/// Add per-copy gate times shared value to BF16 hidden; out may alias hidden, never the projected KV. +pub fn encodeGate(e: mtl.ComputeEncoder, p: *const Pipelines, gate: Gate, hidden: View, projected_kv: View, mask: ?View, out: View, tokens: u32) !void { + try gate.validate(); + if (tokens == 0) return error.EmptyEngramChunk; + const width = @as(usize, gate.dim) * gate.copies; + try hidden.requireRows(tokens, gate.hidden_stride, width, 2); + try projected_kv.requireRows(tokens, gate.kv_stride, width + gate.dim, 2); + try out.requireRows(tokens, gate.output_stride, width, 2); + if (mask) |m| try m.require(tokens, 1); + e.setPipeline(p.gate); + bind(e, hidden, 0); + bind(e, projected_kv, 1); + bind(e, gate.q_weight, 2); + bind(e, gate.k_weight, 3); + bind(e, mask orelse hidden, 4); + bind(e, out, 5); + e.setValue(GateArgs{ .hidden_stride = gate.hidden_stride, .kv_stride = gate.kv_stride, .output_stride = gate.output_stride, .dim = gate.dim, .copies = gate.copies, .masked = @intFromBool(mask != null), .weight_dtype = @backingInt(gate.weight_dtype), .eps = gate.eps }, 6); + e.dispatchGroups(.{ .width = tokens, .height = gate.copies }, .{ .width = 256 }); + e.barrier(); +} + +test { + _ = @import("engram_test.zig"); +} diff --git a/zig/src/families/deepseek_v41/engram_test.zig b/zig/src/families/deepseek_v41/engram_test.zig new file mode 100644 index 0000000000..dd085a8744 --- /dev/null +++ b/zig/src/families/deepseek_v41/engram_test.zig @@ -0,0 +1,247 @@ +//! Behavioral hash, rollback, selected-row dequantization and per-copy signed-sqrt gate regressions. +const std = @import("std"); +const hash = @import("engram_hash.zig"); +const engram = @import("engram.zig"); +const mtl = @import("metal"); +const View = engram.View; + +// Generated offline with the official NumPy multipliers and prime-bucket algorithm, not the native code. +const first = [_]u32{ + 8939594, 28321402, 40265357, 55817216, 71147020, 86924254, 110813108, 121427189, + 128454377, 156247504, 165979524, 188168291, 198564788, 211763979, 239563936, 241631034, + 263018334, 286373017, 297618161, 319241270, 333656732, 339279600, 364226324, 373889199, +}; +const third = [_]u32{ + 13176822, 30752650, 42532473, 56771564, 70133176, 85254602, 108815600, 115334181, + 138863141, 149049926, 167314083, 191090346, 204680718, 211476972, 237672992, 245604111, + 269210034, 282954889, 298312790, 313992267, 329778842, 345458689, 359433926, 374584261, +}; +const after_dead = [_]u32{ + 762556, 18016262, 38494153, 58317688, 64056565, 81970637, 106927994, 122902996, + 129060773, 153088767, 160206490, 178949865, 205667218, 211027556, 235070602, 251994813, + 263310652, 278975004, 292595948, 315407209, 325281398, 348093105, 353903080, 377405725, +}; + +test "official hash IDs preserve order, padding and globally unique layer buckets" { + var history: hash.History = .{}; + const a = try history.push(7, hash.layers[0]); + try std.testing.expectEqualSlices(u32, &first, &a); + _ = try history.push(9, hash.layers[0]); + const c = try history.push(11, hash.layers[0]); + try std.testing.expectEqualSlices(u32, &third, &c); + for (hash.layers) |meta| for (meta.primes, meta.offsets) |p, offset| { + try std.testing.expect(p >= 16000000); + try std.testing.expect(offset + p <= meta.rows); + }; + var high: hash.History = .{ .tail = .{ 1234, 5678, 42 }, .position = 3 }; + const result = try high.push(99091, hash.layers[1]); + const expected = [_]u32{ + 3315102, 30010869, 33829090, 51274514, 77267621, 84401643, 106522093, 127296527, + 137402994, 151593930, 162661867, 180852958, 195297507, 213361283, 228864003, 255457679, + 262256667, 282063670, 300635716, 304966933, 321424067, 350721784, 358839309, 370654406, + }; + try std.testing.expectEqualSlices(u32, &expected, &result); +} + +test "dead image positions block all older lookbacks and snapshot rollback restores the exact boundary" { + var history: hash.History = .{}; + _ = try history.push(7, hash.layers[0]); + _ = try history.push(9, hash.layers[0]); + const saved = history.snapshot(); + _ = try history.push(hash.dead, hash.layers[0]); + const got = try history.push(13, hash.layers[0]); + try std.testing.expectEqualSlices(u32, &after_dead, &got); + history.rollback(saved); + const replay = try history.push(11, hash.layers[0]); + try std.testing.expectEqualSlices(u32, &third, &replay); + history.reset(); + const reset = try history.push(7, hash.layers[0]); + try std.testing.expectEqualSlices(u32, &first, &reset); + try std.testing.expectError(error.InvalidEngramCompressedId, history.push(-2, hash.layers[0])); + try std.testing.expectEqual(@as(u64, 1), history.position); +} + +test "all prefill/decode partitions preserve hashes including a dead token at the chunk boundary" { + var bytes: [20 * 4]u8 = undefined; + for (0..20) |i| std.mem.writeInt(u32, bytes[i * 4 ..][0..4], @intCast(i), .little); + const map = hash.TokenMap{ .bytes = &bytes }; + const ids = [_]u32{ 7, 9, 11, 3, 13, 15, 17 }; + const mask = [_]u8{ 1, 1, 1, 0, 1, 1, 1 }; + for (hash.layers) |meta| { + var whole: hash.History = .{}; + var expected: [ids.len][hash.columns]u32 = undefined; + try hash.hashChunk(&whole, map, meta, &ids, &mask, &expected); + for (0..ids.len + 1) |split| { + var chunked: hash.History = .{}; + var actual: [ids.len][hash.columns]u32 = undefined; + try hash.hashChunk(&chunked, map, meta, ids[0..split], mask[0..split], actual[0..split]); + try hash.hashChunk(&chunked, map, meta, ids[split..], mask[split..], actual[split..]); + try std.testing.expectEqualDeep(expected, actual); + try std.testing.expectEqualDeep(whole, chunked); + } + const saved = whole.snapshot(); + var invalid: [2][hash.columns]u32 = undefined; + try std.testing.expectError(error.InvalidEngramToken, hash.hashChunk(&whole, map, meta, &.{ 1, 20 }, null, &invalid)); + try std.testing.expectEqualDeep(saved, whole); + } +} + +test "raw tokenizer IDs are compressed before hashing and normalization-equivalent tokens share hashes" { + var bytes: [4 * 4]u8 = undefined; + for ([_]u32{ 7, 9, 11, 9 }, 0..) |id, i| std.mem.writeInt(u32, bytes[i * 4 ..][0..4], id, .little); + const map = hash.TokenMap{ .bytes = &bytes }; + var a: hash.History = .{}; + var b: hash.History = .{}; + var out_a: [3][hash.columns]u32 = undefined; + var out_b: [3][hash.columns]u32 = undefined; + try hash.hashChunk(&a, map, hash.layers[0], &.{ 0, 1, 2 }, null, &out_a); + try hash.hashChunk(&b, map, hash.layers[0], &.{ 0, 3, 2 }, null, &out_b); + try std.testing.expectEqualDeep(out_a, out_b); + try std.testing.expectEqualSlices(u32, &first, &out_a[0]); + try std.testing.expectEqualSlices(u32, &third, &out_a[2]); +} + +fn view(buffer: mtl.Buffer) View { + return .{ .buffer = buffer }; +} +fn bf(value: f32) u16 { + const bits: u32 = @bitCast(value); + return @truncate((bits + 0x7fff + ((bits >> 16) & 1)) >> 16); +} +fn float(bits: u16) f32 { + return @bitCast(@as(u32, bits) << 16); +} +fn completed(cb: mtl.CommandBuffer) !void { + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.EngramGPUFailure; +} + +test "Metal hashes chunks with image breaks and rolls back bounded GPU history" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + var p = try engram.Pipelines.load(device); + defer p.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + const ids = try device.buffer(7 * 4, 0); + defer ids.deinit(); + @memcpy(ids.slice(u32, 7), &[_]u32{ 7, 9, 11, 3, 13, 15, 17 }); + const map = try device.buffer(20 * 4, 0); + defer map.deinit(); + for (map.slice(u32, 20), 0..) |*id, i| id.* = @intCast(i); + const mask = try device.buffer(7, 0); + defer mask.deinit(); + @memcpy(mask.slice(u8, 7), &[_]u8{ 1, 1, 1, 0, 1, 1, 1 }); + const history = try device.buffer(engram.history_bytes, 0); + defer history.deinit(); + const saved = try device.buffer(engram.history_bytes, 0); + defer saved.deinit(); + const out = try device.buffer(7 * 24 * 4, 0); + defer out.deinit(); + const replay = try device.buffer(24 * 4, 0); + defer replay.deinit(); + const cb = queue.commandBuffer(); + var e = cb.compute(.serial); + engram.encodeReset(e, &p, view(history)); + try engram.encodeHash(e, &p, hash.layers[0], view(ids), view(map), view(mask), view(history), view(out), 3); + try engram.encodeCommit(e, &p, view(ids), view(map), view(mask), view(history), 3); + e.end(); + var blit = cb.blit(); + engram.encodeSnapshot(blit, view(history), view(saved)); + blit.end(); + e = cb.compute(.serial); + try engram.encodeHash(e, &p, hash.layers[0], view(ids).shifted(12), view(map), view(mask).shifted(3), view(history), view(out).shifted(3 * 24 * 4), 4); + try engram.encodeCommit(e, &p, view(ids).shifted(12), view(map), view(mask).shifted(3), view(history), 4); + e.end(); + blit = cb.blit(); + engram.encodeRollback(blit, view(saved), view(history)); + blit.end(); + e = cb.compute(.serial); + try engram.encodeHash(e, &p, hash.layers[1], view(ids).shifted(16), view(map), null, view(history), view(replay), 1); + e.end(); + try completed(cb); + var cpu: hash.History = .{}; + for (ids.slice(u32, 7), mask.slice(u8, 7), 0..) |id, alive, i| { + const want = try cpu.push(if (alive == 0) hash.dead else @intCast(id), hash.layers[0]); + try std.testing.expectEqualSlices(u32, &want, out.slice(u32, 7 * 24)[i * 24 ..][0..24]); + } + cpu = .{ .tail = .{ 11, 9, 7 }, .position = 3 }; + const want = try cpu.push(13, hash.layers[1]); + try std.testing.expectEqualSlices(u32, &want, replay.slice(u32, 24)); + try std.testing.expectEqualSlices(i32, &.{ 11, 9, 7 }, history.slice(i32, 4)[0..3]); +} + +test "Metal gathers only selected FP8 rows with rowwise scales and gates each mHC copy independently" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + var p = try engram.Pipelines.load(device); + defer p.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + const table = try device.buffer(3 * 272, 0); + defer table.deinit(); + @memset(table.slice(u8, 3 * 272), 0x7f); + const scales = try device.buffer(3 * 12, 0); + defer scales.deinit(); + @memset(scales.slice(u8, 3 * 12), 255); + for (0..3) |r| { + @memset(table.slice(u8, 3 * 272)[r * 272 ..][0..256], if (r == 0) 0x38 else if (r == 1) 0xb8 else 0x7e); + for (0..8) |g| scales.slice(u8, 3 * 12)[r * 12 + g] = @intCast(127 + g); + } + table.slice(u8, 3 * 272)[2 * 272] = 0x01; + table.slice(u8, 3 * 272)[2 * 272 + 1] = 0x80; + scales.slice(u8, 3 * 12)[12 + 7] = 255; + const ids = try device.buffer(24 * 4, 0); + defer ids.deinit(); + for (ids.slice(u32, 24), 0..) |*id, i| id.* = @intCast((i * 2 + 1) % 3); + const gathered = try device.buffer((engram.gather_dim + 16) * 2, 0); + defer gathered.deinit(); + const hidden = try device.buffer(2 * 4 * 2, 0); + defer hidden.deinit(); + for (hidden.slice(u16, 8), 0..) |*v, i| v.* = bf(if (i < 4) 1 else 2); + const kv = try device.buffer(12 * 2, 0); + defer kv.deinit(); + for (kv.slice(u16, 12), 0..) |*v, i| v.* = bf(if (i < 4) 1 else if (i < 8) -2 else 3); + const q = try device.buffer(8 * 4, 0); + defer q.deinit(); + for (q.slice(f32, 8), 0..) |*v, i| v.* = if (i < 4) 1 else 0.5; + const k = try device.buffer(8 * 4, 0); + defer k.deinit(); + for (k.slice(f32, 8), 0..) |*v, i| v.* = if (i < 4) 1 else 2; + const output = try device.buffer(8 * 2, 0); + defer output.deinit(); + const dead_mask = try device.buffer(1, 0); + defer dead_mask.deinit(); + dead_mask.slice(u8, 1)[0] = 0; + const masked_output = try device.buffer(8 * 2, 0); + defer masked_output.deinit(); + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + try engram.encodeGather(e, &p, .{ .weight = view(table), .scale = view(scales), .rows = 3, .weight_stride_bytes = 272, .scale_stride_bytes = 12 }, view(ids), view(gathered), 1, engram.gather_dim + 16); + try engram.encodeGate(e, &p, .{ .q_weight = view(q), .k_weight = view(k), .dim = 4, .copies = 2, .hidden_stride = 8, .kv_stride = 12, .output_stride = 8, .eps = 1e-20 }, view(hidden), view(kv), null, view(output), 1); + try engram.encodeGate(e, &p, .{ .q_weight = view(q), .k_weight = view(k), .dim = 4, .copies = 2, .hidden_stride = 8, .kv_stride = 12, .output_stride = 8, .eps = 1e-20 }, view(hidden), view(kv), view(dead_mask), view(masked_output), 1); + e.end(); + try completed(cb); + for (ids.slice(u32, 24), 0..) |r, col| for (0..256) |d| { + const got = gathered.slice(u16, engram.gather_dim)[col * 256 + d]; + if (r == 1 and d >= 224) { + try std.testing.expect(std.math.isNan(float(got))); + continue; + } + const base: f32 = if (r == 2 and d == 0) 0x1p-9 else if (r == 2 and d == 1) -0.0 else if (r == 0) 1 else if (r == 1) -1 else 448; + const scale: f32 = @floatFromInt(@as(u32, 1) << @as(u5, @intCast(d / 32))); + try std.testing.expectEqual(bf(base * scale), got); + }; + const gate_positive = 1.0 / (1.0 + @exp(-@sqrt(@as(f32, 2)))); + const gate_negative = 1.0 / (1.0 + @exp(@sqrt(@as(f32, 2)))); + for (output.slice(u16, 8), 0..) |v, i| { + const want = bf(if (i < 4) 1 + 3 * gate_positive else 2 + 3 * gate_negative); + try std.testing.expectApproxEqAbs(float(want), float(v), 0.015625); + } + try std.testing.expectEqualSlices(u16, hidden.slice(u16, 8), masked_output.slice(u16, 8)); +} diff --git a/zig/src/families/deepseek_v41/fixtures/hc_attention_rms.json b/zig/src/families/deepseek_v41/fixtures/hc_attention_rms.json new file mode 100644 index 0000000000..d1f1c2b61d --- /dev/null +++ b/zig/src/families/deepseek_v41/fixtures/hc_attention_rms.json @@ -0,0 +1,25 @@ +{ + "embedding_bf16": "WT1bPAO857tZvTS9Cr2HvDY9MT2nvUI867vvPIG9Xr3wPMs8GLwLvCG91b1CPf87mTzuvC69rTzOvM28x7w+PSC6VL0YPOG8Fb1TvRE84zwNuw48pTwrvF89OTxEveU8SDybPBA9Yz0XPKU8Nj33PJK8jjtTvNW8C731PDk8Rz0SvAK9l7xnPB69ojvNuxM9y7yMvDC97Dx3PA89Ib2TO0k8j7orvDc7xbwwvCM9lLydvDS9OD1JPQC9JL2yvBu8ebyCvUE9Qz3+vDm8Bb0XPGG8gryUuAi8ZT1vvN+8wbyQPI47orz/vCE93jzVPHU90DwgvVe8rLzPvDe9Xr1tPZK8ojtCvcG84byJOzq9QD2APGE9Xzu3OwY90LzkPNY9iLsZPao8kz2Ivbs85LxKvZu9AjuiPQO9BL6RvAk9hL37u+08gjoavBw97rxDPXq5kz3vO5e7arp3vX+9hj2dO6m8B7sBPHS9jDywPGI907zYvAy8P7y9PDw8u7zrO1O8rjzLvD28Yz3yu5u8lLz0vCa9kjxuPE+9ED1XvJW7RbwfPDe9Mz1yvRy9Br0CPNc8OL2zPLI8tjwdPQw8Fb03vIM8kTzrvPU8DTwgvdM8YbwJPKY877vVPLo8Y7x6PAw9IbyOPEu9+TxuPfo8E7zVPO48ybzNPOe8EbtlvYi8Kb3VvCI8Wz2rvb88Wz1HvYe9oLzFPMY8pDyCPXO8k7yQPQq9IrxbvS89JjwKvQo7Jb0QvVa9oDxWPLM8rrgMvI46mjwWPdi8nzwevUG9Dz3eumy8K70bPZS9Jz2/vIO9xrxLvZi9W73GOsY9mL2OvCe8BL1DvOE8D70tvbU9mzxPvT082zwnvCy9Ar0+O1s9qTwbvIY9Sb2qvTY9KzvvvCA93TxAPeG83TtbPN07s7vXu2M8RjzyvAe9wb1fvUk9YrwwvE+8nbywPVG8WzsDPZI9XD2RvG09+DybPAo81zx8PdI8kr0JvJI7Xj1GPTS9irycPFS8gTzcvES9wLwvPY+9Bju2vRs9IDziPF09GD3dvHA7ML0MvEa9nroCPbo8yrtcvZg7QT3WvC47aLxmPKa7zbwVvGq7Dz30Oyc9Bz2FPBo9Ob14vVW8ELsDPUc8qL2kvCA8DbzlOz49EbwsvS8907wQO5w6sDzcuo085jt3Pdu6Hr0UPBm9OLzVvOq8LT0wO8y7Hz2/vFa9rLwePSC91LyXPa87rr0xveK8jDuMu8C54rx6uzU7HT3AvLE8Jj37O047/zz1PB+7Ab0QPDa9Oz2JvZW8Lb3guwQ9njtivey8vTyhOuE8Brx5vXC80jxdOw08i70OvcS8MD2qvbu8XbyZPLm9mz1aPV66iTx3vMQ7oTyyPIw9PbuOuzM9HjzgO4c7ubzivS69PDzTvLi8H712u767ZL2dPFk9gTzvvK48BTwOvYA9TbtlPIy8Vj0wPSu8D7x2vZQ88ruvPLy8tjwyPSk99rycOxk91DzgvL87pjyGPUi8Uz1ivBS9hb28OjM9izxevFq7Rb39vK07fL2QvcU8uD1vvPq6TLz2PCk9hLzhvIC7gzyBO2Q65DySux6967zbvA09Iz2cPVe8rrwmvbC8djwtPVe8Pz1qvUi9e71ZvYU9R7okPS49jbpIvaY8QbwiPee8rLwKvbE6WT2MPFa8YbxJvLo7Hr0iPVM94DzMu4s8Ojw5PRs9N730O568Fz27vFK8tL2uvEa8irrtO0K89TtAPVk8vbgmvTk9pzqDvLs7KTxtO6W8GjoWPCc9DjxBPZi9oryoPJS8O72jvBu9Fz3lPKA8bj1lPAS9UD0iPQ48KbxFPJy9J703Pbi8/rwqPVs7Er3Iu3q9qDsvvR28VrxzPVE8Y72UPB+9WbsfvS+8KL2BOu08B71ZPVK6Nbo0PZg9iT1pPYE8RD0IPB89Mr3NPEG8oz2wvLU8QjzavLK7qLwgve88ybxCPV68obxbvbu8f71xPGa87LwRPa08Pj1vvG89mz1WPdk8E711O5C89zwXPOy8hDzxPAc9ELwqvGI9I70SO7Q7Ab3AOzg8abtfvTe8zbzAvLS8ibyVPTu9tTyYPKy857wnPZi9JDx5uz07E70MPVq8ZLysOy67BT00Pda8bTx0PN082T0nvdS8Lztyuzm95DyoO1I9vLv5PBO9FD2kPWQ9kLuguQY8A73LPD27pryUvPq8Rb2Pvf68kjxzvGS9+LyDvOC8TDyRPAG9HDy0vLW89DyGPQg78LqMOza9ULt+POM44zzRvIU9xTyLPQc8zbxtvaY9bLuoPKw79zvMO7U8h72pPAa9ML0BvUK907xDvZC9Fbx3PfA8VDxsvW88Bz0TvDK91DwrPIk9ab2uu0a8iTtBvck8sjz7PCQ9D70JvQe94DrHOhG7hLzvvJ894DwxO2q7mb1jPDY8qTzPvEM9N7zZPHE9IboPPUM84zxbO0y9q7ylOh68JD3/u5c7ujwKPIu7yTzLvOO8Ej0YPdE9r726PR28C732PBW8wztrvZY8eLyVPLU8T7xhPea9i7qvPES7qbsvPI08jj2uPOM8T71bvOC8ijwrPY28G707PeO8JD0BvC85sbwPPA491bzNO5Y6J7wbPX88n71hPA+9wbyzPBi7kj2JPM+7j7sROgG8Zj2nPJ88uzwGvJE9j70PPay7yLz+vAC9Sj19vYe8hz3OvBW9DrswvYM8Eb2dvGy9/TuKPeI7JTwIvUk9mLuLvI68rrx2vUg9izxfPf08ez33vB+8L70lvdk8Sb0lvYY8Pry0POk7yDy/PCA8cDwNPQY8dT2DvYu6FT2dPWy8C7wcPUw9nzxhvKA9t7sWPci8Tr25vL47Vb02vBQ9j71vva+8tzyNPDe9lrm3PAK82jq1O2M7WbuXvCw9krsmvag7tDoIPXM86zzJPAU9aj2bvRG8uD3SOqa7XrxNuy09gDvHPF+93DwxPAu9Ej1FPU69hTuyPOu8L70avQ89zTqzvUk9tjvvPBw7u7zEO7k8Ez0hO0U7NL01PQM83zzJO3y8oDz8PO0797yOPIG9nLx9PLs8sbx2PZa7kT2NPCe8k7vUPJU6Gb0fPL48cDzDOzQ8RLxKPU89Wr36PJY8rDy+uya7E7wMPau6TryIO6O9kDyyvc87Er3hvAg8BLzWPEi8yzxqO5I9vLwWvRs9sryUPCa8trw6PA09EDuNvaa7QDxXvSq8U71fvHQ9M71QvU88TLuYvV89xrxFPXS8/jxAu2E9cLyZPL29+DvHPOq8QL0XvCi9TT0JPX+8prxPPSe9wr7FvG09OD2mPJM8g72NvLS80jytPLo8F73cOz+9CD2LO1Q9Mrx1vA+9WT2UO4K9MT1uPIk96L1Qu9i8cL1oPU69v7rePNg7Kz2ePLA7Dj0oPEU9Mj3/vDY9hL2SO488J7sGvIE8Kz1Xuxs9KDwCPpa8O725vGA8SrxAvSI8A73HvF47n7x7PEA9ejyWPUw8NT0TPXo9DT2ePGu9yjzXu4I9tTsAPee97rsEvSO9GDw/PEq96DvOPQS8Yz1dvSw9cz2vOxq71boZvVM8AD1vvYQ5UT08POm8zjuEvZy8JT0Puw466LyfPfg87zqFvMw6Ujzqulm6fz2muwM9t72pPc87+LzIPci8xLzzO2S95LtoPXu9bDx0vBG8/TzPPHC97zzWPDe9k7wXPJQ78rwavQW9gj2NOFw8/rxgvX28gjwRvei8F7ywvDW9z7s0PUE90rsePV2987yZO8o8Rj2TO8e7mT3oPM47zDs3vBS90jykPBa9zTyNvOM8kTwTvT87Wjz2PLi8rDs8ugS9CT11u3e8Wr0rvNs7ITz3vA09T73bvIu8qjyIvYC9oz21vIC9Z7uAPEs9Gr0MvYe84ztXPeq86TykvAW9FD0uvII8ELxAvK67Sj3GPIe9HrxcPAW9yjxdPSc9RL3kvC48gbysPBg9BL0SvUE83jzCPMI7Cryyu448+rr0PEM8Wj3mvAY8mLyGuyw8Mjp4vcS9AjwlPXw8Rz1BPq89HT2yu9i8Zbz4Ois9S701Oym9FrwIvaI9lz1nPMO8nb0OutA7rT2IPFa96ryrvTG9e73xvKS8ETzsvGs95zuzPC28dz2dPIc9/DxPvGG8tDw+PGo8h70HPCq93TwqPJG6erznO8C8qbxfPBK8iD0EvWw68bwpO2a9lbvSPHE73DoEPGo9YD3vOzY9Q7ouPCo9I7wjPeu65TysvBA7gD2VO5S9BD07vbo8hL2QPG28AD0vvWG9JL3EPFw9q72guw68qzwRPRy9rDybPPG8BL3svGK9nj17PWi7ETwEO+s7Czz8u/E80DyPvFA9fLyHPEI9vT1ovJe7Gb0aPRQ8br0IPVe8Wr04u8c7BTw8vT+9Kb3ZPJW9+DxQPFC8UrzBPDc9Jb2Dvbq8Sz2zPIa9aT17POO8lbzEPL88Ar66O4U8Zj0Xu5W8Vz0NPSQ9tr2dvIM8PL0VvMs8n7wBvXq9sTuCPbu9ezwBvQO9Ubw4PSA9ITyruwm70bwSPUA8hDwPvZ89F73tvOa7Ij3OuZU8AT1svb28P7xIOzu8jz1oPQo9HDwePcG8ozy+vSm9vjyJPEW9ezyOPO28tTzLu8E8Er1mvdK8TD25vCO9fz1VPJu9BbxxvGc8rbwJvYO8bTzKPIo9Rb0yPJo82LzAu9I8Bz1XvGI9RrwMvSE9IjvAvA08FzoUvbQ8PLxbvfm8Zz0zvIK7zTv/PF88Mz1PvJa9IT3JPKW8Sb0QvQW8cjyjPLs737z5O4+807w0vQs9ZD0LvUk9Gj20PPe8I7suvOm8l7xVvca8KrtzvQ89QzyHPWU8ejzUPA+9Aj2aPBY9ebwmPXy7bj1mPS09Z7sLPZ+8KL28PCS9z7xQvW47QTw3vdu7/DvhvPq8X7ySvTq6O72nPUY80LwJvR496TxXPIU7LrywPfq87zz5uwW9/DsEvUI8Qr2wPPE7pz1SvbS8Xjx5vTe8Qj0LPcM9JjwzPZa8HT0zvBw9KjyIPZQ9WLwTPIg8cTutvHm8+Lweveo8k72ivOM8Sz17vYO9VD0Zvek8wjzAvdg8cr0IPFo9mLygPZe9GTz+OYc9mD1NvBs97TyHvTq96bwovXE8zTtIPK08qz2XPCw8MT2/O0m9ezwEPE07lT1uPU29s7zHPAO9CD3Ru2g8K71yPUA8nD3Ku0I8YT0ePU48TbzYu1s8sDwtPY29BD0UPPI74joFPUs9rr0uPD88Cb3CvOa8rjx1vVe8xbzlO6S8yTyBPJq7YLpOvLe8IbyzPDW9C7vkO8s8izwtvQQ9gbwbvIW8HT5PPXo8ID0oPAu8dbstvRq7pzxkvMM87jymPcE6BDyOvSI9w7tiPXW8vbwJvAi8pryJvHa6oz3APPS8QD3wvIE94Ty8vNc8Qz2RPGs9LD3YO4+9r7yhvEO8nzutvDe88bxVPHc8tTwuvV68Ab0NO5w9ijz5O6W8S71UPQG87TyIvAA9KLyWuys9/buVPHG8dL3oPMg8oTy7vNO86DlLvQs94jwUPHK8rTsTvQ49SzwyvXo8XTzbut28aTwQvOS5KbzkvJs7Pbkcvdm8P7x9vek8V70KPb489zx8PVo8gz3lPAE9RD04vSs6Trx6u9M8uDwOPLq7OT0rvSm8Jz2svI08t7yiO5A9X73Au1i9zjxvPES9Qj3zvGw8Tb0+vSw9Vb1pOxu9zL00vYu9QL0fPcs8djy1vS88vD1PPcI8NzyFPIK97zxXvOE8hz3iOg49Gz3XvBA9sr0bvJ884jzkvNy8kbyzO++8U7s/vXo8hj2cvDQ9R72DPYI9P7p/u2m9z7w2vDK8GD02vbc97bwqPP28Yb0xvWY8RjsQvX47eDySvAU9nLwVPVG9Hz3SPIa8f723PEo8Gb28PIW7Nr1cO5M9+rxsPDA9ODz5vIE8Az3QPFu9jTw7PQs9Yj2IvBk9kL2GvAS9zLx6PI+8oLzMuwu9Jj3pvGu8Vr1SvOw7V7yJPLc8G72ovGA8dT1JvKu8KT2CPHU9ET2ZPKM84rn5ug49mD0sPfo8Uz1yu8u8T7kPvXs9qDyrPe27Lb2ePKo8/rswvPS7wb2pvLc7n72VvFe8BzyrPKC8CD2ovHA8oD0ivF07+ryLvTA8mTxtPBS9xTvXO/K8Qb2fPDu8ID2sPLO7Rj0Kve68ibzrus68jrxMOye9ybyFPF88TD2KvfU84Ty1vKM86Lw4vFm9iTzcPCq997wIPBo9TD3MPUg8W7xbPVA93jiwvGA9J70uPbu8Rz0hPM883LxNPR48Jb0HPCu9sDyNuwu8ED3RPP28gr0dPUo8Ej0pvM29tT3dvLy85rw6vZc9k7ynPM+7Q7zeu0u9Rr3tPLK8Fz2FvGs7kbxWvOS7LD2FvDC90jwjvIE8Kb2mvH48PDzlvNC5G7z7PBW9Kb1vPG48oTwkO6e9HT26OzW8QL0APEg9t7uRvA68l73zOaG87LwwvVy7ZLy+vNg7HT0sOyM8vDyevcg7xrx0vNE9wryNvYw7Nz0wvIy9W701vCi8qD2DPba83zx7PSU9+ju5PD69Lj2BvCU9hjlJPQO9sjwdPQS7Mb2ROzm9OjwYvWQ9Ur13O0I9gT3dPKU8xjuVPIk5gjyku1+97rwcvIq9dTsKvVQ88jvLPB894Tv0vAu8yL1XuqC9ibwivSY8RDw1vYw8oTwEvZI8C71xvEM9HztYPXY9FLwdu5G8dDxavDu8ir3ZvPo8tbyWvVu8Kr0pvZA8ZT3Euxe9e7youQ28ZjwHvVk9Dr1OPEQ8/ryfPD+9kb0RPFS91jurvL08pDsFPRS957wLPSa9QLw9PVu8J73+uFK89DxXve68mD2XO4Q9Bb1FvIg8Uz1pPWg9qbtsvI494TwPuzI9hLysu4k9QDvKvPM8bbsGPea7Jb30vBs8yzxavRa9STyWvOc8Sj2KvHy9ZTwqO4y8mLwhvWW8GTxZOzo9wzx+vVK8Srx6vJi8EL2nvCg9AjxwvSM8ZD0uuK88x7zuvDa9UL1SvIe8CD0hvZ088Lymu4I81D2TPIi6OD3aPGk857yOPaA8yTzdvJG8uTsmvcs8ijz8PNE8J70WvG68OL3lu9q8l732vY49Ab1sPG28bbw6vRm9zbxEvUA9+Lw7vH07Yr0QvAo8orxmvHi6FLyavCu9gDyNvBy8zjxAvQY9aT38ux49bTynvAY9ojycvQ080zy9uhi9Zz3bOxC9jT3tu0m8iLuRvYY8s7zQu5g9jT3nvM67LD0nPcG8fTwkuxc6tr11utk8l7uvPDq9Dr2kPVW9Lj2hPCO9Qz1TvY68cL21PD69mLw/O6q84T2JPYO9cz2mPZA8v7xeuXy8hz14vJ08ljxyPZY9Uz16Paq65TmKvGc7OLrou928Ib1mPcC8ub0tPSg7LTz3O6W8/LyIPUM7Trxkvbw9Dr3cvOO8C71aPeS8lrw4vDe8db07PXq5CLzDPMI96DywvAA9KrzrPJY8Dz2jPHG8k7xBvGi9OTyxPMi8QbyIPJu8k7t/POc6rbrWPCi9NTxOPZ49VT0UPSy9Sj1YPCO9JztUPQ68gjwwvLi8jDuRPEw9U70xvcW73btcPbE9/jw/PfW8ND0Kvay8W72wPFU7Iz2bPeG7OTyCPao8nzwUPTI8fLvdO/G6trt1vfy897tBvIa8JzwWPS49uzytPKM8k71VPN+8KT1bvdM8zrxaPD07VT2kPBg9TLyPPJa9xLsjPSe7GT1hvO88QjxSvds8bz2mPGw9xLx8PW89pb0Kuww9jzzAPW+7E7wxvLC8WDxnPAq8sLzNu0C9KD3jPNy8/jvqPKq9I70WPDs8UD27PNE8vjyAvWM9Xrx0vIM8Zjw3vAi9Bj5XvT09j7xvvDm9Rr15PJS8/DuXPKU9vrw0PYm8CzuYPOe8LT0zvQG9aL36uxI8RjxKu667Uj0fO6w8Lb29uqM8hrzmPMW4Xr2OPBm9/zyNPZ+7Lbw4PVi8j72/u4q7tjtJvDO9ED0QPHk9wzybPKI8i727PIe8RjwNPVo9ADwgu1U8ybxJPey8VztTOwc9KD3lO0+9sbwpvTM8gLwQvSQ8SL2ZPNC8ST2pPNC8aT0CPXe9dLuCPGw9YD1fPAY81TxIOye8wrwcPRe9SjwePdw8BT12vQe8Vb0JvYQ96zuTPS09PrzdPJE8rDzmvKe7/Dz9u/K8Zj2IPZ28Ez1SvCc8T7wEvb67R73mO/O8sTvXO3W8FL1Avfs7xTwavVY757x/PcW7GL30vBc91rz0uiY9i71kvIG8WzzQvMc9QrzHPDu9ULvwPGQ9XT26vPy6q7uGvQQ9DT07PCk9mbyHO0S7eD0BPSE9Aj3bOsQ8Fr1dvbO8IT0QPsG81L2UPN28JT0GPYG8lzwsPWy7kLyyPeE8Wru0PBq86btYu/68ibxsPGk8Jj13vMe8jLxqPGg90btGPIO8gT27PJK9P737vF28tro6vZO8Jr20O3A96zw3vZe8eT1yPYw8vDsqPRk7obxnPd87j73lPNs8Hzo2PGW82jxOvYo8vbxPPWU8HD1SPZm8Br3GvKG8ZTylPG89kD3xvH69Ir1+Ozm95DoEPIO8Pj1BPQo9hb05PHW96jxevaC8TT2BuoO807w8vNq8E71JPQy8gz0DvKS8lTyQvTC9vjyUvcG7Qr0WvK+8CL27PcK8Hj25PEo9KD2KvBo9ybv5u628abouvAW96bwpPVG8tzoOO+a7WrxKvcE8xLtQvWs95bqDPa889bsePaY8STkyPbI7FD2HO6W8u7yJvT89FjwOPcG84Tyhvey8arzGOtc81ryyvNs8fT0avYq7Kz26PNK9Szk5vfI8PT0ZvQc8vD1IvaK8Bj2mPAe94bvhPEA9TzyEPZE9zDsKvSI9s71APXS73TyBPKQ8Hr0APQs8bz0zu1e9qDx5vZq9fzybPXa9qr2OvKg9STz1vIs9cLzvPBQ96Tsmvam9/rsMvUo957o6PbA8tbslO3W7az0MPaw8Cr1gPMa8ab1ku828Gb23OwI9b7yYvCQ9aT0WvbG847wevFA7BjwbPRc8hLx5u4i8gDy4PCA9dD2MvFC8iD30PFc9SDyNPHi7xrwyvXW9kT2LPFy9+Tt+PKw8qLwUPe+55DtzvRo9HD0NupG9rjsIPCc7pjpDOwk7Nr2UvMO86zyHvTK8H70lvas9MTwvPZG8hL5UPXg7FzxrvCu9d71LvS69vbwKvVG9tzy2vZe7ED1kvFy8nDvgPIo9BL2DPQ89LL3avEA9Tj1TPYY99r0ZPR27w7tru7Q8O7sBPd+827wqPYM8WDxkPaO8Pjx1Pc26mrzXOyW9mj0Tvc29sjzWuzY9Arx4vUM81Twduhs9hjsCvS89qDudPDM6lj3NuxW7Gb1EPcK8xjzwPAi9Xz0SOw094rkXPdq8ab0/PfI7bbv7uh28GL1YPfw8K7xJPQS9mbxbvMa8xjt0PBK9a70qPSS9HDwHPTO9yTxNPMs8DbxkPCW9lL38O029nz1QvL08hb2wvJS8Zjx6vMC8JTs8PQ28aDwUu1A8Bb0GO8c8Dj1NPFi8G72EPZ48DzxrPbI7ljwYu9e5Nj0TvaC65zyLPU886j0OvZo907u1vCo9Jb0au1E7szwdPSc98Ls0Pdc8Bjz1PLK8FT0Jvmm8l7yzO1E8Uj1BPfu7MD2lvHK8i7wWvTu8TzzlvIm98zvZPBi5gb3ZO4w9xzz7O/C84Tv1vCE8PL1GvRS8Y7z+PMs8nbx3OxG9XrwsvRA927uQvJ+9U71zvDo9oTyVPTS8wTwRvbG8FD3CPYy9vb2QPUC9XD0iPC06PL2VvLa8C72VOz89ZL33vHi8Sr0KPQI9nrxUPQq9xDpOvaO7sjxWPUc8+jxbvco8jDwAPXs8TLyWvPe7vLz7u7y6Wz1Ju7Y8BD0uu2C8j7zbutM7IDz/Oki9YbsoPCC9F7ytvBO85Ls2vb28UDuWO5w8Br0TPSu9ZbwUvfe8dj2gvUA9pT2nPFC9bzy9vLa8+TqCPPY8ibx/PeY8kLw6vfK7vzzzOw88u7wnvcs6mzwwvbu8PLx2PVA6Zj2SvQu9G72rPIy93bxiPQA9Iz3KvJQ9t7zPPX49oTy3vDS7XDzZORY83rwRPeM8/zvVvC+9CLy2vBM9SD3gvB07qzzgvIS8qz2KvOs6iLtIPS+8hT1dvP88lTzUPJq6WL1KvY69WD3HvD+93rzbvAY8HLx3vXK8ujtMPNk5yrxqvKS8nLtevYO9or3lu5C97rwOOma8lT09Pci8iT2KvJI8Hr0Kvcw8Jz0OvPY8kT33O6m9mzubvOq8JD2XOje9Mj1FPZc8pbvYvCK93Tsxu9U8ELomPZo8wbyHPL87+LyHPJ+6c7x2va68i7wqvTa8mj1zvaS8lj23PDC9sryhPSi9sTy9u6C8t7ktvCI9VT6+O7o8vDvfPGG8ozyJPH68tDuQvE69AzwrvPA8YD3MveI74bwDPBO9YD0SO1e9Fj0qO2W8IbvEPAQ9WjwCvb+8s72uOQo6nL2YPKq8Zb2zPJk8Cr0WPGi9Jj2tuxW9izzGPTY81rwEvM28pbnjPbS8gDyIPQQ96bzwPFS9SrqWPJe90zugO+q7G714OkI8vDyeOzW9aLzCPcS8cj1NPci8/rycPLc8Qb2IPPg6irw1PQi9irzkPIk9brzuvAo9cD2pvA87vjyeunK8kL1RvUY8ebrEO/u83DwLvMM8jz2OPVI8FT3evB09PzwgvUs9JT3ovG+8yTvDuGq8urxfvJg9uT0uPY69YzxHvIo8AL0cPSw8trz9PMG9xL0QPdo8CL1bPNm9UjyzO4K907zKu5w8ID0yPBo8s7wwvYu8jrynu328TztyvAQ9YzygvAo9hDxMN9Y8DT2yPc0837yVuhU8kbw3PV07h7wPOw69XD2rPDk7azxXPRQ9Hz0LvVm8zzxkvX+9Gb1GvJE7oL0/uwS8WDxAvYO9JD3xu887Fj3LvEo94zwlPSi74rwnu/Y8ZD2/PAy9/zymOhQ9uDuJvEm8STwLvZS8jj1xPSK9oDygPTk8VT3bPFq7rDsWPJQ8iTwFPQa9ezztvCE9bLs1u/s8bz0fvQe4Q7sfvea8Db0dPca8MD2bPYM8Ir0OPaY6hjwkPSQ6fDsBu1C9zzxNPd65LLuQvVk9ibtMvDS81jxnPWK9SjzTvKi8Sjx3vSg9KT0NPV68Jr0Rvau8DL0IvIk83DxePYK8iz2RO9u8tb3/u6y8LT23PGY9x7wru5E9ojzlPJ28hLyDOx28JLzwvCs9zTzzPNO6gT1xO4m8nr0uu3I8FT1vPFu8Dry+vBG9tbxYvWu8Az1tPKK8HT3CPI49pzxUPQO8sTxmvXK9STxAPLQ9bTy2PMA8tj2GvKu8eLwWvSy9Kjz5uzQ8F733PLK8q7zuO0O9rDx5PYC9BL0kvRu827z8u5S9izz4O5A86Dv3PBE8Gz0dvKc7ozxWvHG9EL1DvZC9KTwNPci83jqevYE8mDzxPNq7Dr3bunA9Tr01vZc7UD2AOhU9dTp0PIE66LxUPFU8GLv4vA89YjxjvaQ8w7y+PAa8eDw1O049Ajw9vMS9R73ovB481zuTvPQ5az1lPcA8urzVu3c95zrSvA47ejzIvIO9WTuSu7Y8JD1sOza9FL2EPMO8nz3yuys8Yz2EOQG9f7zHPEO9LD3QPLA8Xr1MPU88+bxZPKK7dTwNPfS7kTyYu1Y8Sb3zPEA8NL0UPdc8nzqBvJ28gzwTPaY8y7z8vBM8+ryeOiI9vDwzPfc7zbsVvNu8QjxlPfw6ZLwwvJ28FbsCvWW9LT2rveE8HT2FPUg8v7xKPO29VL1dve08Qry3u1E8IbwOPaw86jyuO5g8nz3mvOO8jzy7vAO9BT1CPQq8H72hvNO7ijwvuz+8pL2QvVg9d7sCvtK8U70XPa07IL1rvea8PLugPRS9hDxqvP+8G7ywvHY7OztIvdc627sJvKg80DxFPEQ9dj0Zuwo9uzx2PG87kD1aPac7oj1xvHE7dDyfPMW8Ejucu5e7Ub3UOwc8qDwMPXu6uz3VvF+8j7zEPIk6tLyUPWY7LbypvPW8NbxGuuy6qD1TvWq707wWvfO8Q71yPTQ8Ej1sPU+7wzqdu9c4X7xKvQ88SrzTuxK9qjtZvQW9vjuJvMQ8Ir1gvRo9ojxfPSg9yjyFPIA8A71EO7a8ebzUvIi8FDwUPRe9yr1xPeK6JrsOveC8hzzmO3+8IDySPRa9Jz1MPKE8Dz2SvfQ8xjyWvMC85LyavB28lDwVvQC7TjuGvAK96zyxPIG8VDzkuzq6ZzqsPHY7Az2EvTU9B7zDvNC8mzzsPJE8Kj2JvCK8lDz/PLc8l7xJvFw9Bz1uPIA8sDzXvHA9Zz1Cvam90zyjvAM9qDmGvHo7nDsQPd28bLzZvKg9Cr0bvTE9qLdAPc86Hr2oPKu8drwNvVm8Hr32vKu8cb0ivZU6870HPJk9hbxsPaE8CD2AvF09sLyOPDG9tzz5vK+9nbpBPRc9jT1CPNW7i7xaPai8I72cvB07j7zOO0m9p7xvPYC7pLu3PEy93rwzPQ09EDzEPPM8XbwLO4w9JD1Hu2O9qDm1OxK9yTx8u7y8qryZvNA8FT0Ku5m9TT0LvOO8sbyMPTa9VTy2vKi7Cz2RPJK8JrsrvQ69iL3MPL68F70mPai9NT39OyI8xbyBPVo8yLuBvdm7gTxSO9E8JrwhPBE9RL2fvNe7Hzw+PJy9wTqzPGq7ET0mvYY8NL2TvHU9FD2WPO+88Dw/vWU9VrwavRo9h70RPbS7bz1HvQe727w7PQe7J70ZO0G9Kb4hPK681TswPDO8ozyBPB49Qj33vP47Pr2WvcW8I70UPfw8Vz0qPRW7vrwFOzu9YLmWPEG98bwUPS69Or1cvPc8lTz4OrG7Qb2IPIk9mjwVPLk8zryGvII8U7zdPIW8jLwxvHM8BbyOvPA6Dr3OOS+81rwNvF06tzzRvM08Vj3TPHU7jjsPPXC75rwlPQG88jzaPJS7Ij3ivKw9qjxTPeY8DT0OPTk9Lj1BvEq9RT1TPIy8lroSPdo7ez1AvYy8xzsbPYc9ND25vJQ9qz3mvBA99LtivMG8WLyePLc8xDyrvPC8AjzYPI49V7yuPJo9nT0mvUq8P70pvSi9hbwEvNS7Mz0pPQu7mbxWPIm8u7xrvXK7LzxkPDG9IL0aPS69g72jPD09ELwIPWw9ETytvEA8NTyYPM08xjwPPYe8hjyjO8G8bryGOhw8MTyIvPA7TD06vZy8Br3zPM48Dz1SvbM87b0wuwC9hrxvvTY9VroZPUq8WLyBuxw9sLyWPYi8Eb1SvRc9uzxWu628BL1GvHy9pDuMvAy9A7xPvIU8dbxVuwM9X7wevTC8jL2yvK29ObyYPbM82ryrPLQ6yrwVu068yD2evAu9Bz0qvYm7UjwNPTQ97TuhvFG9pLsHvl+6Yj00PQG817wsvf48jjwiPai7H72JvDM73DyovDI8o7yUvG69qLy0Pbg8VrzOPAs9qzw5vuM78jwRPTA9JLtZPAm8sL0PPRg9HjzoO2g9C7wJvea8VL3IvMa8Oz0Uvby89L28POa8Qz1KvZq8zbuPPDk97jk2vM88/zwnPRu9lLxCPTe9szyuPLQ6mz1vvGG9mb26PA==", + "mixes_f32": "5ertwO6atMABjNfAmaqHQE2ySsG204rBMOp7wJFAr8AAOxE/S0CVuhU2ssD8mqBA2IAxv1nkMT83yZu6uaNIuUHJwbqG2oG/SCKCPzAy3LloCZa7EL8+uv22bLroEMs7", + "scale_f32": "UyCBPWQ7RDxMEXw9", + "base_f32": "ChY2QHj3VcDr8U7AWOUNwHV/hcDK9HzApJWcwXxptb9nNFU+IZzLwZlLo8AJrcU9GSQBP0av4b4WAsrBt7OswV3ty8HInUA+BOflvQtYsMFBCZzB416twTi2s8G+BFdB", + "branch_bf16_bits": [ + 16053, + 16067 + ], + "residual_bf16_bits": [ + 48359, + 48377, + 48359, + 48377, + 48359, + 48377, + 48359, + 48377 + ], + "expected_source_tree_target_bf16_bits": [ + 14092, + 46834 + ], + "provenance": "Token 671, layer-0 attention HC, official model revision 2cba9e42aa026125f3ed06c6d98c1db82f7ca027. Four copies of the actual BF16 embedding; mixes are the independently executed native FP32 projection boundary. Target columns 3093 and 4687 of copy 3 distinguish native old RMS from pinned Torch CUDA single-row mean. Base64 payloads are little-endian BF16 or FP32." +} diff --git a/zig/src/families/deepseek_v41/fixtures/hc_fp64_rows.json b/zig/src/families/deepseek_v41/fixtures/hc_fp64_rows.json new file mode 100644 index 0000000000..5894e27712 --- /dev/null +++ b/zig/src/families/deepseek_v41/fixtures/hc_fp64_rows.json @@ -0,0 +1 @@ +{"eight_rows_bf16": "LD40Puy9RL50vqe+8705vaM9QL3Fvn299zzSPZg+aD5MvQY9gzyXPRa+yD2UPbW95j0XPkW+KbtsvnK9lz2HvhG+AT3IPck9I7wLvPo9sz00vjK+BL5Ivi2+YL5Mvp8+Lj2Mvb09c729PTG9+7udvNi9RL6CPaA+2j0WPr8+jT0Jvvw9NT6RPp09+z3uvR89Cr7cPRU+lL7cvTu8xj5WPrE9izw/vQE+ej0jvgG+PL0Bvoq7Jj4HPpu9KT5nPvS9GD6hvlS+Rr2uvW8+ozlivu49VD6+PI48gj1MPqO7m71cvQu7FT3vO9679j0SviO8KT3HPSe+Br5APjk+jb0PvZ69G73vPJW8Fj2evu29I74ivmM+Br4Evpu+Ub2MPLW8Ej6xPQg+9z1ZvqQ9Kb7iPjO+Yr2KvFC+JL6PPpy9DL3kPWw8bb6NPvU8Ib4tPYW8FT5IPjI+lj6VvjI+zj2IPGG9zL7+vfK9RD4yPq89F76lPmy+5D2evFi94b07vT2+AD5FPTc+8z0APhy8hb04vpi+hru3vcQ8PD66vAs+Tj0OPVC+mD1JvvW9Rb6FPAA9lr76vC6+drxpPRQ+Rz74Pd692TzEvWq+Ar7rvAA+Bj3OPWw+9j18PgE+ib0VPo8+cL0GPh+9UD6Hvtm8h71LvjQ+u7tYPjI+xz63Peu8Yj6rPYq9UL2+vSC+FT6mvpS+8bx9PH29Yb4MPHu+MD6cvZq8pz3nPaE+HjvGvd89or1Qvli+AL5gPpS+jz54PdE7oT5Ovnu6Lr0Wvoe+jTxUvhC+/T2VvR6+zr0SvUY+0z3YvoU+gj1FPmQ8/731vSC+/Tx/vs68Cz4SvoG+0r2jvn6+nr7PvZQ+3r2VPSc9TT2jPuW9s73RvBa+eLuoPgA+lT0evhm92zt1PpO9GT2JvuC9ZT4fvgm+LL5GPIW9/b2hPXc9C77pPPw9uL0hPbo9RD7oPWI9azzPvXg+9T1XuwG+azwWPgk9Kr4EvT2+Vr34vSQ+tj6vvSI+o70CPjK9xTzQvTU+BD25PlY9Kr6rvUk9i77gOwa+PD0JPm++QLvJvTM9JL7ivWc9TL6yO6c8pz1GvmM+YL0Nvh28Lb0qPoi9Qb3Yu/k9tL3YPYi9dD0Tvjc+8DywPNk9Kj4fPaI9wD4mPj8++L3YvSg+UD4NPXQ+grybvRI947z3vQ6+gz6ZPYq9ij2nPA88Cz3CPYE9ijyNvZm8t71vvD89Oz0OvtU9bj0bvYK+jj1SvTq9uT06Ps49XD2HvYa9ub1GPcQ9XT7rPe68n7yBPTu+Dz45vXk+E74Kvua9ZbyVvVo9Ar5pPIi9xT2IPrI90r2rvoo+hb1pvSg9Fb3+PVc+4j0avhg+nz2KPXM+Zz41vf68LT4YPUA+OL50vWY+Dj54vm+9cL0XPnO+Eb0Hvhi+Cz4RvlA8VD6bvge+6zwFPsu96rwVvB6+uz1bPbi9rj5mPn6+Fb0hPb09Vz5HPgc+MD3HPGi+Fz4gPZ89Db+ZPZy+qLs0vte9dT6VPc0+Er7cujG+dj4Pv50+ib12vRk8dL7UPRM+mD7OPe08nb0oPgo+Xj6OPi49lL0/vn6+1zwZvm29nb6zvT++Jr5NPpm90TtaPtC9hr3JOyU+hT3FPRA9T741Pr0+CL6HPne9SL5AvLI9iTy1vaE9Tz6evNM7Ab5mPuI9Ez5UPUm8rz3AvZU7bT5nPUg9pTxyvRc+hD0XvMU9SDy0Pa+8KT4TPTm+7j0kvqk8kL3cvRo+8L1jOrO+AT6XPi4+gLxyvhi9N7wgOm8+wz0+vva8OD6uvVA+Fb7YvUw9D74HPli9d70tPr89fbv1PT0+2r3KPS09Vz61PW08Wb4KvpE9WLy8u6q7iby1vRw9fL59vtE+9j3Fvcs8a76PPdS95b1fPlG99T08viI+jj4ev5a9er2ZOyO9Vj54PZc+VL0Jvok9TT4nvpE8Pr7ivJQ9A757Pn8+J75Lu9a7+T2SvTy+KDyMPOy9Sj7Yvaq+i774PcQ76bxcvlY9Rb6XPSK+FT47PS0+5D2zPO69O773vka9bb6Mvo69zj0Xvlm+Ar6Wvi++iD7dPRa+mr6EPjK8bz49vi0+NT6ivKQ8rrwzvf09nz05vkg80j3TvZs8gD7jvRm+sz07vj83hz36vSk+kr4IvLa9Br63PYc+ib6mPvk8cj6xPb898z2APt69wj0CvfA9Ej3/vVQ+2zlePTs9wbujvdq9M75qvkA9rr1nvaM+DT4HvXy+kr1NPea8570YPQI93jyXvdk+HT0avQk+i76tPDK+AT1QPgU+hr1jvho+jb7IvRO+IL2HPhI+Rr39vWW+CT0qPmG9V72qveI8C74Muuk82L2KvYc+Zr6mvfM9VT2rPoO+HT7tPDq+HTykvCk94zxtvgE+Xb4Dvnu9G7thvVY+dzyFvYE+0L0xPjS90r3SO+w+3b2EPem9jz0EvCE9ZrycPUc+ujyjOs89GT6ZvTE+lL02vUa+G7wBPno+pb3XPWO+RD2UPVm9gz3SPWk5uT4oPEm9wL48PAK9pT0BvuE9Nz6FPaS9XT7LOyI+hr5qPbS9Zj32PnK+nD36uwy+9LyZvTk9xz07vA09Ar7Wvqk9zL3qvd+9Cj7qvYu8nj3EPr+9zTtQvSY+0b6PPXc+z71cvvO8Cr1OPr+8Ir6FvdQ+nz4zvlK+IL2NPd495z3AveC+sL0ivo+9jL4jPpC+w72IPVi9YL2/POU8/L3fvFE9ir0fPV0+sb2Zvia9TT5TvVk94r3zPSK+QzwEPmK+u76BPiY+Sz5JvYG9m7xMvhK92D0RP6m+ST4hPnA+eD4qPbm+Rr03vnM9ij7wPJy9bzxYvpa9vbw+vo0+uL0nvsg8ET4CvRc9Xz60vB2+IL4Gvu49jj60vaS90b1UPuS7WbvovTO+NL6XPSK8z726Phu+YT1MvWi+Cz4Kvby8YT48vfC8/jwrvgI+azxXPcE9m76yvWQ94LwrPja9uz4WPVu+p7ltvnu9CD5Xvju+3jz0Paw9er2CvsG9kj2svRA9pT0nvvi9K77bvuI9o7uEPZO8sLxTPqA8Ub6sPNk+5LvuPaw9vLvMvcy+K73HvZS+Ob6JPoI+nb09PYk9AD0IvY08hD5APgM8xr1OvKM9Lz4RPQ4+NrvdvaE9K74DPYK7Bj55vYE96L0MPqc+B766PZQ+LT6NvUs7Oj5EOwC+5zx6via90j1Zvty9jTzKPow857uPPaO8uD3kPQS+QT03PhQ+k74xPri8Ir3fvC4+0DzIvZ29sD17PMK7E7/EvdK94D2UPkQ+FD5MPRI6gj2UPTa6Kz7/vea9GD7kPSG+Zb0TPfi9db4NPn4+Vb1VPoI+IL4PvOM9oL1EPn4+Mr0Wveu9/77HPUk+sT2jPAu+hr2pvLm9JD0uvgu9rD2/vU6+Bb4ovvo93TwQPdy8rT3GvB6+RD5Hvv49DT5QPqo+wL1qPq++kbw8PUa9oL3RvRk+obuIPUy+mbwFPWm+/T3GPeC9IT2NvsK9XT4avu497rwDOwS+C7vsPSA9Ir4XPlA+TD6iPfU9SL6CvQw+4r0HPXc8mb6/Pn09zT3yu0Y+fr4yvuW9jj0+PgE+HjycvsO8o709vsU+qLsRPvY987yCPjI+zbwevLu9bz5jvqK9rT4rPgy9ij2pvGy+lD50vkE+zL1NvZE9Bj7pPcE9nD7zvR++arwzvs+93b40Plc+fz1UvnA+KL6yvii+Aj0hvWK9zz29vb09TjxXvby8dr0RPXA9PLzCPSU+3T1SvMO8jT1bvdi9Bb2hvTU8lD0FPhK+ebwfPmU+iT1bPbA9J74KPbM9oj71PLa9Kz6ivQQ9FT6Hvgc+/r1KvbE+tL1UPZW9Nj5Mvgc7uD1Dvl49jb56vvu9Qr4TPS8+hD4tPUQ+Sz7Uvau9VL4EPWq+Hr4fvP07ur6Lvqq9hj6OPQk8D71SPh6+erymPVa+tT0KvRc+Oj7APbS9wz0fvR0+mLzvO/o9Yj7EO6u8HbzOvBy+z77MPi8+UDzBPE09GT6WPR2+cb7LPsC9A71hvbU9AL3PPQi9vTxfPlg9e765vaE+FD4wvmm+fD26Pp6+hz1Nvmu+7r2oPkW+5D1OPWc+9z6IvGO+oj3kvL2+qz1Oum2+Kz6hvjY+jz5cvgW+zD2zvTI+Nr5jvn0+6T0wPQi+kz7fves9RL6uvfW93TxvPn27aj0Zvjc9lzxlvh69MT66vdA9mb2nPIe+2r2yPQe9/7zNPJi+qr20vrE9sj3bvcE9GT4kvpM9BD72PQG+DD40vCa+2T25vR++Tb2PPJc9E72uPhq+pb2CvZc9hr5Hvac9Qj6rvdy9xb0nPQ87R75ePpi+Db4dPha98j2OPQQ9GT4Mvhq+P77mPQw+oL5BvJm8pj1SPhA+Kr5cvt28yb08vjS9QT6RPqm9GL7TPsA9Pb6/PBq+AL5DveC7xj3pPMg9hr0YPqw9gj1lvre9JL7svU28cL3wvc+9tD0pPqI+tj25vZS97T1vPjs+w72FPp89H7yvvEe+tz0/vve7ZT62vWU9FT4MvfA+Yj4Lvni9CL6DvQw+iL40voU++70TPgU+jD4TPqa95j39PDY9FT4aPi8+1D2tvhS+1z51PUe+gb6GO5Y7Mz5FPtq7hL32vVG+LL6OPn++Cz4hvDI+Gb08Pfu9Ij4NvWi9nj5SvoQ+tb1Xvp4+tL6lO/W9EL4wvpA8TTvXPR89pL7NPDA+ab4/vuk9bL6nvVu+M70bvbo8cb5tvI8+PTzBve49jD6HPhO+vL4ZPi+8Qr1vPrI9aD6mvTK8wr2+PUq+fT3+vCW+kD2YNx4+173rveK9iD43PUK+LD6+PWO9G77+vr29RL5vPpg9k73KPdQ8ojwYPni9U74yPGQ9ij6oPcM8HD4LvAu+7b2KPRy+ZT4LvRm+JL5Dvg46oL3wvXM9gL2aPiC+Wz7gvVO9Cr5Cvj6+/z0kPne+lT6rve895j6RPaY9XL6qvrc9qr5gPCQ+770DPgU9Cz7fPDo7Uz5+vWM+Qz0LPW69h74vPuI6Nj52vBG+wD2HPGC+sb3YPWU9br0DPdM8Fb6KPW8+/zqtPWG9ND77veq9p71avh29qj2eO0o9jT6PvbK8nD1KPni9UL5cPde9Ob3ePbi98j0kvkG9Kj5CPbK+5T0pvV2+Jr3PPve7lz0jvvG8pD4YvjE9LrxLPri90D1QPZk9h74fPgm+GD6uPNG7mz21PUI9h70aPYy+Uj4JPo+9g76lvds80T1FOwM9Gz4HPm++JT4Bv+Y81bs4PQQ94r2NPq29jT4rPsW8t72Ivm28G76pPcS8Gb7CvUg+Xz6JvrG+ET13uwS+oL1jvIe+s71mvao9cT28PZM+wj2fPRO8BT/lvdw8Zb60vRI+kT4dviI+Kr5QPik+qT3iPdo9Oj1XvYC+/70QPgK+iz0UvqI+1D0Pvl++l731vbA+IT57vh4+bD6kvnI7d7wKPnU+rD4Lvj4+EryWPlw+N74pPny9NLs5vs+9CL52PG69ozwRvuM7pbx1vnO+t71tPt68Ez1+Pg29vj1IPjI+TT2kuxQ9xr0evoS9qD1Hvge+srwavKI9hDqlPTO+gb15PZw+SL2LvIW+ND0vPSu91bo6vhG+9LxNvWk8az6Jvq89472BPry9fb6Kvcq9xL42PmA+Abz+PN49P727PVS9B70ePg6+n72Ivb89oL59PjW+4DyMveo8h70Pvb8+hj6Hvkk9Z77mvF++qj3PvdG9Jz5zvUA+EL7IPTo+zz2JvuM9tL68vUy9R74FPgy9KjuIPWa+pTz/PCw+1b23vRg+Mj2cPb89hrybu127Uz2MvrI6ar1ovqw8Kz6PvCU9+j3JvTQ+Hz50PbW8mbyEvbI9Ab6wPai+iLzlvRO9O74rvkg+SD70vcm8rzyTPYw+XT5SPiu9sz2uPeW9OD7Qvss+ST3/PFC9rj24OsY9zL1Hvso9tb18vjC+27x3PFC+8722Phw9tj5NPrq9cb4bvvG9Vz08PA2+DL4TvZS84D1qPi8+Qj6RvkE8Kz5zvpg9K728vT2+LT2/PdW9vD1EPkq9Bz48PqM9nD0nvt29tL2/vZi9jb1AO3++Kj65PGs+Br6kvq89rL2Pvlc+ijwEvhI+Vj4MPKo9/L0dvaE7Cb7PvX8+ab6Cucu9p72CPSo+Wj7kvVy+Fj6DPvY9Ob3MvaI+PT6vvoI+Erxpvpc8Lj3fvYe99L07Phe+kL7GPZw9e7vkvQ8+6r1SPcI9MT6APR0+trviPYM+P759PTG+hL7+u+S8mb2NvcW91j1hvUC9AD08vpQ++TxuPu49xL1mvSK+4zzRPRU+k7xyPie+lz6avZK+krzKvYO+uTyqvQu+X77/vIo8zjygvIO+Hz5jPj2+Fj56vU693LxkPNA9iz2KPea9pb5APXS+/r2IvoK+Or4WPmy8MD2EvZQ9Ij6lvZC+sDzwPRK+yrxIvl+7oz6Hve4+Mz24PQq+NT7TPWG++j0RvrK9zTvjPSU9qD6svZS9ez7qvbu9UL26vq+9LT6LPqO89L08vgg9Sb02veE9RL6FPt89yb2pPdk9lLtTPT+9MT7/vcq8b71/vk898j43vTy9U74KPvM8L74/Pgk+zLzDvUM++TzBO9Q9N76HvhE+TD6HPqK9fz5bPaW+DT4UvWS7qb0fvIM+kD4zvii9Fr4lPhU+cj0ivsm93D0Zvuu9mD1xvUE+2bwNvjY+AT2tvWa+Kj7WvLk+oD4YvRa+db62PlS+Bb1zvRo9cbxTPWy+FL0XPiK+yD3DvRG+Wz4/PiE9tbsPvoS+brxCPtQ8vrzbOo09/rz2vJw9GD52vVu9Tb35PC+8q7zgvXy+Zz40vY89FL7WvGQ9Ij7mPRY8tD1ovRM8oTyQPSw+Br1qvZa+ZT4nPpQ+IL7MvCs+qDwGPtk7ED4JPhe+mz0Hvp88tj5TO4q8F77FPUw+vL1fvty+or2DPlU+Mj6tvS09ZT4uvgi9VDwuPW49cD2mO3a+dD2MPa+9pL0XvkM90rybvni+qD2gPfE9VL6HPcm85j3+PQ++5b1kPi27yb0tPkc+ALytvae9kj47vjM+rT6nPdi9770/uaS9LbxlPeo9EbyJvuS9Ab2CvYe9DD4sPhs+Lr5pvoU9r70qvgE+k73gvRC8ibwaPqW9/z2jvhy+A70HPDw+Iz02PTO9ZT6/PKg97r3FvjY+x734PbO91z2IvlK9RD7GvkY+Qz6Ivm8+Ej4QO1s+gD4JPtK8i7tcPrq98TxBvYo8Dz6nvQi+Ub2kvXu9o72dvYO+t70ZvVy+7DxAvmM9pz4vPpy+Wr3/vTo+hj2ovMQ8l7w+PJs7Lz3cPHw+Ib4Tvvm9JD2bvNK8g706PhI+LL6Uvi2+rz05vgw+E71fPZQ9Or6KvXs9zz1oPky9uTy/vec9yD0evAO9ar26vU+9mr7dvM69s7yjvQC+5b30PY4+lz1yvF0+0L0IvnU+Qz0avnw+1ryrPhE+pD6jPqY92L22vaG8tLwavSO9Hb2sveA9bDxrvnm7XL5WPGy79r2pPAC+db4DPs49OLwGPu29ZD6Ovbg8fL2vvvA9BD7lvvw6TD2ZPZq9fj0ZPVM+CD5dvi+9cb4WPvW8y7wOvTo+K769PoM8t72BvaS+ST5DvQK+Qb7HPXC+DL5JviW+Bb4xvhC+qL03vYu99r3sPRe+2ryQvY+96r5FPsm9172ivpk9Dj0svTk+cj0ZPtE7mz3KPNi8qz1iPQi+0707vV++hj5GvRw+5D2LPqI9873rPlq+CL6Zvsc9Ur1SPl++hj0EvT2+ab7TPXS92r6QPo0+fj2BORg+Er5Tvms9Lr6JPSa+5b3BvRK+lL0mPkS9gb3ivTi+gz0gvMm8jb31PZ48Sz5Fvqu8nb1LvVO+772yvd88Ab5zvrC9Fj5VPSy+1TxFPvS8BL4Yvim9pD2VPci9vL6tvYq9VT2rPUE+Bb4RPiQ+C72SPZq+ML7FPY29FT6KvZk8I744Pp++XT7+vTY+O759PfE9Gb4Cvs+8MD0yu5c9Tjy9PE++s74SPgk7Bj6wvdA8M74TvRC+KrzdvV89tL4oviA977yFu0I95b05vkm96L0avng847ytvSM+FD7tu5m9IL7Fvck7Tj4Cvgy+qr05vs8+hz3EPSG88D04PbQ8nL0jPhc+eL1tO469rDwhPTE8hTyNvdY7Kr6NPQS+1LxLPaa8Yb4HPe68Zb2MPYo+MT0qPn4+kT2IPL08lj0jvS6+mT3rPUw9Cz6+vCi9gr6rPpy9W7x3vmS8ej5APry92rxWvVQ+Lj4yPq+92Tw9Pge/WLufvos+Az79PC0+Cj5hPv09fb5BO5Y8DT4hPqe8Tzv4ve49n76LPWW+vboDPnS+Fr4bPru+2b0ovbO9Oz7DPfs9hT4vOzu9J73avUk9E70uvh++iL5jPEg+Jz46Ppo9tL2uPKI+AT2qvNe7kbwsPRM9Bb6WuzW+Jb1EvZ29Vb5mviu+L72pPRI9a77pvdo9Az9XvWW+hL1nPso8nToUvrs9A74QviO+Lj0QvoK9Nr6QPqc9M7tWPeA9Nj2sPWk9b7yPPYC+lzzbPf+9XT08vfu93T0vPsO+jz7Bvbe9/j3dPcw+eL0MvaS9I77KPiG7IrwOviQ+CT5HvS++lb6OveG9Lz7PvVs+7j0nO1W9Qj0aPuu+Tr7kvZa9nz2RPna+ib3ku0C+H759vBe+PL5yPr89kr21PTs+g72bvRQ9Mj0vPi++hL6FPYq93zuYvhu9BT07vg49CL0+vuE8qb6WvTO+47wtvNe8KrzKPAG+Ur1Svia+Sj5LvjS96jzZPW67p70rvgK//LwdviQ+dT5PvsK8Ir7yvHg+Yr63O+k9lL6CPcQ8t7w9vo2+/r2jvpa8Nb5zPj2+2z2SvYc9s70cPKY9Cr7lvds8oj6lvHM+vD04PhA+Wz6hPti9Wr7cus+8Nz0nvQg9g70qPSi+lD2Tvia+Kj74Pfs9Cr+3vTu+m70ZPQK+1L0Tu4E9IjwbvRm+ED4KPki+uj6bPm69s72zPKy9C74sPdk9Dr4guwG+ZT6BPIA9DL4wPtK8AD74PS29gzzPvZw90j2Xu1m9CD5rPlC+Xr67O0k+RTwPvna+aT0gPLa+jj1nvLy9q73lPgY74D38vV8++LtgvnS9870NO628V735vIa+ub59vVg+4D3evKS9Fb6PPhW8yD3kPV49SzxFPRg+db4SvoO9Xr6rvFK+Hb5dvYo9Lb3qvZ49yL3ZPS8+/L0EvQC+r7y0vjS9sb3nPYq9FryyPQY+677XPZW+ID4Ovti+Bb0VvRy98L1IvQE+jL02PRA+370lvUS+zL0bvcm9T74Hvvg9bbvmvew8Wzy2PXm9gb0sPMU9y70sPhA+Gj7JPTE+hD1gPpc+Qb4LvkY7RT6SPhw+Nj4JvRS9vj3FPT29Qz60Pqc8SDyovlS+vD1fPg0+Yz0yPLG+ULxJvts9Yr5UPcY8I75uvda9gr7gvLo9KL7Xu6u7Kj5ovjg+mb3GPgc+EL6fvhk+kD5APmQ+Y76NPZO9Mr0BvnI9Or1DPCs+670vvrs9tb0qPG4+nr6BPhS+qT0uvT8+HL06Pg+9VD6kub69HL4BPge+Jj44vsk86b0uPg0+1zyyvkI9Qz1kvho+UD5cvco9CD4wvhA9rr5Gvuq9BzyLvqO8mj62vUy+6bzvPRw+5zwBvo+9zz0bPDi+sD5Jvii+iL5SvFM+vD02PEo+4r2XPSa+V770PUc+Eb5tve+65T0kvkk+Yj6rvls7jz1dPYK7lb1mPu88aL0kvvo9/ju3PqK+Hz2gPKm97b2Tvju+h77evBw+Yj3/vG08FD6Yvhy+gr3wPKE9sLwfPDI9jT2FvTA+ML3IvQG8qD18PFE9cD34vXu+yDyBvbG7Or3HPZW+DL6Eu7C95TxuvUW9sb0MPMI8DL0mvOQ8c74Lvss91D19vRi/8LqDvg+9Xj5lvWM+HT5KvTk+aD6Cvb+9Rz6TPjg9az46PgQ9rLxmPZG+Ob5vPic+Wb6pvXI+LL0JPtg9rzzGPSG+cT5mvsy+xj1hPlY+nb06Puw9dL1qPpu9Vz0WPdm9Lb2pOxY+571vvHA+oz2OPZg+QL44vuc+Db0kvMI9Xz3DvdO9S71qvlY9CD4TvuW8Q70Ivui93LxTPqg+Kj08vkC+ej2XvpC+Ij36PDW+kb60vRG+or0tPkq9fj1QPQO+Vz1Ovpg9qb4Xvaq8yLw0vM09rT5BvFM+rz0hvpC9Eb24vOi9qT3xPa49Ij6bvaM9Cb5OPPM98j0nPRq9cT73PFo90b0julW+GzwYPVi9ET6avX0+JL7UPUc+iD2Evti+ST3uPXo+0T0gvY4+1b3DvhQ+7T11vqS9tj1Zvku9gT0JvRC+Hz1fvZQ8Ab4CPo29hT7YPY0+dTyjPei97z0+PoS+gT2Bva69lbusPYY9mz5uvmw+9r2cPsu8tb0XPna9sj5yvQ89RT6fPR8+572Uvcg9KL6nvce9d74BPoK+Cj4dPp09mr6RPI49zz1YPQq+Fb7GPfe9ub0mPqA8g7zkPd49Fz5LvpG81j3yPIa+Zr58PeC9sL3VPWY+474vPR8+sjwSPuk9Rr6YPh++eL6ZPew9iD3pPZ89VL5Ivu89Bb4hPiq+qT21PZa9Qj4BvsE95z2XPlI7ib1Svru9/b3YPik8YL4Fvo09cb15PrQ9Xz1PvlM+Jj7zPTQ+E76SO0m+i706PtC9L74BPsm90T0vPXy+DT09vjU+1bwcvla9hb1TvaI9arzJu8o98jtvvhY9lz3qPbA9lz0Mvcc+yT2DPfS9Xj5Avsk9mr04vrg9s76PPcu9Yr3LvYg+Tj2Mvgu9sr1QvQI+Sr4Evlg+Qb6vPUi9WTyiPog9p73lvIY+Bj5muhA+EL6EPDa+Bb6Rvnk+gj6Vvb49171IPRM+r73SvDY9hL18vIM9+z2Dvha+VL5cvgw+zb2TPNq9b77PPYk+M76KvZK7mT0pviY+IT0MPgW+/L0pPqM9JT4kvds8AD1kPvc9CT4RPic+vj18vbW+N77evUE+u70SPo49/D0cPaq+MD50PpQ+hD36PZW+Hr5Evs69Jz5aPtc9Iz6UvlM+Nz0wvWu9ljo2vfg9zj2RPUO9rLwxPYY8yz6tvoe7kj2SPv+7ML6GvuK8672+vHm8oL07vui9J74NPu68gj0oPpg9zL1APqo+Jj4Hvk2+Vb5FPCk+Nj7UPda9k726vt08rz23vea9rj6BvCC+NDyMvOq+hT34vbY8pb2wvbs8+7sJvYq9Pj6UvTc4Lz0YPgC+jjzRvam+dj5Ivms9X77cvoo+8LxmPpE+BD6MPEE+hj28vUa8FD5xvBE+oz5SvsI8ID3JPQ4+Nb4VPmA+YTvyvDu+rz4dvoa8HT4kPiu+gD6gu3O+xj1SPsQ8WD0tPoc99z3tOYs+h70WvRO+JD68OFg+rzxjPgk9dT4avkw+7z3KPf89iL1SPeW9qDzyPXa7/T2RPWa9Wz2DvKu+7DzivfK8ar7RPfc9qr0CPSq9b75LvgU+uD3Fu7u9VT4rvsa+/rt1vo0+Ob5iPik+Hr6APYA+Hr0uPoE+vb5EPvq6PL6JvoU+cj5HPjG+nb4+vh8+JDzePH2+Xr2IPXk+cD7lPbo9vb2Bvkg+271ePoa9jb5mPiU8hb3BPZO+Tr5pvs+8iDtsvge+PT7vvI89ib4qvOG+Mj79vSg9ZT4GPoC+L71Kvos90jyWvoi9Mb75uxk+DT63Phs9ID6IPai+oLySPfc6HT0RPVS+K77vvVc+xb27vSW+FD1mvoM77z0Jvn+9fL2HPRE9kz46vGu+nr00vjM7rD2PvWy+qL1BvYs9Y7nTPjG9A75+PiW9Bb4VPko+K7zNuys+8D3JvqA+Vz1mPTe+bT4vPga+tzsqPYy9Vz4wPrs7zL0mvh28Vb5BvQg82r4CPoO+Ez5ZPlc+Tr4zPAC+Qz4jPqy8KD51O4U+1DopvrE9KrygPS69F70Tvmq+UL7DvY29G73JPao+v73JvUQ+jL7OvdO8Fb0LvdM9Aj7gPEE+672kO/K9sj1GPmy+Nb6nPSS9xjqpPRE+h70wvDS+2z2BPWG9iL6Qvnm8Rb7pPLu9k74TPQ++jL2HvoO9ijy2vYK+uz0PvmA8ej1SPrk9cL3NPT09Lr4gPt89br7nPaw+ZD5avTW+SL3hPIc+j75ZPZs+DD7iPlE9lD5IPRa7hT4GPrI9zj0DPqo91j2hvbg9sz06vQc+t72JPt69LjxdvQe+Cj4VPks9LL1PvIC9+7wPvrc9gz0JvXc+PL5jPNu8dD4yPni9Hj2TvWy+rT2xPdG8Pr2WPTY+Mr4Hvgw+gT2yvlU9pzzcvW6+qr4zPae9Dr5EPpa9Sr4sPgW9g70Qvkq8Jb0CPW49ob1jvsW9C7yEPFQ9BD4rvSC+ib3yvEG9kr2NPh4+Q77wPKa9Wr1pPr0+0Do2u0g+TTtrvKS+LT6JPRY+tTzsvX89CT4bviu9Pr6zvCM+v76wvRW6t71CvYw9Yj3NvQg+MD6mvl2+gb6XvLA9M729vY6+8z2gvOq95z0pPeQ+Pb1AvcO6rT4zPYs+Hz1xPg2+FD4Ovos9Pr7jvP+9Nr0cPhy9R702PlM9hL0RvgO+FL2oPis6KT46viM+zj0Zvgo+Hr6RvhS+rD05Pqi8ab4ZPjM9Dj6LPJY+p72KPoW+kT4ZvRQ+r71mPYW9LD4bPZ0+Gj4nvjy94L6Bvuc7aT4pvvA8oz0SPuS9kT7NvaY9Kb3Hva49SL5vvYK9Nj4kvoW+wT2GPnO9GL5tvQi+j74JPrY+mr3vPS2+/zyYO2Y+DL7xvbI9BL7PPSm6AT6Gu6m7IL5Evfu9uT5nPUi+wL1Wvr+9DD6KPog8sr0cvrS87z1Uufy88z2HPbQ8fTyLPkA+Wr6NPVI96b2XvYy9873yvFc9aD2xPek9Jr0gPsk9Vz4Zvqi9tjvIPfo8ZL6LPRo+kD3GPTO9Iz5qOvm7t7wqvje9ij7dvRC+Hz55vZc+Kz4yPau6Wr6SPde8qz76vBe+Lr22PUY9u73qPMC9Rb6EPvW9oD0MPs+9Mz5Wvpg9Kr5APeI7gD6dPCQ+Xr68vLM9Dz2iuxY+8r1FPsc6vDxQvaS9i7xIu5k+M77QPXe8xr2Hvak8BD5IvS2+bj5RPfi9A758vdc90z1kvqg9wr1nPRi+u711PnY+WT4Yvo69szolvvg9jbu/vjw+nj31PAI+Dj7ePHg9/zqrvn89uL3+u948WL6IvYe+gr5tOsw+i7xtPPI8Jr76PYM9Fj5Gvt8+873mPR8+Oz4qPda7Z76uvfy9Iz33vSe96D3/PRO8TL3LvTC+Yj4kvG2+W77LvI28Rb1Tvji9zz2JvQA+Vz0BPg67Z75/vdQ8fr3+u8s9oj2BvXe+Ib4Nvjo8o75cPqo9PbylvY49hT5AvjK9JD0bPhg9kj5APiu+lj2svY09br1zvR69GL0IPQW9KD4fvUA+Qz5ZPYu94jwEPQ+8g7skvgS+Pb71vJU9v76DPve9ZTnsPQU+cb6PPZq+9T3GPsc9Aj7TvVu9KjxAvgI+Tb0QPhS+FDxlPdy9gbyMPm4+lz27PMG8bT52Pvs8lT0RP3g+b74dPiy9H771PVk9WzwDvOe7Wb00vQq9h7w2PTE9p71CPOu77zyBvV698DzLPBi8C7whvdW9Qj3/O5k87rwuva08zrzNvMe8Pj0gulS9GDzhvBW9U70RPOM8DbsOPKU8K7xfPTk8RL3lPEg8mzwQPWM9FzylPDY99zySvI47U7zVvAu99Tw5PEc9ErwCvZe8ZzwevaI7zbsTPcu8jLwwvew8dzwPPSG9kztJPI+6K7w3O8W8MLwjPZS8nbw0vTg9ST0AvSS9srwbvHm8gr1BPUM9/rw5vAW9FzxhvIK8lLgIvGU9b7zfvMG8kDyOO6K8/7whPd481Tx1PdA8IL1XvKy8z7w3vV69bT2SvKI7Qr3BvOG8iTs6vUA9gDxhPV87tzsGPdC85DzWPYi7GT2qPJM9iL27POS8Sr2bvQI7oj0DvQS+kbwJPYS9+7vtPII6GrwcPe68Qz16uZM97zuXu2q6d71/vYY9nTupvAe7ATx0vYw8sDxiPdO82LwMvD+8vTw8PLu86ztTvK48y7w9vGM98rubvJS89LwmvZI8bjxPvRA9V7yVu0W8Hzw3vTM9cr0cvQa9AjzXPDi9szyyPLY8HT0MPBW9N7yDPJE867z1PA08IL3TPGG8CTymPO+71Ty6PGO8ejwMPSG8jjxLvfk8bj36PBO81TzuPMm8zTznvBG7Zb2IvCm91bwiPFs9q72/PFs9R72HvaC8xTzGPKQ8gj1zvJO8kD0KvSK8W70vPSY8Cr0KOyW9EL1WvaA8VjyzPK64DLyOOpo8Fj3YvJ88Hr1BvQ893rpsvCu9Gz2UvSc9v7yDvca8S72YvVu9xjrGPZi9jrwnvAS9Q7zhPA+9Lb21PZs8T709PNs8J7wsvQK9PjtbPak8G7yGPUm9qr02PSs777wgPd08QD3hvN07WzzdO7O717tjPEY88rwHvcG9X71JPWK8MLxPvJ28sD1RvFs7Az2SPVw9kbxtPfg8mzwKPNc8fD3SPJK9CbySO149Rj00vYq8nDxUvIE83LxEvcC8Lz2PvQY7tr0bPSA84jxdPRg93bxwOzC9DLxGvZ66Aj26PMq7XL2YO0E91rwuO2i8Zjymu828Fbxquw899DsnPQc9hTwaPTm9eL1VvBC7Az1HPKi9pLwgPA285Ts+PRG8LL0vPdO8EDucOrA83LqNPOY7dz3buh69FDwZvTi81bzqvC09MDvMux89v7xWvay8Hj0gvdS8lz2vO669Mb3ivIw7jLvAueK8ers1Ox09wLyxPCY9+ztOO/889TwfuwG9EDw2vTs9ib2VvC294LsEPZ47Yr3svL08oTrhPAa8eb1wvNI8XTsNPIu9Dr3EvDA9qr27vF28mTy5vZs9Wj1euok8d7zEO6E8sjyMPT27jrszPR484DuHO7m84r0uvTw807y4vB+9dru+u2S9nTxZPYE877yuPAU8Dr2APU27ZTyMvFY9MD0rvA+8dr2UPPK7rzy8vLY8Mj0pPfa8nDsZPdQ84Ly/O6Y8hj1IvFM9YrwUvYW9vDozPYs8Xrxau0W9/bytO3y9kL3FPLg9b7z6uky89jwpPYS84byAu4M8gTtkOuQ8krseveu827wNPSM9nD1XvK68Jr2wvHY8LT1XvD89ar1IvXu9Wb2FPUe6JD0uPY26SL2mPEG8Ij3nvKy8Cr2xOlk9jDxWvGG8Sby6Ox69Ij1TPeA8zLuLPDo8OT0bPTe99DuevBc9u7xSvLS9rrxGvIq67TtCvPU7QD1ZPL24Jr05Pac6g7y7Oyk8bTulvBo6FjwnPQ48QT2YvaK8qDyUvDu9o7wbvRc95TygPG49ZTwEvVA9Ij0OPCm8RTycvSe9Nz24vP68Kj1bOxK9yLt6vag7L70dvFa8cz1RPGO9lDwfvVm7H70vvCi9gTrtPAe9WT1SujW6ND2YPYk9aT2BPEQ9CDwfPTK9zTxBvKM9sLy1PEI82ryyu6i8IL3vPMm8Qj1evKG8W727vH+9cTxmvOy8ET2tPD49b7xvPZs9Vj3ZPBO9dTuQvPc8FzzsvIQ88TwHPRC8KrxiPSO9Eju0OwG9wDs4PGm7X703vM28wLy0vIm8lT07vbU8mDysvOe8Jz2YvSQ8ebs9OxO9DD1avGS8rDsuuwU9ND3WvG08dDzdPNk9J73UvC87crs5veQ8qDtSPby7+TwTvRQ9pD1kPZC7oLkGPAO9yzw9u6a8lLz6vEW9j73+vJI8c7xkvfi8g7zgvEw8kTwBvRw8tLy1vPQ8hj0IO/C6jDs2vVC7fjzjOOM80byFPcU8iz0HPM28bb2mPWy7qDysO/c7zDu1PIe9qTwGvTC9Ab1CvdO8Q72QvRW8dz3wPFQ8bL1vPAc9E7wyvdQ8KzyJPWm9rrtGvIk7Qb3JPLI8+zwkPQ+9Cb0HveA6xzoRu4S877yfPeA8MTtqu5m9Yzw2PKk8z7xDPTe82TxxPSG6Dz1DPOM8WztMvau8pToevCQ9/7uXO7o8CjyLu8k8y7zjvBI9GD3RPa+9uj0dvAu99jwVvMM7a72WPHi8lTy1PE+8YT3mvYu6rzxEu6m7LzyNPI49rjzjPE+9W7zgvIo8Kz2NvBu9Oz3jvCQ9AbwvObG8DzwOPdW8zTuWOie8Gz1/PJ+9YTwPvcG8szwYu5I9iTzPu4+7EToBvGY9pzyfPLs8BryRPY+9Dz2su8i8/rwAvUo9fb2HvIc9zrwVvQ67ML2DPBG9nbxsvf07ij3iOyU8CL1JPZi7i7yOvK68dr1IPYs8Xz39PHs997wfvC+9Jb3ZPEm9Jb2GPD68tDzpO8g8vzwgPHA8DT0GPHU9g72LuhU9nT1svAu8HD1MPZ88YbygPbe7Fj3IvE69uby+O1W9NrwUPY+9b72vvLc8jTw3vZa5tzwCvNo6tTtjO1m7l7wsPZK7Jr2oO7Q6CD1zPOs8yTwFPWo9m70RvLg90jqmu168TbstPYA7xzxfvdw8MTwLvRI9RT1OvYU7sjzrvC+9Gr0PPc06s71JPbY77zwcO7u8xDu5PBM9ITtFOzS9NT0DPN88yTt8vKA8/DztO/e8jjyBvZy8fTy7PLG8dj2Wu5E9jTwnvJO71DyVOhm9Hzy+PHA8wzs0PES8Sj1PPVq9+jyWPKw8vrsmuxO8DD2ruk68iDujvZA8sr3POxK94bwIPAS81jxIvMs8ajuSPby8Fr0bPbK8lDwmvLa8OjwNPRA7jb2mu0A8V70qvFO9X7x0PTO9UL1PPEy7mL1fPca8RT10vP48QLthPXC8mTy9vfg7xzzqvEC9F7wovU09CT1/vKa8Tz0nvcK+xbxtPTg9pjyTPIO9jby0vNI8rTy6PBe93Ds/vQg9iztUPTK8dbwPvVk9lDuCvTE9bjyJPei9ULvYvHC9aD1Ovb+63jzYOys9njywOw49KDxFPTI9/7w2PYS9kjuPPCe7BryBPCs9V7sbPSg8Aj6WvDu9ubxgPEq8QL0iPAO9x7xeO5+8ezxAPXo8lj1MPDU9Ez16PQ09njxrvco817uCPbU7AD3nve67BL0jvRg8PzxKveg7zj0EvGM9Xb0sPXM9rzsau9W6Gb1TPAA9b72EOVE9PDzpvM47hL2cvCU9D7sOOui8nz34PO86hbzMOlI86rpZun89prsDPbe9qT3PO/i8yD3IvMS88ztkveS7aD17vWw8dLwRvP08zzxwve881jw3vZO8FzyUO/K8Gr0FvYI9jThcPP68YL19vII8Eb3ovBe8sLw1vc+7ND1BPdK7Hj1dvfO8mTvKPEY9kzvHu5k96DzOO8w7N7wUvdI8pDwWvc08jbzjPJE8E70/O1o89jy4vKw7PLoEvQk9dbt3vFq9K7zbOyE897wNPU+927yLvKo8iL2AvaM9tbyAvWe7gDxLPRq9DL2HvOM7Vz3qvOk8pLwFvRQ9LryCPBC8QLyuu0o9xjyHvR68XDwFvco8XT0nPUS95LwuPIG8rDwYPQS9Er1BPN48wjzCOwq8sruOPPq69DxDPFo95rwGPJi8hrssPDI6eL3EvQI8JT18PEc9QT6vPR09srvYvGW8+DorPUu9NTspvRa8CL2iPZc9ZzzDvJ29DrrQO609iDxWveq8q70xvXu98bykvBE87LxrPec7szwtvHc9nTyHPfw8T7xhvLQ8PjxqPIe9Bzwqvd08KjyRunq85zvAvKm8XzwSvIg9BL1sOvG8KTtmvZW70jxxO9w6BDxqPWA97zs2PUO6LjwqPSO8Iz3ruuU8rLwQO4A9lTuUvQQ9O726PIS9kDxtvAA9L71hvSS9xDxcPau9oLsOvKs8ET0cvaw8mzzxvAS97LxivZ49ez1ouxE8BDvrOws8/LvxPNA8j7xQPXy8hzxCPb09aLyXuxm9Gj0UPG69CD1XvFq9OLvHOwU8PL0/vSm92TyVvfg8UDxQvFK8wTw3PSW9g726vEs9szyGvWk9ezzjvJW8xDy/PAK+ujuFPGY9F7uVvFc9DT0kPba9nbyDPDy9FbzLPJ+8Ab16vbE7gj27vXs8Ab0DvVG8OD0gPSE8q7sJu9G8Ej1APIQ8D72fPRe97bzmuyI9zrmVPAE9bL29vD+8SDs7vI89aD0KPRw8Hj3BvKM8vr0pvb48iTxFvXs8jjztvLU8y7vBPBK9Zr3SvEw9ubwjvX89VTybvQW8cbxnPK28Cb2DvG08yjyKPUW9MjyaPNi8wLvSPAc9V7xiPUa8DL0hPSI7wLwNPBc6FL20PDy8W735vGc9M7yCu807/zxfPDM9T7yWvSE9yTylvEm9EL0FvHI8ozy7O9+8+TuPvNO8NL0LPWQ9C71JPRo9tDz3vCO7LrzpvJe8Vb3GvCq7c70PPUM8hz1lPHo81DwPvQI9mjwWPXm8Jj18u249Zj0tPWe7Cz2fvCi9vDwkvc+8UL1uO0E8N73bu/w74bz6vF+8kr06uju9pz1GPNC8Cb0ePek8VzyFOy68sD36vO88+bsFvfw7BL1CPEK9sDzxO6c9Ur20vF48eb03vEI9Cz3DPSY8Mz2WvB09M7wcPSo8iD2UPVi8EzyIPHE7rbx5vPi8Hr3qPJO9orzjPEs9e72DvVQ9Gb3pPMI8wL3YPHK9CDxaPZi8oD2XvRk8/jmHPZg9TbwbPe08h706vem8KL1xPM07SDytPKs9lzwsPDE9vztJvXs8BDxNO5U9bj1NvbO8xzwDvQg90btoPCu9cj1APJw9yrtCPGE9Hj1OPE282LtbPLA8LT2NvQQ9FDzyO+I6BT1LPa69Ljw/PAm9wrzmvK48db1XvMW85TukvMk8gTyau2C6Try3vCG8szw1vQu75DvLPIs8Lb0EPYG8G7yFvB0+Tz16PCA9KDwLvHW7Lb0au6c8ZLzDPO48pj3BOgQ8jr0iPcO7Yj11vL28CbwIvKa8ibx2uqM9wDz0vEA98LyBPeE8vLzXPEM9kTxrPSw92DuPva+8obxDvJ87rbw3vPG8VTx3PLU8Lr1evAG9DTucPYo8+TulvEu9VD0BvO08iLwAPSi8lrsrPf27lTxxvHS96DzIPKE8u7zTvOg5S70LPeI8FDxyvK07E70OPUs8Mr16PF0827rdvGk8ELzkuSm85LybOz25HL3ZvD+8fb3pPFe9Cj2+PPc8fD1aPIM95TwBPUQ9OL0rOk68ervTPLg8Djy6uzk9K70pvCc9rLyNPLe8ojuQPV+9wLtYvc48bzxEvUI987xsPE29Pr0sPVW9aTsbvcy9NL2LvUC9Hz3LPHY8tb0vPLw9Tz3CPDc8hTyCve88V7zhPIc94joOPRs917wQPbK9G7yfPOI85LzcvJG8szvvvFO7P716PIY9nLw0PUe9gz2CPT+6f7tpvc+8NrwyvBg9Nr23Pe28Kjz9vGG9Mb1mPEY7EL1+O3g8krwFPZy8FT1RvR890jyGvH+9tzxKPBm9vDyFuza9XDuTPfq8bDwwPTg8+byBPAM90DxbvY08Oz0LPWI9iLwZPZC9hrwEvcy8ejyPvKC8zLsLvSY96bxrvFa9UrzsO1e8iTy3PBu9qLxgPHU9SbyrvCk9gjx1PRE9mTyjPOK5+boOPZg9LD36PFM9crvLvE+5D717Pag8qz3tuy29njyqPP67MLz0u8G9qby3O5+9lbxXvAc8qzygvAg9qLxwPKA9IrxdO/q8i70wPJk8bTwUvcU71zvyvEG9nzw7vCA9rDyzu0Y9Cr3uvIm867rOvI68TDsnvcm8hTxfPEw9ir31POE8tbyjPOi8OLxZvYk83Dwqvfe8CDwaPUw9zD1IPFu8Wz1QPd44sLxgPSe9Lj27vEc9ITzPPNy8TT0ePCW9BzwrvbA8jbsLvBA90Tz9vIK9HT1KPBI9KbzNvbU93by8vOa8Or2XPZO8pzzPu0O83rtLvUa97TyyvBc9hbxrO5G8Vrzkuyw9hbwwvdI8I7yBPCm9prx+PDw85bzQuRu8+zwVvSm9bzxuPKE8JDunvR09ujs1vEC9ADxIPbe7kbwOvJe98zmhvOy8ML1cu2S8vrzYOx09LDsjPLw8nr3IO8a8dLzRPcK8jb2MOzc9MLyMvVu9NbwovKg9gz22vN88ez0lPfo7uTw+vS49gbwlPYY5ST0DvbI8HT0EuzG9kTs5vTo8GL1kPVK9dztCPYE93TylPMY7lTyJOYI8pLtfve68HLyKvXU7Cr1UPPI7yzwfPeE79LwLvMi9V7qgvYm8Ir0mPEQ8Nb2MPKE8BL2SPAu9cbxDPR87WD12PRS8HbuRvHQ8Wrw7vIq92bz6PLW8lr1bvCq9Kb2QPGU9xLsXvXu8qLkNvGY8B71ZPQ69TjxEPP68nzw/vZG9ETxUvdY7q7y9PKQ7BT0Uvee8Cz0mvUC8PT1bvCe9/rhSvPQ8V73uvJg9lzuEPQW9RbyIPFM9aT1oPam7bLyOPeE8D7syPYS8rLuJPUA7yrzzPG27Bj3muyW99LwbPMs8Wr0WvUk8lrznPEo9irx8vWU8KjuMvJi8Ib1lvBk8WTs6PcM8fr1SvEq8eryYvBC9p7woPQI8cL0jPGQ9LrivPMe87rw2vVC9UryHvAg9Ib2dPPC8pruCPNQ9kzyIujg92jxpPOe8jj2gPMk83byRvLk7Jr3LPIo8/DzRPCe9FrxuvDi95bvavJe99r2OPQG9bDxtvG28Or0Zvc28RL1APfi8O7x9O2K9ELwKPKK8Zrx4uhS8mrwrvYA8jbwcvM48QL0GPWk9/LsePW08p7wGPaI8nL0NPNM8vboYvWc92zsQvY097btJvIi7kb2GPLO80LuYPY0957zOuyw9Jz3BvH08JLsXOra9dbrZPJe7rzw6vQ69pD1VvS49oTwjvUM9U72OvHC9tTw+vZi8PzuqvOE9iT2DvXM9pj2QPL+8Xrl8vIc9eLydPJY8cj2WPVM9ej2quuU5irxnOzi66LvdvCG9Zj3AvLm9LT0oOy089zulvPy8iD1DO068ZL28PQ693LzjvAu9Wj3kvJa8OLw3vHW9Oz16uQi8wzzCPeg8sLwAPSq86zyWPA89ozxxvJO8QbxovTk8sTzIvEG8iDybvJO7fzznOq261jwovTU8Tj2ePVU9FD0svUo9WDwjvSc7VD0OvII8MLy4vIw7kTxMPVO9Mb3Fu927XD2xPf48Pz31vDQ9Cr2svFu9sDxVOyM9mz3huzk8gj2qPJ88FD0yPHy73Tvxura7db38vPe7QbyGvCc8Fj0uPbs8rTyjPJO9VTzfvCk9W73TPM68Wjw9O1U9pDwYPUy8jzyWvcS7Iz0nuxk9YbzvPEI8Ur3bPG89pjxsPcS8fD1vPaW9CrsMPY88wD1vuxO8MbywvFg8ZzwKvLC8zbtAvSg94zzcvP476jyqvSO9Fjw7PFA9uzzRPL48gL1jPV68dLyDPGY8N7wIvQY+V709PY+8b7w5vUa9eTyUvPw7lzylPb68ND2JvAs7mDznvC09M70BvWi9+rsSPEY8Sruuu1I9HzusPC29vbqjPIa85jzFuF69jjwZvf88jT2fuy28OD1YvI+9v7uKu7Y7SbwzvRA9EDx5PcM8mzyiPIu9uzyHvEY8DT1aPQA8ILtVPMm8ST3svFc7UzsHPSg95TtPvbG8Kb0zPIC8EL0kPEi9mTzQvEk9qTzQvGk9Aj13vXS7gjxsPWA9XzwGPNU8SDsnvMK8HD0XvUo8Hj3cPAU9dr0HvFW9Cb2EPes7kz0tPT683TyRPKw85rynu/w8/bvyvGY9iD2dvBM9UrwnPE+8BL2+u0e95jvzvLE71zt1vBS9QL37O8U8Gr1WO+e8fz3Fuxi99LwXPda89LomPYu9ZLyBvFs80LzHPUK8xzw7vVC78DxkPV09urz8uqu7hr0EPQ09OzwpPZm8hztEu3g9AT0hPQI92zrEPBa9Xb2zvCE9ED7BvNS9lDzdvCU9Bj2BvJc8LD1su5C8sj3hPFq7tDwavOm7WLv+vIm8bDxpPCY9d7zHvIy8ajxoPdG7RjyDvIE9uzySvT+9+7xdvLa6Or2TvCa9tDtwPes8N72XvHk9cj2MPLw7Kj0ZO6G8Zz3fO4+95TzbPB86NjxlvNo8Tr2KPL28Tz1lPBw9Uj2ZvAa9xryhvGU8pTxvPZA98bx+vSK9fjs5veQ6BDyDvD49QT0KPYW9OTx1veo8Xr2gvE09gbqDvNO8PLzavBO9ST0MvIM9A7ykvJU8kL0wvb48lL3Bu0K9FryvvAi9uz3CvB49uTxKPSg9irwaPcm7+butvGm6LrwFvem8KT1RvLc6Djvmu1q8Sr3BPMS7UL1rPeW6gz2vPPW7Hj2mPEk5Mj2yOxQ9hzulvLu8ib0/PRY8Dj3BvOE8ob3svGq8xjrXPNa8srzbPH09Gr2Kuys9ujzSvUs5Ob3yPD09Gb0HPLw9SL2ivAY9pjwHveG74TxAPU88hD2RPcw7Cr0iPbO9QD10u908gTykPB69AD0LPG89M7tXvag8eb2avX88mz12vaq9jryoPUk89byLPXC87zwUPek7Jr2pvf67DL1KPee6Oj2wPLW7JTt1u2s9DD2sPAq9YDzGvGm9ZLvNvBm9tzsCPW+8mLwkPWk9Fr2xvOO8HrxQOwY8Gz0XPIS8ebuIvIA8uDwgPXQ9jLxQvIg99DxXPUg8jTx4u8a8Mr11vZE9izxcvfk7fjysPKi8FD3vueQ7c70aPRw9DbqRva47CDwnO6Y6QzsJOza9lLzDvOs8h70yvB+9Jb2rPTE8Lz2RvIS+VD14Oxc8a7wrvXe9S70uvb28Cr1Rvbc8tr2XuxA9ZLxcvJw74DyKPQS9gz0PPSy92rxAPU49Uz2GPfa9GT0du8O7a7u0PDu7AT3fvNu8Kj2DPFg8ZD2jvD48dT3Nupq81zslvZo9E73NvbI81rs2PQK8eL1DPNU8HbobPYY7Ar0vPag7nTwzOpY9zbsVuxm9RD3CvMY88DwIvV89EjsNPeK5Fz3avGm9Pz3yO227+7odvBi9WD38PCu8ST0EvZm8W7zGvMY7dDwSvWu9Kj0kvRw8Bz0zvck8TTzLPA28ZDwlvZS9/DtNvZ89ULy9PIW9sLyUvGY8erzAvCU7PD0NvGg8FLtQPAW9BjvHPA49TTxYvBu9hD2ePA88az2yO5Y8GLvXuTY9E72guuc8iz1PPOo9Dr2aPdO7tbwqPSW9GrtRO7M8HT0nPfC7ND3XPAY89TyyvBU9Cb5pvJe8sztRPFI9QT37uzA9pbxyvIu8Fr07vE885byJvfM72TwYuYG92TuMPcc8+zvwvOE79bwhPDy9Rr0UvGO8/jzLPJ28dzsRvV68LL0QPdu7kLyfvVO9c7w6PaE8lT00vME8Eb2xvBQ9wj2Mvb29kD1AvVw9IjwtOjy9lby2vAu9lTs/PWS997x4vEq9Cj0CPZ68VD0KvcQ6Tr2ju7I8Vj1HPPo8W73KPIw8AD17PEy8lrz3u7y8+7u8uls9Sbu2PAQ9LrtgvI+827rTOyA8/zpIvWG7KDwgvRe8rbwTvOS7Nr29vFA7ljucPAa9Ez0rvWW8FL33vHY9oL1APaU9pzxQvW88vby2vPk6gjz2PIm8fz3mPJC8Or3yu7888zsPPLu8J73LOps8ML27vDy8dj1QOmY9kr0LvRu9qzyMvd28Yj0APSM9yryUPbe8zz1+PaE8t7w0u1w82TkWPN68ET3jPP871bwvvQi8trwTPUg94LwdO6s84LyEvKs9irzrOoi7SD0vvIU9Xbz/PJU81Dyauli9Sr2OvVg9x7w/vd6827wGPBy8d71yvLo7TDzZOcq8arykvJy7Xr2DvaK95buQve68DjpmvJU9PT3IvIk9irySPB69Cr3MPCc9Drz2PJE99zupvZs7m7zqvCQ9lzo3vTI9RT2XPKW72Lwivd07MbvVPBC6Jj2aPMG8hzy/O/i8hzyfunO8dr2uvIu8Kr02vJo9c72kvJY9tzwwvbK8oT0ovbE8vbugvLe5LbwiPVU+vju6PLw73zxhvKM8iTx+vLQ7kLxOvQM8K7zwPGA9zL3iO+G8AzwTvWA9EjtXvRY9KjtlvCG7xDwEPVo8Ar2/vLO9rjkKOpy9mDyqvGW9szyZPAq9FjxovSY9rbsVvYs8xj02PNa8BLzNvKW54z20vIA8iD0EPem88DxUvUq6ljyXvdM7oDvquxu9eDpCPLw8njs1vWi8wj3EvHI9TT3IvP68nDy3PEG9iDz4Ooq8NT0IvYq85DyJPW687rwKPXA9qbwPO748nrpyvJC9Ub1GPHm6xDv7vNw8C7zDPI89jj1SPBU93rwdPT88IL1LPSU96LxvvMk7w7hqvLq8X7yYPbk9Lj2OvWM8R7yKPAC9HD0sPLa8/TzBvcS9ED3aPAi9WzzZvVI8szuCvdO8yrucPCA9MjwaPLO8ML2LvI68p7t9vE87crwEPWM8oLwKPYQ8TDfWPA09sj3NPN+8lboVPJG8Nz1dO4e8DzsOvVw9qzw5O2s8Vz0UPR89C71ZvM88ZL1/vRm9RryRO6C9P7sEvFg8QL2DvSQ98bvPOxY9y7xKPeM8JT0ou+K8J7v2PGQ9vzwMvf88pjoUPbg7ibxJvEk8C72UvI49cT0ivaA8oD05PFU92zxau6w7FjyUPIk8BT0GvXs87bwhPWy7Nbv7PG89H70HuEO7H73mvA29HT3GvDA9mz2DPCK9Dj2mOoY8JD0kOnw7AbtQvc88TT3euSy7kL1ZPYm7TLw0vNY8Zz1ivUo807yovEo8d70oPSk9DT1evCa9Eb2rvAy9CLyJPNw8Xj2CvIs9kTvbvLW9/7usvC09tzxmPce8K7uRPaI85TydvIS8gzsdvCS88LwrPc088zzTuoE9cTuJvJ69LrtyPBU9bzxbvA68vrwRvbW8WL1rvAM9bTyivB09wjyOPac8VD0DvLE8Zr1yvUk8QDy0PW08tjzAPLY9hryrvHi8Fr0svSo8+bs0PBe99zyyvKu87jtDvaw8eT2AvQS9JL0bvNu8/LuUvYs8+DuQPOg79zwRPBs9HbynO6M8VrxxvRC9Q72QvSk8DT3IvN46nr2BPJg88Tzauw6927pwPU69Nb2XO1A9gDoVPXU6dDyBOui8VDxVPBi7+LwPPWI8Y72kPMO8vjwGvHg8NTtOPQI8PbzEvUe96LwePNc7k7z0OWs9ZT3APLq81bt3Pec60rwOO3o8yLyDvVk7kru2PCQ9bDs2vRS9hDzDvJ898rsrPGM9hDkBvX+8xzxDvSw90DywPF69TD1PPPm8WTyiu3U8DT30u5E8mLtWPEm98zxAPDS9FD3XPJ86gbydvIM8Ez2mPMu8/LwTPPq8njoiPbw8Mz33O827FbzbvEI8ZT38OmS8MLydvBW7Ar1lvS09q73hPB09hT1IPL+8SjztvVS9Xb3tPEK8t7tRPCG8Dj2sPOo8rjuYPJ895rzjvI88u7wDvQU9Qj0KvB+9obzTu4o8L7s/vKS9kL1YPXe7Ar7SvFO9Fz2tOyC9a73mvDy7oD0UvYQ8arz/vBu8sLx2Ozs7SL3XOtu7CbyoPNA8RTxEPXY9GbsKPbs8djxvO5A9Wj2nO6I9cbxxO3Q8nzzFvBI7nLuXu1G91DsHPKg8DD17urs91bxfvI+8xDyJOrS8lD1mOy28qbz1vDW8Rrrsuqg9U71qu9O8Fr3zvEO9cj00PBI9bD1Pu8M6nbvXOF+8Sr0PPEq807sSvao7Wb0Fvb47ibzEPCK9YL0aPaI8Xz0oPco8hTyAPAO9RDu2vHm81LyIvBQ8FD0Xvcq9cT3iuia7Dr3gvIc85jt/vCA8kj0WvSc9TDyhPA89kr30PMY8lrzAvOS8mrwdvJQ8Fb0Au047hrwCves8sTyBvFQ85Ls6umc6rDx2OwM9hL01PQe8w7zQvJs87DyRPCo9ibwivJQ8/zy3PJe8SbxcPQc9bjyAPLA817xwPWc9Qr2pvdM8o7wDPag5hrx6O5w7ED3dvGy82byoPQq9G70xPai3QD3POh69qDyrvHa8Db1ZvB699ryrvHG9Ir2VOvO9BzyZPYW8bD2hPAg9gLxdPbC8jjwxvbc8+byvvZ26QT0XPY09QjzVu4u8Wj2ovCO9nLwdO4+8zjtJvae8bz2Au6S7tzxMvd68Mz0NPRA8xDzzPF28CzuMPSQ9R7tjvag5tTsSvck8fLu8vKq8mbzQPBU9CruZvU09C7zjvLG8jD02vVU8tryouws9kTySvCa7K70OvYi9zDy+vBe9Jj2ovTU9/TsiPMW8gT1aPMi7gb3Zu4E8UjvRPCa8ITwRPUS9n7zXux88PjycvcE6szxquxE9Jr2GPDS9k7x1PRQ9ljzvvPA8P71lPVa8Gr0aPYe9ET20u289R70Hu9u8Oz0Huye9GTtBvSm+ITyuvNU7MDwzvKM8gTwePUI997z+Oz69lr3FvCO9FD38PFc9Kj0Vu768BTs7vWC5ljxBvfG8FD0uvTq9XLz3PJU8+Dqxu0G9iDyJPZo8FTy5PM68hryCPFO83TyFvIy8MbxzPAW8jrzwOg69zjkvvNa8DbxdOrc80bzNPFY90zx1O447Dz1wu+a8JT0BvPI82jyUuyI94rysPao8Uz3mPA09Dj05PS49QbxKvUU9UzyMvJa6Ej3aO3s9QL2MvMc7Gz2HPTQ9ubyUPas95rwQPfS7YrzBvFi8njy3PMQ8q7zwvAI82DyOPVe8rjyaPZ09Jr1KvD+9Kb0ovYW8BLzUuzM9KT0Lu5m8VjyJvLu8a71yuy88ZDwxvSC9Gj0uvYO9ozw9PRC8CD1sPRE8rbxAPDU8mDzNPMY8Dz2HvIY8ozvBvG68hjocPDE8iLzwO0w9Or2cvAa98zzOPA89Ur2zPO29MLsAvYa8b702PVa6GT1KvFi8gbscPbC8lj2IvBG9Ur0XPbs8VrutvAS9Rrx8vaQ7jLwMvQO8T7yFPHW8VbsDPV+8Hr0wvIy9srytvTm8mD2zPNq8qzy0Osq8FbtOvMg9nrwLvQc9Kr2Ju1I8DT00Pe07obxRvaS7B75fumI9ND0BvNe8LL3+PI48Ij2oux+9ibwzO9w8qLwyPKO8lLxuvai8tD24PFa8zjwLPas8Ob7jO/I8ET0wPSS7WTwJvLC9Dz0YPR486DtoPQu8Cb3mvFS9yLzGvDs9FL28vPS9vDzmvEM9Sr2avM27jzw5Pe45NrzPPP88Jz0bvZS8Qj03vbM8rjy0Ops9b7xhvZm9ujyOvIe9Lj7hvRk+6jzmvNy9qz3MvIg8JD2JvZM9OT3UvJ28Vr2lvNq9Zz3QvI67eTymvQw8A74rPdS9GD1xvee9Dj7ZvRO9z72lvAI96D2/Paq9lD1JPfk88T1Bvqs9WzwaveC9Db6evTg+Y7xQPVe8Cb4lPDI95T2rPNG9Q73QvE8+3r1xvDI9yT0ZuCa+3D0ivjy9yT2mO5G9ib2bPQQ9ezyKO4c8p72fvD6+ej25PKW9Er0bPh29tbysPPM8rbxevcG9cT0DvNg8D72oOhE9ar10PVo9HD5lPmQ5Q70gPcO8+z1GPj69rj0IvVO9iL16PfY7DL2hvEo9mL17upS9Rz36u0K9qjyVPk89Yz2pPEg+vTxtPes8WT1ivrQ9vLttvbo85b0APdC8Zb6VPWY8vr2YvSI+PzxpPak9DT0GvNk7kr2PPLe9RD4uPnE9Urz1PS+8dDyaPXc7GT04vVY97DtDPZ491z2/Pds8i72BPUa8kTyRvZU6sbyCPAe6Pb4vPYU9sryAO4y8NbycvWk7Y72GvRU4QT0EPlA8/L1ePYi94r3lO7A9270MvYS9hzzrPaO93L23PQA+fL2MvTm9Fj7HvXW9Jr2nPHy7gD2qPf89YT2wPHq9Pj2evQM9IT33vHe9yryLvUK9hj0UvGk8oD2RPZq7nr2xPZC9Wr0gPDU8BTw0PpW9wz27PTe9pD1PvSs9Br2uvZs9nz0Wumu+bj0sPBw8yby9vV88dzwIvm280L0cvEk8ojzWPbY9Uj2VPYC9rbwyPQu9hL2jOvA8xb0tvQU+XTwzvmY+Uz2DPe+8EbzOu0i+Wr3LvWM937vyvZ+9Ir5rvJY8AT2qvGK9Hb6mvAy9tzw8vme9PTsXvem86T2SPP+8LL7wPIk84T3dPao9Mr12PNM9gz0EPho9Bz4iPhK+T71pvAy9qj0mvSu+wj0Jvlu8OruTvfY8EL0ZvZU9nT3CPFi+abwkPbE8Kz0aPSA9MD0zveQ9wb2bOgS9Kr6IPKc9Ar3vPbe9G73WvRC+sryrvXI9Yr6jvYc8yb3evPa8hr3dO/a96TsJPjc8i7z+PTM+Yb0jvA++2r0hvU89Hb1EvQa+4D2jvak7lLqlvG69870uvVI9Cbx6PfU7pr0qvcY9SD0NvME8x73lPYa9lD2sPRG+sb3fvak94zt7u6+9oL2gvP28IL0fvXO9Cr4HvZa8CL2tPdi8OD0APV89nL3pvZu8qT1Lvd28Zr0GPqu9ZjwEPoi9hj00PP49tLyuPQ2+pb0JPuk87j01Pai8Lz2LvBW7/z0GPjW9/j2kPNy93L22PXI8wbyavVQ8nj0xPr08Hj0DPne9DrycPS898z1xPVK98z2tvX09drykPAW+Gr3tPZ+8YT1GvUs8Kz4ePjG9mL2gvce7Cj0bPgG+LL7QPbY9Fz2QPDM+iTy1vWs9Lz0nvua8kb1PPAu9lztBPIA9AD3VvA4+Tb1oPQY+ML62vfm8cjzNvKA9x70zvls9hz1yvZ49sT2lPce9HDyEu9C9CjsUPmi9HztRPSg9nr2MPII8br2PvfA8cTynPaE8JD4zPe49pjw5vH++HD7FvEu+GjzTvTy8AD0bvRM9z73OPDa+eL0hvQO7FT1MPBm8Iz3BPWi9sb1pvTs9NT6CO++8b7vbOzE9t70Jvj0+hT08vG88Lz3AO909QDyLvY69nD1JPRs9Dr55Pd+9Nj49uzc9ar65vcG6UTziPfO9Mz2+PQc9pL2vPQE9w72mO2u8UTyhvNy9C76HvRc+Qz4ePtM8sz3Su8G8jT28PYq7nzxYPcO9pz3FPA06iL0yPcS8BT0cuok9lr2sPa49/T1bvau9oLxQPau9nzztPNy8jLytvWi96b0ZPUY9WL5Cvei9QzzkPAm9pr0APRC8cr32vcY9Bj6APR8+EjwIPBQ+2D3kvYw8P7wzOwc9jb28OpS9rzxBvou8oT0EvpY8F76tvNU9IL6JvbQ9B76pvfO9Sz5TPc+927z6vQM93r0MPtQ9ELymO+Y8Hz1QPbM8LDtoPSS+rD2EvTy9jb3HvdA9BD3fvZs95T1svCW8uzsQPsw9ZD1JvYk8Fb1mPQq9FL6FvA6+0z0/vcE6mr1KPQm+YT0IPhE+pz3xPGg9+z2mvfk9I729PSk9Gj6pvbK9hL2CvWC95LxCvsQ8QT4SPCm8B7v3vai9LTtEPR49WD1+vBa8f71uPcY9/DwOPlC8O70MvWc+4Tw7PPe9IT2avV26SD7dvBE+NL5EvYk9T7zrvLu7+T0LPr29wTwZvX89Bb6SvVM9FT12vcg82D3vPTe9Ab0UPSI9uL0GPmC7sruuue29kj3vuxk+PL3UvT29TrzhvZy9D70MPaC8Lb4aPo29pDydPZ090zoavV69BTzmPYG9lTw4vvA9sz3xPJO98DyZOxu+WL3CPTC9kT0qvVW9Cb7DOqk9gr3lPIQ7kT0Xva89i70LPYm9yj3Svag9grzKPNG8OLuWPeW9Nb3YPAu+drzRu5C9fT26PT88ZD3BvTi9Fj6svKG9cT2KvLk8xby8PUm7gz1oPMm9wzzZPII9vD0oPjO96j2MvS89ub3ivPq8vL0TvTw++7vsvc09C70RvWs99j3ZvTu9Q75SPdE99zw8vn+8BzqPvVW8JD5Bvdo9Oj2VO0S7cD06PSA+aD6KuuA8Lr1ePYa9rz0vPBQ+cb4GPli8yj2FvZ49LL5APYo9LT6HveE9BD2RPRc+Fz3mvTS8bj2bO5e8zLwovXe9RDwCvg89Yz2uvR++9borvUG8kT2cvVm9Cb5MvQe+tL1PPvE9xz1dvUu8mr0bPNc9gD55PNC9LDtzPgk9F7uMPEW8lzt6PZE9Ij7BvOC9RjtAvh++FD6vPYU9Ub0FPV09bToRPQC+Tj0YvCO+7z2IPX69pr1hPRo+0TyrvQO9j7wHPga+IrwwvoA9Tr2TvZI9+jyxO768DbwgvUk9qTz+vFc88L1DPHy8xz06Pie9ZDy1ve09rb3EvKg9ezt0vZ89TD2xPXG9W77lvRq+Ij4ovjQ9Gz0+PZ09lr3oO1A+pb0LPKK9tz0/PZ69lD0TPgA+DT55Pei80r2svYq93jwwvkM+tD3+OwG9Oj1wvYG8Sz3TvfK8Cr4+vay9Zz6bvCQ9Qz0Fvu08Iz5jvWu9jD0FPuy8OD0TPt27LDv1vMg9HD6qvdy8i72NvaC7Q71WvZA9jD38uyc+Cby8ves9Wr02vUE9Az7EvWc9DL6YPYi5Yz2mPV69RbygPNY8g72KvQM+jzymO+k8UT2cvdG8bT0avtu8UDzSO7y9NL1Nvns8xD3GvT8+rL1Pvio9KT6TvXC9ab31PcC9Sb03PLM9L71rPak9TL2gvay8tz0cvUK9Xr3xvCu9YD0GPti9BDtNPfO8/rz1vQA9hD3iPYk9E75+PW899r3SPSG9UL48vDw91Ty5va492LyXvfe9EL1MPRi90DzfPfQ70j20PQ29Hz7CvLy9yrwlPTc86r0tPmu9Ab2EvCE9I7wdPXy9H74Pvou9kz0YOxu8HL5svZE9hb30PG49LD3NOyg9nT3mPN09Dj5XvOm8Nj01vH68YbzcPTG9WT2NPT4+Sb4SPjw8Kr6EPQy+k70EPZI9+rw3vTG9BzzDvVg9Jz04vSo+er1AvBG92byJvRY9v71mvB68NL1UPtc9VD4mvaw9rj0vPhe9nzxMPke9Qb7fO7K9Bz7Gupw9vTxKvD494j0SPAG+M74bPbm73LzOvX88JbzTPcA9Zj6CvZK9uTzCvGs9BL3bPTw+gbzXvBo+fbzrO7E7mz3bPQs9Tr49vfu7F72hPWO6+73TOzC+6bx5vU8+Tb2pPPY9l705vmA9ETyHPRu8573pvYc8xj3ZPTK9nTy0Pe28fLxqvY48Ib5KvPy6lrxqvTs9jT2Bveq8Fz2+PGk98j2hvfW7UL2nvfy95bxVveQ8xb1JPQS9ur0xvQW8vD0gvV49MT0rvbE9ST4zO2a+pb19PQo9fT7IvEq91b2NvZk96L0aPB09Tj0fPKe9TbvePDE98Ty2vUa9Wr6APRA9LL7gO+o9wL2IvrA8wD2LO3E8Dj2GvU490z0bvQc+JT2nvQS+sjxHvCe9Vb1DvQm8Ez7xPLy9c735vYQ+tTw/PG09Br67PS49zb0YPc+9Kz0JPXO9jLwPPuI95z1gPMm9kz0JvgG9wbv7OxC90Tw7PR08hD1HPas8kj1APaI82juoPZM9Db2SvRQ9IDxzPdY8pT0ZOxs+Xr2RvVk+tb3QPcG8nLyfvRa++zzBvSW+5bw/PU89orzmvK49n7zmvIo9qz0BvLA8sj0jvaU9JL7ePRw+aD2tPQc+T71yvNU9dLvVPam9nD0UvgO9ajrEvNg7i72LPdy8bD0LvUI93D05vXs9nD1bvQ89o73KvcU9iT0VPgq9FT2bPXO7xD3fPX09kD0dvdy7EL1VvaO9wzwXvce8pDwCvk8+ZT2yPQe8Hz6xvT69mrzVPRm8ib07vE+9oztAvR+9/jssOlC9n7vdPCI9RbwFvrc9hD7ku+W77b3OOwg+lb0bPps9zrxROwE9XrzZPWE9mL3Qvdu9/T1Dvdy8Eb7nPQ68db2QPOS9br17vRU9I70BPak7iz0rPEa9TT0Tvd27j73FvGo9TL1jvDO+yr06vW+7B7yVveK9nbzCveU9qz3APVG8Q70Bvcm9Gb2nvDG7SD23vCy+HD04PoC9xL0euwU94D2rvQA+uz16vQy+GL3pvF2947wJPkY93b0YPpM9hb0xvhe7BzyzPMW8ajuVPUW9Jr5zvfi9GLtavVu9HD4Svga9oz26vYa7Ib29vdw8bD0IvRU97r0rPiM9Oj1rPLY9xDxmvM+9iD17PVa8qLzEvQE+cL3XOoE9DD4BPls+lD2xvHq9YD1GvY09I71xvNy7ET4Fvmm9KzhRPcs9+Ly8vPa98r1mve49Jj2wvdg9AD7CPNK9rTyKPca9jzw1vXG81zxqPYW9oDsEPrW97TslvZC8l70zvaC9r71nPSW+Hj71vVK9eb1NvQE+872FvQ+907tNvRg9ED1mPvE9jr3ZvZE91T0Sva69Xb2xPRW7IL2FPL69SL3Bu3m8Ez3mPWY9N7xQPI69671qvC+9Czu5PdM9pzxYvds9pbzDuvQ9kr2lPbk9yL0jPrm9Jb2+vYo9tzziPKK9W70Gvi+8mD3BvXM81L04vOI7OD18PfS9bb0PPsY9ir2QPSk9tj1ZPKW9HD1TPWu9I72FvVO9sj3iPRY+HjzMvVe9D7y0vW89Kb0OPca84r0zO4k9Wb06uym+kD1qPVm9Jr1jPMY9zbzMvZA9qbuqPTG53DyQvQ++nb23vQq+67s9vdE9JT2QPIw9rLzVvY69m72TvRU+h7wIvuW81TzqvR29x70IPSK+Jb23PQ+9Jz2TPUY9ib1ZvSG+T73xvOi9hTxSvZq9fzwBveI96D1Ovpg8yL2ZPXE9o7ytvQO+PLwqvFW8Jb0OPrq9nDuevXK8fj3HPZ09hLyRvRc94b24PZI8mj3KPX49Gr6UPZS81L0yvgq+mT2fPDo8Xj1jPb+8E72lvQs9qjw8vW69Qj1lvc69arzlvNU8CD1xvQk7KD4fvRY9vL2VPRW+zr3uPek9pT00vgc9lb2Iu9M9ZD0WPVO9Cz5/vc48Jj22vXI9QD2zvI69Rj1aPo+9pzyEPYk9Or6JPTe92D0TvoW8NT02vg+8PDxWPaW9Mr1BPTi9ib0Jvjk+3TyvPXu9jL0Gvh0+Sr0evpi8nLoGPoW8b73LO4+92z09PUw9ujuZPXS8Pb2DPYg90D00PQO+nb36vXY8tr3rvUS+vT1Nvbe96DwMPhc60j3lvWO9hL2IPTo83bytvGS9DL6wPG69Kr6RvVo6nDyMPRE9gbzsPZK75zqFPfU9Eb6BvT07iD1Vvfy9Gb2DPe49kD0bPp29nj1KvAC9nr0rPoS9Dz0VPv48Rz0lPT2+TTtZPI29BD6wPTY92jxBPOw9VL0DvgI+uj0CPf07rDz0PU697z0cvjQ9+buJPYI9iT39PZK9Ir2HvUQ9yb3gveq8/b0UPDq7hD0vPZq9Rr0EvlU9DT4JvlE7Mj3DvWS9Pj08vU29LL7iPVy9JbxUva299DsiPv+7Pb0zvRs9aj1WPSa9Vj4GvpM9Lb7nvVe90r03vvW9zrx5Pdm8ETyiPWM9UL0nvgK+Nz4ovag8Bj5UPco9Bb40PaO9Ib3aPZi6jD3/vAU8ZT3RvQm9FD0BPpU7BTx6O8E9jj3FPRW9tjuwO14+8DwzPYi91z0ePb89/D0RPrQ9gT0avQQ+AL6mvMA9sj3yPIe8/rsoPSO94rsUPBo9gLzBvYS9uj3TPfm9LT3FO6g9yr2uPG6+lr2BvA++qT1FvZe9fr4HPhK9WLzXPTK9vT0Uvbs8Fr1kPQ89671ZOei8fL0ivoa9a7wovdm9Dr4avsy92jqqPJK90z2NvbA9yTzEvMW8fbwtvfO9PD0VvhQ8wT2vvN48ND5Wvqk7f70+PbC9kb3mPSe+Ir4YvWA6Er2FvYM8Wj23PLu9G7zGPbi9Cb1avcI9dbywO+m9T75aPQk+Zz0/vcw8BT37vCI7wL1ivHM9sT1vvaA95bxPPAc+Ij1KPiS+gr2RvY29BbwWPeQ9EbzhO6A9Gj4ovXY9bDtSvgY6A74tvv+9wT30O0s2lz0OPg89Yb3lPZQ9x7zfPAm+rTwFvZI8m7z9vfA9oT37PKs88L1mvf88iT3GPb48SbzXvUK9bj0GvgM+nb0oPh8+FLtRvW+9Hb2MvLE8YD2DvIA8cb33vPE8ub21ve077L0QvQ28F71NPe091z2YvWG9vLxCvGg8/jysPBy7qrw1vuM5ajsrPUo+y728vbw9QrznPQG+mj3hvYq9mDuoveg92z2SvKm9Kz2IvQI+cL0PPgi+qryPPOE9az17PSq68D1Yvaa8qr3iPFy+jD2Bvba8xz21PRO9H76KPYS9pL0Avfa81L07PQa+mDzLPQc+9DygPVQ8nr2dPRu+9ru9PK09iL4CvaK9vDxUvfi8ELw2Pck8dj2YvXY9RT6IvGq9QjqyvYQ9iL0KPZm9MD2uPMa8uL0Bvja947t2vRS+FT6DPTs80L34vZw9aTyNPCC8fzz1vXM8SL4tvKQ9Fb2YvA29+r0lPZO8Dj3SvX0+fL0kPCu+Jz3NvR2+az0Cvqa8Ur0mvYu6Hz1luQo+er0WPQg8Xz3KPJ085T3WvFC84L00Pm49GDrpPJi9Cr1dvas9j73qvRA+ebyvvZs8Fr0TPRU8SDx3PYG89zxNuy89gT1PPW49or3OvEc8ubyePWC8Hj0gPag9DL7cPDc81j17Peq9cLsIPlC8lD3+PFk8Yr0XvIa9uj0IPrg9ST1zPeE9Mj2DvYA9o72Evfk9Rr16Pbu7mb1QvSE7Gj6DPfE98r0Yvmm7kr1kvhy+fTvjvUu9kr2XPUM9UT0evnC8z72ePS09kb3fPbI9RD1LvcI8Kz4tvTm9x70LPdo6OL6hvcy83jx8vX48wbvvPfs9er3mPI49Sj3NO6W9kz37vd89ur1lPDI+p7wgvko9KL1YPaM7dT06vvS9RT7cPSa7qzwtvS2+iL0uvQI91b0UvQO9PD2LPAw8t72FPGS9570mvge+Hb01PRo+jD1AuxA+qb1qvOC8Vz2SvME8Vz0yPTS8Wj3LvI49Gj5iPRu9zDy0PXI9DD1OOQw+FLvaPRk9Ub0DPYE8AD4tvLa9JL29PDM9FT68uxq+mLvUOu89Aj2jPTM8qz0Rvg+9EToQPTK8Mj27Pcq8pDfjPQe+gL0xPd+817wNPgC+bz1KvDi9eL6BPYw7SD2DvB09YryivQg8lj2WvSO+SLvuPAK9mjxBPtw7gj0DvkW9wrlSPls9mT0Hvua79z1avfu9Y7yTvLk9x7yOPD49s71ZPQM8rrzrPBY8Rj0qPcg8n73bPRS95j2nPTY9y7x1vXU8Kb6JPS49uL0YPsu9orwhPu69cb2MPdy9nL1zPdg9oD3PvSq9Ab7puwu+GT1mPa69bT3zu/28JDwfPaA8C7uwvQk9M73nvQW+WD0+Pdm9n7uqvXA95L1sPPk9Nb1ovc+7Cb73PZc9MD2mPaW9Yjw7PWm9DD0jvfS9YD1AvVY9rjurvLc8BL62PHi9J72fu9E8ED07PcO96D0vvXm94D1fPYC9XbwGPqE9bj0JPY48STySvYm9frw9PDK9KT0yPS08+7z3PIM95bu8vb85zL1sPVC9XD2RPTq+Ez1ZvSk91zzhvJa9E77jvN88Or14Pac96DrevJW9Gz3iPFa9D77xPY29YL2wvSC+hL3DPQo7Gr2OvYm9Db1QPEc8oT2/vEW95b2tPd89Uz3KvYC8wj0zvsA9ir1pvdw8UzwOPlo6uz2RvLu8bTxaPWG9Bb6JvV09Iz0yvTG9Ob26vfU8jLwYPIA9yr2qPSK96rwyPha8uL0KvlI9yL2wvfa9hT0fvmA97DwEPZe8hz1LPZ69JT2UvZK9jL0GPfg9V7wyvQo+AL34vN68Zb2SPTe+hT0NvXo8ID6AvJY9PTuDPDI9Hzu/PfE8oL1mPEK9I74bPVC7jj0UPQ8+ib0jPTU6hb07veE9Sr1WPTg+oLy3PU09Fb00vIK9Kr4JPTM9D76qPYS8Fr3qvQ6+L73xvQI9hT0uPb09WjxnvdW9ib0LvYy9Kz4aPjM8hDsBvse7XDwWvTU7gzu7Pb29lTyDPRG9rz15Paw94z06vpU94b3wvCW90T0ZPXk9kz3LPZ29wD0FPl69mj2UPXY8Dz7bOoE8HrsCPQe8BD5sPKE9sDxTvTS91TyaPUW9bD3ivSq+nj2QPf68DT2bPZ69rL33vNk9Bz0bu4W9G77xuyW9uLuJvXQ+IT5/vbi7YL7GvQg8uj3lvfG9072iPKa9BL6ZPbu93j2EvQ69wj0nPrM9mL1IPbg9uL0NPpo9Tb0Dvoy9F72oPQ4+FL7Zvfs95T1hvRE+br3su7K9tjyUvGM8Fr6OOvQ8wz0uPq29AzsSvT+9Cj4KPu28tLzuPAO+Cb3TPfM9sbz2PcM8mLyOPaM8kL23PTM9PT7ouxq9iDy6vQY80r2huZc8vrwAvXY+9bsOPfm9IT64PSy8+TvcPc09qL2CPYw9Nb7OPfq8D71xPZK9Bz45PEe9CT6ZvZa7mr29uya8ir0ZPSG+ir0SvQO+wj1KO0M977zYPXu96zzyPMu8sr2WvQG97zyfvUy9A703PWk9N72BPcw8iT2wvB0+QL1+PSs8kzwcvo29+LwFPQa+jb31PQW9UTxPu7A8972yu5290j0Dvbk8x7yrvPC9oD3evUA+1z0GPQE97b3IvJ890j3NvYK9AzwHPpc9KrwGvho+OLzCvAg+tr00PM88Gr4yvjo9Nj4Avlm80b0xveI9g70AvmI86j2kvcU8nbpZvpC7pLsHPbW9Xz2MPMo9vbuZvfm9HD4kPoa+hz2avBe8Dj7WPXK9Hz4cvp29bL0cPqw9AT0zPUc9hz0cPaM9kr2DPaS9Ub22vUu95jwQPWO8Mb2FO6G9cr2xPXY9Cj0OPLK8Sj7RvRM9Er5vvTO9BL4PvZY9Wr3hPOu7Nj34PW+9j7wZPEc9SD0pPhU9XT3CvFu8Ub1KPGq9fTz6vfg8Pr1GPiI8YL2nPeO9jr3RPa68Xz0Avl69cD0JvjI9DjwjPJa8MD2JPS49gD1/O8y92ry0PZM9P718PXa97T0nPjQ9ubxdPBe+kLskvYS96zlfuyY9Lb2xPMi9hT3IPGO8A75XvbI93zxvPUc9cz66u7e99TywvYg9QL6IPJK8e73SvB89nTyQPo29BL7Bvdc9nLwcvUq8PL2bvVy9AD32Pfg9jL1rvXu8d71NPVK9zDwuPoI+t7wlPRw+Fj0pPYG8cT1zPTQ9DD4kuxM9kb1IvpQ9Rb3GPUE9EL26PaK8qr0RPaW8LD4JvgE9k7w7Pu89ED6Wves9Ej7MPCo94z1TPfW9Rz4FvYa9wb3cPYQ7OT3cPb488DuBPUI+3j2UvQY8ajuqPOC9hDsvvUQ8CjwKPQC9KT3uPC09/jwsPY4+Hz0UvjS8xb0VvhA+Kj6AvIk7/jy8PfQ9AT6nvLS81Tv7uj4+Vz1Bvb49EL0KPjQ9cz2Ru0M+5TzPPGW9jDufvRo9GzwwPD8+Cr5TvEW9oTyRvao8qb36vFS9s73BvJO9Z70ovDQ9Db6oPZM84byCPBO9Iz68vco84D3tPe09tryMvYw9ub2CvCe947wcPS8+KT7WvZY9m73MPXe9KrxJPI89lz1rPfo92D1NvXi9ozxQves9AT2VvQO+uT1kPb09IryhvTi9mL24vZK8WD0+PQu+rr2TvMQ9KbyWvfA7MT04Phm7Nb50vMa9YTyKvLO9M7s4Pb89q70sPc08eDzUvFs9Kj0EPbA9Yz3yPZC9rb30PHi8h72aPE49hjv6PRi+cTw6PoA9ADy6PWA8SD0WPV+9Ir3GPQo9tr2hvae9XD0AvqM7s70HPaI9HL6IvbW9MLw5vbm9sL6HvYa8AD6avaq9OD01Pkm9e73evbY8Cjy5PTI9DT5pvRm9pj0tvsK8Ab6Qu3U8qr0avua96Lypva478b1iPaO9JLqJvUc+Fze6PAo+pz27vWy9ZD0CPVY9vb3XPIc95b0LPRA+5jwWPeY9Cj2nvZ+9SL2DvSa9EjwtPOG9qT20PY69mDyhuqW9ij3Jvao9Cz59vJY9QTw4vr28PbwFvb29/L1+vSI8z7z5vRK+w73hvCc9pz0ivYE7tD1MPTg9bz0KPhi9sT3vPZu9TL6XvN89t7z7PWI8YTufPSC+orwLPmi9Bj2Mvcw77jxGvKe9NT3Huwe+gr34PDg9+jyOPQy8KTwavms96L0KPbc8e729POy9fDybvA07CTzHO9K8GT0zu4W97ryQPbe9+D0cvZ89uj3pvDO+z71nPeu8ID7/vMc9n70WvHQ9Dj67PY49nzwrvfo9kzxzPSo8/b1sPX49ZT3TPKG8nb0OPa87Tb65Owu+lD2ePeC9jDwYPcK9L74OvTa9O7tuvcK8P71ovBs+FrtKOk471L2bvR8+mb2KPZG9sj2Mvey99Dx8PUm7Mr3nOwk8frwSvt09aTqnvXu8ojyIvSU9xT3LPI08mTwAPiG9K70ZvRm+3j3KPC29JD2qPRc+Nz2LPdi9nz2uPVY9qjsvvQE+KzwsPtS8GL7vvI+9Sj33PE88D747vTg+oz2mPbA74zswPqG96z3CPQs95T3UvGy80jsNPcG9DL0LPbU8er3EPQW+Cr3WvV49rL2OvVi9gz2fvfo8S75TvaW9fr3RPUo9er0Wu2i9kT1+vce8i739PWy9jjvRvZs9g7spPra9iD2zPGy9lDzJvZe82D2tvIi8rjuBPX89oL2VvSo+272evdC87rwQvcA8LT0zvQW+QD0NPb+7l73wPAe9/b2gvQ++3L23vR++xDyXPLS9nLt+PSQ+ZLzVPbK9tr0kvVa8ob1gvbA9Yry4Pay9Qb0XvTk9h70IPKO9NL3+vdY82jmfPZI8G7xhvfU9BLwvPI698L3UvSC93j2IPRY9vb1yPdy9dD0AvoE9XL01uUG8PD2LPXM9Gj6gPeO91zwlvvm9QrzrO6i9Tr7VPCM9hT1Lvf+96zs2PCq8ZT6xuki95Tx/vda98b36PQ+9zbuHO6E80zstvUk9oT0kPqC8DD1BvMe81j0rvLa9mT3YPeQ9Hb0CPu697zz3PNM9kr0jPRI8Kz2yvBG9ST3LPBS9/bwwPi87jjwuvVI9DT0pvXu9zDzWvUM8Cz23vE2+7TzuvEm9jz1KvSk9oD21Pc28l70JPQ2+CzxOPSI9Ij5dvYo8hD3kPa69TLzQO0W8z7uEvPw9ST2tuzm9Fb5kPVK9bL33vc28TT1RvuS8pz21PdU9r72jvVy9jrhXve28+7sHvs+9B77XPdO8ij0LPIc8y71oPS28fT0nvRg9Sj2bvAu9Pb1dPL+8jj0/PSu+Tz66PVE9KT0zvAG+vryCvea7fb3wPfw9wrwqvdq9lry5PbA92zsKvle9Mj0ZPYG7h7woPbi92L2iveA7Db1pPNI8hb16PBu9ujyBPZK9kb22vaG7pT1Fu0s9/7zgvcc9v7w+Pas9OT0+vdw9xLx3veY857xyPXq9Z7y5PGw8pL28PJ69ubsEPQy+O72mPd29mb13vLI9cz2dvLO97D36vGM9370TPRC+LL5kvSW+kb2XPbW9ujw6PYK8G75vvSS9cb6ZOwY8iDy5PZ48QL2Svb+9ED22vIO9Wb4EvKw9bz2EvSG9L76pvAc9MD3zvFW9Czwqvba9fj3zPAo9Gz6FPWY9B76SvH89/rzEPcy92z0dPge9mb29PHg9Jrz/veW9/juBPcK9DT4jPSe9Hb0Mve495jyovTw9bD2sOwi9CbyGPdY6Rz2CPNG8tb1IvKq9cj2SPYC9zLxKvaC9+r23PLq92r2YPZU9gL3qPae8Gb1EvTu9aL0Ovl6+Zj2uvfC957obvfK9Vz0yvMk98Lz2POi8S70GPte9Mr5HvXs9B76Dveo8fL1lvPM9Rj0oPdM9B772vQ29hT1DPsw9yLxKO129gz3RPYA9R72IvdI9+zoHvXe9Kz0BPnE8yTzWvRY9Azr9vGO9bjxGvkE+Lj6IPTC9772xPS+9aT6cvRG92zzqPGq9F70dPLe89D0jPBc9Oz1QvXy9cD2uvAW+WT6XPT4+Er16vMI9przNPWI9sjxGvg68fjsUvr094j0lvEq+oz2tPPi9ML6Zu2G9Jryovfo8kD2uPYc9f70SPbi8pL3juYe96z1evNi9Ob0BPia9oLzvPVk9ED2cPBY9IT1+O6A9qDxoPSq+rr0MPni9FTyPPB49mL1/PYU9Sjy9vQI9I776O8k8vDzePYM9QD3TvFI9Ub7eO6U9WL26PY690r2+PeI9qj2OvZw88L0QPq+92b3Wvb89Hr4dPXs9qD2MvYK8YbyOPrG9EL5zPTk+0zxkPUY9M725va+8jLxIPQM+17vFPUM+Oz4APkW7cT3xvHY8fz0yPts9cj7YvBS9yD1mu8Y9qjy3POM8YDxxPQa9rT0ZvH+7Kb0Zvbg89T31va89trwJvNa9WL0hPak9FL0+vZ28JT55vSS+W70pPgK+SzcJPgu92r1QvjI+Eb0Kvq69TD3IOoW9gj2dPX89wb3zPDk9dr6fvYE9mT1xPY08Kz7nPIW9/T2mPSa9GjucPSQ9+7yDvdY9wzyRvNI94r0zPd296z0TvSs94TwPvRO8dj0fPqE9rj0+vYK8QztGvCs91rw+vA4+pL26vTA+0D2Wve29Jzr/vSi9b72RvMG8gD39vAu+pTyCPVK9hb0Ivae99TxGPFm9+71MPEY8C71cvYG9JLwJPvm7+DysvUi8fj1XPfi8Hz1Wve09H777vD+8uLzYvds9fb2+PYw9jb3tvY+8073evd+8AT68veW8yr2wPYo8MbuIu9C86bxuvTe9kL27vVS8CL61PMM9FD2MvLY9Mr6vvCW+lb2rPVa8Fz1lvce9nr2RPZQ8iz1ovaG9MT2OPfc9Hz7vPW69oD2FvQY8kL2WvgW9oL1oPXO8+zxmPU49r71pvck9q735uz69Cj2lvH49FL6tPZS80r0wvY+82j0yPLQ9Jz29u5C7gTwIvSU9FT0Zvdi8orp4vaK8Bb05PYS83zxDPRC8nTxlvJG9Ur2RPSO9krzGvHW81zwEPaM7sTyZve+8nL0Wu+E8ozxqPJi8Az36PH+777xnvYY8sbzgu5+4yDxdPHG8HD04uNo8AD1DvMg7OzxGPT49Zbwyvcs8CbwAPDY9cz0ZPS087jyKvCQ9ib3ZvBq9oLsRPZq8Trwruz+9JjuNPTA9ujzMvLk8hrwvO9C84zt0PL88YTxmO0E9ujsyvO+8aTuKO0I9H7xjO1M6AD1GPV09ij1Xuh69qLrlvM+88zssOwU92DwzuwE9nzstvek8PryiPJG90Ly7O3S8k7wDvaO8szzuO+w7kzysPNm8o7w6PNI7rbyeOx49WbwWPT68B701uxo9Pr06vYa9L71LPW28Dr7FvFI9ALxxO9g8xjzEPD28izuXPJ68fLrZPE69yLu3vG08Wj2hPAW9DT02OhQ8CT1VPRy9mDxlu927XrtQula8Cj0DvRi9CL1cvdE8wbyLup+9CTymvRG95DwjvZi8CTy1u6e5dbyCPPa8jbyAvE29MLywvPy8Aj3BOli9N71OPey8eDsRPQi9Gr0fvec8RbwuvBW8oryJvUo9Orw9PRg9lLwzObs8GjzevBu8gLyjOx09ozuNvPM78Ly8PDU8AD0FvQG9Lb3XPJI7FTvNOye9mz0qvfq8AbzgOlK8nTyou5q8NDyQvKO8AbwrvWg91jmfO6M81rt5vAK9PD19PL08jzxNvSu8gD2QPfC6tTz0u029JzyTvOw8CT1fPaK9NT0Zu4i8z7zvu9a9bTxDvOU8nDzquwE9PzwRPMs8YLyRvP88ID2FvVK8lbwdvA+99LvhPK27MD3ePKq8ib2WvbA89rrrPIK8ljsuvC48jD29u6w8xrz/POQ72TwqvdQ8vLxwvE28ArzrvQM7BLyBPNK8WT0XPe88hj3cPKs8Rz0VPdS7Br1LO3K7B72yvNU8pT2zPBU83Ty3vC49QDwRvIQ5FL17PR69iD1QvEM9Cb0ZO7E8ozzgPJU7Jb1XvSI8pLy4O708wjwSvRg9kTy/vMq8RLvWuoO8YT2/u9u8qDwAPZ+8kDxcPLs8y7wdvUW80rxbPMM8hbz9vKA7hbsNvWE9hDoJPWE94LyOPDq8hbyavJi85TsSO3m8j7vyOxs9GL3kPPg8Xj0qPRe9fjynPBW9b7wCPYQ8IDwWvOa7VL0mvRy9ujoZPSU92bzxvFq86zxdvbc8pLsoPQW8dzu9Ozq8gL27PE68JjsDPem8MTwQvIE8K71Jvao8yDyUO8g8bzwSvdK8M7yfPGw8urs5POC8nDzQvDG7uTzTvJe9Wz2mu8w8sLz0Ogq8ybxcvO+7fz2ivA285rs3PKY9lryxvaW8SDutPCa8EL0RPdy8wzwGPQ49zjvAvGS99Tw8PJm8rruHu7w8LzzJPde8lDx/PO08BryUPde8Wz2XPO08Ar1rPJO8Vz2Pu4G8pzyCPJe8fjudPEa7yry0vHI8sTufvUY6xjxlvNQ8QbojPEC9hLsGvTm9aLxwPLw7tjxQPAe9njyFO6W8Sz2ZPBC8Ur2nvFu7njwmPG89Cbwgvcs8CT34vFw9mjvNvEi9Dr0ZvFA9OTyevKi8d7yKvM08bDyFPbw84jyCvPG8Xz2hPCY9LruavFA6qbyCOo08GT24vGM89rwIPag8wbz9PBM91zziPAU8a73gvLc8mDyevAa9Kr3MPJ48BTymvJw8rTwrvOk7FTw1PTu9FLoAu4c6bDyOPZA8JTwPPfm70zqbO3a7vbwAvdw747sYPLQ7Qz0cvFu9Ab2EPDe87TybPPC8sbwMPcm83jtpvS29lrzrOSi9CL1fPUM7oLuSvKe6y7xJvVG7STwFvXc9iroVPfM8TrzWPKY8BT0fPTa4Tz08PV48f7zXPNO7DD2QPDG7TLoWvGI8R7u1u5q8XzxivUS8ar0YPZY94L04udy82jzQvIY9FT38us08Jj1SPa08q7yGPOS8hz0MvIy51jztu2A8rTwvvUk8MTxLvbc8tryqPfW74jyOurg857zUPD+9hDxrO/c8/rw5vC09wzzZumW9OTv6PGw8vDw3O2y8kbtLPDK94jvGOpI7DDw9vMu6CjxLvTA9vLtZvY08Fj1cPIg8bLvkvEy7CDwEPS89Az2AvJg8LbzqOyY8zTwSu6I6DL2kvf+8TDy+PGy8QLsCPZq9IT0HPYA8AL07vem6mTr5u/k7sLw8OkK9mTuFvKy8KDxyuzk9xbwGPaM6Cb0HPGU8rb32PAc9CzwpPCs8WD3/PFi88LwVO8M7PryUvKi9HD3MOzM8Ab2TO5s8PLwVvRu61Tu7PFk9Br21vBW93rx2vKq82LwdvSU9Qr1TvQM7E7wDPf+70rytvKo8wjtouhy9Fj2QO329yLvCvDQ9B73CvIQ7WzxYPBg9LjxnPF69czwQvRu9PDyVu968fD0bvDA9MT2gPKs82jxtvSg9Fr2wvCU8G7tlPNQ81zsFvaS7O70MPRU7hb2/vAS9Nr0gPY48Hz0+vI08uzyoOZw8+zs5PKs5Vz2cPDU90Ty3vB86STzkusu7qDupvJS7N72kvO48Qz1QvD86vDwfvBI8iDzQvBs8Jz2uO6288bz9PPM7lz3EOze8GD0NPHo9Pr0vPTm94rx8vAI8tjxIvYW81DwzPfK8vzt4veo8ebxHPIW9hbvKO+a8Mb3/ugs8Sz2yvF09Yb3JvLk8mTx6PT29KT0qvFs8B73fOw89K72GvS49STxyPd87grs1vBA8sTyqu5u8Rz37vJ27hzxCPPa8gjwuO8a8gbsWvNs8oLzTO6i85LygPMo8iTwnO7g8LL2VO7S8FD0OvX07n7wdOzW8xrwmPRm9wLz+u3O7Jb2nOxs9Dj2NPE27GrxRPSM8rzwSvWu8CTxhvCa9gLw9u6o8VbwHvQG9Gj0gPca8lL2DPcE7HjsbPWq8eruFPMY8wDxpvZU9vrvQvJM8c7yuvD09cDxcvI48/7sjPdk8DT3Duxq89jw1PUQ8P7yfPIi9KL0zuy88hjw5va47Ijz8u607Lr1hPRI9gT1NuzE7FjtwvEI8nzyvPPA7ybxnO4a7hLxFvVm9Rb2wu0G9S70fPPI7iLx8vQM8C71xPJq8GT24vHA9oj0HvI49kr27vG49Zr0FvRA9CjwgPQi81LzgOya9Tb0DPTE8hDzDu2c8xDsIPAm9DL0xvMk8vTsBPek7ujreuz895jvVPK88RzzBPDw9WrwBvSC9AD0iPXi8+jwNO1q73L65PAQ9ljoAu9E897puvGw8sTwxvDo7NjxjPIm7Gz1sPDA8urwxPEI9Pj0qPSq9W71yPH882znDPK88Fb3vOWG7gDxqvGA9VD02PVc8KjyWPCI9jbwTvBE9pjvLO1e9wjz8PBc9Kz2YvOs8Ib0mPWe7Kr0NPcW7Gj0mPCu9K70LvBG9njyIvdy8Yz3YuwA9Ij2xu5K7iD1TvRA6aj32PJU7D7wfvbW9gLynuyE91rynPN67vLiRPb88A73uPOA8PDxTPce8CzyNvM68KD1Iu4A9BDyDPRQ9ATywvQg9ST32PDy8F71ZPRE97jsTvWG8Kz3xPO28XD1nvBI8kTznPME6IbudPfc8Ury8Oyy9cjyavCm9gjwjveg7gb06OwS9ED1OPQm9nLyMPAC9Dj28vEy8rTxxO/W8rrz4Ovm8c7sYPaS8xjwQveS8d7y8PEs8/bz6O0O9e72ru3G9Gjueu1U62jxWu3+9bLnFvLw8Cj1OvAy9Gj1aPNg8TT2EvTs9rzlbPZU71byrvOY8fryCurM8cj2gvOK8OL01Owy95bxGvJU7Qb2guzm9oz2LPTq9vLrHvMQ8KL0+PeW8Kz2RPK48fTwKOW48CT0xvK+7Rj2nPE+8Yz1KPbq8xbsSPcy9UTtEPTk8pLwnvd68fjzQOxO9cL2uPEE8ij1lPf278bscvaa8Przlu8E8kTyMu488hDuZPd88WTx1PJm9TDyKvBY6yTwvPng8Bz2LPX+90zygPGA8OL3Uu9g7Gz0XPCg8rLzaPBk9xL2guxI9fj2TPIK82ryyPCi93bq1Om08vTucvbI7Vbwdu488Rjy3Ox89pbwnOw65IT3FPAk9Ib1lvJG8lTunuqE8irteu5Y89LzJu0o8Yjw4PBw8Kb1yvGO80byPPaA8QT3MPAK9cTvCvNc8ZDwiPAi8Eb30O087ujwBPK886Ds1uyC9tDqyvIg9Cj14vQc9ebvRPAm9F720PAY9srzavKO8wDxpPBS9B7yiPMW8z7zkO0u9ljxTvKY8pryWvBC8C7smPWs9rjywPYY8Z7yyPEc9Qz0Ouwo9JL2jOi49iL3TPBk99LwMPUG9ibzYvG28hrxJvZq9UTznvAu8rDylPL88vbwDvYC9HD07vfU8ozyavBq8xbrdvD89Ar7AvJK82TzpvMI6ADzRPIC897t3vJg7KLwYO8W8ujzBvMm7Ur02PAu9dr2rvPy7wDunO289Mzy6OxG6CD2jPGQ8Qbq3vBc90jyFOne9Hz2/vM48ljuYvO27oDwVPaC8az2kPJE6AzyBPHC6Bz05vJg8zjwAPPK8PjzPPau84Tw6PYk66jy3u2s7QLyDPKy6OT0Cu2e9MTyDO1I8yLx2PIs8i7zrvBM9kr2YvBI8N7t3PMa8nD0avTc9QL1ivOg8K70OPC09rL2bvcU8QLyPOze98jzlOqc8oLy2PeY8t7yLux69JbzUPIy8sDw+PJi6Ej1FPcS8TjkEvDk8pbohvSk8hTuRvP08Rj0zvC68CL3avKy63bwKu4m8oTsovS08t7wRO6i7/DtNPd+8OT0sPI48Ir0yPT691TzVu+48RLyyvI29XbxGPTa8aDyLvde7ATudvBO9qrxRu6K8JT0pPOs8bzt1PUY7hjuKvXQ8jTw5u8a7R70qPae7HT3jPOq8/TmrOn48hbxHuu08RT1RvK88Mb2XvLc7ybwiPcc9+7uuua88UrwFvZS7F7yxPOM7FjyUvH09sjz5POm7mrzGvH27oL3pvGO80LtHvRa9WT2DPHw9AzxFvci8NLzMOx89YrzjPCW9AT1pvJS7WTxKPa68xrvNvRe9obuAvPu7j7qXPJ+8AT1Ou6G7bTyUvN07/TvqvJI8xjz5PA88BT1AvA+8oDztuk+7X710vMa7NLwaPTo9oDvBPB48Bz2evVk8Pjx2Ow89GL1dPFK98zwsvWo9GT39vB49erwEvRI9nzxJvck8zro6PSq92TyduwW8QLz5O9g587uYPRC9ozy1uBQ8NDxavQw8hbrnvG48Dz3yPK67fjs7PUw9s7xsPPW8AT2ivK689jxXPAw8OD0AOxM9dT1mPSu9ZbyJPNe8Wr34u+A7o7tcPCY9Tj3HPBe8i7zQPCg8xbwVPQe8kj3PPM+9s7yTvJY6Kb0MvNC7lLzgPDW7aj39PL68YL3QOlg8Kb3fvNs8eDzcvIM9Pz2AvWY827wCvRQ9DjwEOgK7Z72ju0g9F72JPNo7E7w3vOk8uLwZPby8abwsvYG8IT2+PPO8Uj1MPRo88jzjPB09MbwLPK88h71XvZ68Wrqouka8jLwIPFo9p7zLPCo4ID0lPEw9gzwpvJo9tDwqu1g9ary+u5E7ErxdPDE9zjuQPAw9o7tHvQc9RL1rvDq9zbxCPD48pD1DvW08Pz0CvN67bLyZPIO9Dr3UPJ+8j7yTvPO7ibyDvXy9RbyzvBg9OLuYvKi5STzevBe9ZrsFvX89lTqiuhS9Jr0LPdM8kbyYujW9YT1TvRk9yzxcvMM8PD0ePYY9Nb3vPMm8ljwivH082ryovB69tbxpvZM9PL3yPPC87jspvb88JbmivAC9pTvzPAI7mDqCPX691jzXPHg8j70YPY28dbtpPMo7hTpuPQ496bzGvJk8mToiPKs8Hb2HvCy9z7vzvKW8K70bvf+7krvjvCC8tDxFPT07nbrdvMk84jxvvTw8yTu7PAQ72TwdPAY9hTwtPRO8Iz2bPA+9kj1GPeo9FbwXOmk8E70YvMS87Ts4Peo84zz1PJG6sbx/PBo8lzuAPFe8nbrlPIa8YbzCPBo9xr07vDK9ob3Vu5M9NzxnvC29ez0Ova28/zxUvLs8Nbw5vae8xjxKuvc8F7zwuws9nL0avEC8JbzzOz69HztJvBi897yRPeW8Ib2+ubQ8h72WPCE93Tv4vC49Bjx0u8M8XLz7PP+8m7xQPPc737wDPKY8+TzMPP08aTyZPP087Lyeu/s80byTPSu8pDxdPJw7l7ydPDc8XjzdOuE8nzy6uxA8xTsgPb28Rr0vPSQ8jbtFvJ+9aD0mvAI9LD04vY08sjyTPPS7zzxsvNk7Nr1fOwS9ijzJu4i81LwSvfg8brznvE08tDwuvMe8Or2JuwO8FL0XOa28lDzzvDw84rwMPYU9OT0hO0u97zvjvP67kL3NvKW8Brz5u1a8ODxVu/W7HD2avH09hbwJu329+bs4PYw9RD3+vOY7n7tCvTw93jzLvDu7iboZPbi95TyiPCy9uD0UPeW81jymPAa9Kz0ePBs9xzsoPDi9nTz/Obm9nrwRPY88lzy+vMw8Ljz4u+s8WL1CPVC7e7sHPPw81by9PMW7hz2eu+M7ebyPO8u8ujxCvZW8wreNPHa7uTwOvI07q70HvRy9jLycvI49JzyJPIK7XT3QvLS8CD1VPM088rsOvIM9fj1APS697ryKOvu7JbvCPMQ7QrwEvDu8Gr0Cvac7Dbw+PUu8ZL18Pem7Kj0SPQm9IL3OvPE8Hb0DPVw8vLsuPXg7Gb0ovFe9/zw4PZQ667xkPUK8dTt/PRO9T72EvM08irv+vJu8zDwCPZc9eby8vEg9sDw7PWY9Kb2CvP48oT30vG09Ar3Gu6k8Djsyvcw8DD2MPK087TzQvE68RLwQvRu8RrrIu9w8WDwgu/i8grsPPWQ7GL35vFo8tLt5vTU8NjuBvZi8BL3EOsk8erwNPYI9/Dy1vF681rtZvF89Or0Lvfa8n7wxvCy9xjzjPIa8RT2kvOE5wT24vPC7Jz0wvdI8Hr1QvVW8Rb2EvIO7XT2vvKM8EL3kPBQ8Mbw8PWS9FLzmPBO9KbvUvNs9sDpivM86nLsrPPW8Qb0vvZY87LxKvPm8lT0FPLk897yEvIO9Gb3RPBc9nLz8OmU6Wzw0vQA9aLouvW28kjwJvD496brMvJU9QjyqvCC9aT1bPE+9WT2APdI8w7wOvRO9DL0vPDw9gDwAPN473zyePbe8yDuDPE+8Nr1bvQM9lbs2vEG9DT3qu5m8GDyMvTy9TDw8vTk8OTwsOty8uzz/PHq8cz37PLC6szz5PNe7dbzGvNI8Jz2RPWU9Cz3xPLs9UDw5PVW8Aj3jvKO7djsNu+q8Mj01u9O8gb2cPGA8Gr0UPRS9XjsXPRw9hTyavJw9g715vSG9JbvkPHe79TxTPSe6mrwDPWO9Cr1/PBw9EDugOwA9xLuDPQ+8Kry7uwA9Bb24O369x7x9PZ+8Pz2EPcQ8ULzJvFM8mzzXPN68YDv8PBY8oj2kOzO9DbxmPJW7pbw7PZg8sjxAPSk96TzuvC49Gb3CvOM8pzyjPX89Gj0VPYG8mT2RvEe7Ur3GvFG88jwhPQ489TuRvAu89Ty1PIe8vzwPPSS9Bj3iPNi8ID30O8U8O712PBs917sTvRO7d73Bu3G8KLx7vZQ7qbqfvA88/jyGPC09UTvxPAw96zzFPcQ8Nzz9vJI8AbyEPOE8Zj2Lvcg9JL2vvLk6fj25PJU81zyqPco8VryuvE28sbyXPZY8krwnPAo8vT3Fuq87KbxLPE29mrwSPPo70TwQPey8ybz+PJE81jy+PGE9Hr0Quhy8gz2EvTY8kTsFPJe9mb2evDk9kT3tPCY9MjzrPFU9K70SPYg8Kz2Suvg81jthPP68kD0QPDk9gDr6Ole9Rr31vKy72LxBPCO90jwgPQa6gDxHPKI84zygPB28grtkPEs9mjy9vOq8DrwLvQo9nDyAuv+8Bb0rvJ+7/zxHPfk8sbqRvFk80joCPem8lzwqvS88AbyOPEK7X7sRve28Mj1yvLs8Xb3yPD69kzwaPbE8ez2RPQG9pjyTPIo9MT25PIk8Cj2GPB29fDwcPCU96jtbPQM8hrvHu1+8lr0BPV67cb2YPYA9UbxJPRu9mb38PJy8/bzLvI283zwbPcK8JLxnPTy9KLwTvCE9L72sPG287LxuPFS8RLuwPB+9Ar3lPNQ8Ij2FPM286ryhvPM857wZPGY9obyhvKO8pb1bvWo9GDzmvPC8C71mvFk9czz5vBo6bD1XO408fr0TvYc98rxZuhI9GTzlO2Y9oLr4PME9ML00PUa9HLtDPv+8/L0iu666gLy+vMq8kLy1vBg5P70fO5g9F71GPVU717x9vAk8u7uJvLa6iD2NvEy9qrwDug09Pb1vvJS7cjxnPLe8ML00vba8BL0WvPA8Vr0CvYc9/LyOvBi9qjw8PSS9sTyKPO48VrzvPDU8hb1sPYA8mDrMvIc8k7yzPMI6HjwLPSs9zjwsPcu5TLoIvVU7/DtvPTw9Mj0aPGA91jqhvJa8Kj27OhK9AT0+O/46tTuOvIW9YjtnPDO7oDtPPR+9m7zmvEW7DT2MPOO8MzwDPC09wzwKvV27b7tsvLq7Ez3Suqi8gbx5PE+84DyavDA96jwHPAU9tLy3PAe9Gz2SPBC9RjwLPTu9tztmPdi8AT1hPBU9yTwSvZA8RjwJvMW7yzzju+o8nbwuPSI86jwrvSE9Trssu648ajyvPC28RTsTvXI89rwFPU08N70DvSU9Gbz/Oyq9vLw/Pbu9KTtgOBc9ZT1WPb88UjyIvfu8qzzGuqg6Tj2DPYK8mTyGPKE8ibqdvJI8tLwNPYE8rLstPaY89bvuvCS9LLwEPQK7jrxgvTU9hTwoPB69Qr2vO3o89rt2vWc9CDyIPII7t7z9uTy9KrukvDQ92Lw6Oyu8Nb2RvCq9RTtOO7W8hLysPBq9qL2cuyS9VjzOPNe7M7wbu4893zu4PBc9yDwgPbW8qzz1u6+8nbxpvaw8f7tJPYc8Sz0TvdA76DxRPLG8DDxkPN682LvPOgk9JT0XPJm8PL2UvVg9Qz0RvL48sTowvZK8DLwyOkK9kLwSuxg8wzyavBM9cTwmvIY92TyHvCi9kDzzvLw8Qz2gPLy8QL7JPKe8mjwCPB06Pj2buw690LyFu4K9OTwgPUo8Er3bOw29jjxYvUW8ZLyLPW29L71fvFy8FjwmvVk9t72LPKw817wRvUg8CLzROrq8Cz1tvAE8yTwnvdy8XrxaPZa8KL3fPO88xDuuPP67VzwfvUC97rwUPfO7mjyzPDu9Bz0evGU8LzwiPKm7kbyKPLo8vryDvC69Sz0CPac8hDtXvZU8hjyjPPi8ojvyPEe8Zzx3PIs8t7uEPZi8Fr2YPU27Nr3UvM08CT3OOw29/LxjPE28Az3uPOw7O71jvOc7b7yoPBS9TLyIPIq8RT0VvMG8Nb3hu5i8zTokvBW9eb0APd88cz0KO+s86Ty5vO489zilvLQ7ML2Hus48qjw/PRQ86jyKPJk8cT3XvJ+8iDwSPYw88rzZvGw8Eb29PNg8SjtTvQo9vjtUPSW9l7vkPC49QjwFvWG7Vj19vW886Lztupc8ETxVPME8GzzEvIe7/DqCuwO8e7vkvLm95LuRvCc8EL0ZvUg9CzqVvQG99rwxvYq8uLybvLU80byCOwA9Xr2vvYi9VLwhvVE9Nj0dvJO89jzCOwO8jjyYPX+8qbyju+Q8OTt3PbO8M71ZPIa9FjzyPCS8DTwfPSQ8CT0NvdQ8e7w5vdu8l7w6PUk8GDw+Pag8zbz9PA+9jTvNO2k7zjtPvZc8PD1UOjA8a707Pb28KjwRvSA9WzsqPXO7VTxLu2+8lDwgPRS8Dz0ivU+7TD0tPa67CryXvIy6Wjz6PEc8cz0HPRU9mLy5O3w9DzuuvC+8LTttvEe8NjuuO7W82rw2vW+9y7zUvPE8u7yzPQE8KL0LvWy93TzEuwG9tLpBvEo9zDuuvb08Aj0CPo08Rz0vvRA9NbwRvAq9wDyxPGc9ubyRu0+8F7ygPQa9HLtAu/Q7fztBPKE7HTwDPbk8p7yVPB08pjzNO0y9D72PPDm8bD0QvD68kz0/PH07jrrxOga9Cz3jPFY8tjxIvFU9e729vOE80DulPBe91TxmvHE8RjtVPGQ7njy8OzI7U7uduxi9N72WvBm9Pb3LOrO9OT0yPXo9Nbx8PPe8HTxRuxY8przdvEC9Fbueu9M7C7rouxW9NLyPvNu6bz1IPQK9O7uoPKY8UbzsO4u9cLyhPBo9Bj1dPAA8Ajz3PLu8ir16uke9hzzVO7u8ZzvRPFy8kz2XvOC8i7xVu6C8IL0gPRm9qjyAuwO9hDrFvAq9VT4vvX67BTyIPZ88YzwDvWA7qrw8vZe91jw2vQA8OLyavMu8zrwpPFg9pDzovMW8CbwoPNw6Kj0JvHe8Eb2PPAe9Pr0WPR69zrxRPWG7E70GPUk9gj0VvVc8Rz1HPBe8Ar3hPYu7PrwlvfK8KD2oPSG9tLzhO7G8JrysunO9Kr0wPTC8CLwivMe8Ljz1PP06orwivP284rwzPRm9Ij0cOzS9jbzwvCG80TwwvZC8yzuXPRu9t7ylO3U9+7xoO5k8CD0zvZs7y7w/vLo8D739uyG9gjvHvBa8sTyTuwI9Hz1vu5g8Br2Hu688Vz0svVa9Qz0lvQA9VT35PAE9CDw2PYo8mTxvut28I7tUu9A8eLx/PYa9z7wUPea8r71gPHo707rzPEm6kr28PMW7s7zJPMo857zcPIa8gjxPPaU8AzydOyY9bLvYPE09cjtuvAY9vzx0PKI8EL0nPZU8Pb22vI89Ej3sOgy8ZL0/OxQ9HrqaPAM9ED2bO5c9jTymvKk8Bj0zvCQ88Lxku8m8LL0LO/c7srufvWW9VTyjOt07jrwKvJ07nTyBvRQ8f7zivCU87rozPUa9CLxCOwA9/LumPZM71bzovMm6mDxePIu9Ir1xPSi9jLwwPUo8/7tqPfU8KbwaPYK9Pj3huu67nbwQvFI9iDxWu4w9ybumvJM8b7wyPb670TqGPLU8pztUPBw9CrwDPI48nDw7PNm8z7uZPGa9CD0IveA8rzlmPDa7Dj2BPPO8BD00Oz49uLgivdg8JT3lPDy8O73lPK28L72OvRW94zvlPHs7TzwZPDE827v9OzI9bz0YPJW807w6PV89EL38OxO8X73ZvKk8mrw3Oxk9irxYvRM9f7slvQE8Dr3IO/881Lx+PbM8BzzyO4w7Bj1ivGI7vTyBvIc8ULrwPAY9Jj0zO9s8JD3tvNC86jxrvVU98LowPca8gD26vHu9/DxuOyY9xjzlPHI7crqnvBa9sbykPMO7hjzcPKM8wbxTvWy8KrwCvEK9Yb0wvOY8abyJPAC9e7oUO6A7PLzavKK8rb2UO548mrpwPcE3IDwbvTi89jxPPcC8JTz7vBQ9vLwQvZO6CTyXvEi9W72fPPg70zwPvLk8LrxIvGA9rLxnvY68ubsdPJG8X73pvLw9EjzWPEu9FTukvCs7nTymvGQ9MT1NvPO7KLwZvYk6lDzBOwg8AT2IPEy8Ij1ePQE9eDwrPWU8ybxtvE688DyLN9u8ab3xvL681Lz0PJM4ET0ovHI7TzxRPAU9nTz8O4I6/rwou249+rymvNO85LzxO9q7Xr2HPSW9VryMuzS9g72BO847+bwMPaM9qL04PT08eLuhPCO9HLwyO8+8rjwcPAU96zt7O+o7FDz8u3S9NT3DvPo8PzuevE48b7xqPFa9Ur0Svfg8MbuLPNk88LstPae7Sz3FvZ08Ljy9vMa8hbwPvR69rj2ivHQ84rqFvOY8hjyZPJg8E73dvIg9qjxTvA29wL1ZvAW93rtPvJw8wToePQy95DqGPVg8CDuCu9Y8Aj2bvI07szxyvS+83bwVO4U77zxaPCY7ljxPPdY8Oz2YPJK7vD24PLQ8P7wfPdO7gD3lu6w8Mr04PJi9qzwhPWG9SD2CvfU8zruKvLY7zLy6PFe9gL1VvTi8fb2MPBi9iDx1OeY7mLxYvZg817yAPTc8gT38umS9Kz0EvfA8TjmWvAq9nzylPOI7I7y3vCu8Nj0UPTk95juTvZi8ETwSuxI9Fb2qPGI8rDz6PD69Bjx9OoU7NLwRPQA8r7xjvSQ60zsmO4K8zzw3vbA7gTwlvSu8Kjx/vI68sT0mvYC8EDyGvbY8Fr0BPbA7CjyVOxy8wLu5PBu9PbyNO4y8Wj3+PAo9GT2tO708JT2zu6s837skvRO9LL0MPJa8SD30O6E6qTxxPOW8MzwTvU08rDstO/a87bzFPA09iLzyvJ68sDxqvWE83jw7PAs9gbyQPb286DzVvKk8pbzuuzI8BT0UPX09CT2VPI88gbuWvJq8BjtVvS47TjuKvRw9NL2bu6M8Cr2DPCc73riIPIM8u7wUvXG8drzBPNc8L70BvaA8Zz0ovTg8nrywPWq8vrzku+27DryJPGc9SL2husc82bszO9q87jvYPLo9Bb29vFo8Jb3mvEg87Dw9O106X7z8O5a8NDvQPF694TsBvNw9ITz1u+Y7+DpZO6+6oDw2PTO92rwHvJ88G72sPB28AL2kvBy91bx+uqU827xOPa67ozwuvXM7vTuePY+5XzyIOso7Kr2ZPFA8e7whPUI9g72TPJo8pbz4urM6IbzxPFm9ZbtSOo88Bjxuurk8kz3wvPE8qT0ZPXO86jySvGs9ALwvvBe8UjxJvb08czybPeK71bzIPB48Ir6fOwm8tDwFvTe7sjzAvLo72jzKvLg89rz+vDS9nr3vvMI8ID0+u1c8+zxBPDI9hryDvDa9RLssPbs6jrxoPVg877xSvc+6r7wPveE8xTx1OrK8wLySPao7uLzcu8q7Eb3mumK8Dj3svJY7VTpxvJy68Ty/PN68Qz0nvAs8ojznuwi9IbzhvOm8GDwuPfu8gbx3uzk9Nz2dPNQ8U7wCPZI8rDwNPQU9rbzOPLa8DD2bPUA9ljyMPDq7jTyMO/w8kD2rPRE8jzygPNw8nTw3PBE9DjwhPCC8ML2GPAk8zbxrvPG87jyOO3M9hLxaO7Y8uDx8vdM6frqvvcK8vztRvC+8Hz2AO/O7Ab1Bve48yTvyPDg9lLyyuz67qjumPFQ8v7vtOww8Nj0ovfE7qjxHPXK8WjpKPci8Bz1BuoA8aT2iPPK8WLxTuzQ86zskPdO7lD0MPWc7QbzZPVq8pjwBvKq7Sr1dPDO8B70BPMU80LxoPYm8SL3nPA0817tvvQK8vDyXvGk8gjy8vJE8i72LPKq8AL3du6s8lLxMPNi8kLylPNi8o7zpvBm6UrwavRQ7p7oaPZi9DjxNvfU7BL1IPfy7mjwivU698zwMvE+9ozsiPWw9OryWvCM8obz5vaU8lrw8PBk9bD23u6c7ez32PLE8e7wCPbM8IzwKvdW8jjzFPNa8sLwSvYw8QL1IPKW7XL0JvtA8VLwquig9oLy+vBi9hLy3PKa8e7ysvCk8FbyvO+K8bL1tOVk9Cj0QvUu9yb0mO1C91j3QvLs8OjzAOmA9Ij0YvT+9g70CuxK8zrtqPfq8LbsWPTi9SD0OvTC9hr2IPKC9FT10PJO9xD3vvYq9Eb0yvXG9ir0yPqq+4b1LPNs9wz0qO6O90L2kvVq9ajwKPpu9Gz1HvTq9nb3FPdG8hL1HvKs8tjyZvAO9qj2Fvew9pz39PaK8vb2tvTU9KD1zPQu9hD0CPea8Fz7ovBQ8UrqMPaO8YL0DPmQ8FL66vNA9Hj02Pbe9eD0Futy9SL0mvcS9Br7APUK8Ob2mPa086r2OvRw9tL2rPZi9/7z2PZC4073tvHc9iz2rPZ+9Uz28vc27rL3aPay9tD2PPcY91D2WPMQ6jD1FPQa+grxsO9U7RLyuPG09yztMPSU9xD0qPVO9kb0KPiA9a72nPAk9rD0ZvMy9tDyMPa08/z1SPQo+kL0Qvhy+XToDvVA9vD3yvFQ957zKvV29Nr0fvdO90bw5vv289TtFPYo8iD35Pc49Br5HPU+9gr0Xvp68JT4fPpQ9HT36PWU94Dzou3Q92DyHvdO9lz1oPaC7aLySvXS96b3TPQo+nLwFvRk9z7w4vcg8H72vvey9Jj2ZvSC9YrziPOQ99LyDPMy9B73rvA2+8D2KPQo+E723Pdq8CD7PPay9/r2WvSA90D1ZPXM9Nz1wvcO7b729vUY9iD2Rvaw8GL1PPm49AL61Pb+9HjwgvTS8xLywPAm9oD0evae9or3PPBW9/b3+vcY8jL2NPQw+xLsEPv89jr2dvVG8GL0RPUA94L2tvYm92bvhPf682D0CPBg9Ij6wvRE9UTyvvUM+k7swvg28Rr65u9m9gL1qvc89BL3qvTU9wTvNPfO98rz6PO+8kz2VPNi9CD4IPUQ9PzyyPOi9ur3QPe49/L23vfS9oL09vu26xLv5u8g9pDwjvD29Hj38O0C9zL0Hvqg95r2sPJY81D2gvea9Rb5LvDM9XD2qPQS96LydPUo9Zbu2vRQ+wb3LvJy9kr2+vA29/r2vvPq90TwKPGa9kLzZvTw9mj3UPA09ET1zvHY7gzy2PbW9zr2PvcO9QT3iPa0947zfOdi8TT10vZy9zb1GvdS9B76XPcQ8hLyePIA9Oz3avAM+7z3PO2e9Gb6UPTO8QD0hPdM9Kz34vHq94rxSvN27gz33O6c9zz2ju0M9Nj7/Pf+7q71LvaY9HD2+Pem9Xr73vNu8DbtkvAS+Fr0TPkQ8RD0hvGU9obsmvvm9ID2evNS9rTyiPJo81L2pPSM+az12vdI89rx6vAA+3j2Zuom9AL6JPSS9xL0DPg4+HL7WPZW9oj1evfI9yr0/vYE9ZDyJvts8ijqEvR6+mDxavN26ID2GvVW9pbytPaq9cD0vPpI9VL30PPg8Bb5dPeE8B77lPNQ9JD7QO0+8TL7TvKk8uTz9vC4+2D1+PPG8prsAvi29pjyUPD4+rj0PPQg9FD1nvZ08nL0MvV09XT3pPAK+Fj0ZvCe9RD0PPVM8Sj1uPQi+Sb0IPRc9WL2+vYm8pLx3PAW9Ur3Evdk9iT34vXE9YjxdvFA91LpJPYM9XbwzPom9Ej7bOhk7hby4PUS9BjyHPDY9rr15O988hT2aPT6+fj3gvZS9171+vaw9S72yPaI96TyaPaq9mj2TvY490L3svbA8sTwRPjQ+8zrfPJK8kDwGvb25mr15PPc88D3avai98ryivUc9Ez1tvFk9TTzdPT2+lr1CvL48QztBvsY8iD3xvMm80LxhvD28uz12PCy+HL5cvai7xz00PaM9dLwYPh4+lr38PPm8aDyhPUM+ib2OPQA9ZL0+vgk8WDzMPRQ+AT0MPJ69fb0APTK9Kj3Su8+9WT1PvBm+yT0tvQe+ozyLPcO9Jz2TPGK7hDyzPKK8T71pvci9ALvVvRq9Ir16PdO9W7wjPZe9DD3lPY88CbycvTs9i70Cvse87z2DPLC9azwzPU29trsfvoi9Fz6sPIQ9tL0WPdS9ND52Pdu8az3kvMS9hT2Ou9M9DD2QPea9QD5SPKW9Xr1TveG9BD4vvdS8dr3XvWm92TxXvQI+vL2OPV8+wb2jPem6Aj6uPe099zozPQO9ED2EPRg+uD2KvdS9tjtoPS+9gD2kvXq9kT3+vSG+OL7NPQo9FD2cPEO9zzryO/69Cb4cvhU9Iz2OPZU8VT3kOym8xb0wve+7MLkzPae7kjqBPcS8D76avcG76jx/PCY9A77+Oyq8rjyoPVI9zbxWPQW+Qby0OxE+yb3ZvT6+1bw7PS877j1IPYs9GjyjPT694jvnvYs9BL4lvfA93b3wvQo+E70oPpc88z3XPfc6CD06Pf+9Wz2JvC6+gb0DPpa9FT2gvLq8lj2kPfE8GT5APNG9tz15vEI9m7t6vbI8PL0VPks9FDxUvRC9iz2DPDC86TyXvBM+Dj5du7E8Sr2IvDe997yWvvM7Rj6DvGi9sLxivYy89TwqPMu8Ib2NPJk8n7wFPoG8cr1aPXs9aD3yPNA9ob0CvLO9yLyQvbG97j3hvRI7K76VPZ69Kz3oPbM90724PP+9CjwzvY09gL3LPbc9DbyZPYg83LwVvcc90jzFPSE+B73KPAo+yT2VPYw8n71SPYK8Cj0evWm9njxoPik+nzzgPJk85zwQvra9Jb5vvdG6rjwhPo28rj3APZq98LzNuzS9Z7xeOnq9C70WPc+8izsKvcY8Gj1IPSw9/r2TPSo96zwbvTC+Xr3fOuQ8Wr3BPYu9h7oNPhY9TTwuvac9hjmKu8o9kzzjvDg9DL04vIm9Rz0AvaI8oj2avZe9ATy2PJW96T2tPJY8Vz2HPeg9jj0dPG+8dL3IvQe8yzuLPV+9pb3ju6a8rD2QPeA9Jjyxve67rr35vZw6+L0avki9tT15vTo8br6rPX68NLr4PEy9iD2Cvd695LwUvZk97j3nPCg9lz2rvYy9izsDPX89TT1RPRY9kL2GvO08rD2EPQy++zz2uwu9oDyTvUG+mD0SvsE8sr1cvXo+GL2aPHA8LL2lPd893z3FvXM9x71uvOc8Aj6oPJe8yzxjvIu9Lb5KPKs9aryGvPy9Qj4TPT09lr2tvTi9JD3CPYM9H77BPZO9P72+vQS9/7yMPZi9rb0lPsC9Ib2GOhu+oj2yPLe9z70OPme8uz3VPLs9qzy4vSq9fLz5vCg9jLw7vO+9FL0AvV+9rruEvAw+jrwqunq8/7tgPiK7er0FPtW9sL26PAa++70EPRA+lr3zvCo+l71CPn+9HD4rvSU9ub3ZvI67tj0qvCa9x7w/vcI9hzyzPCm9ujwpvHs8Z729PXc9UzyMPRQ9DL7Nuua73j1FPb69DD0hPam9dz2ZPTQ+Xb0pPb49qD39PA++zrwTPYe96r2bvP89Er6DPVM9ibxHve260j2YPZY9nLsMvps9vr1XPRa8FD4JPhY+370MPgq9Bb4qvXy9oj2OPcs99r1WPj89vb0QvHA+wD0YvaG8r72MPGe7KD6NPQi95L28va+7Sb46vVM9Gr2KPS+9LbztvQI9xb3YvRg+Nb3gvQQ+cr0APr66Kb2ZvT49RL5uPR+9Rb6uPSs+GD6bPfU8Cb1GvTO95D3WPHk9dr3WvUK92T2iOpg99r3hvRI+4jt3vKs8bz28vZU7Aj6bvDs8uT2IvEg9lj0QvoE99T0/vdw9AD3VPcs9NzwqvXc9dz10PZ49Qr3Bu+W8xLxPPSi9Wz4ovrY8dD2Ivbq9qT1JPNI9v72JvUG9rjwEvaq9CL6qPPE9Er3/PRW90L3cPUE+Aj6aPeE8W70xveE9gbxYPWE8hb7qPAg9aj2XvFO9l7ydPRS9ZD5CvC49nLoJvv86rD15PFA9GT5/Ooy9Nj2PO5K9BDwovLu91b2zPZ09Ub3uvZa9Cj1TvQW+9z0ePI88sr0vvYc9k70jvi099z1tvIi9jrzqvOI9izwbvrg9fr1mu/Q8+L1iPbK9j71pvYQ8RT6Kvds8b72DvS89Dr1EvKi9tz3Ru5c9ED3mPXY7ez2qvMC9Ab4bvoE8Rb3GPGC9Vb2/vDW9nj3iPfI83r0NPgY+tr0vPV6+Y75dvNu74DwavsK9OL0bvY+7KT2LPQ49Aj6DPce9yrz0vfU9vr2aPKQ7pjx1vcw9ob2cPZg9Lj6gvQO+vztDPbC84b2CPRs+zrwoPGY9jz0hvU29lz2gPQA8o71nPSM92z3zvSE+3L1xPfy9Ob0Fvsk8OL2MO6o9cbzevVC9vL2bvCI+or22vQU+Y73LvQ89pTxsvIk8PrxMvXy8gz0LvYc9Zj3eu5S84zw9PTq+/TyOPR09NL02vc89L70CvIG92LzVPNy9T717vbC9sDxAvYa93b2YPc49cjxYvQe9dT1Cvlq9lj0hPXC9QT23On49vjxtun09Ab5vvWK9F70evuC8Nj3GPYU8db2mvJs9bT2kvC49Vz2WvYK92j0Ovjo9lT2DPHu9Gr5+PTO+5708vRq9uTyuvIo8V7xmvby9oz2dPFc9pj2ZPDQ9/jzIPcK8TL3hvUu+ub2cvBA+7Tzou4O+F73gPTq+5b0cPMC9JbxavXS8Er7RPOU92jpyvVi9Dz22Pb+74rzMvJg9K70xvRC9or1cvgs+zbvYOw++rLzaPQO90TwKPQc+ST1QPHE9SrsBvbk9ED07uq29obwmPnq9QL2fvaQ96D0lvQq8B705vsC90DvNvQO96juBvP+8vzxmPaS8TT0vu+K9wLwGvhy9OT1vvSk9Hr2mPZK9/r1qvQG9nzxPPnQ9TD2WPaw9rjtqvU49Kb25Pbm9Gj6OvCY+3z2Fu8O8CzzCPLA9rLvRvGM8IT0FO+a9oz2ZvUg9wb3dPSQ7iTyqPd298T3CvXk9L7zVPLS9Xj1cvTA9ET1dvRC+lL0NvoI8j7y8PLK9yr3nPK49xD1rPSc8+LzKuZA97z3tPbE91jwXvms9wb0XPpO9lL11PQI9FTwsPQI9ezw6PDC8hLwlPdC6yz0HPao9Fr0LPLW9oL0Hvsi8OL2rPXY8ijqdvWa9Ob1dvRq9170RPa48DL6MPMS8IL4gPoI9IL7Uvc+8Yj3kPc89gT2zPQ4+5D0KvSs8q7ylvUa8aL0pvdg7j720vW89gr1fPOy9oT0evWE9D72kPQC9sr1IOwU+fr0vvco9Ob2JvVa9pj0IPhC9Iz2HvG89yj2ovO48Qj0HO7E8z7wAPq29gL2/vdu8s73YvVW9Mj1Cveq8vbxqvbA9kD3APW+9vD1rvWq9jz3YO/K8vb0TPfW8YTxqPYU9dbskvAU+Srw5vqO9dbzUPcm9Fr6jOo090z21vRQ+c7z0PDS9Dz47vdE92j07vQO+rr0HPsE8yD0gPYa8KT0kPte9GD1hvhM7P70ivja9lzzlvYo7GD30vdm7gby6vXW7sb3AvS69cT3zvLE8ST7cvA6+9b3fvPY9vr3dvIc8pbxgPoU83jyZPfy9tD00PBe9NT2Pvfc8Hz0DvEM9GT5Yvic+qTyNvZE9irwSvhY9ZT2OurI9JT0Hvi28G75rPUa96r3OOyI+yD3Mvem7zr2QPSg9Bb2uvcq9Ub1avZe9LT7vvQ48zj2TvC29dbzWvQq+8z3gOow95L2nvXS9DL7pvG89vj10Pba9jr19PVq7jb6mPbc9Nb2CvCc9B76Su709gD22vcW7kL2+OxM9Nr2JvaM9Dj2NveA9Db7+PZ07cLyLPKA9mLx2vd89gD2xvUk9NTuEPQo+xj2avJk98L1YPtM9nD2bPMC8zzyYPTM9Tb1zvQs+Rb35PYi8FL2CPTi9eL1Dvl+9wz30Pb6977wTvBM+Db5APUo94D3bvZ69Kj6dvR097jyEvJM9iL3UPIy9tr0PvBa+Yj0gPg+9xbxEvSM8Pr1Muxw+gLyjPRC9qb2KvVi9uTxoveK9EL26PaO9IT2GvBK+Gr7nvGS9vL03vba8mr2UPEU9wTz9Pae9SLz0vB69aD3SvZq9cj2QPW09F70kvm498D3nvHq9jD1Yvf48l72IvVy9j73JPBQ+Wb2aPeY9E7wpPT29or1TvP09kL0DPIK8AD7TPdW9ar4wPFm9N72/vUk91D0yO3y9oD3xPQK91Lx+u+A9Kj4lPQ29Er2hvPo8fjxxPRS9SL5KvNY9hr3RPRM8ET7kvUo9m7t6vC88hTzkvPc8J71gPQU9mTuPvP28JDwQvk29VD1QPcA92D3ZvQg937xbPUy9x7wsvba9VLwHPsC8Dr3mPbk90L0vPcQ8xju3vQG877s2PL896jzEvfs9wb2MvQc77L0Tvny9PT38PSO+MD02u/68Pz2kvVi9s70PvnW9Rj6OOxa9oD3lvVS9sT1RPOU9Y70lPSG99ToAvW08hz1Zvbc92T0oPVw9vL0ZPca8lL3yPLC8MTvdvYC9sT29PQa9Wr18PbQ9Tb0xugc+5r1hPQk9u71LPaY9jTyfvTe9Vb7tPRa+r73vPOu9wz0uPZO9A74RPto9/7wuvt29EL0xvuc9ebwTu6o9VT4EuYk9xr1uPhc+Nb4dvq49zT2WvYA9BrwrvQ09PD1MvGO9nLzHPQ0+sbz5PRy+jb2HvUk9trxAPfI7try9PKc8Wz22Pey7ur3zvLg9HL5evXc9bj0pPiq+Fr4lvlm9ar1FPb092j2pvc68wTwVvQu+aDy4PVU9lb2DvYS9sDwYvZK917wuPSG96TxmvXy9+D2zPek8rz0RPZG9Iz0qvhy8K745vQC9p73TvTG9CLscPcS9+z2IvVM+sTzDvIq+KT7zPZ29M74tPk09Ij0MO1O+ij2nvaO9FD2FPTG+Cr4jPZ+9rLyUPUk9ZrupPTu8jLyTvIs99T1PPAy9AT7PPTW9CT4cPge9ITw1vOs8BDsBPZi8lr2jvVM7c73PPBq+yr0NPuw9P72hvNE9or1GvPK8h70Evq09wT2Qu+K9pj1SPKa8bjwpu1W8FTugPFk8/j30vRi94j0pPDw9TjwaPcC9mb1PPq27ML2FvE29yD2WPKw9Oz5HPd67lz2fuRU+0b1Cvgg+djzAvCc+fb2nvbK8Q725vZ29yzq4vaE9SjzQPdm7mL1GvQc+mz3kPV89nDw+Ot08w71+vTY+Ej3Avf89lj2zvCM9wL2bvYC9xL0Rvha9Fz0YPbs9Gj0nvRo8uD3lPXK9Uz3cvZ08uz0Mvb+8EL7uPT496DwaPMI9C77cvZA8Xj2DPcQ9ZrygvYY9pDuLPaI9xT1lvYQ96byNPaO95bxePb+9UT2dvSY8mr03On49ojwsPO28WT2IPJm9gr2CvQ68PTvovJw9or2JPf+9M72GPES+kDzvPEi8iD0avr69Wb20vRG+Y709vR0+B7tPvac7g7zxPLU9r71dPXa9m7zNufa7ozwTPMC9nD1ZvYA7LD2WPDA+vD3WO0K+D71fPTa6TT35PAa8PD2GvZc9Cjx8PXq9cz5DvSA9Sr2WvaO9CL4QPqa8c7wTvZS8mz0jPi29zr2xPCG7bjzcvcs8vLwFvhy+vL0dPWu82T03vdQ8W72aPSU9aj6Fuqu9nb3qvaa9sD2VvZC8yztGvRc9pj2CvZ88xD2hPLs9Ar2Wvc68ybwevsE9c72hvPu7mD0Kvum9yD29vCm93z1hPb29Cb7fvYM90TzsPcU9Dr0RPgo8iL18PSa8Qz6mvQm9UT78PSe98b1LvWQ9mz3/vDM9cL2gvR+9FT2pPHO+Hb4vPtC8gL2NPCC8kj1hvNM92bynvNq7Bj2fPbS97b1Lvte9273wu8G9XDz3vIy8x72sPYA8zb0XPvi9171BvXQ7eD2KPGA9xzyOvY48qrwlPaY9br0zvV68EbwUvVW7jb3WvYu8yj0OPl892j2ePIM9I70EvZq927wQPnq9ar2hPWK9uL2cvNa9oj0vvUe+qzxSvW+9Lr0BvtQ9Yz2OvGq8Db2GPcC9rbw0vps9/j13Owe+ZL3YvPi8C71TvZu9RL0AvsY9Wb18PKo9BL49vfm9/7qWPWy9Jb0EPo28ZTxxPZO9Kz6DvDO7gLwSvYe7LrySPLs8OjxyO8u94b1qPsw8/j3SvN07DD3hvfo8tr05PZu9Krzlusk8zT2wPJE9gj21vIU8fL1UPT+95T3MvRW9dDtWPrS9DT3dvc29Fj6TPZu9KrwtPTe9H766vIO80L35PU89Wr2/vUK8YDwxPuw9/73yPfy9kT1Wvba85T26PCw+WD3fvXQ99z17vAw+L7zCvQe+D72bPR89obyZvSc9ajycPM68sj1uPZ886L3QvYw9ELxePZ29KT1JviK9HLyoPRE+Fz2qOwQ9nzxoPsG97LyPPWm9ML0vPRa+fzkOPL89K74rvaQ8Nj7+PPi94bzvPLq8k705PLs94rzYupQ8lLwHPtM8AbwkPVG8Vb1bPPe9Cz55vZk9x70zPgy9uz2bPAc9rb3XvVw9Rr0cvTE9qrtlPVu8WL2yPcs8+DyOPUE+4D0Avga+zT39vQq+NTxrvM+9cT1tvcy987ytPa69Nj2sO509jj2EPdS9jT3vO2G+Bz0yvoq95b0HPbm9qz3gPcK9Mz2JvSS7BL75vTS+rj3KPQm+kjz9PTS77b3UPCi99T3CPem9jbxKPag9T76pPM295z1Pvck9mD2PvYi8fzxmPV09v7quvTW9oLpDPKm9sj34vBU9/LyLvZa9tL1+vXY+Kj36vDa7rb1Wvqc9jLrAvH+9bj4XPSu9KzwnPd49aD22PMu8Qr08vtw8rjtWvmA9LjssvW69P70APAE+iTzIvVc9zr39vSW+pDqdOEA7Gz2tPae9WDy6Pd+8jzuwPeK8fT0xvZi9C7wQvUM8ij39PP09kr0iO3O8RrsrPpu9hz1DvYK94j0tvsI9YD2UvMK9iLyHvdM7FLuSvQs9Gz4PvSg95juwPCA8/zyEPeO9ebuGvNi9QTymPT49er3XPOy8xzz1vaw9Rr2huz88wzzgvWc9Ab4QO1K9NT48PWY9dj2fu/w9Nj2MPcW88DziPbI5Dr51PQM9jTv6PTS80TxsvcC7/jxMvSy+iLwUu728Nz5GvlY8FLoBvtU49D2IObm9ljw7vD+9UzyEvZY9b71KPOA8hDyGOyk9wr3lvZw9uz3APeI9Fb6mvME7OD4GvSc92b3XPDu9nb2bve29sTtfPey8Djy+vf88iDyUvds96D3CvRa+hT3rPb490zy+vQ69oL28vaI8vzzDPIs8kr2UvY09/z0SPd+7kb2kPeA9uzwjPaU9mr2UvHU7g73EPWA9yDyyPYK8WT2xPXa9SD2IPZq8Dj3hOiS9/7wHPg47zbsOvT28xzwBPds9Iz1Rvcy8LLwHvWU9z73ZvQK8x73gvKG9Qry5Pdo8mz0SvSk9uL2Gvam9RT5KO3A9Ab0/PHC+hDyVPAy+hb3mvV89wb3jvSW9m7xSvqI8P75JvSi8tL0QPUQ+Fr6MvJe8Vr0NPKg96z2jPZq9ez10O6s81b0dPbi9Qz3hPTc8sT32Pbg8W70Gvoq9JL2PvbG9yTzvvI68pb2BPfW9QrtKvaO9Sz14vaq8Pj3vvCM94z0lPY27y70MvYM95DpjPaQ8lT1YvJY9kT0ku8w92jpUvbO8vj3IPMo8qzqFO8O7sLpvPRu6kr18vRk8Xj0hPsU8/j0hPQo8iL1xPRc92L1GvWq9rz23Peo9wz1UPfC9zzyXvaq90T27vZS9yz0QOgW+6D1KPYE+sT3wPQS9Ar0Rvng8nz0ePtO9Nb2WPbG91TxlO6K9uTwJPYk7ir0rvcc8Rj0TPfa80b1dvF48GT3pPO28Cr4UPBC+IzykvYy8Yb3Zvci76D0+vBa+wD3+vGo9Jz1cPcO83L3HPb+8Czzvvbw8jb1ePHW9Mz6PPee8jTtyvSm9T725vce8PL1/vS0+17sTvRO+Fr0KvZW93z2uvR++9DzvPIQ9XT6YvV49VD2RPWA8GL2QvS+8BT30vQm9Nz22PRm+rr3dvd29XDy6u7m9QD0BPgO+zr22PTc9rryhPKu8Zz06vUU8cLs5vku9B7yaPZk9vL2oPU29qbxtvLe9Iz2SvRc+K7yvvZs9cjsZPre9qL2IPUw+zrzxO0E9yT1dPXy7572kPNW8rr2IvSI9LL2DPdO9xjydvRy+Hb4TPqw9gT5quzK8cD2NvEA9T70fPae9qD2ZPdG9eD2pvW29gz3VvQo+OD0DvTo9lT2qPfm7aDzzPZM9zLwuvKg8Y72OvbI9Iz4dPIS9ET2TvbM8wj2rPDK957yDvAa7NjxBuki9yzxmu6i95bx2vIE9rDwrPSC9ST3UPbM9oLzIu5695DtJPfM9z7z1PT+9Eb7mvcu9ozulPaw9+T2UOhy+C751vVg9fD1QPLi8BT64vSK+BzwSvn49eLw0vWW98L2FvBy9U70evEC8LT2bPG+9AT6qPT6+j7v3PRk+17ysvOA9q7yyPYo8HTyVvPS96DskPQe+fz0sPUg9bbyaPP68mr3MPb29Fb4LPZe8ar7CvKg83L2QvSw+FT0aPfa9dz0Lvrk9NbxPPrK9A72/vTg8QjxTvow9P7xKPcI9tbk/vmy+Dz2KPXW9wLrBPaA8gD1mPR49UL1dPeq9jLwTPUy8mr1avKC9O73FvcS96T0lPp282DzTOpm7CL7aPCy8wD0fva89+TyOvG09F723vUc+Ez26Pau9Aj4jO5I7LT0LvSo+Hj7WPNA9nby3vf68xT27PYW9Lz13vc692Lwevd09i70pPdQ97bx/O4E9oT2bvQs9p719PVs8gL25vMm9aLwsPZC9IL3Tuzo9iT17uoI997pKPZq9wjzmvbq91b0cPeY9AL0FPYw93D0bvss9wDxyvHA9JrzUvYI8A73HPbs8Vr04vn69PD3IPAE98zxMvps9WTxVOSA8FT3wPEq8KD4JPJY9Yr05vLU92L2SvTW+Fb2fPJ2+Oz7xvYU9ojtMu3U9sb39vFa+uDwYvBK9Y70WOtU9SL3ZvYi9K75rPYg9hTwTu5I8Gz39vVM6HT0HPqq8Tr3evdq9ST7NvIQ9urwQvT2+uL2tvAI+Ar6TvSK9ET7HPVu9XD27vfG8Aj2vvA2+Sb3ePcu9Cz0dvR29Ar76PFG93L2lPUM9vryUvc89CL7KPMu9aj0RPpO8xT37u6I8tD0svVO+Bb16vSQ9rr3YvQK9sLyovUC6KD0RPW28Yr2uvDA8Er2MvS89lT1mPXy9vL0Jvqm99bw5vri9HjylPdg8TD5DPoc95bzXPMi9oL0tPYe8Uj2rvaK9l72GvYM9cj0QvjO+a7xxPVq9Wz08vn89Yr0ouio9xD2sPRI+mr1CPXe9sD36PO09UT2CPAI99zy7Pby91LyYPX69nT3wvcq8uz1iPVw+0L0vPJG9Pz31PKi9Vj3Ru327vT0LPnW8Bz5FvEe90b0JPkk9Ej0wPbm9770bvEQ90j08viS9ET0oPJg9rz3KOg49TD0pPQy+Qr0jvH48jLwRPfi8CL6/Pdo94DwNu9W9Kz5TPeq9kT0Cvi08kD3PPYi8uz1ivBy9Jj1FvSA6oTxXOyA85rwGOw48AL2tPS+9tT1KPek8a72+vQO+uTubvb49Pb2/vDg9e71hulU9yL3QvAe9lLyyvYY9E77JvMG8ej0hvEy9jb0zvDU+nb2dvVW9GL2Dvc+8XD1UvTo9h7yVPVK9kDwnPlC+azyvvY69Dr7CPBa9TjioPAO+8zw/PaO9sT1sO2g9YzyTvY89Gj1dPIc99TwdvDs9IL5uPfo8kDvhPMQ9Ub14vSe9mL2/PSU+xb0uPBo9jL1UvJi9Nr7tPQM+5z1cvVc96zzgvRS9Sj2QvVY94TvhPek9xb3IvC++Hjy2u/I9UTxUPVO+Fz1hvb09Rr06vYM9sb1kPTm98b08vhK+gTujPRo9bL2hu2s9jDu6vIY9BDwNvUa9PzvIvTK+iL0Xvli9x72HvQk9Njx8PAq9A70MPNE8mr0bPe49NL2MvI0+qb3PvQW+Dz3lPQA9MjzKvds9bD2XPUO82T0CPbI9PL12O8A9uT23PQS+mb0+PqW9obxyuwk+9b0evn89vr2/PNC8wTwQPoI9VD0ePZi9IL2cPRU+AT7BvQO+cT0bvqY8VjwUPQ4+Cz6QvYa9HL3/u/+8XT0wvdQ9UT4OPgG8hb3XvGa9sb3QPVK9S7zZvPg8+73/PVe9oD2DPQc9GzyfvBA+xTw4vac9Zr30ve+8kz0xvZG9Fr7ZPRc9gj04PcK9ujwhPve9Rz1aPVE97z2zPPi9E73wPC28Dj32POC92TuGPQC9DDzJvYo79Tynuqk80TopuhO9Ib1Ou6O9vr3GvG49ujsJPR89Cb4EvS69srxzvZG8mDxjPQS+PD0GPXA99TvdPS87HL2fvbu93r3svdY9njwNvbe8y7x8vOI9lTwDPZO9E72xvee9Lj4SPkU8OzsUvs69Gr1TPhE6eL0Pvky+fDwBPB+9UruhPZi93b2YPXu9Rb1YvZE9Bz2LvJ89OzzLPQI+eLyuPcy9STsrPRK9Ez6Cvdu7/r1PPkw9kr0Dvtc9Zb1oPp08IT6XvJ09Bj64PBQ8Zr0KPU49yrxcveO8272SvYc9cryUvf88mb3KPBK+zT3jO4u8M74fvqi92j2ZvZ+8Hb1Iu0a9FD75vR49E72FOzw9Jr1CvgA9cT1jvOk7kzz6PLi9wr12Pd06wT2VvXi9+zznvUa6CL77vCW9MT28PJ+8BD6OPb07Tj2yvKo9hD2Cvc+9br1UPSk+CD0ZvDA9br0cPUy9Az07Pac9iz3svSu8gz0GPDo9NT4KvgY9iL0BPW085Tzfu/08VD7avbm8FD3Ruw89SL51vYu6YD2TPge9ib2Ivm49Ij0qPYM80r1Lvos6Zz3BvIo8Oz35vLu9nT06Pa29ET5SPaC70r3hvZy9kL2rvYe8bD0cvgM+oL38PbQ8Ujxevto9G768Pfi9qTwkPXQ9E74KvCc+Nz0nvLg8srzAvfy9kL1/PQc9ArtWvGo+d72NPRU8cDr2O/29ozwavru9Rr3KvUY7Xj2bPYq9JD6VvQs+sbywPVu9WLwLPiy9Ar7fuwI9O70SPcu9JT7SvBe9072BO0I9bT6Cvd89hD28vcY8Kzz7O4Y9Wb3AvdK9Iz5fPLS82D0wvWM9sT3ou/o9/7yPvAc9BzwqPTw+fj0vPdI9dT1rvVg9sb0xvLI9vLpKPgS+Ub14PV69Yr0Jvn68ST5WPTG9Ub22PY+9pL39O8G8ijvVPTG8xb0Fvuy9iTxzPGa9rj2KPJQ94brGvBA+Vr4TvYU98j0avUA9RD1NvRa8y73UPEW9TL2tPVw+pz0mvdo80j1mPYK9Eb7tPZc8sb0qvGg+h70SvKA92zovPc49nr3pvKI9eDy8vSY+2zthvEg97T1ivfO8/7xkPq89v70JvtE9ibwEPr89Oz20vbE5Ab2HPRc+wD0wu9C9WD6zvGi9VD5TvdQ9Tr0QPnM9bz1EPKq9VD6xvBs+wLw8vpk8nztnvZ+9b738vWa9YDsGPC69v7tEvVc7rT0zvdO8Ir4zPSo+T76PPQm+tj1DPOA8szwAPlQ90j3LvII9gz1SPSg9HL4rPJy8Ib1QPa28ur3UvRC4xzsrvRM+ljzXO/Y9lL1zvQC+yDxqvYO9JT1HPSK+2TwGvTs9kr4ePiw8OL5lPTo9nr3uvZq8Mj0Nvfm8mLxmvAM+8D0ZPOM9aT3PPK09Mry2vY6+iD2BvG+9ar1Xvb09djyivMC8TT6gPD27Ab4UvRa7WTzdveA9hLurPPu8eb2FPYs9yj1NPD69OT0fPQC9Lj1IvES927zDvFa9x7yfu847YL0KPA09Dj1Svc28gr1GvZs9FzwUva68IT1tPfg7V7vsPCe9/ryHvUM82zpAvNk8Y7x9vC68zDuovFO9mzzjvFI7Dz20PCO5vrwrPAE86jziPI48yLxgPE684ztLvUO9Gj1DPaE8zj10PQ09lTwMOj+8RT3LvPi8Jb0IPSQ9MLzguRq6cb2sPDo7Ej0ePIW8TD24PE88e70WPKU9ij3gPGM77Tp6OhQ8KL2UOz29GLxZvfq8bjy5PEY9kDwoPak8Dj01vd27Eb1TPG08dLwxvYw9CD1EvYs83DwLvYW8CbvLupI8kTs6O4q9vLyEPG+75zyKu4O8Bbx4vJY82LsNPDy8jDzcu189w7tLvW48Lj2NPHm9ar0lvVA9yrzkvb86hD14Ou68WzyvvJU7/TxnvA89+zyePIo9Mzz0OwO9HL0PPao7hbwQPTE83bwUPHw9GLsXvJS8/rxMvVY8Dr3tPBC9g71xuqO8ezqgPJ09mb3NvIe947uTPK+9fb1PPIg8ub3JOkA9Ej1+vKu7jrzju/E8CT0aPD88Cr29PHk9ljyLPOg6mrtjPNO74TzgPBc90DwdvPa81Dw1ves7gj3uPLQ83jsxPc+8nr1VPDy5vzw1OTU73zshvYA8AbxdvBa9S7xJvbY8gbsgPcw7FDynPcq9Cb2iOU+9Lz07Pak8aDyFvei8ZzvQPAe8hz33u6m81zwJvbW8iLw9O7Y7SD36PC+9rTuOPM08F73oOuc8Ir1vvGS7+DzVPJU8qb15PA+9sbuxvFe9kL1mvJQ7nTwBPV29ojwrPZi7+jw5vYe8xTzyPAG9crr2vBs8M72UPNu8LjtdPcA7+TzvvKK93TzuPD89bL2jvJM9/7wePR293DwovOW82zoWPVS8Rj2QvF296Twlvb+9yryWOuk8wDtSPQs9Rz0cPQK9mT2pPFo8QbzhPBM8ybxivA87Az2fPbg7jzyEvGO8tTwTPcm8Jr0nvYI97Lwjuwy7LD2PvA08gzvNO9C8jjulvCO9a7ojPJC7zDtVPV+9PjzKvHq8Bb0HPcq8JTwcPYc8urzCPAo8yzwou4q73zxfvYy8ert9vNu8hD2vvFm91jzZvKG7pjrrOzA8pD2EvBy8VTybPEs8ZTyLO+U8Eb2nO5s8WTz1vPQ6Dj1ePde7fjsbPcU8Gr2qvLu8rjwivIA9GbwavY+9Er3xOyG8uD0+vRe9fzpDvby8Ij01OwM9ZTwhuwI9srxyvHa75LwuPXI9rjzOPFW9wTstvW69Wz2MPGu8Wby2PPe8BL2ru1G8gDzIPAi9ETziPBS8hrwXPRS8cL12PcW8nzzTvCC8/DvFvCY8ibzHPMg8Jzz9Omq62T2HvHq9Q7wVvQK9VrxqPIs8yLzJvMA8dz0/vc+83bwyvZU8nTwSvYs8rjyIO089NL13vG69OTwzvXQ9grvEu2A8Xj1kvOu8jL0RPUI8jLxoPBy55ryqPM08gz2ovLe8Djx6PaW8w7t8PBq9MzzYPL28M70TvLI8E72YvYK77rxQvWg7r7y8O7o8BT2dPV68ALwPvZK8EruFPBo9kjzIvLe8Vj2dvDs8k7wpPVe9jrx1vGO9ez0fvQK9w7ykO3K4ND28u2A93bocPbq8Rb1ePbc8wjwlu3e9s7yruVQ82zw8PKA7HD0+PVk9NLxfvFo9HzsIva08k71ZvSu9Q7ySPLA8hrwPvB49OruqvEe8Bb3YPLo84zyIPDo90LykPDg8nTsYOE08lDyrvOo8Vr2qvNC8bLtUvN68Fj1TPKQ6MD2TPNu8gb36urk8kjwsPIo77Ly8vKs8RDpyvR48CL1HPP67xbxqvPY8CT2ZPGs867tuvQK9yjwBPSe7jDy4vH88AT1jPK475T0QPcc9Aj2xPbw8h7wFPdo80zzDPFk8Fbu5PLW63jxyvOS8aTxnvQW87bySvek6sDyFvQ07BD0KPaK99zw3PRS9dzyVPSk9RbxQvd68fL1VPeY88Tz3PCy8hTs6Oyy9aj1uvYq7Bzy0vIg9CbwuPXq8Pjxsu0I9jr1EvTg9GzwoOui8Mr3LO3g8PL3EO0I7Ej0OO488qLuku4688rxFvJM7lLygPFe8bjxKPbW7lLtxvK28ULwGPTY93TxPPMG5fbpvvDg9qD0fvRq8LT1UPHe73rs/vO88Lr2EvSK9srv8PF88CL3sPEo9lbuHu9w8erwovY29HL27PDw8GT0RPMS8NbxOPJU7F70ZPK47gDyjvEc9Fzrpuwi9zzymvUM9Hj1FvQ28b7z/O4C7LL3EvB68+Tu0vHE8cL1YPZC7gTx0Om08gD2qPGO8sjs1PDa8rDuZvRy8yrwLPSy8oboNPVK7Bj22vZ29gLv0vF09lLyrO0E73Dw7vY68Er2LPLs7Hb2nO7K8Fz0pvYm9gbxKPSc9v7x5uws9kL39Oxw8vLzIPGO9j70lPWg9wDyFPQO9Yj2NPQa9NT18vae7Fr0DvfO6S72CPI08KTxjvQ89Fb1Ivaq8Sb2CvF89xLvCPP07i7urvJ08tjw7vUI8vTz9vOa8cz2sPGM80rx8PCE8vbxFvJq8Jr2cvPi8k7woPSK9JT1kvJg7hbz+u8q8EL3yPG88Hj3PvCE7jTsHPLI7lLz8PJW8Vz0mvTY9frwovfc8wTuCPBO9rTtCPSI9G70fPdW8zTssvBi9l70JvF49+zugvC69ID2EO8s6tz01vJm8lTxvPEI91bxTPHK8djtjPPu8Sz1CvRy8hj0ePYs8GbwRvQi9bLzEOzY9nbw/PMG8NDySO6A7xjyavO88Hz2fvN68IT2EPd68njsEuzY8BTtnvLU85jwtvd281bwou1O9wbwdu2G8hrxfvR683bxCPA49lLscvSS8BT0PPcI8L7yuvIY7M7ySPTq97byPPbK8q7x4vY299DwrO2C9Zr0JPdC8Cr1cvBM9+7wjPRo9IL2hvXU8Lr2RPBm9jj0ovYA9oLuMPGm9cD02vI08STyUPHo82bvuuz48/Tt1vJ48rDwxvJ07Y72cvQS7DDwHPRS9MjxBu0q9Ar04vT09+DxqO0M7rzyfvDa96bwhvb48rbogvAs82TzOvME8B730O9a8Eb0Ivc48hjylvKi8njz8vB49s7sGPCK8Frx7Pfc7FD0AvZC8xDoEvQ48Dzz9O7S7NzrQvOw7lL0kvZc8kDwkvfa7OjzavKE8Gj2WPC69FD28PE49ZLwzvfq86Dy7uwm97TySO0E9mbvTvKK9V70nPJo8qbxjPbs7wDzFvhU88zyMPL88IDyCuwO6ATwfPBs9oryIu0O8pbxpO1m9nDx3Oz28Jj2OPL67Ir1ivW26LTwNvZ+8rbzhvJA8Ab0VPRi9oDyBPJA7vLgpPU49jz1cPfM8Wz2PvIw6KTypPIw9GTtmPHy807xoPJQ9BrwSPcM7sLyau3m9gb0BvWW8lDzcOz29VjyKPX48aLxHPQy8Rzy1PCu8rrziPKy8BD1HPT49lr1EvTQ9+TvWu4I9kDx6vIc9SzwJvYs8tDwBu9E8sDzvvK+8BT2rPMI8ITyaPRE9/jyXPAG+JD0ePfE8Hz2fPFA9Iz2zPMK8WLxWvCk89zwRPVM9kbwBPFK6ebzZvJM9Zrv7PHO84byyvAc8JLzqPGi9RLrHvNw85TtEPQI8kbsgvA49EzsbvTW9q7yMumE9Mb0HPX69Ir0KvTE6R72WPDa9D72IvI89Zrz2vKM7Yr27vJU7lL2CPDo87rw6PZS8kr0AvV48y7xgPJM8gr2qO948s7xjPRG9SbwPvUQ83DyEvRW9EDx/u068mTtrPIQ8YjwsvSs8Pr2evHc8FDwBvXw7nL2xPTo89LzUvLU84zyzvOw7k7wBvR871LqKPeq5ATtBPaa8oTyxOxE9yrqrPZQ8RLyqO748Cr1ePOA88zsavcK8xLzWOu482zyAvUC83Tx3Pas8arxPOZG9Nb3xvJK8jDxhPSM9MTs1PJw9Hz2GvKw5yL1Eu028aLyCPB8+ajzIPLS8OrwzPNA8grxnvR29czurvDe8/jxavAo9QbuovUQ8PD13PbG86LwYvem8p7zLvBy8rTzePCe9Pbx+vHu8HT0JvQA8Kb0avNU8MLx9PAk8X7w4vbY7m7xivaK8SzwrvSs8Wj0rvPO7ortjPYU7CLxZvIm8Urw+PPc64jzruws8n7taPD+9NzsoPWI9vjofvFy9ezwOPcO83Dz6PLy8mL1RPJC9Zjw/PGW9WD3bO4q7q7wxvMo9+ropvC29KL3VPB09SL22PDY9Qrw/vPu6Ir3SPPw8zjdDu1i9Fb29u1U9UD3JuzY6kTqUvNU7RT0yPLC8yjxYvWA8Bb0EPQk90Tw2vZk8uLyFPXu9WLyjvDe8KL2yOza9SL0APcs8sjzBvK+8R73mPGe93jyXPOe8Rr1NvRq8RD0Mvgy6B72cPJm8+bpDuwq8Jz0LvJc8b7zZvDG9jTxXPHu8Wb3BvI05yL2FOki90zsJPV08Bz16PZ28AL1evRA9YT14O9U7wj3pvLW7Cb1JPJm7Sz3lu0O9Gzx7PUC8Gr14Paa7Pry6u/I8Tz0HPb68Rj0ePRG88TtqPe08RL0JPJy8WLyTvL68xDxpuzG8Cj0yPXG7L70WPX87DT0bvQY8kLysvQs7Uz0nvRq8ML3fOye9Fb1ZPEs8bj0NvU69Dj2tvd+7xDzkvE+99jzJPJC7SL2vPHS9mrxJvYE9nzwjvA68J73ePOC6Ib03PGe9e70hPJc9Mz2ju/g707rXO9c7ET3tPM68PT3wPCe9HD1tvEM9DjxEvMc82bwIPXS9i7vhPLU8rzmvPMs9U72bPUI9Bz18vMw8qDz2PBq81jwTuvi7YLwJPRY9HbzvvOS8gTwnPRG947xAvSW9HL22PIm8zDyBvK89hrzMO629wjypPKA85rs+vZc9Kb0jPfk8j7ytOyk9RD3EPK67bzxfPZi9Wry4vMG6yDwDPAU9fj2euqu8jrw7uy29hTywPDc907sVPdm8ED3uvJK8Pb1VvU+9vLxOvV26mDydvDW9FbzGPcw7gz0SvfG8Kz3EO9Q8Ab3LukA9kL0gPcu8BTxzPBA8q7wFPTW9ub0EvTK9jLxDPBQ9DL18PY+8IbwvvNO8VDuKPeo8d7pOPZM9uTyzvHw8jzw1u/C6Nzk+vT68FLw9PQY9NrzDOqK8TTsdvS69LLwLO9O8RbtmvVO8U714utW8xTySPG69ez0evAQ9Vz24Owy9Pz1uvLi8d7xSOla79rtSukM8JDrSu5E9LL2VO+O6Wzz+PNC80TyGPEa8wbyBPLE92Twlu2u81jwlvQY9E70rPS87EjwePae8grtzu4y8G72GPUg9Rr1HvMa8PTvZvG48BD3sPIY8kjydPYc7tjxTvRU8Gj1ZvPc8QrwmPZY8tL2API88kjx5vMk48LygvAg9NrybPOM7ujs9vQ46Lj2EvdS8Or0SvF68hDxoPak7Fz37vIi7bDxvPV28ITn9vN287TsmvUk8sby9OZy9Bb3hu6w9Ez0vvUu9Fj3GPLW8Vb3uPK49+zyoPNQ7TD36uiS7VL1pPCG98zwhvTi9l7ySvHm6jrwuPc68VDsmPdy8HD3HvCC9ij1oPN88oDwsvQG8D7wXvVU81DzGPHI9lj2MOwQ8EjwevZ28TrwnPZG9Szx4PVA99TzuPHe7Kjs/vdc84byEvUc92bxPPAQ9nzsSvey8TrwjPQi9xjwdvAy8AT2KPB69DD20PPQ6Az31Ooa9wDzlu1q8sjvqu468VrxKPcm9RD0jPaC8zzzuPDs9NT2FvIg9QTwjPfu7tztePGY98zwKvDq9iD1hPAM9WL0+Oxq9hzyCOxK9WD1IPDs8BbxtPFU8rb21vMw8qjzIvcA8QjuDvBU9Ej2sPOM8xTxcuzu8CD28vBm9cjxqOyC907wBPKm73bl4O8i9Rzxqu2S8JbtVPGs99TyGPDI8STxKPB69Vr01vG490jxLPDa8sTzwPGo69DvOPEc8N73sPOK8hD37vOI77rvQOy29QbyMOw09pjykPEC7s7vQPAk8DD0LPXY9OzydO2+8AryWvKw78zy8vWy8R71fvR88mzzAO7k8u70MPbC8Z71UPMi8ojyVO++8ujzKPCY9Dz0CveM8qjxIvXa9ejyHvL+80bx5vCO9ObuZvBU87LyqPHo5ijuQvdw7fju0uxW9QTwBPV08RD0cvYA8kL1QPJa8djyBvRU9Rr13PSg9ZDwpu4Y94TxEPYU8BT2JPD894bytuzo9GTzhvBw8L7sMPZ26Fj1cuZS9prx0vCk98LyUvSI89Tz6POm7ab0LPZy8+Tw2PR29hz3oPNA8kjwsPZK7Bz0zveE8kL0pPJy8mzwuvYe9hjzQPAy84TyqPJ67zLv1vAm8I7wGvUO8GrxMO7K8TTxIvaM8bj19Opi8Ar2dPB67D72qvXS9pDzDPAe9CT2NvO+8trvuO9U80T3QO6Y7k71DPBe8Mj2mPcS8DrwGPVS9lT1EPKq8mrwrO8Q8er0XPFw927ygPaQ8UTsIvdA8jLx8PTQ9gT24Oma8Pzz0vJq6uL0WO6E9ebySPAw9hr22PBQ84rzSvI89Dz1MO9E8HT1hvS67srwTPfM8rjy4vDO8Qb3WPO28HD0dPBC7Wj0EPQg7HDyHvX67Nb2PvN+8Cj2OPAM96jwdO4s8xLo7vI68WjysvKC8iT0EPe66QjwfPLq8iTwIPOo8qbyTOuW8dTuCvZq8HTvCvPW8Vjy1vM48w7xcPI07PL1SvMu7DD3OvBI9Bzwdusw8eLxkvYO8ar1Vu/C8WrsTvbM8tLqmPF89O71cvTe8fj0DPe86gryAug89pD0QvVS8Kz1FvWE9xjw+PNS8UD01PSi98TyEvN87FjxbPRo68zzvvNy8RDzovC47+ruZO868Cj0Huss72jyKPJs7i7xMveE6jzyHPGK8mrzQPDw8dzw1u4q8sLyzvK884Tz7PMU7FT2bOcq8KL0LPYo88jyRvcW8dTvovGC8Q7qEPB49GLsnuwq9PDyaPZy8ozsxPek8wDynvJi88Dwcvci8zrwUPdY6Pz29u4k9irxtvMo8E72Du0g9Yr0kuz296z3xuog8JD0rvUO9Wr2FvAu8gDwQvP+7Lr14POO8oT2tuoK6obwDvR09lz3BPGe8wryrvKy8nrsNvUW9wrsNvc+8o73gOxA7ZD3QOj48Xr2dPSQ9Mb1BPMs8KL1FvI48ITmNPFM91TwNPIK6lb3YPJc8q7xjvV48+bs6veG8zLsivTw7EL0jPbw8ubzPPIG9hL2GPGC8yLsfPJW9Tr1/PIe9c71wPaQ8jz2APVY98jqFu5u8CDw9PfY8+bpCPWA94j37PBU9bjxrPZQ8vjyfOzA9Pb2vvPE8k726vLS8MT0cvRE92ryJOzM9iT0+vGq79T0hvTK9cr04vLg8TLyhvBa9A729OzU8Kzwivd68/zzjPHo6kzzWvE49ILz1PD68Ez3Du088CL1Xu4Y9WbwIPCM9UT0/vLu6+DtHvHE8lz3xOjQ99LuYPVq8gL26PBm8nbylvIM8Cr0FvAa7Oj00Pb47kT0BvZ+8hDyBvcE9WT0DPO47tLw6PeY8AryBvWs8pLu8vPs8CLpdPFk9JDxoOvi8lDzgvIs9J7zOvAe8fLwmPJI7hjvWu609Yz2DPOq8VzygvdE8UTyju2O9bDzUPDE9J73lPNA81zxZPI487zyVPNw9Kj0vvDa9PTytO4c8MDqaPHq93DsYvS87GzviOya6zjwvPcY92DyIvCQ8iztGPCc957wAu4i8HL2zPS47TTz0vAc95bzYt0Q7krx7PQM8zztBO5Y7AbxgPdm8Bz1guym9M71JPZO9Yz0BvMg7nb2cvbC8iryjPWk8kT1YvVc9ST09O9M73zvgO0Y9wLtSvQ89Mr2KvDU80TyjPKS9Bz1QvZM7FbynvNC6AjwqvaO7Crz5u148Sb2NPWi8pDzSvbu8RD2iPGq7cb1XO2s8r7s8PQw9Nr2AvTK7T7z9Ohs807uwPFq9GD1zvf08sTswvN67tT07PfW8F7eqPGe86bsqPdC8Cjw5vW49jb1kPIo8QrwtPTI9H73SO9A8HD0bPYS8QLn2PKw8bryYume8hDqcvGY8ZjyKvJe7WLt2vVg9qTxDvUM9xj1Mu3w9RLszvQO9C7wYPHG9bL2+OhI9hryIPEU9MDqbPPc8TT2kvdG7R7wHPRw82LwXPd67Q73xPOU7OD0+u7M7SL1AvYu7jzwbPAW9rzzdvOS8AL2JvQ69ujivvVa90by8vKw8Az0IPZK98LqFPKU78zlyvQc74jzDvBU7Fj3ZPIO8l7wxvIo8RL2Wu/C7pr1kOzI+a710vWc8pLyAO/48jzziPAM8QL2Eu5E7kzxJvS47yLonvCU8Ub2xPda8lbwcPaI9Jr0iPAi8FbuJvAa87LyFPdY8XL3wvLq8Mj0JvF886TwdvTm6bT2evFa93TwPPUo9L70FPDM9iTyTPOa7Ez0UvIW8sztgPAm90LpMvAG94DsFPDK7CryVvBw9vrovvSu9gTzPvIM7uj32PK67xzwMvRu90byovM+8+TuVuXC8YTzWO9c8wrxavBo8trycPEI9qryzvLm8prmEPN27/LleOni9z7w5u4a9VLxwPN877ry5u8g75rosu3E8drx/PI28DDw9PAK7Ej0hvdm8Lb3sPHI7JL0vvRO8sryPPJo8ATxcuo08hjyWvF68Dzz8PKc7Kr1VOxU8xrsKPHc97LxuuyG9MDwQPdI8eT00vFo8j7wavGw8DL1CvEc8bLymvP+8VT0kOjI80js/vMo9BL28PJ08dD0avYU8Bz37PKS7/rw3PFA7OTv2u6Y8m7y8uyu8mDxvPa08vTwwOgA9MTtcvWU9ozzCOt48mrsPPfk8qTvAPJW9WbykPK68Pb2JvP06+zwLvNW8GT0EPQw9iLzmvFq8E70GPKo8Cz3kvBg9K72fPEo8OL0cO0A8Gb0OvVA9i71NvK+8bTz4u8G8t7xHvDu8sT0nvAO8QzutPA49xrxXPIm8fzwVvfu8Qzw+vJU8iTw7Pe08EruWPEA9+jsOPJm7Gr1OPAW9hDwjPLq7ybrVvVO9tTyyPKA8Dr0MPZq9rjwTvX68gr0GveO8CLwRvb47+bshPA89pjyQO8y8Jb3tvJc53DsRPbG8MrxivgM9UjzBPGY8Wb0jPTU8Br11u0o9PTx8PGa8+DuWu++6Gr1yvJm9LrrBvIc9+btJvYS8Fz3GO2e8Lz27vT49WDw3vRu9lLy+vN26rDvTPJC85DtROqQ8CrvFPHw9KD0DvaI9kLrbPNa6XTzdPDq9E71KPP08Bb3pOuI6pLzvuz697TzAPB89kLriOww9zjqKvP87DbxtPb48Gj3APKO8hLsBPDE8qDyQvB89Ib3nvCs9ZjzOO8A8PT2bOwo91by/vDG8wDuhOxk9p7zCOwY8vDwBO5o9sDy+vPm8VD1BPMC8mrqrPFs9JzwjPRa9V7t5vQ497rwNvVy9Fb0mvaq8Wz0hPSI9sD1qPR69lrtbvHS96DwnvTQ9cDxFu1U8uTwNPa88zDxGPaM8HzyiPDM9ZzwPvTi9wT0avTO8g7tzvIK87LzOvKo89zypvDc9GT1APRq9GjwDPWo817shvA48mjxePQ49ArxfPYa8V7y/vP+8u7sUvAw7VL1ZvLK7HbtZvbC8tToqvBU8PL3FvG29oTxoPPW7zzvivOy8Dr08vey8urxwPMi8Zj0cPVM9Qr36PJe74jo2PFk9QzygvE68jrxZPSk9rbzvu2Y7X70kPRk9+bvQvKS88DnDPG29izyMvD29Rr2sO3E92DyKPEE9MLysvK08yrw0PRY9AT1Svb67jLpRPIe86jq9PIg96LtSOte8ADu2PKI9LTySPOE7M7z+O1U9lTsDPXo8VbwnPR899Dx8vWw8kTwYuys9rDqGOu88KD12vbI8Mz1Tvfm7iju4PM28EjwOu1s9mLxzPKW9lbx7Oyy9kLsVPK49+TsoPf68Dr3QPEC7gLyxvA29zTw0O8y9AbsWvOw9KT3BPTK98TzFulG9xrx1PdU8Ez0fPAi9vzx6PLc9OrxRvK08OjsDOzI8gT05ve48KrwPvBO9yTwwPYG8I70avRY9QzyHPfK8G724PbK8mjzcvNg8S7wIPR687DwvPH48hT2pvSG9KT3fPKw5qL0jPEa9FDzdOqS8HL3rObq8Kb3FvMg8KrzfvHs7yTt6OyE8h73hOzA8Gz3XvIo8vLsUPXM7Fz1AvUC8BL2+PEW8SruNuye9gr13PGg8Czv8PGs9U73vO488ujyjPD49OL3tO728XT1oPNk8S7wDPQI6vLuAvac8p72ju5O8dLsVPdc56DwHPQi9xLw/PQq9g72SPPA89LwuPeg7GzwAvdO5qDtrPqo8TTzNPNM9IzzqPAq9Kjs2u6C8hL1QPam8az3jvLK9krwFu2S7Dz19vSC8Dr3XvLa8n7sFPaY89TsIvJ282LxQvV091LvhvGM8hLxqve471DsPPTu89LwePQw9jT0FvdA9oLzNPD68XT1PPDw9nryoOwg9izwTvaw7Bb33vKU8Tb1VvNq8rrxuPVA9z7yMPKo8hbwuvQw9ubyOPSy9pr3Xuhq88zxgvUO9qbwIvKI9pL0FvYw8pz3oO3K8Qj3LPMA8N707vEO8qjzavIM7Tby+vDK7wDyiu948Kz2fu6+8m7upu3C9nbpwPZ+8cjz0ura8kLwxPAo9Hz3Iulg9tDyFPJy8xrzFPB07CD0YPUE9UL30PBo7gr27vWs8EL2avVc87LtDvMs8Pb1dPD884zxkvLo8Sj2KvFe9ejxCvXs87ry0vGW7ijwDvSI8Cz14PAI8GDs1PF49ITw0vTi9Wj0+uko9CTwgPLA8Tj3vO/u6VjyyvBQ9gj38PNC8/7srPDG9Kb0LvIe8qDsgvee8LzwhvZW977xXPDI9PjxLuwy8oTv0PIC98buvvKK8CLwNvSo9D729Oo88IjwSvUU9/rxrunu8orwhPIk8q70kvTI9bT3ePIw7Obx3PPM89bytO+w8Ob2UvIy8nbs+vQy9xju0PEo9Jz1FvQ87Q7yVO308wLz3PK09IDuNvHM8Db34vHo8J7w/vB073ry0PBo9Cr0MPPG8jT1rvRE8gjzqPP88gbuhvAK8bzuqPIa9dztzPV09ST2XvN+8Hb1YvWW8sLwJPKs8vTswPIu7hDwcvfO7trwmPRY9hLyIvDg9+TyhPBe8V7uZvRE8Fz0kvKO7tzsIvI+9M7wEvRg8YTy9veY8aDuDvA49LT24vEq8Ir0dvAu9Fzt+vPU79DsSPC49Bj1sPVU9y7uoOw+8BLzJPOu8MT3jPCI9a7xJPL48rLydPHk9hbo/vHA9qzshvIS857w3vB48SbzRvIk877slPUc8iLwQvEy9Dr2qvEE8hzzmPGQ9UTz/u728Z7n4vP+7QL18vSW7lzw0Orc6ujzGPIi8Kz19PRI9jjuUvew8Nz2EPBW92zoKPWO8pLwlvZq6xDw7PJe8iTwJuzU8FD1YO6i9SDxhvQk9pbwbvT29nj2pvM47Kr3iOsW8a7r5Oz29Mj2EPD49dT2QPRK83jtPPOa6kbxAPIE9T7w5vOW5Uj1hvO48Q7y/vMI8gb0tPQS9rTs4vRy97rwGvQw9ILwwPb+7YDz/uz+8Fz0VvYS9bTzau009BD3yOyO9NL21uyi9c73/vAQ9WL2NOww8C71wuyI8H7wVvPK67z0SPLw8dD2evME7sDyFPLo84bw2POA8UD2ivBO9ljuRvLK9bL2+PGm967uLPNK8lDzNOgE91bw1vcu9SbwZPK48G70EvUM9QTxJPXO9yLyTvCC8p7x+PDo8/rwePcm8BT3NvEG9EDxIPNY63ju1vGC9jj0iPXO91byGveU83LxLO427Iz3MPPQ8Or2kPK08Mj17vJG81zrTO4O9ubwPvJC9jLtAOYM6n7w8vam3MD05u/48tz2mPFQ9xLuTPcE8hTyUu8M7Fb3tPGq8Y7ycvXA8iL03uyg9TL04PRG99LuYO1+8Pb3dPIg8Fb3VvBe9Mr2uvL28M7s5PPG8zTq3PIG8lTztvF86zDyNPBM9b728PPi7ujtMvIa8w7xfPHC8ED1Qvcu8WD0bPR89hj3HvI+9Dj3BPBs9SD1yvIo9Fb2hPO26c72zPKy8n7wLO9Q8Krysu3y6HzzxPBU8ir3SvAm9F71DO5y8wzzZOsa8GTwPPec8FTzRPBK8jzyFOg49s7xSPBu92bsyvT47tDsPvZ48nLx3PX47ATwxvIg8Sry8ud87ZTwDPJC9UL36vLI7Or0gPc09gTx2PQK8ojweOyq98Dw1O0A8TLoavQU9cz38u+m8kb1FvFW9wDwtPEQ7kLu6vAk9qryJOxi9ij2NvV68tbzQO+C88zxKvHU8zLyBPA+9qjwZvWW9wbrdPLO9lDxrvdi8lTzLvJw86DtRvLC8+jxYvfo8jDwrvS08WbzJvIW7lD2VPBO99zz8PIo9hbtTvYC8w7wePco8ZDslvU48Nb2JvYs9N7xVu1I8Pz3qO+68DbyUvMk8Dz0cPRG9W7yHvW88jLxhPXE9Cb3dPNS8lT07vXC8MD3AvPa7iLwZvGs9s7wrvQO9qzuCOq28wDz2vNm7DL07vfw77jwkvA29MLz/PGi99junu8M9nLySvL67Cb10vZe7Yz1GPLi7kz1ivXm82ryGvBu9yjwzvRI9Hr0Su3o92rySOxG9xD01PSO8tLyvPbg8uTqyPO+7DryyO+G7lLwqvYe8Bz3dPI095Dq+u3o8ZzwLvuG8Fj1mPf68uzyNPX28YjyiO847G7yLvFq9Yb37vKG8JT33uhC9tDv6PNQ8ETqSPHY8vrwePdO6gryRPOC7Dr1wu9g8CD2BvZO8qzyhPM68WTy/PFo9qrynPDw9TLySvL08JLwFu1m7E70Evas8RLyrPNe7jrwxPfg8H7ymPJC8+jwMPVg9Pb1BvYs9hzrpPMe8Dr2NPTo8JD3XvI09gjtXPRq8Aj0CvPw8Jr2UPcU8LT2Yvd08PT0NPTK94zxzPbw8zzuXPcy8VT1zPSy8urz/PEY8hboIvaU8Fz28u5a8Mr2/vL+86zv2vNC8ID1ZPNa8m7xpPDe9BTwzPb88Hb3wPKw8o7zGuqS8lbzFvJ26UrsKvR68wLwKvUo9vDzWPPO8sDqDPbc7mT1TPIY8Db0EuUU9oDy/PHm9vrtAPQ09Bb1HvC09YTwOPTY87zyKPQa8irxvPJ09JLyEPVm9SjwuvdQ77ztVPCo7Aj0nvLs8nDoYvQc9HrwhOxy8Rz3OvEY7AL0SvXs8/ruEvag86ToZvQe9xbupvD89zDx/vOI8MLwavQy9Hr10vC+9BjwTPLw98bt0Pba87TwKvU096jugu429AryDPFY7Br1+vDQ9tTyFvDu74TxhvQy+Pj0eOqE8yrvSPIW89TqKPSM9DD0UvAe8Lj0LvVE8g7yRvIw8sDyLvCm9WzymvCa8Kj3gvDC+rjwGvDU8MT3TvIi9V72fO4o7Ub2fPKy8Hz1DvOQ8SDxxvUy8H7rkPGA7JL2zvbs8CL1/PZQ7lTwbPcK6aD1ePc28QL2xu3q7JL12u6g8Tr00vEE7P70+PZW9A72NvRw82b0bPeY9eD1oPjU8773Pu9+7rry0vUK9SL6APQm+Rj64PVs94boVvli9gL3NPTs+47tRPaa9qLxlPUQ+5r0Hvto9ir3cPJC9Kz0tunG9gDzYvFQ+77sbvkg9lLwGPVE7Oj0+PVE9gb0dPpY82z25vYK9xL2Qu4s8IrsmvuQ9sz09PoW9xb1xPfI98b2wPIU9ML0Gvv48AL5sPR+4/LxCvQu+Dj6tvS09AzwUvr49vTw6PWA8vT1jPCc7pLxTvMu97z3MvMY9db4JPnI8nj2HvLq9FL0jPsW87L3yu4K9IrxZvVs9TT6QPcw8+7xWPZY9FT3bvRw+WLuZvTe9STyYPfw9Ib0pPQO+kz1+PXU+/z1IPCu+WL4+vl49Fb1DPpi7+D3DPDC9Zr29u4W9BLwwPU2+Ib22PUo+9L2gPek9njx3O0G9Bb3vvdw9lT1UPhI+nL06PbA9Vz1IvOM9uj05veU9zL3IPU++XT1APIO8+71JvTA+BT6YPDW+XT3DPbm84Tw5vKW9hT35Pdu82b1Gvfw9E7zxvVm9cj2hPYg9Br7DPQo+jz3dveg9pLwgvT49LDtNvTa9Pj3fPRa+Cb0oPBw+lDsvvYK9Qz1CPbC9yz0Kvi8+J73zvb27G74uvp68bD2RvYy7ML76Owe9Lb4hvgQ+QL3OvFS+Bz4Rvq+8yTztPaa7bj3OPfi98D2NvX+9jj0EvKe90L2PvZQ9uzzEPQa+2rtgPsq9670wvEa+1z2lO6O9oL0Qvjq9HT46vg++8j1TPYa+1D1vuwQ9UT1/PH46jD5wPds6Kr2VPBK9Zz1zvQQ+GTwLOoa9Ab7YPQG+Kb7DvCi+jL2PPQQ+UD4RPtw80T3jvFc+DL1UvSS8x71ivbI8nj3KPF2+Cb7jvd+96z3BvXK9DT2Jvco9vT0SPVG+oD04vSc+YT0svhm+Xr3eu8S9Gr5yPQ28DTwoOyg8m70zPvo9ar1CvPM7Gz5KPL08cb20Pbk84bsrvm88FT3FPPu9mz1MPXW7hL2mvQ686DkfPZ882Ls4vUQ+oj2Lu608LT68vJw8mT3qvfE9tT1RvRi98j0XPey93r1pvCC9Lj3EPRS+g70BPmQ9Pb2CPTG9FL6kve29sT1cvA+8+b0fvpe8gTwgPVg9Ob3/vbg8Tz17PfI8CDulPVC+kL56PI09j72QvJm8Br68PJs87T0TvYO9QT2yvRI+1D2zPUa9q71nvni+2bsKvLq9Kj38vT4+W71Muza9ijuMvXc7wD3JPJW+5z2kPOm7kr3OPXq8B76Uvbw9hL0Ou2Q6HT3qvHW9eb3BveY9hTzJPAg9uTwjvuk9+z0iPky9Ij5lvck9Dj26PaQ9iT4uvTU+tLzlvAK+B71RvIW75jtoOcC9Mz2KvJs9Wz0JvV48QD3uu4+9tr0JvQo9P77YPTQ5IT0FPXO99b3PvZI9k730vQ67w7wfu8g9CD63PCe+A75IPp69Hz4SvQk9Vb2zPJ88pDzOvcs9aL21PS2+br2uPCs+vjxGu8g9zTsOPqw93TwdPtc8hL1jO8W9F74OPY+9rjyauzo+nryPPGs9RzvVPdS9Pj35vdG9pjz9vbU9MT2GPQS9Sr23PdQ8lbzrvbu7O75yPdO8BL2ovVg91j23PdY8Lz5TvuQ9zb38vFs9HL7IvAe+C72ivfY9tb0Wvgg+Ar5PPZm7+b3XvGu+Iz7LPRO9jD7nvEU+uj0bvVO9uLzPPeC7CD7QvZg99z0+vcy9cLwOvQ69Wb3DPa08Iz4svpU9rr1APc68qDzVPYq8Rr6EPWO93r3ePR0+Wb4Jvne9/bzcvaW7A74cO5+9lr0rvHa9tD0pvgq7PL4qvMO8kL2UPRY9Yrxevda9Szx8vIM9o72SuXY9rr29Pcg9Ur0APD69jb38PBq8d7zKvS48pb2YPma9sT0mPCK+JL7uPec9+Tw1vG492b0iPti81b2PvVu7Ur2sPXW+Ar7HPRu+E70qPha+Nj6BvSM9kD17PZk93T0mPhE9j71BPNQ9IT1GPrq6lz0FPgC9zb0IPug9hr1dvNK9BrxxPZC9Yr2YvTs9TD6CPZk9OLwFPhM9/ryMvDq9XT1KvJ89Ab3DO4U9wb13Pb27Sr0vPTw96rsYPkI97r35vME9o7xUPMS7mb3lvQ69Lj22vRc9Ez3DO5a8/L2IPco9xzpuPHu8k72OPZY89r3oPJM9d726Pda9Er73vFy8+LmHvQ49OL7NPTy+2jvxPLy9V72LvFs9Pj4tPcK9AL4DPTW+ID0GvrY93r2RPZa71D0gvO48NDwvPk49Er7ovco9ADyevYO7pL3xPQY+Ij75PRM8iL2Jvrg9mbyXvJa9DT11Pck9r70VPtS9iTs0Pci9Qr0mvY08o7yXvYI9jbxoPTC76bsDPSs9+zz7vSE9B72EPbs8qDyYPYK8+bwcPcq8M74TvW+88ryFPbS99L3ZvWU90L0mPnM9qTw3vlo9Mb2BPQE8Nz0qPUE+5T0CPSu8JL2gPQC8iDxEPve9KD4VPsm9kL3lPe09Jb4TvrY9rz18vE+8ub3QvcE9vzwZOww+Ib2iPek7gjwovsC9dLxBPTW8Cr3/vbM9hr2/PBK+wD2ju8u9J7wQPg+8pLzlvb294T0ZPlK7l7wgvYS8BD4RvR09NDyfvKA9Pz00vrw9nL3oPSs+kTyIvai8pD0SPAc+nT2JPbO9CrxPvS69y73cvIG9Kr0GvhO8G76QPC68H77fPIO6Lj4kPla8MD56Pdo8172DO6+9dD3APas8i7yvPHk7TL24vQQ+Jj5aO869tDqovdW9Fj0QvmS+f71AveS8sD1GvTw+ib1CvG+7Gz60veW9O72NvYe9vzwfPsU9vzsZvZ67cb6CPLY80j0PPpQ9iLxyvRA+AD7YPIQ8fj0vPs29iT0ruyi9s70MPvW9oT0gvk09lD6hvSM+nL0jvg08zLwCPqq9+D0GPma+lb1EPUw94D1IPGW9SL1uPHa9u72cvSc6nzwrPm28iD1uvWq99LpRPqo8TjxgvRo7N75xvB49jTydu4q93r3wvUq9j73BvQk+ib3Lvcm9G74KvhU9uz2kPUM8sjxIPJ+9Tb3IO4+9iDyPvbw9d71rvVU7UT1jvXg9dT2WPZk8S72cvYc+ij1MPcu9/r1kPeW8ur0tvkW6C75IvgU9Mr72O9g9Fr1MPsm9tT3mPak6Yr21vZM92r1Yvay8sz1CPhE9Yr4JvaI9q7wQvjC9LbyzPX29kb07vf28jb2qPQq9wjwNPjg8Bjs5veo9gT0ivcG5JDyIPRs+t71uvdi8lj3evbm8CD41PJ490r2qPaU7Tr0oPqS9gj0lvrs9xbxUvp+9ULzEPQ+9Jj6ZvJ49ljzFvO89DT2KPdK9kT3KvUI+vD3fvf083j2gve69Ab65vak9Jz0NPpe9hj3+PQg9qb0wvj29lT1kPdY9VbsFvt69Ar6AOJg9lj1Bvsu9Iz6zvFI+37xuPHG91j03vi092b0RvQ49Pz5pPME9BD6uvXq9Fz4hvKW8M73AvYu6n722PCA+LT0rvfC8GD79vSG9Db3yPSO++jwgvaO9JD2EvSc+AT2uPGC9rD26PVM9GD33PR89Nb2nPdi9Z72CvfE7HT3kO749lr1OvCK9Pb1vPgO+KD2EvJ29Cb3YPd677j0JPb69mr3xvLE84z3gPPU8fD74PDc9n72SvIo9jz5dPZq94T1IvH48zrw2PoU9l72VvdA7lbwBPqQ9N7w4PrW8gD0OPQ09BT2iPVu+zDqZPf28XT0BPBo+lj13Pt89rD2bvHG9sL1nvoI9qj39vIu8Dr4vvYu81b3WPPK8c72OvZ+9zbzFO869Dj4aPiq9v7wWPmQ9hryzPTS+tryYvUO9pL3SvZc9L74hvZg76bxnvbC9Xz3/PLu8X70vvqK9RL3RPJ89pT1AvDC+Tr0HPcq9973GvV6+GD4cPSK8Fr14vOq9BL6ePeU9Kr1xPHC9hbxmvco8Cb68vbw9LD1WvJk9kD1kPGS9Mz5rPv29ND0dPo89ib0CPbS86D3cvHK9WrsWvSm99z2cvKo9AT5wPgq8W73OO+094b2LvT09rTy/vCo8BzxNvTU7L76/PTE8qz0SvoK9Bj2UPcm9gTyhPAo9vL3ivEy9Rz2/vZ49PTyxvBC9C75fvga+YrzJPae9CT0Dvn6+OjyEvOC8Uj2iuwo9sLv8PTY7Oz2HPbC9mT25vE+8L764veE70jxUvT09UD1ivBO+TDqMvYG8nb0jvTC9mrzWPZS9v73JvRK9xD14Pe29Qb69vbe9pj2dPjS9nLxrPSy9UD25PI88Zb3svMi9gTwuvba6fz2YPd89r7xWPbu8qT3ju9Y9sT1Yvbq9gj3fPWY8Az19PAU+nz0IvhW+NL4cvaY9jrzGO3q8nrzAPaO9Br55PhI++j2CPRm+7TtHPf27pT0zvWs9j72zvPy9Fz7UPRo92T0bvXI9D75WPcE9aTy3uy2++T3tvb+7XL1OPZC+jTy/vaI9ND24vee9UT4oOw262b3Yvd49rztQPEs9ZL7Tvam9hT0lPt89HT7IPdg8wb2IO2u9cb3CPVU75b2TvJk9jT1nPUO9Mj6HPZm8ELyJvS29sL0hu6m9Az25vbQ9D74tvco8Rbw6viM9yby7vRs97b1BvD89Jr0wvFC9lDstvbW8VbyUPH0+kD5JPMe8Y729PUa84D3LvWc9y73avXm9OT5jPpy8jzy9veY88b2bvZc8lb2IPeC9Tj3ivOo9srvnvQM92D2qPZI7oL0qPYu9Wj6lPF29Pz2SPQi+mTsAvCa9Qb06PXq9Q75kPeY8770IvVo8bT2JvBk+ej0Ouvm9+T3/Pfg9A71JPu68uj2GvQU+RL4cvYI9KT0IPQQ++b0nO7U9NT0Mvse9vr09vFG9YTz/vQ88ez08vQe+4z1Fvlo9tD3+vL+96jwWvje93b2qvWg8tTyOPJa8+jo4vl8+8D06vs69Kb1YPqY8lz20vJM9hD3FPSM9KT3APQU+0j3RvDG8Qj6fPZI7Sz2XPQE+HL4JPlo7JTuvvYk90z3bvay9VT4JPew9TT6DPdC9sj3YPUo+Fb05vRi+nz0OPoI9tbziPT680j2svYo93zxHvSu+KbzgO8q8KL6RvHi95zyrPRg9mDz1vAu7UL3JPaK9Rz0hPjC9kb3bvbu9P73tPZY8sD2kPQO+dL06voi+D74zPd487D0uvvM9PjxiPgm+nj0yPnA6Z7u/PaI8Cj3gPag9Eb6wPP490LzMO6M9nLwTPPM9Or0WPPq9Mr56vSS+/rwIPiS+mDwGPqg9pzvNvaM9tT1jO5o9drvwPDu+2zzUPfy9qb3RvUE8mzzovbi9m71EPQw9pz1YPl49oL03PiA+zLsLPjK+5r2SPSa7cT0cPCO+zD2Fu1i8QL5mvYm98D3vu329qD2pPUe9B72MvEg9XL0BPcU8ET0Cuha+/zzTve49sT0Jvf+8+D04vlq9ar0MPl+9uj3GvOi9Ub1BPXG9jT0TPlY5Dz6ZPAu+WL00vRO8nT2FPZA99T2RvTC+EL6Kvrg8P72evS09VD1Fvkw8MjyHPSe9JL2lvZw9kL1ovdE9CD72PaK9/T2HvSU+cDw+vFq9lLxZvWq9hz0ZOyC+LTy3vQ+8qj0jPpw9Tz4IvqU9Dj1dPEq9iL2avTQ9zj17POO91z3EvFi9HT3pvWA90Ly+PeY99L1tPZM9Kb4kPPw7AD7XPBu9Z71kvvY9STx6PsI6QrxNvgi9ej2fvRk+Db2FvY+9Rj3GvYk+KD2lvYM9/LxQPLI76D3IvAo9Hr3RPQ29wDxsO5E9Tb0bPUW9Cr6BvYq9qb1zO7O8DD4kvk89PL7XPKA9lL3dPWw9gz3gveC9uT0jPVm+cb2aPZw80L2mvWK9Nz5fPfi8Ar7rPYa8hT0Cvqk8BTzNPcU8pD2fvHU8/buBPRo91L0APde9Dj5YvcG9mb22PNY9Kb5GPWI9fL2SvWu9pD1LPis9Yb7Bvfc92jzQvSk98j3BPWS9fT1UvO08Cb4vPUG8ML0JvaO7eD0aPUg9hjybPaS9EbzqvYY98b29PGc8Bz7MPf+8pr0Mvg89jzzsPca9pb0kvHM93j3ivHa+gD17vQk9WbycvOC8Gj2vPSs+sD3UvAE+Hz7mvTg+S7ykvBa9+rvqvbI9vTzFvTo91TyfvQO+/DwBvbe9SzyivA0+Gb4FPTE+er5lvNe9Gz2gvZM9dL3yPQk9BrzovaG8br0TPuE9lz2cvAK+1rwDPpc9EL3cPZq94z00PtS7qrzGvBc+HroHvpO7Eb56u3S9Sb4HPhc+PLyjPf89YzySu/a8uz0Svvy8gz3xvR485zsGvL68Jbv3vbE9SbwMPa09AL24PUk8GD6dPAc9Cb6WPWi9Nb1aPZK+Bz7UPBw81z04Ps87OT0kPf89qzwcvsi90j2mvES9iTykPLi9uzwIvXA8ET7YvBm9fb2dPho+n70YvJm9XzsJvTq9yL1RvYc9dL1TPvw9xL1dvau9HL0svp+6AD51vJ+9fb64vWO9WT3rvZc8MD6HPcu9Rz1mvd69zz20Pfc9kT0PvTU93b3CPAc9372hvQ28W71ZPBK+Fz1oPOA8Lj0+PHG9wr1kPey9mz0wvjY82T3rPCk8h7smPZs6dbzbPbW9wj20PZU7Xr6mPXI9hz29vds9xDwkPbm87r0BPZm9zL2ePSU+Ib4YviM9Ej3ePYI9d71uPts8drwcPrI8VD1OPue9nb1MPcQ9xL0wvbI9qb36vQG+jT0CPNK9FD6BPeC7r7wkvgw9MDwlPY09GL3OvTS+lbyHPNy9CzzPvM08sz0oPVi9qL1gvpC8ET0cviY+i73OPdQ9E7xnvau9pD0VPgq+yj19PSa8QrvbvQS+gDvovcm9nzomPQo+yT2hPY68072fPJg9vj3QvZw8173aPc490zzAPYs8JT1ZvI09Gr25vYa99z30PBi9nj2VPSi97z1vvJo9lbyfvdq9Nz3dvZA9MT7bPS6+kD1DPU88zL0GvQu7GD1Mve69sb37vdC6Cj0gPai9XD3WvSI+MrzjPX+9HD0DvqK8hb7gvBG8Vz15PQc+/z0IvQ6+orwQPQ2+Eb3IvWU9FT26PUi9ErwYvYq+FT6xPag9g7xnPds9PD2Pu4y93b20POY8vTvmvVE9yTzWvU87/z2vvXK7dzs6PfM8Ir78O+w7aL79uyY+TL2nPXw9JL0SPX6+fryCPSg9jLwqvTW9AD4tPBa9XjyJPYq9/z2Uuyo9OL40PQc+FT6rvdu9sL2ZvV69Mz4JPsY9/D2pvLg9/L2dvbU9wr2fvMo9zjxwPdu99D2wvZ89TL2XvDm9zr1QPKM9f73MvUc9RTs6vV+9Kr3rvBw+0z2tvLw9gT0nPQS+6b19vS2+iL7vuiQ9oD25PIc85r3VPYU9Fj3wPdE8Ib5OvZQ9Cj2XvEu9a70KvuC9bL3qPQg+6bymPUY9bD3tvY69wb2YvbU8ELv/vX29Gj5hvYY8eb4rPhC+3bw1PF0+E74lvYG9iD1XPKM9KT1KPYc97LxevVg+fz1wPfs8X76/PdE9i7zGvYA8Oj23PYE9yj28vO09KL4DPe48YL74vZQ9UL26PRK88LyOvSI96rxcPSQ9Yj2nvDe9SL6SPUo9x72FPXC9Sr3DvOy8QbyaPd493j2ZO/C8hr1/vcq9CTwQPT49xT1mvYG9hT3GPPg8CT2UPMO8v71lvYK7ET4GvZO94j1APvs9+j0APqK9DD4vvrq99TvdPIm9Aj6Dvec9yDwHvoO9qb3DvIw94L2qvUm87r2qPUO+AT7JPZE9oz0Qvhw+wbwQPky9bjzVPci9Ab2QvBw8HL7VvYM9MLyhvIE8Wj51veu8GT6gvSq+gzwFvmy9m72UvYI8br0Uvg0+lL1Nvac9aL0FPtA9xz1TvAe9NzsFPuo9NbxHvjs+y73hPUg9grzYPd68lD0tvaS6EL0ePZW9vDxhPSc+3z02vs492D0ovCc+9L02PuW8o72qPX49djycPP+8zryPPQs9pr10PVU9Gr7kPOG957wxvfs9hj01vha+0b0Mvgc+nT1DvRo+A745PR69gzxhPom9tT38PcG9L73HPaQ9iL2+vBM7Rb2zvUO8wbrYPYk9uD12PUc9ij2BvOC72D1OO7M80z0cPde6oL2ovL481r0xPKS7nT28u9o9fzwgPtA9l7rRO4637L18vSO9Er49vRE9Gz0wvgK9qDyCPSc9TL6TO5c+Yj2yO5S9Tz2VvWm9BT3lvIE9Jz7tvUy9GD1gPcm9xL3OvS49Gj1xvbU+Aj64vA++mD3Cug2997uKvuc8KT2tPAw8Kj4uPT+9vz1ZPYw8GL1qvfy9Nb6cPdG9LLqBPWg8tr3IPeq7eb5yvMc9NLtlvYQ9Tj7jPAg+gb0VPAy+ib2QPT2+0L0CvYI8qLvYPe89CbxIPTA+BT3SvQa+HL6BvTc8G75BvbA9Mr3QvQy8H72OPYI90j0VPbO7xT0fvRE+ODxgPfM8DD6WvYu9cr0EPf097T0yPQW8Nj5NuzO9Zb3Vux880D2+vAo9MTzaPAe95j0ZPMG9sTzovDK+yT0FPj497zs/Ph8+Y73ZOwK9hj0XPrM9Ib1QvAC+OD0EvfW9Y70UPpw9kL3evSI+GD7Zvc69GTwMvha9Z74xPRG+jz0ePmQ9Fb6EO2g8hD2jvc290by5Paq98bzqPM+9GrwHPgu9Lz6gOx6+jTwSvf89QTzcPW69Bb7IPYm+qz0avuu7pr3EPRU+2L2LPOU9krupPSE8pTysPTI9LL0iPBO9g71NPf28qj3SPUK8yb0dvhw+EjwXvYG8q712vo+9Bzy2Pde9lj1BvqE8GL3UPNs9rTzGPC2757yDPQY+hj3AvDK9FzwsvdU98r0XvDI+Fb4HvP68JDxFPSu9Gb7SPSy8RT2+Pd69zrwovDy+bT0EPqM9gb23vEO93LuNva29zjwEPki9H76tPRC9Ej01PNq9ED4UvvQ9gD0UvgI9wbxHPpK99z2FvjM7H73OPV+8ITuEvae89DzEOzK+MrzwvEk82D3APDC+/70Lvpo9+z0QvSC8Z7w5PEg9pLwNPDi9UD1TPXO9GL1+Pjg8G72dPSo9YD6XvcA8gTzRvT2+BD0jvjq62zwhvVM8y7xqPQM+hDsiPoY8AzyKPG893ru0PQy9NT3fvUG+T73YOw0+jrujPa48xb0hvdU8pz21PVy8AD5ovdw9T77VPYM9Hj6jPQc+573ju7w7VD0cvVK8gz1NPI49WL6WvZg9H70zPXC+9z2ovY68v7xQPSS+VL3zvQy+9T2avQm8nz63u6u9tT1bOkY8Tb30PXy97TxnujM9hr2Gvbi9tL04PmI9Gr0VvYI9gz0rvbq8Lb6QvQC9vT3UPYc9UL6Jvim9PT0avim9mj0xPT88FrySvfS8NL0hPgo+3zw0vjy9Yb1oPUU95T1Gu5e9hz7pPUm8Kz21vXM8kD0wPi69ZT1DvQm+FD48vaU9cr1zvde9+L25uyQ9Fr5NvBO8y70cvpq9cT3JvXC92zxDvRW7jz3RPJE9ETtAves7/738vfi9A764PRK9IL7uPS29rD2JPpk8UT1iPQ27xjwCPsI947x3PWc9Ej2DvQ+9C72TvQY9JbwrvfE9wD2oPXO9Jr0xvhq8Db2CvJc8SDxevSe8l73CPUG96byiPQC+zL3Vu8U84L2EPZu99D0rPYM6kbyPvdM96T0nvS2+WD7AvTW907xXPQa9tbx5PQI+ib2CvTy8cL2rvbW9uDy1PAS9bb62vVA9wTxxvQC+Ob4SPgK+tT3KvOe8mj28PeU9Ij4qPQg9mj3bvXC9nbzsPTM+nL0aPgO+rL1evbK9Mr6JPtA9Fb4wvYg9Pz0Dvp49cT1jPnm8hjyFPRG+Sr3QPS0+JT4ZPW09Q76BPM695T0MPpu+2j2wPea8/TsWPp08y700vNo9Fj4dPto9Qb6gPcs9lbvOvSU9SD1jPe89grxdvf+9jjypPGE9A76yvZw9PbxYPtG8WTr7PYe9+zvxPfK8LjwWvTs9QTyLPca9gj3RvHQ8iD0rul47l709Pae7WbvnPUc9tD3lvJu9wbyJOoE7Cz11Pai8Tr0NPZs9Jb6cPgc9Yb2hPTU84TyfPe09Kz2kPXC7V75EvQY9jjz/PVM9aT6hPlU+rz3CPa090DzHvdW9CDx9PDS9Ij2JPQW+mryLvS+9dj3tvas9pL1mvqI7OL2svR68qL2DvNG95L2bPfW8pztWPVO8Gb7rvA+9Zj0avtc8xL09PSq+srwKPhc+Kb0uvYA89T17vV28rD3nvb89pbzAvBy+Frxpvf88c70PPcY9FL2FPc48Q74evnk9JL7XvS89f72Rvok8+7swvhe+Ir0OvW8+Gb6PPQm8QT4fvbk9ub3IvJE9Fjw6Pc+9Ab1Zvay9OT75PPi9gb37OzG5pj1Zvbc8gT00vSk+3zyVvU69Ab6lvUu9xj3qvfA8lr03vom9v72cPdM9TT2QvY69tTxpvjY9gz1yPaO9Ez6lPYW9Kz1BPZy9iD6nvSC9aTzTPXS7+DxMvX479jv8vFY9Rz6bO4K9870KPhS9Or7jPXu9wr2PPOi83D0FPYA9kD2pvDY8VLzZvDu+eT1LPSo95bvpuqs9Ir1Tuhi9PL2Qvdi8Er6CvBg+Dj7mvXO8sb0Vvku+e7yxvfM8Uz4FPZs9t7xnPTw9pT2DPS6+Bj5xvbi9zj2Zu1+8lzzDvLE5xzzAvXa7c720vAW+gb3WPWC8UbyKPJc9cbskPn09xrz/vBo+0j09vhG9nryIOQm9EL4oPQS+BbyLvSc9jr0cvsq9vr1bPYg9uL2svKe8Jzs9PaG9g705vUg9JT25vSI9hb2RPQa+nT1fPUi9Ir40u2q9Ar2sPVM9kD3dPMW9bL0BPeG8Fz5nvQe9t70qPgY+WzyLPAq+BL19vaW9OL05vBu8j7xyPBg+MD2vPDQ+iL2pPBy8Jj60PVc9Bb6/vdy8R7ygPXU9SD36PBe+Yz1ZvaA8eL2JvZO9tTwCPW49gb37PM29A7sDPlO9uDxNPP+9yz2vPSq9bzuHPY29jLxQvRW+Vr5rPZa8Ab1RvbY9xT1wvc+8lTy0PQ89BL0BvZq8hLtgPtG8sb0qvnc9jj2iPUi9370GPYk9Hb4hu+E81T3Kvb49RL5FPm49lj7Avbu9UrxHPZK9xD3avQg9Nz2fPb49jL0TvRY+J74YPpY8d7yHPR09HD5Qvpg8kL1Cvus9172rPB68b71EvYM+qLv6u4e9+jwOvi09cT2/u489SL59vYO9z7xEPRU9lr3JPbe8T73fPTs80Lo8vK49ZL0VPie8YT0LPky9iz0ZvYQ9bD0XPvI8yzxSva89F75VPV29BL4LPsI9oTwkPh4+VTzLvBy+Qb5+Owk9OzziPNY95ryMPXo9yr0cPdI9VD2LPck8Ar6Pu0u+W73VPJC9dr2HvT8+AT7jvf+8qz1HvaK9wbs/vue95T3gPfG9h70XPYU71D0avus9z70Qvjy9Fj0jvMa8Oz3nvMM9/bsNPOs8sb0TPrC9FD2fvZ09kz2XuYg99Lt5PS89Cb7SvPK9nDyou589Tzw7PSS+8Ty3PZs9Jb0dvpy9Zz3PPQi8qzhmPYU96D2mvYQ8/7tBvhy9Rz2+vfa6Cr7tvR8++j3SvDS9cDzQPZW8vr0OPr88CD4Cvoo9mTyxu448y73rvV+90T0PPQO9ML4mvcy8frsqPE69hj2rPBQ9ML3Tvc+9kryjvIg9+T2AvLU9Cz1fPi09kz2jPE09HjpHvXC+F751vn6+NbzevKq9N75ePUQ9LT5LPsu9wTwHvha8BT0Uvo+9ST5bvJi9hL7+Pb288j2PPHY7Kz3SPZ48kLuHPcg9mzxuvaq8yLyjPCY+rL3QvaY9z70FPQS9Yzssvm49tD3svd285TwHPUw8kz0vvXc9uL0Hvke+jz2YPMa8vb2APKW9vL3VO7c9qT6FPYU9ubw7PfQ9IrxhPRY6CD4lPoY+Cb60uhU9Yr2wPRE9cDvxOyQ9OzwEvcU51rzYPZI9mb22uuw9nz3XvHe9zb0FPpe8Kr28PRK98j2qvRk+KT0BPj49pzv/vLc8z73Du049Dj5APmY9Ob1IPcU9Cj17vTU9t71zOto8wzyGuz+9tj0BPhc9Dr6BvdM8Rz5LPRY9mL3+vay68j2Cvbk9JzzYveK91LxevUY9qbxwPO89wDuUPQc9EL5cuyo+KT2bO/S8/ryFvSK+Fz1hvS297b01vuq9S70Evky81DxKvoG9Ybz6Pac9G70OvD+9ULvMPRs9mD1hvQ89673nur888T0GPZe5sb2SvTO9MTxMPdi97z2KuoK9Pz11vSQ9dD1APbk9yb0oPEw8Or4tPiI8jD6PvFU9/LtmOyG+oj1JPac+BL55Pew8nLnOPZe99TwDviq9Ur0kPaw8az0dvvO7Oj2PvFi9kj23vZc9Crq/PaG9k70bvt+9mr2zPRs+cb2SvQE+Gjz8PCM9Kz7GPXo9Zb19PQO+lL10PKK+Cr6cPT67GD1MPGi9t72kPTS+zLyQPCy+3701PQ296L1pvYS9Nr6Gvbg9p712vB6+bT11PQ6+2rz8vK08tD1HPdC8TT7UPcc8wr1zPMc9Bj1MPdC9O74tPjI9Az2cPXW+jD00vZM9ybyLPGQ9ND1zPmS8A7pzPSw+EL4Rvhc9mb1Qvcc9RD2cvcS9db0mvQK9ED2evKI9qz1jvbu8Ib4MPcO9/b3nOzY+972VvHi8QbxDvda92j0BvbQ7K77oN8m97r1rOzs+wz1aPQ2+TT4kvsw9ADzFvQq8zj2/vaY8Qz4APlS9+zwkvcO9lzoVvXg9vD3qvCy9OL1xvOM88b2SvDk92L1sPRu+7rwNvUi+Fb4RPhA9FL2UPQC9RTywvQq9Nj0DPmc89T3NvM497TwrvEs9Y73RPEe9jr0nvYM9lT1WPga+KDzZOmY92bwcPY09+zyWuyC+vzy1Pam93T06vbw81bzEPfS9Bz3pvEg9q7yrPZk9Cj6BvDO8gjvVvBC+TTxwPQU+6jylPYA8Fb4pPhW9nD2tvdi9/rsGPug8JTwuvvQ90jl9vLY9+LwWPu49Rj3NvXC9BDsQO7Y82L1dPC28GL3UvOM8Fj4gvv+8EjwKPoI9zr04PZ09FL7VvYM9ZTzzvfu9Gj1KPta9kb0ivQw+Gb4xvXu81zwzvbM9kz6evAo+3z2OOfQ7Mj7qvSu+vT3QPYy9eDw6vQe9xb0ZvuM9Az6svRw+OD3uvMO9Tj2hvAu9ED4TvYc8mzx8PW0+ED4cPkI9Ir0NPhW9/zy1PXm9FD4Evky9zLsTPWo9cr1BPsK8mD2gPA2+w70XvR09w7ypuzU9l71JPAm+b72oPXy9Wr2fPQ29lb19POI7+rwovuW7hL7XPNw8Yj3tO4C6uTyxPf+7OT36Pd28Ijw2vQW9mLyJPfW83bxZvjC+GD1gOsi7/z26PQw9VT2NPBQ+Xr5Jvd69vjsUuwI+0r2xvaW99j2Avl08zT2nO389rz3OvY6+pjzCPRS+RLwXvYC9UD4pPr48GD4uPZ86+z1zvVS+ir47PBg9dr0bPBk9Lj50PfM86Tv7PUq8XDvmvVM9nD3VvDS+hj1sPQK9Ej3JPYW9VT2IPas7KTzAO488V70zPOo7Mrx4vME7t7xUO/e7zDy2vIU6gLueO6K8d7yuvCq9bz3RvI48yLzWu9M8E7sWvCY8srwoPfa8LTySu6w8YjwfvBM7vTwcvN68VrzGPCW9TLwfO9Q8kbyWO5k8n7yxvI68DzyGPH88nzrZuya8C73kPIM8c7vauxQ9Cz2sPE08LLx9PIK9BbyyvHM7Aj09vf+7srvOvA88Ez1HPPq6AzyvO+k7xjzcvEU8Jjz/PHY8YrwfvN07nLtFvAW8tzlWPJe8Ub2uvGq8hLyDvDk9+TsHO0+9yzt7vCe85DyLPMQ7VzwxPNk7A739u3O7LTsCvfo76TuQvP46OL1VvK88xbynupU8+zm1PI68XDtEOkw8MDrmPMq8ojyfur08YzxuPOC8djwRvSQ8FD0KvCS+XbxHPIa8R7rgPLm89zyvuqg8prswPAE8xTyGvH68sryqvAk9szwzvOY88zzBvKM8izx2u169ITujvBK96zzMu5i8C71wvMi8w7snPSC6lDyXvMi8Hr28Oxm8gb0svSw8tjtZvLc80TuBvCg8i7zFu7M6EL0vPNO8Cj2NPI28lzxjO8O7TzwWvIc8OruYvJu8oTz2O7i8D7yDvIC7DD0hPeY7sLwSPeA7J70NvYq7Lj2XPLe8uTytvEs8Dj2CPES8WbzLPBY7d7wLPOI8C71Du1c9+7yUPAe9JbxAvYM8TzzdO/G8+7ysPH48iryPPTK7+rx8vB08Cb2DvYE7Dz2GvIs8oLwGvQI9TTx5PKg8Sjwxvf671zqwPLG7DrydvMk8lbwlOaQ8HzxxvAu9JT0FPQO9vrwIvfS8nrnBu4e8B72LvAI9fbxhu068aDwuPJC7LryGu7Q8frwPvDq8Nb32O2C9Rjy+O9o8RDwBvNc8gLvbO507D7sMu5Y8lbwau4y8k7xjPDm8Ur2cvI68mjwEPRM9fjyCPYM9zjytOyA94zyxO4s6gjzFPE+9KLz9PLU9DzoHvEg8Xzz2uze8xLx4uw29Lj1au5+7y7xAPYe8ojxMPMo8PTwGPbG85ruqvBC9HbwEO+s6Db32u0O827xiO8+7Wzydu688Nr0Nu5I8J7pEPc479rqPOyK8gbuBPPY5WjwYPTc7fbuxPDM7pzvlPMa8NzraPJq79byHvC08mbwevIU87zyQvNi7bLukvPq8dzqDO1s8HT32vGY8gjw1vXQ7P73GvI48rzzHPCa9n7yPOp68+zyxPHq9XjqnvDe8qTvduxC8gzuLOo67djv4O4g8pjwUvM88ODwDvcK6WLxPPQM6y7kQvRQ8urw6uw09xbwAvaC5yrwmPMe8grunvI68CDwJPVC8HDycvTA8lTwBPSQ8wrt2vJk8ezzTPE262ry8PLq8h7vNvMK8Sb3XvD87Z7xHvEc7kDzgu9k8RzyhPIG8VjxSu2i7nLw7uoY8Rbz6uLs7GbtwO2u7tbtEuy08pDwxvNs8DD21PMe8r7uNvKE8+bqIvN08izwyvak88DwJuy69Abt7PDg9CT1KPEM8hbqNPJ865rzjPFk8hrvYO2o7xDzbPMM8pLu9OyO7u7yMvBc9rjyWvMo4cbwFvBA97zw5PJG8XLziOx49qzyDvMq7SL1FPNK8P71/PZY7DryIu/+8gDwAPfy8Ej0JPDO8Br0DvfE7ybzhOwA9ajyYPGO8ljyEvDW7UryTOqg8LTzjPKa6Qz27u8M7LjyOOzK8zrv2O7S8zjtRu+Y69bxZPJ08Gbw4Okc8i7x/POk8yDvIvBE8Wbs+PKs8gru+vCu98jyhu5K8dDySvNQ7wboJPQI98Ls6uxA9ODwhvTW7Q7wpu6g8ZbtDvQK9OTybPJQ8VrwxvHc8dLxUOg69Zjy6vO+8CD2evCC7u7z4uhI7hrxcPKq8JT3puvG70DzmPAw8/jzEPCA9gz3TvJ28tjyJPDI9wjyePCk8lDw3vCc8n7zsPIc65bm0vEq9WroKvDK9irwBu6c6xLxMPI27X7wwPD09PjzuvAS9hzzOvMW89DoYPbe7S7uSPME8srwtvEO9o7x1vKy8+T1IvAc9+rugvOW8VDhhPPe81DshO+07jryrPAo7/7vRvPe8ETrHPOA7gbtZPI88sLypPOk7Ljrqutg8kDuzO+g8oryuPA08uLwQPKK7DjzEPBI7i7ZAu4U7Iz3qPNC7NzyKOwQ8d7wOPJG8XDpXvFQ78rwfvRo8tDxAvCM6Cj3CvP082bvKPJS7Ybx9PO48rzxxPAG9DDsNvfO7iryGvHU7ET0kOm88rLwNPQk8F72BPUq9GTyoPFi8GD3yuyO9kDwPvU69PryYu3K7Vrs1vS87lDvlPFU8k7yYvCk8ozuMu608tbzuPOe8Bb0qvSO7Q7xxPMy8HbutuiW9lbiyPEa8DD0jvKW6JzmXOw04PDxgPCs8ojyROzM9S7clPTM7O73wOiE8qzyXPK08pDzyvKM8HT0hvfA8O7xnOyw8ljzrPCU9xbyKOk08rbwGPNW8vzsKvA+847zZPKg7b7odvJm7WjtNvIS8BT1wvDW8JDzdPAs9u7w9PDy6mDxjvDO9MbtUO5a7qbv6PCm6MbwYvBu80bzFvNq8sTnIu966frqoPP48ubzovCG9R7yJvPk7x7tIvMI5dDy1PJa8b7ybPAY95TuTOvU7iLwMPRi9nTylvC+81Ds0PCW7Dr2Gu6a7pTwrPI48xryHvKa8N7wpvV+7XbxkvFc8Lr2xPGC72bzcPGq767wUPKM7Cz1JPAm6pDxIOku7xLwUPV28cbxUPWs8CT3wO2k8f7xuvJ88WjzNO/I7ML3wO4g8+jy9vEQ88ztSPMc5IDy8PMI8VTzAOke7STvMO1q8xbybvFe9f7yWvBg8SjoDvUw7GDwoPDS8pDzUu3M7YTrdO+i7i7rEOwc9Sj2mPAK9e7zpO508Jb0kPE88R71BOm88p7uCPCY9nLmxvFU9TrwvvTw7f7v7uz085jwivUm9urySPAk8A71EPSG80jvru5u7iryfvJu7ZzqOPP46TT37PCo8sbz7vEk8kDwdvBq8RzxgOgK9Hr3MvIa8rbyHu0287LwjvCU85TufPG68aTnxPMG8+LwfvD86iTudvOS8DTzLugw9szonvRi8oDy9O6+8ND3kPMk8gbx0Oze8K7sHvYw8nDy5Omk91DsXPRi9GL0OPOa8hTwku5G8hjxYPY+8DTzLvD88u7v+vDA9TzwtPKa6cDw7O0e8FLxKPLo8lTwZvNS8QrzbPHq6sLtSvFC7qTuQObu8Rr3cvAA8ijwUvcm8DD0/PcO+XTwTuoM7FrwyvIM7vTuIOzw8jDwaPM27B73BvKk87zsrPBC82TooOwQ9MzzQvCq82DuiO0+9hTylOmC96TzcOx88HD2uugs8QTyHvBg9ED01PKe7g7w4vPO8pDsjPIu8AzwVPKc8czuoPFK8Zj0LvAw9YDzUOp08krywvGy8p7yjvEI8GL2qPIc8RD0oPLE81zyoO+M8jDwbveM7hbysPGI8N7xLveG8fDzivMi5cTyHvG25yDtrPKi88rowvI+8aruLuoi7h7wAPeg7DrysPIi7bzzcPIq8gr3buQw91zsOPKO8HTyMPLK8LL1aO7o5oztfvCk9vTwmPI87gjsSPCm7Mj0kPLk7lTy+vCK8ojv+Ojw7RzymO4s8rLoQudo8M7vrvIu8Pzx/OzU8hTwkvLI8ZruCuxi9XbxRvGi8ELwjvaA7djylvNO8vTzbvLa8JT0HvRg8Ezl9vCs95rsrO8U8cbvevMW8G7q7Owo8kbzNvJo6ETx5vLA86bwEPBY7UbyYO+a7iTtKOxE89zuHusI7A7zXvMu7qzzSvBe8OzzkvA08aLtuvOI8wrt/vHA3GLzbPB+8GTw7PIa78zzcO0E8yruIvLK6A7sku0u8ZbvFu6M8VD20vNG7vrt9vaQ8az1yPMa82rxWu0g8FzwGPRO8nbwEvBM9Mz2zvPS8Ar2SvJ28urxLPOQ7qDw6vea78DwTPVo8wLyKvU+8LjvZvEo8Fz67PJU8RTevu9S7ND3gO2O8+zwLvLK6jLxePKM8nTz4uye9iTthPFa85bzavHE7Mr3SvCC9ODzIvI08mLuKO927qzsHvXK8qTzQPNe8VbxxvMQ867qdPPq7+zrTvPm6e7sgOoS8gzzlOc68Xbu+PEc8k7vNO9u80buPvFg5HjynO+U8/7qcvFe8F7yQvBw9gjyru5W8ujuhvIE8nbyaPDQ8gDytOuk857yNPL28hbueOho9XbyuPHa8Hj0yPCk8IDwIPAW89TyNvCQ8RDwNvG07r7zBu548ZjyvO9K64LxUup87Uz04PZK73DymOyQ7TbqwOXw9XDxOPAq8lTyMvAy8GLwdvJA6dLz2PB88qrqpu+k72DuJvQq8zrqqvI878Tw1PQy99LwGvdM8Er2wPIw6zry1vKC8lry4POK9oTtDvXk91rtcPEk9U7vBPAS87jz8Oio8aDw/Og+8p7wGvC68ljwhvd47IjyTPAu9q7ziPIA7YryFPMw8BD26PJ28sjz0PCC9DLwGvdQ8tLzTO5o8Vbx2vEa8sjoGvW099Dvcuci7wbwIPew5ULuGPB49WLw9vak8Pz3yuzC8SzzQO08747xxO408Nzvnu0o8wLlBvUU9WzyyOuO87Ts8vFm93bwXPSm9zDklPNm8ibqWO+88QryvPKG8Ab0zPdq8lzxevMM8yrxpO0I8KjsBvLO7brwCPb28+TwiPMK8gDxUvGC6NrzdvLK8Hb1Ju4c83TwbPJe7urymPOC6Djy2PIQ8bbzAuqo8XjzEu0m8EjyyvCM8c7rJPMI7ML0DOx88xzxFO188ljzmOps8XjxEPKK6Mz0zvNk8K7yjPEO8fzyHvPA7KT0KvD+8gbtBPE88E7y8uyM8a7qLvAo8qbstPD68AD2rvKY8C70DPU+8W7vnOsq8Cj10OzM9KD0bvYk7sLpGPbw897sePC08ULyuPMs8DDzePFc8Mz3jPL+7XrszvBg8sbzCPPw7BD1TvLK8ZTyvPOO6nbxNvNG8sbxovHm9ObxmO4I8R7ulvIE9kzzXPHY8ErzcuxS96DzmuY88TTwTvZa8pby0PBM9PjzjO8i8cL0ivVG7PjvDvJY8mDwTvA68FT1lO9S7sDzkO2Y83DqSPN081jwIOt88QTpAvKA8Mbx/OFy9lbxovCC82bsmPPs8tDyVPHW8g7qBvBw8wbuMPIA8hjyjvAW8Cr39PC69JTukPLy8mzyXPE07JbwGPeI7yDzauug7XjuDvAS9azzTu1K8dz1lvJc8ADw6PKU8hLtUPNQ8ALz/OBE96DwgPDq7wjmzPCe84jvOvJ07GLwOPCs75TvDulu8dbwvPIU8ID2DvLo7k7tjPOk607vcPDc8b7w7vBk7X7sjPeq8q7xzPMy8Czwwulc92DxUvSs8ETwnvQU9JztdvK88nzycPNM8cbx5PAK9Iz1QPVG8HbsRvaK8eLyNvP48FTojvVm6T7wovJ87yLx9O7e8qTtdPeu79LqVPJw8+ryLPNo6Aj2hPOa8CbxfPCk8WTgjvRs8TT2tOoU7cbqjPAi9hjtsO2Y8Lb03PBK9ML0VPBS8ujxuvAc9ID1tO5s7IbzRPO07azyePME7gjytPIS85bx5vAe7oDzRPIk8ibsbPIk8drzcPNW8/zsIvYi8gzynvHQ8Mr2vPPK7SDw7vK+8NzwdvDe9jLzLvJo8FT3Eu4S877xZvNw8o7ydPIo8oLupu2i7iru6uxk847w5vK66SrxcPEq92TyPO+87BrwivAY8TL1XPCY9NLxevKm4LDwlPNe8Pz3duiI8r7x1PLM64TyUO0Y8I7zJOwa9yTwDPJK8SbwxvCC8UzzPO/k8Vjy8PCG9ErorvfE83TvpOlw7AT0evPC8ozyMue28xLyxPMy88Ts8vAq8Mb3YOwm6mbyhOx+8JrwYvCU8mLxlPK27trx7vCE7iTwsPLA8LbyFvKE8D70gO/y8gzx0vA88DLvJOoY8A7yfvKU85bvQvJY8CDyAPeS8lrkiPFO93TwXu4o7lT3/PIe7BbnauhI8SLq6PGA7OTw3PCu8XTp6PLu7DDz8PPe9bLzIvGW9H7wEPQi927sbu0k8gDzuu6Q8CzyIvOC8sLx2OT484rwYvFw8BLzVPPK7c7ybu5E6iLzAO4C7EL0jva+7SrzRvMC78DxXvBS9D7xOOwu9m7y/PHU8vjp2PWK897sjvZ48yrz6O728nTxVPNo857uzu7m7lDz3O+c7jbzLPGs8kDyivEQ8F7snPMG8Fz0VPYO7bDwCPEC8G7olvJ47Dj33vBq8Yzs6PYU8wro+vfQ8fDteuwE87by7PPE7sDzmOxW8p7ztO+u8G7v3vFE8pbxRvAO88Ly+PBM9cru4vEM81DsvO9u8+zoXvNW8LbygvH28Zz0TvN+5Mro7PRe6Mrytu4g8rrxxPDC93btnvBy9qbyivFk7qzvZPDm8CLtXu1M8STwNvco7Vrr8PNg8XbyOPAe9q7s7PSK9T7spvFg9nbwBvS498DyEukA9nDy5vMi8rDyjOw882rvdPJk8XbxJvLm7CrsGvUU7rTyrPG66eDyXvJs8F7w8O5C8Dj06PXs8o7zkuoO8CT2pPFQ8FTyKO7U8I7skvQo8KL25uRC837uRPNQ8bDkkOki9ujvhvBO9Tb3MOso7F72bPFa8ozvgvMG6qbxhOwE9Ejy4POU8g7oGPFw87TttPLe8obsTvKO8GbyrvBK8Xb0yPKq6SbvJvK27lDkDvN087jvfu7q83rzoO0e90zz2ux+8frybu+Y7zbxrvKQ8nrqru9O8bLzCvE48QzygvH+86ryDOwg8xjsPPAK8XTwIPeU8oLsXPaQ80zwgPY05vzstPYI7cruePD699bvAvHo8y7wnvJU8abyoO+m6rbwQvBy78LxCPKA8VbwxPCo9pzyjvJu6h7ywPCq8WbvNPPy7JTyuO6U8RLuyvEK8lTx5Owe9dzxXPSQ9p7xdPAg8ULvDuhI8sbz7vAy8j7sCPVC7oLsKvM08iLuFvB09obxEO+08wbsMOnm8/ruuvHC6rbwUPAQ9sTwqu9A8GD3auVM8GTwEvC88wTwkvZW8Wr0sPZa72DuzvGg8Br01vei7jrx4O8O8wjtxvCw8ADxlPDm8jr1GvS29xTpzPJ87vjsKOww8mLsBuzu8qrv1u5U7Eb0zPe88GzvQu5k8nDwVuyE9ljy3vHA8sryuvFe8try7vD+9L7wiPTw9sDmAvJI8NT0MuSe70DwDvJO9FTuyvMK6BjyavOg8Az0AvLQ8orubuWC7Xr3hO5k8jLsHu7u4GryvvLs8zjyAu5K7xTwMvGu8QL28ugE9hDt6PPA77zrhPIE6vDxCvO87o7xtPDI8XTydvAk8xjwgvWS93TvPPM264zznOvo8Uj21vAU7+TuVPOm8Q70zvZi8B7zhu5O70LsTvIY8CT0Nvf28ebr/PJ28HLwaPFO8ujyEu988sTuUPAO9Tbs0vXO7szwiPBS89DztO6I8cbwhvA49QjzQvLG7abzHPAE8jzx+vdI8ijuPvEo8Fz1qPD09+TwOvDE9Gryiuxa9L702PdU5/zwhPQw9BT2OvB09FT1NOrm8Aby/PAo9vDw8PNG7CT1BPOi7cLthvAi9QDwQPAO9SLtQvGa8nbxpOY+8QDz+PLE8AjxjPE69zrpRvDQ8lDsFvJQ3ZrueOy83JT1+u/27NTzVul28zDxfO109MrwLOle7rbzEvP08Wb1APRS9vDsBPPk8X7xBPMU8jj0TvJq8ET1KPC+8KLzrPPu8MrzTvGM9iTyvvGm8ADwRvfG8hje6u2m8ETybOLa8r7xuuk47q7zrO/C5YLsmOgA9Jb2lPC69yjucvAW9Tzz1uZg8CzwgPUQ8IrsDPBg9ljyQun88rzyWO7u8FTyNO8S8hjtCPTK7CrvsvCu9T7ysvAy80jrhvFS8zLsIPOc8LDwjvIg8+DxJu1a8Cr2OPBE7DL3TvPW3prwJPdM8rDsUvb+86LzTu5U8GLtAPb68kDonvA69qDwYvU69rjwbPbU8oLuLPJk8wryOvCA8DrxQPIW8Kj18vRe8lzwfOjY9Cj07vZU8sLz4PAq9vLwFvXI8fDwlvYu8/TooOyy8YDwNvJe8DryHuym9lzyKPOe7sDypPGC8CbtZPEO9ojnJPI88NTyvvBQ9vzzWvAQ9gTwOuGs83LwMPEO9EjzqvI66Y7vOvM08azzdOQI9I7zXOdy86jurvB+9Ib3+vOU7Pbx4PHO7bzzePCW99rzUPCq81Dvpu4K7ajyuPAM9urwpPAY9djtMu488Pbt3PGS8WDsqPbi6Dr0MPXk7cDyiPIO8rroLu/m7Hj7BO2a9FL10vBi8IDz4u5K8zLwHu/u7AT3aPCu8NDysO4u8PTzcO588ZzyiPG070jzdvOM8F7wlPRS8irvavB49SrwFvc68xLwIPcG8mTwjOwK9gTvkPMS8JLxou2K8PDu2vAq40zzhOz68yTu3u2e8CrwYPJc8JbxBvDG7KTzXPGI70Dy0O3w8jT2xvKK86DvQvBu8Aj3GPLw8ALzPOvq8M7xxvI26rrvTvBM9yDxYOlO8ybz7uz68dzopvEg9hDwavNI7MrwOPIU7GDwvvRg9p7zfu5E7H725OwY9dTtzO+A7sLwpOj677Tt/vBc71bzqPAc7cbzyO3I8trirPI88rruxvHk6QTxhu5A797tfvCC8iLtMPIG89bsuPDs8IbuAvLy6pLyXvAw8+DwEPFc6zrpCvHu7NzwTPW66PjyivL68uzvOO6y7vTydOWA80TwXPCK747zqvCk98bwevR+8sLwgPYI83Tyou5E8o7xOvLU8drz2vAA92jp3PLw8rrw9PaE7XbyePPY8f7vYvFk7tjz2Opm8mDy8PHG8aTySvBA9Tb26vOk8gTxivKy8BbzJPKy6djxmvFw9UT3Quie8irwivAg8GL3IO+Y8I7xhPCq9cjzUuzg8trvMPKy8Hj38vGy9nLy2O6+8sjuzu/O7V7wgPBY9jLvxvLq7+zxBvIe8CbuAvJq8e7y8POu8ODzMu1E8ZbvMuzc9/zyzOtW73DuNvIs8xTqkPNg8gjyPPIm81rrfOzY8RTxTPNc8Jr1bO5c8qrs0vEe8FTw8O7081zxEPIY8PryUPNI8Mr3tvBm8ubwQPRU7wzyWPG6+KT3BOs87CT33vJk8OTzbu+Q70bwHvVq83rzTvNc8E7svvQ09hbwlPVq8fD0yvcs8wrw2vJ27vLx0PKS9gjyuPDC8WruiPBw88zu+Ogk9ijx4vDs85LzKO7W8yTuKvPu8oLzcPOo89zq3u5S8SrzcPNS7Dr0yvK+8cD1iO5w8ML2pO/c7bjwmvaI7ZbwIPMW8Mrs3vGU81TwCPJ+8zrwDvJi8qLzhO/Y86TymvBu89bydvPi7NDylOqK8brvQOxa91rwQuya8uDtbvQU7KD3iPNo7Fz2XvJS8JzzsO3K7J7zBuse8KDwbvA49PzxnPEG74bunPAO7XTs5vFS8IDwDPbw8F7ylPI88QjtTuwO7prwevZS77TwyOze7RbuMPDI8Ij2QPEo7vbvoOzQ8rDwZvE091rz9PPc7E7tHPGC9uLvLvKO7/jzfPMS7g7yAPNK4ObwbOx89IL1LPN685LutvDC8PjyoPE+62bxFPFm8wbxAO3q7fjwOvUa8mbzDO3O9lzybPEa8F70FPNa8Q70UPDA8+rwJuqg8DrqoO6G8wjwJvOG8WbvTPDA7eDw1vAo81zs5Oxc6uzxXvIa8Pbw2uzY9TD1Lumq9nTwivLe7Ez2guyO9o7yXvCU8U7vTPIE8rbzYOqO8mjwRPIo8Mz2nPJu7XzytOwi6KbtNPPA7rbyOPEI82ztlvDS8LTxZPCm847wePK87+TuUuCS88zmEOw883jy0u565PL3iuyQ8fjwyuk287rxTu6M8aDydO3K7XzwPPag8iLy4vNk76zktvIQ8i7y/PL48ZrxFPMM7z7w2vVg9GTmQPKG6Rz2ZPMC8uryIvCM9irxUvIc79LzYO8g8u7yzu9K7yD2HORw957yDvFw7Pb0YvG68uDyxPJw7QjwBPOE8cj0YvJU55LvfOwe8ibywPMa6EjyqPA44ADmJuyQ9UzuZvDi9sDzFOiY9ertLO7I9frubPCy9oTyju0S8MjswPQQ9wTzePH69LL3Hu1k6tjwcvR28w7utPAa96jxGPKE8A72avJA7d7xwunq8zLvevFO8cbpavZm7gbw4PI87uTyluXg9rLvzu0i8c7zdPKI8x7xYPGc7M7zju0S6RTykuy49PDzivLC8irwBPKg8fbuFvbA8Mzx7PMW68DtAPAA8krzuu0O9yzwFvZo7RbssPLs7nDxVvFw9hrsEvZA7szxGvMG7oDwlu9Q81LxOvfa85ry0Ozc+ljylO5I8Hz3BvNo8dzq/u5g7cL2ivAK9rTwVPZw8vrvaPIK8uDq7PCe8srwtvJ+8yTtEvJc7nbzGPDM87zsWvPm8KT07PEq8mrzuvJg8qjw6vIo8WTmEuws9pjwUPLm7Cj0YPIU7Mr0mPMM7qDxRvEM8NDx6Oo289TsQvWy8g7wQvTk8dbzZvDs9iTzeOyq7EbmMvFy8Ij0IvT08IbxcvPs7brw1vNW7JbzruwS9KD2mvDA8nzxRPVA7P7zIOyA9XzwzPL68Ir1svPa6RLrbO327dLzQvBY9+zy7PNQ7PDzOPMa8+rxQvHa8ATtMPNo8urzBPD87VDzlPLY6ajuYPIq7HrzuvI27J7y5OxS9QbwqvZY8tDz7uVC96ru9O1S9W7uSOmY8hDzKPEA8TLuuvDA7FLysujI7nrw2PMi7TDyPO9C7JzyjPBS8GL3rPAk9Sjw1vLW8+jw5PCG9g7wTPZ68eTzAu0G8HLxVvAM9oruCPBm8rLqKPS685zsSOxG7DLwCvAq8brybu/O8YjoOOw+8S71iu4A8vbyovI87B7pUPKe777yHu6y8OjywO7+7ljwKvOE8G7mOPBC9QjxvvHo7CL3/PN88RTxgvbC8BT2DO/g7qLtOPJI7xzyRPLM86jyLvRC8v7yQO4Q8NTnqPGI7wDsNPb+8CjzNO0a8Qz0UvL48Aj2SPL87LD1/O4E8Jz14u528aju+vNy83jupvMa6t7xEPdK8NbxDOya70TzIvLm7CroKvGY6bLzkPBI9IzxGPIC8UbstvIG8ITzKO8I8ET3vvC+6Hr2/PEy91zwqPXs8gjxDvN48tjyTPLI8SzyjvBi9/rwYvWM8nTzZPB+9X72CPAw9hzwsvTO9ZDwAO4m8cjwiPd08x7rovNG7v7vXOki80DulvEe8Z7yqPLq7dTyZPOo7VzsivaS8Dr3LPIA7eLz1unM8rjsivbK7UTzMvMO72zyWPME8QLqpvDw9dDs+vKE7bTy0POA8Gjyzube8+LwXvT06jzyCPLW8ArvVPLQ8izvyPIG8JDx/vE68Wrw8PDW7tDsxvMM847yqPLO7tDwJPEm7RLyIPNG6DjukvIy8uDqEOhK8obvqPLC7Oj2rPDa9Iz3svAs8gr2jPAK9hbuXPG072bzoPBo8NL0Uu/m7tLuZvJ87bLwFPbc76zw1Pcq7eLwFvGE9RLsNvEY8rTwivak7grzWPAY9MTynOgS9czocvT898juLPK+8nbwAvGO85Tz/u7A7wLomvJu8Jb1XPNG8U71tvf2307vsPJ+80Lycu1C8n7qBvP26tTxsvaK81jsmPBu97DpVvFU8pLzePJK9DT0tvI27njsIPVk7STwqvGe8QDxpPHU8Z7sePNK7vru5vNK6J7ySPK87VbvgPDi99zw/uwK9Kr0wPDa9Hz24O8S7zTwavEU9o7xBPKg86bwgu2o80zwLvSE9lTs2O/w4KLuWuyy8fzzxuqe6IL0XPZw8WrxnvCu9CD2bOmI8Db33uw895zy5vP87zzyuO728zTpSPCE8OLyiu7+7NL00PPA63jzFPFs8lLsUPX08hTxUPVE8Cj34vAs927tPPRS7vzwhveM8rLxtPAy9mboIvd27dDw7PCs8qTqFuxc8LrzWu428pzwDvay7AL1SvKK8iru9vIA827xRu++7F7z/uqe86zwVPHa8rzzXuwY70bwEPIe8hLwMPJQ80LxZPG+81rz7O2Q8ZbxCPfm8IL0aPaQ7LTrIPKc8Az2gu6A64TsjvSu8pbwrPHw8Ej2evAq8JzyCPJq8i7oOvLW7I71hvAU9vTxWvCC8orzOuUI9Rbx7vNi7s7yzt4u7YbtoPAM9ebyOvIi8XzzSuyS7Jr2FvKg7urs0PFa8aDzCuy88Azy/Opu8mLzAvAK8Cj2UvGA9QjzjPE28ujwkvZE83bwqPJm747yTuiM8SzwKPB28x7sWvYg8ZTimPJ47njv/vBM8MjxPPIU7TbxruQg9xbwmPGO8w7s8PLi8Djx+vJG7IzxMvEm78rs7vDa8vLzkPN+6Ubk4PNK7/jwFPBw8RzwzPGy7CTyivDC82rySO967obzgPFQ8NjwsvM87qDvFvAc7yTzsO5S8jzyrOxO8STuePJc77bsrvEC8STwIPcO8x7wCPZm8s7ywPG88kryXPBS9SjspvAo9+DyuPFK8+7yfPJ28kLt6vEm8GDwUO3K8Mj14vL+8rbw/PDW5C7zEuyy8J70pvCi9XLwMPIm8LDuXvOE7m7xzu4e7Ej2NPCe81DmPvI68iDyavBU9xDpfPK87trwqvJe88LqtPE28i7sUvFI7n7w8vIC8V7zxPBw9hLydOyg9H7yXPAA897w5u5i82zxdvNE8ITzJPOG70rshvB48/7uNvDa+pDtgvAy8s7tbvK88UDxPvLc8wLyOOt68NTznvIO8RDzGPKg8QzuRPMM8E7u3vPG8ATxTuoU8lLziu7s8CD1RvJO8jLwmu688zLw+O1I99Dr3vP07UzxqPE88ELwJvew7+DyEvDq8Ebv2O6E7sTyZvDK8Azz5ul48Bz2XO5g8gTzIvEo6drwOvRe8GT0VPEA8jjyNPNY8nrrKO+87ID3tPAE9CDwKPa08UrxRvdI8crwAPXq8Lj2GPG+8qTreulg88Dy7PPq7wjyGPHs8tzyBvDc8bDzDuuG8Tjvzu6u8aTynu0U8CD1jPLO7/bsePT882zupvMk8I7wVvLk8gTxHvEM9zTwxPEK8SDsyvKi7NzzjutO7xzmzvNY8gTz/Ow88IDwvvBM8MLu/u7m78DsjvM08uLvVvGk8ZTzxu208TrwXPCU8DjyRPLY88jy3PGc9dLw4O446dT2/OzY7XLwQPKy8UbynvCa9TDxCvLa84rsXu868zzuBO5e7JrxJu3e8JzwgPKe8yDyBvWG9bTyNvG+7m7ziu0Q7pjzkO6G8wbxTO9i8AbwKvX+8yrxHvIG87jv5u9O88byLvK48LjygPJk8D737vNI8lbqtvMy8mjy9O0A8F7pivLE88b0fPCs9PD3DPFQ8rrzevMA8x7pZPOq8FTwvPbA8QLwUvH28kTw+vGC7Y7y2O9q82LuXPI+8572qO7S8srsgPUU8Br0SvSI85zujvMy7OryBvLw8jTz9uwu9j7wzPHG87bzWPES9NrwSub09M7x8vOc8m7vxPAU9pjtuvKe8Wbwduxk91zw4uv876jwIvfc7k7wsvYK9FTw=", "ffn_stream_bf16": "WD1XPAG867tYvTS9Cr2FvDU9Mj2mvUQ87rvsPIG9Xb3xPMw8FLwMvCG91b1BPfE7lDzxvC+9qzzSvM+8xrw+PeG5Vb0YPOK8Fb1SvRY84DwQuxA8ozw0vGE9QDxDveQ8RjyVPBI9Yz0aPKQ8Nj38PJK8kTtLvNO8C735PDg8Rj0NvAK9lrxfPB29rzvHuxQ9y7yMvDC96zx4PBA9H72nO048qbomvDU7xbwvvCI9k7yhvDe9OD1IPQC9JL2vvBu8ebyBvUE9Qz3/vDq8Br0YPGG8hrzJOQe8ZT1qvN68w7yWPIk7obz+vCE93jzSPHM9zjwgvVC8qbzRvDe9Xb1tPZK8nTtBvcC83byCOzu9QT2CPGA9cTuxOwY90LzjPNU9k7sYPas8kz2Jvbg85bxLvZu9ETujPQG9Ar6QvAg9g73ru+48gzoVvBs977xFPR+6kj3pO6S7NLp3vX69hT2YO6W8DbsEPHS9ijyvPGA907zXvAy8QLy7PEI8vLzyO1K8qzzKvD+8Yz0AvJq8lbzyvCS9lzxpPFC9Ej1evI27QbwXPDa9Mj1xvR29Br0GPNg8OL2zPLU8tjwbPQs8Fr05vIM8jzznvPY8CDwgvdA8Z7wCPKQ8+bvSPL08Zrx2PAw9HLyLPEq9+DxuPf08DLzUPO08yrzLPOe87bpkvYS8K73WvCg8Wj2rvcI8Wj1FvYa9oLzFPMk8ozyBPXK8lbyQPQu9KbxbvTA9ITwJvR07Jb0RvVW9oTxRPLE8JLgKvJ46mzwWPdi8oDwcvUC9ED35umq8LL0bPZS9Jj3AvIO9yrxKvZi9XL2hOsU9lr2LvCW8BL06vOI8Dr0tvbQ9mTxNvT884DwmvCu9Ar1aO1s9qTwavIU9SL2pvTU9SjvwvCA94Dw+Pdy81DtYPOM7prvWu148QjzvvAe9wb1evUk9W7wtvEm8nrywPVK8UDsDPZE9XD2OvGs99jybPAg82Dx7PdE8kr0EvJI7XD1HPTK9h7ycPFS8gTzbvEK9w7wuPY+9+zq1vRw9JDzfPFw9GD3bvG47L70JvEK9oLoEPbk8yrtcvYY7QT3YvFs7bbxuPJ27y7wUvFm7ED3+OyQ9BT2EPB09N714vVe8CbsCPUo8qL2lvBU8EbzmOz49E7wtvS090rwmO3A6rjzeuow81Tt2PdC6IL0VPBi9OrzVvOq8LD0eO7m7Hz2+vFW9rLwcPR+90byYPaY7rb0vveO8jTuBu4G65LxhuzI7Hj2+vKs8Jj32OzU7/Tz0PBe7+7wPPDS9Nz2JvZS8LL3guwU9oTtiveu8vjwpOuM8Cbx3vXC81DwmOwY8i70MvcO8Lj2pvby8W7yZPLq9mj1ZPYm6ijx2vMk7ojyyPIw9XruLuzE9IDzeO2Y7t7zgvS69PjzNvLi8Ir2Cu7i7Yr2ePFg9dzzvvLE8CTwPvYA9TrtkPIm8Vz0vPTC8Drx2vZU88buwPLi8szwxPSg9+LyoOxg90zzhvMk7qjyGPUW8VD1lvBa9hb3kOjI9ijxjvGO7RL39vJs7e72QvcE8uD1svNO6Srz2PCk9g7zhvG27gjyCO0I64Tycux696rzWvA49Ij2cPV68qLwnvbK8ezwrPU+8Pj1nvUi9er1YvYU9KboiPS89vbpHvaQ8OLwjPeW8rbwKvQw7WT2SPE68abxBvLw7Hr0fPVM95TzIu4o8Ozw3PRs9M73xO5+8Fj2+vFK8s72tvEm8Y7riOz+8/ztCPVY8BLknvTk9EjqBvL07LDxnO6K8SjoNPCY9CjxCPZe9obyqPJO8PL2lvBq9Fz3oPJ88bj1fPAO9UD0iPQg8IrxGPJy9Jr02Pba8+rwpPWM7Eb3Fu3i9tTsuvRm8Vrx0PUo8Y72OPB69W7sevTO8KL2MOu88B71XPcS5A7o2PZc9iD1oPYE8Qz31Ox89ML3MPD+8oz2vvLY8RDzYvKi7q7wgvew8x7xCPVW8nrxavbW8f710PGS87LwPPa48Pz1pvGw9mj1WPdc8Er1NO4689TwXPO68izzwPAg9E7wjvGE9Ir3sOsE7AL2eO0A8drtdvTO8zbzBvLW8ibyVPTu9tzyZPKy85bwmPZi9Ijx/uyU7E70LPVm8YbywOy+7BD00Pde8bjx3PNw82T0mvdS8Kjthuzi93DyqO1E9urv5PBK9Ez2jPWM9jLskugQ8A73OPHK7p7ySvPu8Rb2Pvf28mDxvvGO9+7yGvOG8TDyRPAC9HjywvLW89TyGPRU7B7uTOzW9TruAPC055TzUvIU9xDyKPQU8zbxsvaU9YbuoPL077TvLO7M8h72qPAa9ML0BvUC907xDvY+9D7x4Pe88VjxrvXE8Bj0SvDK90zwsPIk9ab20u0m8nDtAvck8rjz4PCQ9Dr0JvQa9rjq3OhK7hrzyvJ494DwmO1C7mb1rPDc8qjzQvEU9NrzbPHM9I7oPPTs85TxJO0y9rbzLOiK8JD0CvJU7uTwIPHG7xjzKvOS8Ez0XPdA9r725PRi8DL32PBq8zjttvZk8dbyWPLY8TbxhPeW9VLqyPAG7pLsvPIw8jT2uPOM8UL1VvN+8ijwrPY28Gr07PeG8Ij0EvEu5srwNPA891bzRO9E6NLwZPYE8n71cPA69wbyyPPa6kT2GPNm7i7sUOgW8Zj2lPJ88vTwKvJA9jr0PPaq7yLwAvf68ST16vYO8hz3QvBW9AbswvYI8Er2fvGu9AzyKPec7JTwIvUg9pbuHvJK8sLx1vUg9ijxhPfk8fD32vB28Lr0mvdc8Rr0mvYI8Pry2PNQ7yTy8PBc8bDwNPQg8dD2DvZu6Fj2cPWy8DbwcPUo9oTxkvJ89vLsVPce8S725vMI7Vb0+vBQ9j71uvau8ujyNPDa9hrm2PAS8/DquO307eLuZvCw9j7skvbQ7eToIPXg86zzGPAY9aj2avQ+8uD3mOpy7XbxJuyw9gjvIPF293TwqPAq9Ej1EPU69hTu1POq8Lr0XvQ494jqyvUo9vzvvPBs7vLzRO7g8Ej1GOys7NL01Pfg73zzPO3i8nTz8PO47+ryRPIG9m7yCPL08s7x3PZW7kD2MPCy8ibvPPIc6Gb0lPL08bjy2OzY8P7xLPU09Wr39PJQ8qjzCuxe7ErwLPa+6TbyKO6O9kjyxvc07Er3ivAc8A7zYPES8yDx6O5E9v7wVvRk9sbyTPCO8srw6PAw9HjuMvai7QzxavSa8U71hvHM9NL1PvU08PbuYvWA9yrxDPXG8/jw/u2E9cbyaPL29/DvGPOe8P70bvCi9Tj0IPXO8o7xPPSe9wL7DvG49OT2mPJI8gr2LvLm80DyvPLw8F73TO0C9Bj2ZO1Q9Mbx5vA69WD2dO4O9Mj1sPIc96L1eu9O8br1mPU+9wrrhPNw7Kj2ePKs7DT0rPEQ9Mz3/vDg9g72HO408LLsNvII8Kj0suxk9LjwBPpW8PL29vGI8Srw/vS88Ar3IvFw7n7x0PEA9cjyVPVM8Mz0TPXo9DT2cPGu9yTzau4E9tzv/POa95LsDvSO9HTw7PEq97DvNPQO8YT1cvSw9cj2rOxe71LoXvVE8AT1vvQk6Uj07POu8uDuDvZu8Iz3qujM667yfPfY8BzuHvNo6TDzHuoe6fz2guwI9t72oPb47+LzIPcq8xLz2O2S93rtoPXy9aDxtvA28/DzOPG+98TzVPDe9kLwXPJI78rwbvQa9gT05OGE8AL1fvX68hjwSvee8IbyvvDa92bs0PUA9zbsePVy977yLO8g8RT2XO8q7mD3rPNQ72zs2vBO90zygPBa9zTyOvOU8kTwSvUQ7Vzz0PLq8qzs8ugS9CD1mu3e8W70qvOI7ITz3vA49Tb3bvIm8qTyIvYC9oj2zvH+9Z7t8PEw9Gr0MvYW86ztVPeu86zyhvAa9FD0vvH88D7xKvLC7Sj3IPIa9HLxTPAO9yDxdPSY9RL3jvC48gLynPBg9Ar0QvUI83TzCPMc7AbyRu4s8Frv0PEw8Wz3ivAc8k7yMuyY8Kzp5vcO9/jsnPX48RT1APq49HD29u9S8YryhOis9Sr0gOyi9GLwIvaI9lz1jPMK8nL1QutE7rD2HPFe957ypvTC9e731vKO8EzznvGk95jusPCy8dj2aPIY9+TxTvGK8tDw7PGU8hr0GPCu94TwoPFG6d7zqO8W8rrxdPBa8iD0DvdY587wJO2a9nbvVPIc74ToCPGk9YD3mOzk9AropPCs9ILwjPfe66DyrvBA7fj2PO5O9Bj07vbs8g72SPHC8/zwuvWG9I73CPFo9q72Xuwu8rzwSPRq9qTybPPK8Bb3svGG9nT15PWG7DTz2Ov87CTwGvPQ80jyQvE89gLyJPEE9vT1lvJC7Gr0ZPRU8br0IPVy8W706u9U7Bjw9vT+9Kr3ZPJS9+DxWPES8U7zAPDg9Jb2Evbi8Sj2vPIW9aT15POG8lbzDPL88Ab6nO4k8ZT3hupW8Vj0NPSQ9tb2avIA8Pr0VvMo8n7z+vHm9njuBPbu9fDz/vAO9U7w5PSA9JTyru/y60LwRPUA8gzwRvZ49Fr3tvN67Ij1/uZU8AT1tvb28QbxwOz68jz1nPQo9JDwePcG8ojy9vSa9vzyJPEW9gTySPOi8szzJu8M8Eb1lvc+8TD25vCO9gD1RPJu9AbxyvGQ8sLwKvYe8bjzHPIk9R708PJw82LzGu9Y8BT1VvGM9SrwMvR89LDvAvA88MzoUvbA8OLxbvfe8ZT00vHK7yTv6PFw8Mj1KvJa9IT3GPKW8SL0QvQG8eTylPL8727z9O4680bwzvQo9ZT0MvUs9Gj2yPPW8H7ssvOq8lrxVvcS8FLtyvQ09PzyHPWU8eDzTPAy9AT2ZPBU9c7wlPYK7bz1oPS09Z7sLPaK8J72+PCS9zLxQvXg7Pzw2vdC79TvjvP+8XbyRvcO5Ob2nPUE80bwHvR496TxOPI47KLywPfm87Tz+uwi9/zsDvT88QL2xPAQ8pj1RvbW8Xzx7vUO8Qz0KPcE9Ijw0PZq8HD0svBs9LjyJPZM9W7wXPIg8bzupvHe8+Lwgveo8kr2gvOQ8Sz15vYO9VD0Zve08xDzAvdM8cr0DPFk9mbygPZa9HzwdOog9mD1PvBo97jyHvTm957wmvXE8zTtQPK48qj2XPCs8MT3JO0e9cDwFPD07lD1tPUu9tbzHPAO9Bj3Ku2M8K71xPUM8nD3Tu0A8YD0dPUs8RrzJu1s8rjwrPY29BD0WPOk7CDsCPUw9rb0wPD88C72+vOe8rjx0vWC8xLzyO6O8yDyFPIi73LlRvLi8IbytPDO9/7rjO8s8jTwsvQI9gbwlvIC8HT5QPX88Hj0mPAq8f7suvQS7pzxfvME87DymPc06BzyMvSE9y7tgPXa8vLwJvPm7obyJvJS6oj2/PPK8Pz3vvIE94zy6vNY8Qz2TPGo9LD3eO469rrygvD28lTurvDS88rxTPHM8rzwuvWO8AL0aO5s9jDzvO6i8SL1WPQK86TyGvAI9KLyQuys99LuTPHO8dL3nPMY8oDy/vM+8sTlLvQw94TwSPHG8qDsVvQs9TDwxvXk8VTzIuty8bzwSvKq5LLzkvJk73rkdvdm8PLx8veo8Vr0LPb488jx+PV48gz3mPP88Qj02vYs6RLxvu9Q8uDwTPLy7OD0svSW8JT2vvIs8t7yeO5A9Xr3Au1e9yjxuPES9Qj3wvGo8Tb0/vSs9VL1eOxu9zL00vYu9P70dPco8dDy1vTY8vD1NPcU8NzyIPIK97zxSvOQ8hT3hOg09Gz3TvBA9sb0jvJo82zzlvNm8jryyO/C8gbs/vX08hj2cvDQ9Rr2CPYM9RLqAu2m9zrwuvC+8Gj02vbg98LwePPy8Yb0uvWY8WjsNvWc7czyQvAU9nLwXPVC9ID3SPIe8fr2xPEw8GL28PIa7Nr1XO5M9+bxwPDA9ODz1vHU8BD3QPFq9izw8PQo9Yj2GvBY9j72HvAS9z7x8PJa8nbzCuwu9Jj3ovHm8Vb1NvOw7XbyIPLY8G72pvF08dD1KvKm8KT2CPHQ9Ez2aPKI8w7kMuw09mD0sPfo8UT1Yu8+8OLoOvXs9pTyrPfu7LL2aPK0897spvPm7wL2qvLw7nr2TvFu8BTyqPKK8Bj2nvG08oD0kvFY797yKvSw8nDxsPBa90jvbO/K8RL2hPEG8Hz2uPLW7Rj0GvfC8jLwau9C8jLw7Oyi9x7yAPGA8Sj2Ive884Ty3vKA85LwzvFi9iTzdPCq9+bwKPBo9Sz3MPUQ8WrxaPVA9K7izvF49J70tPbe8Rj0jPM882rxMPRg8JL0HPCu9sDx/uwm8ED3TPP68gr0dPUg8Ez0rvMy9tT3dvLu85rw6vZc9kryoPMu7Rbzbu0u9Rb3rPLG8FD2CvGE7kbxYvM+7Kj2EvC+91TwfvII8Kr2mvII8OjzkvHe5Grz8PBe9KL1sPGw8oTwyO6a9HD2wOzS8Qb0DPEc9ybuQvBC8l72SOqG867wwvW67Z7y9vM07Gz0iOxg8uzydvdg7w7x0vNE9vbyNvY47Nz0yvIu9Wr02vCy8pz2DPbi83Tx6PSY97ju4PD+9Lz18vCM9/jlIPQG9qzwfPdq6Mr2NOzi9PzwYvWQ9Ub1eO0E9gj3dPKM8zjuTPAQ6fjyju2C97LwcvIq9gTsKvVo8+DvJPB89yzvzvAe8x71auqC9iLwivSE8Rjw0vYY8ojwEvY88Cr1rvEM9ETtWPXY9ELxLu5K8cjxYvD68ir3WvPQ8sryWvVe8Kr0ovY88Yz3Euxm9f7wTugu8XDwGvVs9Dr1OPEk8+byZPD+9kL0QPFS95zurvLo8oDsFPRW96LwMPSW9Prw8PVm8J71YuVa89zxYveq8lz2BO4M9BL1IvIo8VD1pPWg9lrtyvI494DwJuzM9hryeu4g9PjvFvPQ8ZbsFPee7JL3vvBc8yTxYvRW9RzyXvOY8ST2IvHm9ZDwyO4y8lrwhvWm8JTxmOzo9wzx8vVW8S7x0vJi8D72pvCY9/jtwvR88Yz3Rua08w7zxvDS9UL1NvIW8CD0hvZw88bybu4I81D2TPLO6OD3aPG0857yNPaY8xzzcvJC8wzsmvck8jjz5PM48J70PvGm8OL3pu9m8lr32vY49AL1tPHO8c7w5vRi9zbxFvT899rw2vHU7Yb0OvAU8nbxhvDC6EryZvCm9fDyOvBq80DxBvQY9aj30ux89czyrvAg9oDybvQg81DzAuhi9Zj3fOw69jT3ou0m8fruQvYU8tbzLu5g9jD3qvL+7Kz0oPcC8gjwfu9E6tr0kutY8i7uuPDm9Dr2jPVS9LD2gPCO9RD1TvZC8cb2yPD29l7xMO6m84T2JPYO9cz2mPZM8vLyfOHy8hj14vKI8kzxzPZY9Uj14Pay6JTmLvFU7Abrmu928I71lPcK8ub0qPUU7LDz6O6S8+ryIPTw7S7xkvbs9Dr3evOS8Cr1ZPem8lrw2vDe8dL06PdA5BbzBPMI96zywvAA9LLzoPJU8Dj2jPHK8lrw8vGe9PzywPMm8PryHPJ28kruAPM86wbrVPCm9MjxNPZ09VT0VPSy9SD1dPCO9EjtUPRi8gTwtvLq8jTuQPE09U70xvdG73LtaPbE9/Tw9PfW8MD0Jvaa8Wr2tPFs7JT2aPd+7NTyCPag8oDwWPTE8g7vMO/e6srt1vfy89bs7vIm8IjwUPS49uzyuPJ08kr1QPOC8Kj1dvdE8z7xXPDs7VT2jPBY9S7yRPJe9w7sgPTS7Fj1hvPE8QTxRvds8bj2oPGw9x7x8PW49pb3jug09kDy+PXy7FrwxvKu8WzxiPAy8sLzTuz+9KD3iPNq8BjzoPKq9Ib0ZPDc8Tz25PNM8vTx/vWM9ZrxzvII8Yjw5vAe9Bj5XvT49j7xkvDi9RL13PJC88TuXPKU9wbw0PY28ETuYPOS8LD0zvQC9aL38uxE8PTxRu7K7Uj0yO6w8Lr2uuqE8h7zoPPS5Xb2PPBi9/jyNPaK7Lrw5PWC8kL2+u4e7vjtKvDO9Dz0SPHg9wTyYPKI8i728PIm8SDwPPVw9ATwhu1Y8yrxJPe28YTtXOwU9KD3mO029sLwpvTw8erwOvSA8SL2SPNO8ST2mPM28Zz0CPXa9gruHPGk9YD1dPAc80jxWOym8wbwaPRa9RzwdPdw8Bj12vQ28Vb0JvYQ99juTPSs9OLzaPJE8rDzjvKG7/DwBvPK8ZT2GPZm8Ej1YvB08TrwFvbi7RL3mO/K8rTvjO228FL0/vf47xDwavWA75bx/Pce7F73xvBc91LwJuyQ9i71vvH+8ZTzRvMc9PLzHPDu9M7vxPGQ9Xz28vOK6vbuFvQQ9DD06PCk9mryAO1C7dD0BPSA9Aj0BO8Q8Fb1fvbG8ID0QPsW8072SPNu8Iz0HPX28lzwuPYG7ibyxPeE8TLuxPBe877tRuwG9h7xvPGk8Iz1/vMS8jbxpPGk9zLtFPIW8gD24PJG9QL0BvV+8oro3vY68Jr2yO2497Tw5vZO8eT10PY48xTspPSE7orxlPdw7j73jPNs89jk4PGS82TxOvYw8vrxOPWY8HD1PPZm8BL3DvJ68ZDyoPHA9jz3zvH+9Ib2LOzm9GDsEPHy8Pj1CPQk9hb0zPHS96DxevZ28TT16uoC81rwxvNq8Er1IPQ28gz0AvKW8lDyQvS+9ujyUvb+7P70VvK+8Cb27Pb+8ID23PEg9KD2LvBk9y7vxu668V7o4vAS967woPVG8sjoQO9C7ULxJvb880rtQvWw92rqDPa889rsdPaM8sjguPaE7FD2KO6e8uLyJvT89EzwQPcS85Dyhvey8arwEO9Q80LywvNo8fT0YvYS7KT27PNG9xDc6vfI8Pj0ZvQU8vD1HvaG8BT2mPAi94LviPD09RjyDPZA9yTsKvSM9s70/PWG73zx7PKI8Hb38PAw8bz0nu1e9qDx3vZm9fjybPXW9qr2QvKY9Sjz0vIs9frzwPBM96jskvai997sLvUo95bo5PbI8wrsdO3+7az0MPas8Cb1mPMa8aL1iu8u8GL2wOwA9cbyWvCQ9aT0WvbK84bwkvEU7BzwaPRg8hLxRu4m8gDy4PB89dD2MvE28iD33PFc9QDyOPIK7xLwxvXS9kD2LPFy9ADyAPKw8prwSPey53jtyvRw9HT2HuZC9uDsJPBw7dDpPO9k6M72UvMa86TyHvTS8HL0kvas9ODwuPZC8hL5UPXI7GDxovCq9dr1LvS69vLwIvVC9ujy2vZ27Dz1ivF+8pTvfPIk9Ar2DPRA9Lr3YvD89TT1TPYU99r0aPTO7ybtgu7Q8Orv/PNu83rwqPYE8WDxlPZ28QTx0PbO6nLzbOyS9mj0Rvcu9rjzQuzY9ALx3vT080jwJuhs9fjsBvS89sjucPPA5lj3PuxC7Gb1DPb68xTzxPAi9YD0EOww9UboWPdq8Z71APe47ZrsRuya8GL1YPfs8KLxKPQO9nLxhvMO83jtxPBG9a70pPSW9HTwHPTK9zTxLPMc8C7xiPCW9k70APE+9nj1LvL08hr2tvJG8Zjx9vMC80Do6PQ68aTzZuk48B73+OsI8ED1TPFS8HL2EPaE8DTxrPa87lDwku9+5Nj0SvZ+65zyLPVI86z0NvZk91Lu1vCo9JL34uj07tjwcPSc97rszPdg8DTz0PK+8Fj0Jvmu8lryuO048Uj1CPfS7MD2lvGu8jLwWvTm8TzzkvIm97jvTPMG5gb3DO4w9xjzzO+682Tv0vB88PL1EvRO8aLwBPcw8oLxuOxC9Y7wrvQ894LuRvJ+9Ur1xvDk9nzyUPS68wDwRvbO8ET3BPYy9vb2QPT69Wz0bPIs6PL2WvLe8DL2PO0A9Y73xvHi8S70KPQI9obxTPQu9rzpNvam7rjxVPUE8+jxbvc08izz9PHY8SbyZvPu7vrwHvMG6XD1Qu7Q8Az0ou1S8jbzWutM7IDwgO0a9crsgPCK9HryrvAq86Ls2vby8QDuaO5o8Bb0TPSm9Z7wTvfW8eD2fvUA9pT2qPFC9ajy/vLW8mzp9PPc8jLx8Pek8kbw4veu7ujzkOxM8vrwnvaw6nTwxvbm8PLxxPUA6ZT2RvQu9HL2rPIy927xhPQE9Ij3JvJQ9tbzNPX89ozy5vES7XjyVORY83rwSPd88ADzTvC+9Dry0vBY9SD3gvBg7pzzcvIO8qT2IvAU7jbtHPSi8hT1XvAA9mDzUPKG6V71JvY29WD3HvD693rzZvAg8I7x1vXG8qDtHPJo5ybxqvKG8mbtevYK9ob3qu4697byKOmW8lD09PcW8iT2HvJA8Hr0Kvcw8Jz0LvPU8kT37O6m9oTuavOm8JD3TOje9ND1EPZQ8rLvbvCG94zsau9M8NbolPZY8wLyGPLs78byFPJ66eLx2vbC8jLwpvTC8mD10vaS8lj25PDC9sLygPSa9sTzSu568VLk0vCI9VT7PO7c8ujvgPGW8oTyHPH68rzuPvFC9AzwnvPA8Xz3Mvdc74LwEPBK9YD3+Ole9Fj1RO2S8JLvCPAM9WzwBvcK8sr0ruU45m72bPKS8Zr2wPJk8Cb0aPGe9Jj2puxa9ijzHPTo81bwBvM28ajniPbS8ezyIPQQ957ztPFO9P7qVPJa9zTuoO+G7G70/Ojg8ujybOzO9Y7zBPcK8cj1PPcq8/byaPLk8Qb2KPOY6i7w0PQe9jbzkPIg9b7zwvAk9cD2svCg7vDx9unm8kL1RvUk8sLrFO/283DwNvMQ8jz2NPVo8FD3fvB09OjwfvUw9JT3pvGq8zDstuG68urxgvJg9uT0uPY69YTxBvIk8/bwcPSU8tbz7PMG9w70RPdg8B71SPNi9TjyxO4K91rzNu508Hz02PBI8sLwvvY+8j7yhu4G8WDtovAM9YjydvAs9hDzwudk8DD2yPcs83byAugg8krw2PUo7hrwjOw69XD2rPDw7cDxXPRQ9Hj0JvVe8zjxivX+9GL1HvJU7oL1auwS8VTw/vYO9JT31u9A7Fz3IvEo94jwmPQK74rwIu/c8Yj3APAu9AD2+OhI9sjuDvEm8TTwKvZK8jT1wPSG9ozyfPTg8VT3YPF67tjsdPJU8ijwDPQW9gTzsvCM9Zbs3u/o8bT0gvQO5Ibsevea8DL0bPcW8Lj2bPX48Ib0OPWk6hjwiPUU6aDv/ulG90TxLPcA3JbuPvVY9l7tMvDK80jxmPWK9TDzUvKa8Rjx3vSY9KD0MPV+8KL0Svam8Cr0LvI082DxePYG8iz2YO928tb33u6u8Lj22PGY9xLwxu5E9oTzlPKC8gryAOx68KLzsvCo9yjzyPAG7gD19O4O8nb0mu3c8Fj1vPFO8Eby7vBG9trxavWq8BD1rPKS8HT3EPI49qDxUPQe8sDxnvXK9QTxCPLM9cDy1PL08tT2DvKu8dbwVvSu9KzwEvDA8Fb32PLS8rbzpO0K9rTx5PX69BL0kvRq82Lz0u5O9jDzwO5E84Tv4PBI8Gz0fvJk7oTxVvHC9D71BvZG9LTwMPcu8lzqcvYE8kzzrPN27D72vunA9Tr02vY07Tz1OOhM9hzpwPGE66bxYPFI8G7v3vA89YTxkvaU8w7y/PAe8djxZO049+js5vMO9R73svB882DuTvKU5az1lPb48vLzfu3c95jrTvAg7fTzJvIG9YDt3u7I8JT1xOza9Er2HPMG8nj32uyc8YT23OQC9grzGPEG9Kj3QPK08X71NPU0897xbPJy7eTwNPfa7jjypu088Sb3zPE08Nb0UPdU8mjqAvJ+8gjwUPak8y7z6vA48+rxvOiA9uTwzPQM8ybsNvNu8PTxlPQU7arwlvJ28G7sBvWW9Kz2qvd88HD2GPVE8wLxPPO29VL1cve08Pby5u088HbwMPas86jynO5s8nj3jvOG8jzy9vAS9BD1DPQi8Hr2gvNi7hjxou0C8pL2QvVY9dbsCvtC8U70YPaU7H71rvee8VbuhPRK9hTxxvPy8E7yyvHc7TztHvds61LsLvKo8zzxIPEU9dT06uwk9tzx6PFE7kD1bPao7oz1pvG87dDyePMS88DqYu4u7Ur3QOwc8pzwNPYy6uj3YvGG8kbzFPHg6r7yUPW07LrymvPS8OrwRuum6qD1TvVm70bwVvfO8Rb1xPTU8Ej1rPUO72Tqeuys5YLxIvQk8QrzUuxO9qjtYvQW9tjuKvMU8Ib1fvRo9pTxePSY9yjyDPIM8A71BO7a8drzVvIi8CTwUPRi9yb1yPQW7KLsOveG8gzz7O4G8HDyRPRa9Jz1RPJ48Dj2SvfE8wzyVvL+847yfvBu8kzwUvei6VjuEvAC96zyyPIG8VzzVuxS6iTmwPG87Az2DvTc977vCvMy8mzzrPIs8KT2KvBm8mTwAPbY8mLxJvFs9Bj1vPHw8sDzWvHA9aD1Dvam90zyhvAI9AzqHvI47pjsOPdu8b7zUvKg9Cr0avS89RLlBPbI6IL2oPK68d7wMvV28Hb31vK28cb0hvZI69L0FPJk9fbxpPZ08CD2AvF49rLyLPDG9tDz3vK+9mbo/PRY9jT1IPNW7i7xcPaW8I72avBk7kbzjO0q9qrxwPX+7obu1PEy927wyPQs9EjzBPPM8YbwbO4o9JD0xu2S90zm1OxK9xjxlu7y8qLyavNA8FT0cu5m9Tj0OvOW8r7yLPTS9TDy1vKO7DD2TPJS8KrsqvQ29h73MPMC8GL0lPai9Mz0DPB08wbyBPVg8yruAvdC7gTxKO9E8KrwkPBA9Q72evOK7Gjw7PJy9qzq2PGm7ET0mvYM8M72RvHU9Ej2YPOq87Dw/vWQ9WLwZvRo9h70RPa27bz1IvTG72bw6PdK6Jr0YO0C9KL4dPLG81Ts3PDa8oDx9PB89Qj31vO47Pb2VvcO8Ir0UPfo8Vj0pPRG7wbz5Oj29r7mUPEK97rwUPS29Ob1hvPc8lDz0OrG7P72IPIk9nTwZPLY8zbyEvH08UbzgPIK8jbw0vHA8BryNvA07Db23OTG81bwMvGE6tTzRvMs8VD3PPHk7gTsOPXm757wnPQe87zzZPIa7IT3ivKw9qTxSPeQ8DD0MPTc9Lj1IvEi9RD1QPIq8uboSPeQ7eT1BvY68yDsbPYc9ND24vJM9qj3nvBA96rtivL28W7yhPLg8xzyovPC89DvWPI09UbyuPJk9nT0mvUK8Pb0rvSi9hrwAvNK7MT0oPfW6mbxWPIu8ubxqvYK7LDxuPC69Ib0YPS+9g72nPD09ErwIPWs9EjywvD88MjyaPMw8wjwMPYe8hjyhO768c7x+Ohs8LjyKvOg7Sj07vZS8Br3vPM48DD1SvbI87L1SuwC9h7xwvTU9xrkaPUi8VbyRuxo9rryXPY28Er1SvRk9uzxQu6i8Bb1FvH29sTuMvAy9+btXvIo8fLw/uwE9XbwdvTO8i72wvK29O7yYPbc82ryrPKY6ybwZu068yD2evAu9Bz0qvY27UjwLPTI9AjylvFG9obsGvj26Yj0zPe672bwrvfw8jDwhPZ27IL2MvBE71zyqvDE8oryUvG29pby0Pbc8WLzMPAw9rTw5vuY78jwSPTA9NbtYPAa8r70PPRg9GTzzO2g9B7wJveO8U73KvMS8Oj0Svb288729POK8QT1KvZW81LuPPDc9xDk2vM48/jwoPRm9lLxBPTa9tDypPL86mz1pvGC9mb28PFk9WjwCvOi7Wb00vQq9hrw2PTE9p71DPOy77jyBvV698DzLPBe8C7whvdW9Qj37O5g877wuva08z7zOvMe8Pj0TulS9GDzhvBW9U70SPOI8DrsOPKQ8LrxgPTs8RL3lPEc8mTwRPWM9GDylPDY9+DySvI87UbzUvAu99jw5PEc9EbwCvZe8ZTwevaY7y7sTPcu8jLwwvew8dzwPPSG9mDtKPJa6Krw2O8W8MLwjPZS8nrw1vTg9ST0AvSS9sbwbvHm8gr1BPUM9/rw5vAW9FzxhvIO8ZTgIvGU9brzfvMK8kjyNO6K8/7whPd481Dx0Pc88IL1VvKu80Lw3vV69bT2SvKE7Qr3BvOC8hzs6vUA9gDxhPWQ7tTsGPdC85DzWPYu7GT2qPJM9iL26POS8Sr2bvQY7oj0CvQO+kbwJPYS997vtPII6GbwcPe68RD2yuZM97Tubu1u6d71/vYY9nDuovAm7Ajx0vYw8sDxhPdO82LwMvD+8vTw+PLu87TtTvK08y7w+vGM99rubvJS887wlvZM8bTxPvRE9WbyTu0S8HTw3vTM9cr0cvQa9AzzXPDi9szyzPLY8HD0MPBW9OLyDPJE86rz1PAw8IL3SPGO8BzymPPK71Dy7PGS8eTwMPSC8jTxLvfk8bj37PBG81TzuPMm8zTznvAq7Zb2HvCq91bwkPFs9q73APFs9Rr2HvaC8xTzHPKQ8gj1zvJO8kD0KvSS8W70vPSU8Cr0POyW9EL1WvaA8VTyzPJW4C7ySOpo8Fj3YvJ88Hb1BvQ895bprvCu9Gz2UvSc9v7yDvce8S72YvVu9vDrGPZi9jbwnvAS9QbzhPA+9Lb21PZs8T709PNw8J7wsvQK9RjtbPak8G7yGPUm9qr02PTQ777wgPd48Pz3gvNs7WjzfO6+717tiPEU88bwHvcG9X71JPWC8L7xNvJ28sD1RvFg7Az2SPVw9kLxtPfg8mzwJPNc8fD3SPJK9CLySO109Rj0zvYm8nDxUvIE83LxDvcG8Lz2PvQQ7tr0bPSE84TxdPRg93bxvOzC9C7xFvZ+6Aj26PMq7XL2TO0E91rw6O2m8aDyku8y8Fbxluw899zsmPQY9hTwbPTi9eL1WvA67Az1IPKi9pLwdPA685Ts+PRG8LL0vPdO8FjuSOq883bqNPOE7dz3Yuh+9FDwZvTm81bzqvC09KzvHux89v7xWvay8Hj0gvdO8lz2tO669Mb3ivIw7ibsMuuK8c7s0Ox09wLyvPCY9+jtHO/489TwduwC9EDw1vTo9ib2VvC294LsEPZ87Yr3svL08jDrhPAe8eL1wvNM8TjsLPIu9Db3EvDA9qr27vF28mTy5vZs9Wj1suok8d7zFO6E8sjyMPUa7jbsyPR483zuCO7i84b0uvTw80by4vCC9eru8u2O9nTxZPX8877yvPAY8Dr2APU27ZTyLvFY9MD0svA+8dr2UPPK7rzy7vLU8Mj0pPfa8nzsZPdQ84LzCO6c8hj1HvFM9Y7wVvYW9xzozPYs8X7xdu0W9/byoO3y9kL3EPLg9brzvuky89jwpPYS84bx7u4M8gTtbOuM8lbseveu82rwNPSM9nD1ZvKy8Jr2xvHc8LD1VvD89ab1IvXu9Wb2FPT+6JD0uPZq6SL2mPD+8Ij3mvKy8Cr3NOlk9jjxUvGO8R7y7Ox69IT1TPeE8y7uLPDo8OT0bPTa98zuevBc9vLxSvLS9rrxHvIO66jtBvPg7QD1YPNG4Jr05PY06g7y8Oyo8azukvCc6EzwnPQ08QT2YvaK8qTyUvDu9o7wbvRc95jygPG49YzwEvVA9Ij0MPCe8RTycvSe9Nz23vP28Kj1dOxK9x7t6vaw7L70cvFa8cz1PPGO9kjwfvVq7H70wvCi9hDrtPAe9WT0zuie6ND2YPYk9aT2BPEQ9BDwfPTG9zTxAvKM9sLy1PEM82byvu6m8IL3uPMi8Qj1bvKC8W725vH+9cjxlvOy8ET2tPD49bbxuPZs9Vj3YPBO9ajuPvPY8FzztvIY88TwHPRG8KLxiPSO9Cju4OwG9tzs6PG27X702vM28wLy0vIm8lT07vbY8mDysvOa8Jz2YvSQ8e7s2OxO9DD1avGO8rTsuuwU9ND3WvG08dTzdPNk9J73UvC47bbs5veI8qTtSPbu7+TwTvRQ9pD1kPY+7z7kFPAO9zDxMu6a8k7z6vEW9j73+vJQ8crxkvfm8hLzgvEw8kTwBvRw8s7y1vPQ8hj0MO/i6jjs2vU+7fjwCOeM80ryFPcU8iz0GPM28bb2mPWm7qDyxO/Q7zDu0PIe9qTwGvTC9Ab1CvdO8Q72QvRO8dz3wPFQ8bL1vPAc9E7wyvdQ8KzyJPWm9sLtHvI47Qb3JPLE8+jwkPQ+9Cb0HvdI6wzoRu4W88LyfPeA8Ljtju5m9ZTw2PKk8z7xEPTe82jxyPSK6Dz1BPOQ8VjtMvay8sDofvCQ9ALyWO7o8CTyGu8g8y7zjvBI9GD3RPa+9uj0cvAu99jwWvMY7bL2XPHe8lTy1PE68YT3mvYK6sDwyu6i7LzyNPI49rjzjPE+9WbzgvIo8Kz2NvBu9Oz3jvCM9AryOOLG8DjwOPdW8zjumOiu8Gj2APJ+9YDwPvcG8szwQu5I9iDzSu467EjoCvGY9pjyfPLw8B7yRPY+9Dz2su8i8/7z/vEo9fL2GvIc9z7wVvQq7ML2DPBG9nrxsvf87ij3jOyU8CL1JPZy7iryPvK68dr1IPYs8YD38PHs997wevC+9Jb3ZPEi9Jb2FPD68tTzjO8g8vjwePG88DT0HPHU9g72PuhU9nT1svAy8HD1MPaA8YrygPbi7Fj3IvE29uby/O1W9OLwUPY+9b72uvLg8jTw3vZK5tzwCvOM6sztqO2G7mLwsPZG7Jr2rO6U6CD10POs8yDwFPWo9m70RvLg91zqju168TLstPYE7xzxevdw8LzwLvRI9RT1OvYU7szzrvC+9Gb0PPdM6s71JPbk77zwcO7u8yDu5PBM9Kzs+OzS9NT0BPN88yzt7vJ88/DztO/i8jzyBvZy8fzy7PLK8dj2Wu5E9jTwovJC70zyROhm9ITy+PHA8wDs0PEO8Sj1PPVq9+zyVPKs8v7siuxO8DD2suk68iTujvZA8sr3OOxK94bwIPAS81zxHvMo8bjuSPb28Fr0aPbK8lDwlvLW8OjwNPRQ7jb2nu0E8WL0pvFO9YLx0PTO9UL1OPEi7mL1fPce8RT1zvP48QLthPXC8mTy9vfk7xzzpvEC9GLwovU09CT18vKW8Tz0nvcK+xbxtPTg9pjyTPIO9jLy1vNE8rTy6PBe92js/vQg9jztUPTK8drwPvVk9ljuCvTE9bjyJPei9VLvXvG+9aD1OvcC63zzZOys9njyvOw49KTxFPTI9/7w3PYS9jzuOPCi7CLyBPCs9S7saPSo8Aj6WvDu9urxhPEq8QL0mPAO9x7xdO5+8eTxAPXg8lj1OPDU9Ez16PQ09nTxrvco82LuCPbY7AD3nveu7BL0jvRk8PjxKvek7zj0EvGM9Xb0sPXM9rjsZu9W6Gb1SPAA9b72rOVE9PDzpvMg7hL2cvCU9CLsYOum8nz33PPg6hrzQOlA84Lpnun89pLsDPbe9qT3KO/i8yD3IvMS89DtkveK7aD17vWs8crwQvP08zzxwve881jw3vZK8FzyTO/K8Gr0FvYI9fzhdPP68YL19vIM8Eb3ovBq8sLw1vdK7ND1BPdG7Hj1dvfK8lTvJPEY9lDvIu5k96TzQO9A7N7wUvdI8ozwWvc08jbzkPJE8E71AO1k89jy5vKw7PLoEvQk9cbt3vFq9K7zdOyE897wNPU6927yLvKo8iL2AvaM9tLyAvWe7fzxLPRq9DL2HvOU7Vj3qvOk8o7wFvRQ9LryBPBC8Q7yvu0o9xzyHvR28WjwEvck8XT0nPUS95LwuPIG8qzwYPQO9Eb1BPN48wjzDOwi8qbuNPAS79DxGPFo95bwGPJe8iLsqPDA6eL3EvQE8Jj19PEY9QT6vPR09tbvXvGS84DorPUu9LzspvRe8CL2iPZc9ZjzDvJ29ILrQO609iDxWvem8q70xvXu98rykvBE867xqPec7sTwtvHc9nDyHPfs8ULxhvLQ8PTxpPIe9Bzwqvd48KTyGunm86DvBvKq8XjwTvIg9BL1JOvK8IDtmvZe70zx5O906BDxqPWA97Ts3PTG6LTwqPSK8Iz3uuuY8rLwQO389kzuUvQQ9O726PIS9kTxuvAA9L71hvSS9wzxcPau9nrsNvKw8ET0cvas8mzzxvAS97LxivZ49ez1muxA8ATvwOwo8ALzyPNE8j7xQPX28iDxCPb09Z7yVuxm9Gj0UPG69CD1YvFq9ObvLOwU8PL0/vSm92TyVvfg8UjxNvFK8wTw3PSW9g725vEs9sjyGvWk9ejzivJW8xDy/PAK+tTuGPGY9DLuVvFc9DT0kPba9nLyCPDy9FbzLPJ+8AL16vaw7gj27vXs8Ab0DvVK8OD0gPSI8q7sGu9G8Ej1APIQ8EL2fPRe97bzkuyI9uLmVPAE9bL29vD+8Uzs8vI89aD0KPR48Hj3BvKM8vr0ovb48iTxFvX08jzzsvLU8yrvBPBK9Zr3RvEw9ubwjvX89VDybvQS8cbxmPK68Cb2EvG08yTyKPUa9NTybPNi8wrvTPAY9VrxiPUe8DL0gPSU7wLwNPB86FL2zPDu8W734vGc9M7x/u8w7/jxePDM9TryWvSE9yDylvEm9EL0EvHQ8pDy8O968+juPvNK8NL0LPWQ9C71JPRo9szz3vCK7LbzpvJe8Vb3FvCS7c70OPUI8hz1lPHo81DwOvQI9mjwWPXe8Jj1+u249Zj0tPWe7Cz2gvCi9vTwkvc68UL1xO0A8N73Yu/o74rz7vF68kr0iujq9pz1FPNC8CL0ePek8VTyHOyy8sD36vO88+rsGvf07BL1BPEG9sDz4O6c9Ur20vF48er06vEI9Cz3DPSU8Mz2XvB09MbwcPSs8iD2UPVm8FDyIPHE7rLx4vPi8H73qPJO9orzjPEs9e72DvVQ9Gb3qPMI8wL3XPHK9BzxaPZi8oD2XvRs8BzqHPZg9TrwbPe08h706vem8J71xPM07SjytPKs9lzwsPDE9wjtJvXg8BDxJO5U9bj1MvbO8xzwDvQg9z7tnPCu9cj1BPJw9zbtCPGE9Hj1NPEu81LtbPK88LT2NvQQ9FDzvO+86BD1LPa69Lzw/PAm9wbzmvK48db1ZvMW86TukvMk8gjyVu0G6T7y3vCG8sTw0vQi75DvLPIw8Lb0DPYG8HryEvB0+Tz17PB89KDwLvHi7Lb0Uu6c8Y7zCPO08pj3EOgU8jb0iPcW7YT11vL28CbwFvKW8ibyCuqM9wDz0vEA98LyBPeI8u7zXPEM9kjxrPSw92juPva+8obxBvJw7rLw2vPG8VDx2PLM8Lr1gvAG9ETucPYs89jumvEq9VT0BvOw8iLwAPSi8lLsrPfu7lDxyvHS96DzIPKE8vLzSvNk5S70LPeI8FDxyvKw7FL0NPUs8Mr16PFs81rrdvGs8EbzUuSq85LyaO4K5HL3ZvD68fb3pPFe9Cj2+PPY8fD1bPIM95TwBPUM9OL1IOku8d7vTPLg8Dzy7uzk9K70ovCY9rbyNPLe8oTuQPV+9wLtYvc08bzxEvUI98rxsPE29Pr0sPVW9Zjsbvcy9NL2LvUC9Hz3LPHY8tb0xPLw9Tz3DPDc8hjyCve88VrziPIY94joOPRs91rwQPbK9HbyePOA85LzbvJC8szvvvGC7P717PIY9nLw0PUe9gz2CPUC6f7tpvc+8NLwxvBk9Nr23Pe68Jzz9vGG9ML1mPEw7D714O3c8krwFPZy8FT1RvR890jyGvH+9tTxLPBm9vDyFuza9WzuTPfq8bTwwPTg8+Lx+PAM90DxbvYw8Oz0LPWI9h7wYPZC9hrwEvc28ejyRvJ+8ybsLvSY96bxvvFa9UbzsO1m8iTy3PBu9qLxfPHU9SbyrvCk9gjx1PRE9mTyjPNm5AbsOPZg9LD36PFM9a7vMvLC5D717Pac8qz3xuy29nTyrPPy7Lrz1u8G9qby4O5+9lbxYvAc8qzygvAg9qLxvPKA9I7xbO/m8i70vPJo8bTwVvck72DvyvEK9oDw9vCA9rTyzu0Y9Cb3vvIq8/7rOvI68Rzsnvcm8hDxfPEs9ir3zPOE8tryiPOe8N7xZvYk83Dwqvfi8CDwaPUw9zD1HPFu8Wz1QPYk4sbxfPSe9Lj26vEc9IjzPPNu8TT0cPCW9BzwrvbA8ibsLvBA90Tz9vIK9HT1JPBI9KrzNvbU93by8vOa8Or2XPZO8pzzOu0S83btLvUa97DyyvBY9hLxoO5G8V7zeuys9hbwwvdM8IryBPCm9pryAPDw85by5uRu8+zwVvSm9bjxuPKE8KDunvR09tzs1vEC9ATxIPby7kbwPvJe9KDqhvOy8ML1hu2W8vrzVOx09KTsgPLw8nr3MO8W8dLzRPcG8jb2MOzc9MLyMvVu9NbwpvKg9gz23vN48ez0lPfc7uTw+vS49gLwlPac5ST0CvbA8HT37ujG9kDs5vTs8GL1kPVK9cDtCPYE93TylPMg7lTysOYE8pLtfve28HLyKvXk7Cr1WPPQ7yjwfPds79LwKvMi9WLqgvYm8Ir0lPEU8Nb2KPKE8BL2RPAu9b7xDPRs7WD12PRO8KruRvHQ8Wbw8vIq92Lz4PLS8lr1avCq9Kb2QPGQ9xLsYvXy8y7kMvGM8B71aPQ69TjxFPP28nTw/vZG9ETxUvds7q7y8PKM7BT0Uvee8Cz0mvT+8PT1avCe9GLlTvPU8V73tvJg9kTuEPQW9RryIPFM9aT1oPaS7bryOPeE8DbsyPYW8qLuJPT87ybzzPGu7Bj3muyW987waPMo8Wr0WvUg8lrznPEo9ibx7vWU8LDuMvJi8Ib1mvBw8XDs6PcM8fr1TvEq8eLyYvBC9qLwnPQE8cL0iPGQ9E7muPMa877w2vVC9UbyGvAg9Ib2dPPC8o7uCPNQ9kzyUujg92jxqPOe8jj2iPMk83byRvLw7Jr3KPIs8+zzQPCe9FLxsvDi95rvavJe99r2OPQG9bDxvvG+8Or0Zvc28RL1APfi8Orx7O2K9ELwJPKG8ZbxkuhS8mrwqvX88jbwbvM48QL0GPWk9+rsePW88qLwHPaI8nL0MPNM8vroYvWc93DsPvY097LtJvIW7kb2GPLS8z7uYPY096LzKuyw9Jz3BvH88I7tgOra9X7rYPJS7rzw6vQ69pD1VvS09oTwjvUM9U72OvHC9tDw+vZi8QzuqvOE9iT2DvXM9pj2RPL68C7l8vIc9eLyePJU8cj2WPVM9eT2rur05irxiOym657vdvCK9Zj3BvLm9LD0wOy08+DulvPy8iD1BO028ZL28PQ693bzjvAu9Wj3lvJa8N7w3vHW9Oz2FuAe8wjzCPek8sLwAPSu86jyWPA89ozxxvJS8QLxovTs8sTzIvEC8iDycvJO7fzzgOrO61jwovTQ8Tj2ePVU9FD0svUk9WTwjvSE7VD0RvII8L7y4vIw7kTxMPVO9Mb3Iu927XD2xPf48Pz31vDM9Cr2qvFu9rzxXOyM9mz3guzg8gj2qPJ88FT0yPH+72DvzurW7db38vPa7P7yHvCY8Fj0uPbs8rTyhPJO9VDzfvCk9W73SPM68WTw8O1U9pDwXPUy8kDyWvcS7Ij0ruxg9YbzwPEI8Ur3bPG89pzxsPcW8fD1vPaW9A7sMPY88vz1zuxS8MbyvvFk8ZjwKvLC8z7tAvSg94zzbvAE86TyqvSK9Fzw6PFA9ujzRPL48gL1jPWC8dLyDPGU8OLwIvQY+V709PY+8bLw5vUW9eTyTvPk7lzylPb+8ND2KvA07mDzmvC09M70BvWi9+7sSPEQ8TLuvu1I9JDusPC29ubqiPIa85zxOuV69jjwZvf88jT2guy28OD1avI+9v7uJu7g7SbwzvRA9ETx5PcI8mjyiPIu9uzyIvEc8DT1bPQA8ILtVPMm8ST3svFo7VDsGPSg95TtPvbG8Kb02PH68D70jPEi9lzzRvEk9qDzPvGk9Aj13vXi7gzxrPWA9XzwGPNQ8TDsovMK8Gz0XvUk8Hj3cPAU9dr0JvFW9Cb2EPe47kz0sPTy83DyRPKw85bylu/w8/7vyvGY9hz2cvBM9VLwkPE+8BL28u0a95jvzvLA72jtzvBS9QL38O8U8Gr1ZO+e8fz3Fuxi987wXPda8/bolPYu9Z7yBvF480LzHPUC8xzw7vUi78DxkPV49u7z1urC7hr0EPQ09OzwpPZm8hTtHu3c9AT0hPQI95jrEPBa9Xb2yvCE9ED7CvNS9kzzcvCU9Bj2AvJc8LT1yu468sj3hPFa7szwZvOu7Vrv/vIi8bTxpPCU9ebzGvIy8ajxoPdC7RjyEvIE9ujySvT+9/bxevLC6Ob2SvCa9sztvPes8OL2WvHk9cz2NPL47Kj0bO6G8Zj3eO4+95DzbPBU6NzxlvNo8Tr2LPL28Tz1lPBw9UT2ZvAW9xbygvGU8pjxvPZA98rx+vSK9gjs5vfk6BDyCvD49QT0KPYW9Nzx1vek8Xr2fvE09gLqCvNS8ObzavBO9ST0MvIM9ArykvJU8kL0wvb08lL3Au0G9FryvvAi9uz3BvB49uDxJPSg9irwaPcm797utvGS6MbwFveq8KT1RvLY6Dzvgu1e8Sr3APMi7UL1rPeK6gz2vPPW7Hj2lPCo5MT2tOxQ9iDulvLq8ib0/PRU8Dj3CvOI8ob3svGq82DrWPNS8sbzbPH09Gr2Iuyo9ujzSvRo5Ob3yPD09Gb0GPLw9SL2ivAY9pjwHveG74Tw/PU08hD2RPcs7Cr0iPbO9QD1vu948gDyjPB69/zwLPG89MLtXvag8eL2avX88mz12vaq9jryoPUk89byLPXS87zwUPek7Jb2pvfy7DL1KPea6Oj2wPLi7Izt4u2s9DD2sPAq9YjzGvGm9Y7vMvBm9tTsBPXC8l7wkPWk9Fr2xvOO8ILxNOwY8Gz0XPIS8bruIvIA8uDwgPXQ9jLxPvIg99TxXPUY8jTx7u8a8Mr11vZE9izxcvfs7fzysPKe8Ez3uueI7c70aPRw98bmRvbE7CDwkO5o6RjsBOzW9lLzEvOs8h70yvB69Jb2rPTM8Lz2RvIS+VD12Oxc8arwrvXe9S70uvb28Cr1Rvbg8tr2ZuxA9Y7xdvJ874DyKPQS9gz0PPS292bxAPU49Uz2GPfa9GT0ju8W7aLu0PDu7AT3evNy8Kj2CPFg8ZD2hvD88dT3Gupq82DslvZo9Er3NvbE81Ls2PQK8eL1BPNQ8GLobPYQ7Ar0vPas7nTwjOpY9zrsUuxm9RD3BvMY88DwIvV89DjsNPQu6Fz3avGi9Pz3xO2u7A7sfvBi9WD38PCq8ST0EvZq8XbzFvM07czwSvWu9Kj0kvRw8Bz0zvco8TDzKPA28YzwlvZS9/TtNvZ89T7y9PIW9r7yTvGY8e7zAvBQ7Oz0NvGg8CbtQPAW9BDvGPA89TzxXvBu9hD2fPA48az2xO5U8G7vZuTY9E72guuc8iz1QPOo9Dr2aPdO7tbwqPSW9ErtMO7Q8HT0nPfC7ND3XPAg89TyxvBU9Cb5qvJe8sjtQPFI9QT35uzA9pbxwvIu8Fr06vE885byJvfI71zxYuYG90zuMPcc8+TvvvN879bwgPDy9Rr0UvGS8/zzLPJ68dTsRvV+8LL0QPdy7kLyfvVO9crw6PaA8lT0yvME8Eb2yvBM9wj2Mvb29kD0/vVw9IDxKOjy9lby2vAu9kzs/PWS99bx4vEq9Cj0CPZ+8VD0Kvb46Tr2lu7E8Vj1FPPo8W73LPIw8/zx6PEu8l7z4u7y8ALy9uls9S7u1PAQ9LLtdvI682rrTOyA8CDtHvWa7JjwhvRm8rLwRvOW7Nr29vEw7lzubPAa9Ez0qvWW8FL32vHc9oL1APaU9qDxQvW48vry2vN86gTz2PIq8fj3nPJC8Ob3wu7487zsQPLy8J73COps8ML27vDy8dT1MOmY9kr0LvRu9qzyMvd28Yj0APSM9yryUPba8zz1+PaI8t7w4u108xjkWPN68ET3iPP871LwvvQq8tbwUPUg94LwcO6o837yEvKo9ibz0Oom7SD0tvIU9W7z/PJY81Dyculi9Sr2OvVg9x7w/vd6827wGPB68dr1yvLU7SzzIOcq8aryjvJu7Xr2DvaK95ruPve68MzpmvJU9PT3HvIk9ibySPB69Cr3MPCc9Dbz2PJE9+DupvZ07m7zqvCQ9pzo3vTI9RT2WPKe72bwivd87K7vVPBq6Jj2ZPMG8hzy+O/a8hzyfunS8dr2uvIu8Kr00vJo9c72kvJY9tzwwvbG8oT0nvbE8w7ufvKK5L7wiPVU+wzu5PLs73zxivKI8iDx+vLM7kLxOvQM8KrzwPGA9zL3fO+G8AzwTvWA9DTtXvRY9NTtlvCK7xDwEPVo8Ar3AvLO9TTnkOZy9mTyovGW9sjyZPAq9FzxovSY9rLsVvYs8xj03PNa8A7zNvC+54z20vH88iD0EPei87zxUvUe6ljyXvdE7ojvnuxu9aDo/PLs8nTs0vWe8wj3DvHI9Tj3IvP68nDy4PEG9iTzzOoq8NT0IvYu85DyJPW6877wKPXA9qrwWO748lbp0vJC9Ub1HPIu6xDv8vNw8DLzDPI89jj1UPBU93rwdPT48IL1LPSU96LxuvMo7pbhrvLq8X7yYPbk9Lj2OvWI8RbyKPP+8HD0qPLa8/DzBvcS9ED3ZPAi9WTzZvVE8sjuCvdS8y7ucPCA9MzwYPLK8ML2MvI68pbt+vFI7b7wEPWM8n7wKPYQ897jXPA09sj3MPN68j7oRPJG8Nz1YO4e8FDsOvVw9qzw6O2w8Vz0UPR89Cr1ZvM88Y71/vRm9RrySO6C9RrsEvFc8QL2DvSQ98rvPOxY9yrxKPeM8JT0du+K8Hrv2PGQ9vzwMvf88rToTPbY7h7xJvEo8C72UvI49cT0ivaE8oD05PFU92jxbu687GDyUPIk8BD0GvX087bwiPWq7Nrv7PG49H71yuDq7H73mvA29HD3GvDA9mz2CPCK9Dj2YOoY8JD0tOnY7AbtQvdA8TD2euSq7kL1YPY27TLwzvNU8Zz1ivUs807yovEk8d70oPSk9DT1evCa9Eb2qvAy9CbyKPNs8Xj2CvIs9kzvcvLW9/busvC09tzxmPca8LbuRPaI85TyevIO8gjsdvCW877wrPcw88zzguoE9dDuHvJ69LLtzPBU9bzxZvA+8vbwRvbW8Wb1rvAM9bDyivB09wjyOPac8VD0EvLE8Zr1yvUc8QTy0PW48tjy/PLY9hbyrvHe8Fr0svSo8/bszPBa99zyyvKy87TtDvaw8eT2AvQS9JL0bvNq8+ruUvYs89juQPOY79zwRPBs9HryjO6I8VrxxvRC9Q72QvSo8DT3JvMs6nr2BPJc87zzbuw69z7pwPU69Nb2UO1A9cjoVPXw6czx5Oui8VTxUPBm7+LwPPWI8Y72kPMO8vjwGvHc8PztOPQE8PLzEvUe96bwePNc7k7zeOWs9ZT3APLu82Lt3Pec60rwMO3s8yLyDvVs7jLu1PCQ9bTs2vRO9hTzDvJ8987sqPGI9kjkBvYC8xzxDvSs90DyvPF69TD1OPPi8Wjygu3Y8DT31u5A8nbtUPEm98zxDPDS9FD3WPJ46gbyevIM8Ez2nPMu8+7wSPPq8kzohPbs8Mz37O8y7E7zbvEE8ZT0AO2a8LbydvBe7Ar1lvS09q73gPB09hT1LPL+8SzztvVS9Xb3tPEG8t7tQPCC8DT2sPOo8rDuZPJ895bzivI88vLwDvQU9Qj0KvB+9obzUu4k8P7s/vKS9kL1XPXe7Ar7RvFO9Fz2rOyC9a73mvEO7oD0UvYQ8bLz+vBm8sLx2O0E7SL3YOtm7CbyoPNA8RjxEPXY9IrsKPbo8dzxnO5A9Wj2oO6I9b7xxO3Q8nzzFvAs7m7uUu1G90zsHPKg8DD2Curs91rxfvJC8xDyFOrO8lD1oOy28qLz1vDa8N7rruqg9U71lu9O8Fr3zvEO9cj00PBI9bD1Mu8k6nbv6OF+8Sb0NPEi807sSvao7Wb0Fvbw7ibzEPCK9YL0aPaM8Xz0nPco8hDyBPAO9Qzu2vHi81LyIvBE8FD0Xvcq9cT3tuia7Dr3gvIY87DuAvB88kj0WvSc9TTygPA89kr3zPMU8lrzAvOS8m7wcvJQ8Fb35ulA7hbwCves8sTyBvFU84Lswujo6rTx0OwM9hL02PQO8w7zPvJs87DyPPCo9ibwfvJU8/zy3PJe8SbxcPQc9bjx/PLA817xwPWc9Qr2pvdM8orwDPcI5hryCO587ED3cvG282LyoPQq9G70xPYu4QD3HOh+9qDysvHa8Db1avB699ryrvHG9Ir2UOvO9BjyZPYO8az2gPAg9gLxdPa+8jTwxvbY8+byvvZy6QD0XPY09RDzVu4u8Wz2nvCO9m7wcO4+81DtJvai8bz2Au6O7tjxMvd28Mz0MPRE8wzzzPF68DzuMPSQ9QbtjvbQ5tTsSvcg8dru8vKq8mbzQPBU9D7uZvU09DLzjvLC8jD02vVM8trynuws9kjyTvCe7K70OvYi9zDy/vBe9Jj2ovTU9/zshPMS8gT1ZPMi7gb3Xu4E8UDvRPCe8IjwRPUS9n7zaux48PTycvbs6tDxquxE9Jr2FPDS9krx1PRM9lzzuvO88P71lPVe8Gr0aPYe9ET2yu289R70Tu9q8Oz3+uie9GTtBvSm+IDyvvNU7Mjw0vKI8gDwePUI997z6Oz69lr3EvCO9FD37PFc9Kj0Uu7+8Azs7vYG5ljxBvfC8FD0uvTq9Xbz3PJU89zqxu0G9iDyJPZs8Fjy4PM68hryBPFK83jyEvIy8MrxyPAW8jrz8Og69yDkvvNa8DbxeOrY80bzMPFY90jx2O4o7Dz1yu+a8JT0DvPE82jyQuyI94rysPao8Uz3mPA09Dj05PS49Q7xKvUU9UjyLvKC6Ej3dO3s9QL2MvMc7Gz2HPTQ9ubyUPas95rwQPfG7YrzAvFm8nzy3PMU8qrzwvAA81zyOPVW8rjyaPZ09Jr1IvD+9Kb0ovYW8A7zTuzM9KT0Gu5m8VjyKvLq8a713uy48ZzwwvSC9GT0uvYO9pDw9PRG8CD1sPRE8rrxAPDQ8mTzNPMU8Dj2HvIY8ojvAvG+8hDocPDA8ibzuO0w9Or2avAa98jzOPA49Ur2zPO29ObsAvYa8b702PTa6GT1JvFe8hbsbPa+8lj2JvBG9Ur0XPbs8VLusvAS9Rrx8vac7jLwMvQG8UbyGPHe8T7sDPV68Hr0xvIy9srytvTq8mD20PNq8qzywOsq8FrtOvMg9nrwLvQc9Kr2Ku1I8DD0zPfM7orxRvaO7B75WumI9ND39u9e8LL39PI08Ij2lux+9irwqO9s8qLwyPKO8lLxuvae8tD24PFe8zjwLPaw8Ob7kO/I8ET0wPSm7WTwIvLC9Dz0YPR086ztoPQq8Cb3lvFS9yLzFvDs9E728vPS9vDzlvEI9Sr2ZvM+7jzw4PeI5NrzPPP88Jz0bvZS8Qj03vbM8rTy3Ops9bbxhvZm9uzxZPVs8A7znu1m9NL0KvYe8Nj0xPae9Qjzru+88gb1evfA8yzwYvAu8Ib3VvUI9/zuZPO68Lr2tPM68zbzHvD49ILpUvRg84bwVvVO9ETzjPA27DjylPCu8Xz05PES95TxIPJs8ED1jPRc8pTw2Pfc8kryOO1O81bwLvfU8OTxHPRK8Ar2XvGc8Hr2iO827Ez3LvIy8ML3sPHc8Dz0hvZM7STyPuiu8NzvFvDC8Iz2UvJ28NL04PUk9AL0kvbK8G7x5vIK9QT1DPf68ObwFvRc8YbyCvJS4CLxlPW+837zBvJA8jjuivP+8IT3ePNU8dT3QPCC9V7ysvM+8N71evW09kryiO0K9wbzhvIk7Or1APYA8YT1fO7c7Bj3QvOQ81j2Iuxk9qjyTPYi9uzzkvEq9m70CO6I9A70EvpG8CT2Evfu77TyCOhq8HD3uvEM9ermTPe87l7tqune9f72GPZ07qbwHuwE8dL2MPLA8Yj3TvNi8DLw/vL08PDy7vOs7U7yuPMu8PbxjPfK7m7yUvPS8Jr2SPG48T70QPVe8lbtFvB88N70zPXK9HL0GvQI81zw4vbM8sjy2PB09DDwVvTe8gzyRPOu89TwNPCC90zxhvAk8pjzvu9U8ujxjvHo8DD0hvI48S735PG49+jwTvNU87jzJvM0857wRu2W9iLwpvdW8IjxbPau9vzxbPUe9h72gvMU8xjykPII9c7yTvJA9Cr0ivFu9Lz0mPAq9CjslvRC9Vr2gPFY8szyuuAy8jjqaPBY92LyfPB69Qb0PPd66bLwrvRs9lL0nPb+8g73GvEu9mL1bvcY6xj2YvY68J7wEvUO84TwPvS29tT2bPE+9PTzbPCe8LL0CvT47Wz2pPBu8hj1Jvaq9Nj0rO++8ID3dPEA94bzdO1s83Tuzu9e7YzxGPPK8B73BvV+9ST1ivDC8T7ydvLA9UbxbOwM9kj1cPZG8bT34PJs8CjzXPHw90jySvQm8kjtePUY9NL2KvJw8VLyBPNy8RL3AvC89j70GO7a9Gz0gPOI8XT0YPd28cDswvQy8Rr2eugI9ujzKu1y9mDtBPda8LjtovGY8prvNvBW8arsPPfQ7Jz0HPYU8Gj05vXi9VbwQuwM9RzyovaS8IDwNvOU7Pj0RvCy9Lz3TvBA7nDqwPNy6jTzmO3c927oevRQ8Gb04vNW86rwtPTA7zLsfPb+8Vr2svB49IL3UvJc9rzuuvTG94ryMO4y7wLnivHq7NTsdPcC8sTwmPfs7Tjv/PPU8H7sBvRA8Nr07PYm9lbwtveC7BD2eO2K97Ly9PKE64TwGvHm9cLzSPF07DTyLvQ69xLwwPaq9u7xdvJk8ub2bPVo9XrqJPHe8xDuhPLI8jD09u467Mz0ePOA7hzu5vOK9Lr08PNO8uLwfvXa7vrtkvZ08WT2BPO+8rjwFPA69gD1Nu2U8jLxWPTA9K7wPvHa9lDzyu688vLy2PDI9KT32vJw7GT3UPOC8vzumPIY9SLxTPWK8FL2Fvbw6Mz2LPF68WrtFvf28rTt8vZC9xTy4PW+8+rpMvPY8KT2EvOG8gLuDPIE7ZDrkPJK7Hr3rvNu8DT0jPZw9V7yuvCa9sLx2PC09V7w/PWq9SL17vVm9hT1HuiQ9Lj2Nuki9pjxBvCI957ysvAq9sTpZPYw8VrxhvEm8ujsevSI9Uz3gPMy7izw6PDk9Gz03vfQ7nrwXPbu8Ury0va68RryKuu07Qrz1O0A9WTy9uCa9OT2nOoO8uzspPG07pbwaOhY8Jz0OPEE9mL2ivKg8lLw7vaO8G70XPeU8oDxuPWU8BL1QPSI9DjwpvEU8nL0nvTc9uLz+vCo9WzsSvci7er2oOy+9HbxWvHM9UTxjvZQ8H71Zux+9L7wovYE67TwHvVk9Uro1ujQ9mD2JPWk9gTxEPQg8Hz0yvc08QbyjPbC8tTxCPNq8sruovCC97zzJvEI9XryhvFu9u7x/vXE8ZrzsvBE9rTw+PW+8bz2bPVY92TwTvXU7kLz3PBc87LyEPPE8Bz0QvCq8Yj0jvRI7tDsBvcA7ODxpu1+9N7zNvMC8tLyJvJU9O721PJg8rLznvCc9mL0kPHm7PTsTvQw9WrxkvKw7LrsFPTQ91rxtPHQ83TzZPSe91LwvO3K7Ob3kPKg7Uj28u/k8E70UPaQ9ZD2Qu6C5BjwDvcs8PbumvJS8+rxFvY+9/rySPHO8ZL34vIO84LxMPJE8Ab0cPLS8tbz0PIY9CDvwuow7Nr1Qu3484zjjPNG8hT3FPIs9BzzNvG29pj1su6g8rDv3O8w7tTyHvak8Br0wvQG9Qr3TvEO9kL0VvHc98DxUPGy9bzwHPRO8Mr3UPCs8iT1pva67RryJO0G9yTyyPPs8JD0PvQm9B73gOsc6EbuEvO+8nz3gPDE7aruZvWM8NjypPM+8Qz03vNk8cT0hug89QzzjPFs7TL2rvKU6HrwkPf+7lzu6PAo8i7vJPMu847wSPRg90T2vvbo9HbwLvfY8FbzDO2u9ljx4vJU8tTxPvGE95r2Luq88RLupuy88jTyOPa484zxPvVu84LyKPCs9jbwbvTs947wkPQG8LzmxvA88Dj3VvM07ljonvBs9fzyfvWE8D73BvLM8GLuSPYk8z7uPuxE6AbxmPac8nzy7PAa8kT2PvQ89rLvIvP68AL1KPX29h7yHPc68Fb0OuzC9gzwRvZ28bL39O4o94jslPAi9ST2Yu4u8jryuvHa9SD2LPF89/Tx7Pfe8H7wvvSW92TxJvSW9hjw+vLQ86TvIPL88IDxwPA09Bjx1PYO9i7oVPZ09bLwLvBw9TD2fPGG8oD23uxY9yLxOvbm8vjtVvTa8FD2PvW+9r7y3PI08N72Wubc8ArzaOrU7YztZu5e8LD2Suya9qDu0Ogg9czzrPMk8BT1qPZu9Eby4PdI6prtevE27LT2AO8c8X73cPDE8C70SPUU9Tr2FO7I867wvvRq9Dz3NOrO9ST22O+88HDu7vMQ7uTwTPSE7RTs0vTU9AzzfPMk7fLygPPw87Tv3vI48gb2cvH08uzyxvHY9lruRPY08J7yTu9Q8lToZvR88vjxwPMM7NDxEvEo9Tz1avfo8ljysPL67JrsTvAw9q7pOvIg7o72QPLK9zzsSveG8CDwEvNY8SLzLPGo7kj28vBa9Gz2yvJQ8Jry2vDo8DT0QO429prtAPFe9KrxTvV+8dD0zvVC9TzxMu5i9Xz3GvEU9dLz+PEC7YT1wvJk8vb34O8c86rxAvRe8KL1NPQk9f7ymvE89J73CvsW8bT04PaY8kzyDvY28tLzSPK08ujwXvdw7P70IPYs7VD0yvHW8D71ZPZQ7gr0xPW48iT3ovVC72LxwvWg9Tr2/ut482DsrPZ48sDsOPSg8RT0yPf+8Nj2EvZI7jzwnuwa8gTwrPVe7Gz0oPAI+lrw7vbm8YDxKvEC9IjwDvce8XjufvHs8QD16PJY9TDw1PRM9ej0NPZ48a73KPNe7gj21OwA9573uuwS9I70YPD88Sr3oO849BLxjPV29LD1zPa87GrvVuhm9UzwAPW+9hDlRPTw86bzOO4S9nLwlPQ+7DjrovJ89+DzvOoW8zDpSPOq6Wbp/Paa7Az23vak9zzv4vMg9yLzEvPM7ZL3ku2g9e71sPHS8Ebz9PM88cL3vPNY8N72TvBc8lDvyvBq9Bb2CPY04XDz+vGC9fbyCPBG96LwXvLC8Nb3PuzQ9QT3Sux49Xb3zvJk7yjxGPZM7x7uZPeg8zjvMOze8FL3SPKQ8Fr3NPI284zyRPBO9PztaPPY8uLysOzy6BL0JPXW7d7xavSu82zshPPe8DT1Pvdu8i7yqPIi9gL2jPbW8gL1nu4A8Sz0avQy9h7zjO1c96rzpPKS8Bb0UPS68gjwQvEC8rrtKPcY8h70evFw8Bb3KPF09Jz1EveS8LjyBvKw8GD0EvRK9QTzePMI8wjsKvLK7jjz6uvQ8QzxaPea8BjyYvIa7LDwyOni9xL0CPCU9fDxHPUE+rz0dPbK72LxlvPg6Kz1LvTU7Kb0WvAi9oj2XPWc8w7ydvQ660DutPYg8Vr3qvKu9Mb17vfG8pLwRPOy8az3nO7M8Lbx3PZ08hz38PE+8Yby0PD48ajyHvQc8Kr3dPCo8kbp6vOc7wLypvF88EryIPQS9bDrxvCk7Zr2Vu9I8cTvcOgQ8aj1gPe87Nj1Dui48Kj0jvCM967rlPKy8EDuAPZU7lL0EPTu9ujyEvZA8bbwAPS+9Yb0kvcQ8XD2rvaC7DryrPBE9HL2sPJs88bwEvey8Yr2ePXs9aLsRPAQ76zsLPPy78TzQPI+8UD18vIc8Qj29PWi8l7sZvRo9FDxuvQg9V7xavTi7xzsFPDy9P70pvdk8lb34PFA8ULxSvME8Nz0lvYO9urxLPbM8hr1pPXs847yVvMQ8vzwCvro7hTxmPRe7lbxXPQ09JD22vZ28gzw8vRW8yzyfvAG9er2xO4I9u717PAG9A71RvDg9ID0hPKu7CbvRvBI9QDyEPA+9nz0Xve285rsiPc65lTwBPWy9vbw/vEg7O7yPPWg9Cj0cPB49wbyjPL69Kb2+PIk8Rb17PI487by1PMu7wTwSvWa90rxMPbm8I71/PVU8m70FvHG8ZzytvAm9g7xtPMo8ij1FvTI8mjzYvMC70jwHPVe8Yj1GvAy9IT0iO8C8DTwXOhS9tDw8vFu9+bxnPTO8grvNO/88XzwzPU+8lr0hPck8pbxJvRC9BbxyPKM8uzvfvPk7j7zTvDS9Cz1kPQu9ST0aPbQ897wjuy686byXvFW9xrwqu3O9Dz1DPIc9ZTx6PNQ8D70CPZo8Fj15vCY9fLtuPWY9LT1nuws9n7wovbw8JL3PvFC9bjtBPDe927v8O+G8+rxfvJK9Oro7vac9RjzQvAm9Hj3pPFc8hTsuvLA9+rzvPPm7Bb38OwS9QjxCvbA88TunPVK9tLxePHm9N7xCPQs9wz0mPDM9lrwdPTO8HD0qPIg9lD1YvBM8iDxxO628ebz4vB696jyTvaK84zxLPXu9g71UPRm96TzCPMC92DxyvQg8Wj2YvKA9l70ZPP45hz2YPU28Gz3tPIe9Or3pvCi9cTzNO0g8rTyrPZc8LDwxPb87Sb17PAQ8TTuVPW49Tb2zvMc8A70IPdG7aDwrvXI9QDycPcq7QjxhPR49TjxNvNi7WzywPC09jb0EPRQ88jviOgU9Sz2uvS48PzwJvcK85ryuPHW9V7zFvOU7pLzJPIE8mrtguk68t7whvLM8Nb0Lu+Q7yzyLPC29BD2BvBu8hbwdPk89ejwgPSg8C7x1uy29GrunPGS8wzzuPKY9wToEPI69Ij3Du2I9dby9vAm8CLymvIm8drqjPcA89LxAPfC8gT3hPLy81zxDPZE8az0sPdg7j72vvKG8Q7yfO628N7zxvFU8dzy1PC69XrwBvQ07nD2KPPk7pbxLvVQ9AbztPIi8AD0ovJa7Kz39u5U8cbx0veg8yDyhPLu807zoOUu9Cz3iPBQ8crytOxO9Dj1LPDK9ejxdPNu63bxpPBC85LkpvOS8mzs9uRy92bw/vH296TxXvQo9vjz3PHw9WjyDPeU8AT1EPTi9KzpOvHq70zy4PA48urs5PSu9KbwnPay8jTy3vKI7kD1fvcC7WL3OPG88RL1CPfO8bDxNvT69LD1VvWk7G73MvTS9i71AvR89yzx2PLW9Lzy8PU89wjw3PIU8gr3vPFe84TyHPeI6Dj0bPde8ED2yvRu8nzziPOS83LyRvLM777xTuz+9ejyGPZy8ND1HvYM9gj0/un+7ab3PvDa8MrwYPTa9tz3tvCo8/bxhvTG9ZjxGOxC9fjt4PJK8BT2cvBU9Ub0fPdI8hrx/vbc8SjwZvbw8hbs2vVw7kz36vGw8MD04PPm8gTwDPdA8W72NPDs9Cz1iPYi8GT2QvYa8BL3MvHo8j7ygvMy7C70mPem8a7xWvVK87DtXvIk8tzwbvai8YDx1PUm8q7wpPYI8dT0RPZk8ozziufm6Dj2YPSw9+jxTPXK7y7xPuQ+9ez2oPKs97bstvZ48qjz+uzC89LvBvam8tzufvZW8V7wHPKs8oLwIPai8cDygPSK8XTv6vIu9MDyZPG08FL3FO9c78rxBvZ88O7wgPaw8s7tGPQq97ryJvOu6zryOvEw7J73JvIU8XzxMPYq99TzhPLW8ozzovDi8Wb2JPNw8Kr33vAg8Gj1MPcw9SDxbvFs9UD3eOLC8YD0nvS49u7xHPSE8zzzcvE09HjwlvQc8K72wPI27C7wQPdE8/byCvR09SjwSPSm8zb21Pd28vLzmvDq9lz2TvKc8z7tDvN67S71Gve08srwXPYW8azuRvFa85LssPYW8ML3SPCO8gTwpvaa8fjw8POW80LkbvPs8Fb0pvW88bjyhPCQ7p70dPbo7NbxAvQA8SD23u5G8DryXvfM5obzsvDC9XLtkvL682DsdPSw7Izy8PJ69yDvGvHS80T3CvI29jDs3PTC8jL1bvTW8KLyoPYM9trzfPHs9JT36O7k8Pr0uPYG8JT2GOUk9A72yPB09BLsxvZE7Ob06PBi9ZD1SvXc7Qj2BPd08pTzGO5U8iTmCPKS7X73uvBy8ir11Owq9VDzyO8s8Hz3hO/S8C7zIvVe6oL2JvCK9JjxEPDW9jDyhPAS9kjwLvXG8Qz0fO1g9dj0UvB27kbx0PFq8O7yKvdm8+jy1vJa9W7wqvSm9kDxlPcS7F717vKi5DbxmPAe9WT0OvU48RDz+vJ88P72RvRE8VL3WO6u8vTykOwU9FL3nvAs9Jr1AvD09W7wnvf64Urz0PFe97ryYPZc7hD0FvUW8iDxTPWk9aD2pu2y8jj3hPA+7Mj2EvKy7iT1AO8q88zxtuwY95rslvfS8GzzLPFq9Fr1JPJa85zxKPYq8fL1lPCo7jLyYvCG9ZbwZPFk7Oj3DPH69UrxKvHq8mLwQvae8KD0CPHC9IzxkPS64rzzHvO68Nr1QvVK8h7wIPSG9nTzwvKa7gjzUPZM8iLo4Pdo8aTznvI49oDzJPN28kby5Oya9yzyKPPw80TwnvRa8brw4veW72ryXvfa9jj0BvWw8bbxtvDq9Gb3NvES9QD34vDu8fTtivRC8CjyivGa8eLoUvJq8K72API28HLzOPEC9Bj1pPfy7Hj1tPKe8Bj2iPJy9DTzTPL26GL1nPds7EL2NPe27SbyIu5G9hjyzvNC7mD2NPee8zrssPSc9wbx9PCS7Fzq2vXW62TyXu688Or0OvaQ9Vb0uPaE8I71DPVO9jrxwvbU8Pr2YvD87qrzhPYk9g71zPaY9kDy/vF65fLyHPXi8nTyWPHI9lj1TPXo9qrrlOYq8Zzs4uui73bwhvWY9wLy5vS09KDstPPc7pbz8vIg9QztOvGS9vD0Ovdy847wLvVo95LyWvDi8N7x1vTs9erkIvMM8wj3oPLC8AD0qvOs8ljwPPaM8cbyTvEG8aL05PLE8yLxBvIg8m7yTu3885zqtutY8KL01PE49nj1VPRQ9LL1KPVg8I70nO1Q9DryCPDC8uLyMO5E8TD1TvTG9xbvdu1w9sT3+PD899bw0PQq9rLxbvbA8VTsjPZs94bs5PII9qjyfPBQ9Mjx8u9078bq2u3W9/Lz3u0G8hrwnPBY9Lj27PK08ozyTvVU837wpPVu90zzOvFo8PTtVPaQ8GD1MvI88lr3EuyM9J7sZPWG87zxCPFK92zxvPaY8bD3EvHw9bz2lvQq7DD2PPMA9b7sTvDG8sLxYPGc8CrywvM27QL0oPeM83Lz+O+o8qr0jvRY8OzxQPbs80Ty+PIC9Yz1evHS8gzxmPDe8CL0GPle9PT2PvG+8Ob1GvXk8lLz8O5c8pT2+vDQ9ibwLO5g857wtPTO9Ab1ovfq7EjxGPEq7rrtSPR87rDwtvb26ozyGvOY8xbhevY48Gb3/PI09n7stvDg9WLyPvb+7iru2O0m8M70QPRA8eT3DPJs8ojyLvbs8h7xGPA09Wj0APCC7VTzJvEk97LxXO1M7Bz0oPeU7T72xvCm9MzyAvBC9JDxIvZk80LxJPak80LxpPQI9d710u4I8bD1gPV88BjzVPEg7J7zCvBw9F71KPB493DwFPXa9B7xVvQm9hD3rO5M9LT0+vN08kTysPOa8p7v8PP278rxmPYg9nbwTPVK8JzxPvAS9vrtHveY787yxO9c7dbwUvUC9+zvFPBq9VjvnvH89xbsYvfS8Fz3WvPS6Jj2LvWS8gbxbPNC8xz1CvMc8O71Qu/A8ZD1dPbq8/Lqru4a9BD0NPTs8KT2ZvIc7RLt4PQE9IT0CPds6xDwWvV29s7whPRA+wbzUvZQ83bwlPQY9gbyXPCw9bLuQvLI94Txau7Q8Grzpu1i7/ryJvGw8aTwmPXe8x7yMvGo8aD3Ru0Y8g7yBPbs8kr0/vfu8Xby2ujq9k7wmvbQ7cD3rPDe9l7x5PXI9jDy8Oyo9GTuhvGc93zuPveU82zwfOjY8ZbzaPE69ijy9vE89ZTwcPVI9mbwGvca8obxlPKU8bz2QPfG8fr0ivX47Ob3kOgQ8g7w+PUE9Cj2FvTk8db3qPF69oLxNPYG6g7zTvDy82rwTvUk9DLyDPQO8pLyVPJC9ML2+PJS9wbtCvRa8r7wIvbs9wrwePbk8Sj0oPYq8Gj3Ju/m7rbxpui68Bb3pvCk9Uby3Og475rtavEq9wTzEu1C9az3luoM9rzz1ux49pjxJOTI9sjsUPYc7pby7vIm9Pz0WPA49wbzhPKG97LxqvMY61zzWvLK82zx9PRq9irsrPbo80r1LOTm98jw9PRm9Bzy8PUi9orwGPaY8B73hu+E8QD1PPIQ9kT3MOwq9Ij2zvUA9dLvdPIE8pDwevQA9CzxvPTO7V72oPHm9mr1/PJs9dr2qvY68qD1JPPW8iz1wvO88FD3pOya9qb3+uwy9Sj3nujo9sDy1uyU7dbtrPQw9rDwKvWA8xrxpvWS7zbwZvbc7Aj1vvJi8JD1pPRa9sbzjvB68UDsGPBs9FzyEvHm7iLyAPLg8ID10PYy8ULyIPfQ8Vz1IPI08eLvGvDK9db2RPYs8XL35O348rDyovBQ977nkO3O9Gj0cPQ26kb2uOwg8JzumOkM7CTs2vZS8w7zrPIe9MrwfvSW9qz0xPC89kbyEvlQ9eDsXPGu8K713vUu9Lr29vAq9Ub23PLa9l7sQPWS8XLycO+A8ij0EvYM9Dz0svdq8QD1OPVM9hj32vRk9HbvDu2u7tDw7uwE937zbvCo9gzxYPGQ9o7w+PHU9zbqavNc7Jb2aPRO9zb2yPNa7Nj0CvHi9QzzVPB26Gz2GOwK9Lz2oO508MzqWPc27FbsZvUQ9wrzGPPA8CL1fPRI7DT3iuRc92rxpvT898jttu/u6HbwYvVg9/DwrvEk9BL2ZvFu8xrzGO3Q8Er1rvSo9JL0cPAc9M73JPE08yzwNvGQ8Jb2Uvfw7Tb2fPVC8vTyFvbC8lLxmPHq8wLwlOzw9DbxoPBS7UDwFvQY7xzwOPU08WLwbvYQ9njwPPGs9sjuWPBi717k2PRO9oLrnPIs9TzzqPQ69mj3Tu7W8Kj0lvRq7UTuzPB09Jz3wuzQ91zwGPPU8srwVPQm+abyXvLM7UTxSPUE9+7swPaW8cryLvBa9O7xPPOW8ib3zO9k8GLmBvdk7jD3HPPs78LzhO/W8ITw8vUa9FLxjvP48yzydvHc7Eb1evCy9ED3bu5C8n71TvXO8Oj2hPJU9NLzBPBG9sbwUPcI9jL29vZA9QL1cPSI8LTo8vZW8trwLvZU7Pz1kvfe8eLxKvQo9Aj2evFQ9Cr3EOk69o7uyPFY9Rzz6PFu9yjyMPAA9ezxMvJa897u8vPu7vLpbPUm7tjwEPS67YLyPvNu60zsgPP86SL1huyg8IL0XvK28E7zkuza9vbxQO5Y7nDwGvRM9K71lvBS997x2PaC9QD2lPac8UL1vPL28trz5OoI89jyJvH895jyQvDq98ru/PPM7Dzy7vCe9yzqbPDC9u7w8vHY9UDpmPZK9C70bvas8jL3dvGI9AD0jPcq8lD23vM89fj2hPLe8NLtcPNk5FjzevBE94zz/O9W8L70IvLa8Ez1IPeC8HTurPOC8hLyrPYq86zqIu0g9L7yFPV28/zyVPNQ8mrpYvUq9jr1YPce8P73evNu8BjwcvHe9cry6O0w82TnKvGq8pLycu169g72iveW7kL3uvA46ZryVPT09yLyJPYq8kjwevQq9zDwnPQ689jyRPfc7qb2bO5u86rwkPZc6N70yPUU9lzylu9i8Ir3dOzG71TwQuiY9mjzBvIc8vzv4vIc8n7pzvHa9rryLvCq9NryaPXO9pLyWPbc8ML2yvKE9KL2xPL27oLy3uS28Ij1VPr47ujy8O988YbyjPIk8fry0O5C8Tr0DPCu88DxgPcy94jvhvAM8E71gPRI7V70WPSo7Zbwhu8Q8BD1aPAK9v7yzva45CjqcvZg8qrxlvbM8mTwKvRY8aL0mPa27Fb2LPMY9NjzWvAS8zbylueM9tLyAPIg9BD3pvPA8VL1KupY8l73TO6A76rsbvXg6Qjy8PJ47Nb1ovMI9xLxyPU09yLz+vJw8tzxBvYg8+DqKvDU9CL2KvOQ8iT1uvO68Cj1wPam8Dzu+PJ66cryQvVG9Rjx5usQ7+7zcPAu8wzyPPY49UjwVPd68HT0/PCC9Sz0lPei8b7zJO8O4ary6vF+8mD25PS49jr1jPEe8ijwAvRw9LDy2vP08wb3EvRA92jwIvVs82b1SPLM7gr3TvMq7nDwgPTI8GjyzvDC9i7yOvKe7fbxPO3K8BD1jPKC8Cj2EPEw31jwNPbI9zTzfvJW6FTyRvDc9XTuHvA87Dr1cPas8OTtrPFc9FD0fPQu9WbzPPGS9f70ZvUa8kTugvT+7BLxYPEC9g70kPfG7zzsWPcu8Sj3jPCU9KLvivCe79jxkPb88DL3/PKY6FD24O4m8SbxJPAu9lLyOPXE9Ir2gPKA9OTxVPds8WrusOxY8lDyJPAU9Br17PO28IT1suzW7+zxvPR+9B7hDux+95rwNvR09xrwwPZs9gzwivQ49pjqGPCQ9JDp8OwG7UL3PPE093rksu5C9WT2Ju0y8NLzWPGc9Yr1KPNO8qLxKPHe9KD0pPQ09XrwmvRG9q7wMvQi8iTzcPF49gryLPZE727y1vf+7rLwtPbc8Zj3HvCu7kT2iPOU8nbyEvIM7HbwkvPC8Kz3NPPM807qBPXE7ibyevS67cjwVPW88W7wOvL68Eb21vFi9a7wDPW08orwdPcI8jj2nPFQ9A7yxPGa9cr1JPEA8tD1tPLY8wDy2PYa8q7x4vBa9LL0qPPm7NDwXvfc8sryrvO47Q72sPHk9gL0EvSS9G7zbvPy7lL2LPPg7kDzoO/c8ETwbPR28pzujPFa8cb0QvUO9kL0pPA09yLzeOp69gTyYPPE82rsOvdu6cD1OvTW9lztQPYA6FT11OnQ8gTrovFQ8VTwYu/i8Dz1iPGO9pDzDvL48Brx4PDU7Tj0CPD28xL1Hvei8HjzXO5O89DlrPWU9wDy6vNW7dz3nOtK8Djt6PMi8g71ZO5K7tjwkPWw7Nr0UvYQ8w7yfPfK7KzxjPYQ5Ab1/vMc8Q70sPdA8sDxevUw9Tzz5vFk8ort1PA099LuRPJi7VjxJvfM8QDw0vRQ91zyfOoG8nbyDPBM9pjzLvPy8Ezz6vJ46Ij28PDM99zvNuxW827xCPGU9/DpkvDC8nbwVuwK9Zb0tPau94TwdPYU9SDy/vEo87b1UvV297TxCvLe7UTwhvA49rDzqPK47mDyfPea847yPPLu8A70FPUI9CrwfvaG807uKPC+7P7ykvZC9WD13uwK+0rxTvRc9rTsgvWu95rw8u6A9FL2EPGq8/7wbvLC8djs7O0i91zrbuwm8qDzQPEU8RD12PRm7Cj27PHY8bzuQPVo9pzuiPXG8cTt0PJ88xbwSO5y7l7tRvdQ7BzyoPAw9e7q7PdW8X7yPvMQ8iTq0vJQ9ZjstvKm89bw1vEa67LqoPVO9arvTvBa987xDvXI9NDwSPWw9T7vDOp271zhfvEq9DzxKvNO7Er2qO1m9Bb2+O4m8xDwivWC9Gj2iPF89KD3KPIU8gDwDvUQ7trx5vNS8iLwUPBQ9F73KvXE94romuw694LyHPOY7f7wgPJI9Fr0nPUw8oTwPPZK99DzGPJa8wLzkvJq8HbyUPBW9ALtOO4a8Ar3rPLE8gbxUPOS7OrpnOqw8djsDPYS9NT0HvMO80LybPOw8kTwqPYm8IryUPP88tzyXvEm8XD0HPW48gDywPNe8cD1nPUK9qb3TPKO8Az2oOYa8ejucOxA93bxsvNm8qD0KvRu9MT2ot0A9zzoevag8q7x2vA29Wbwevfa8q7xxvSK9lTrzvQc8mT2FvGw9oTwIPYC8XT2wvI48Mb23PPm8r72dukE9Fz2NPUI81buLvFo9qLwjvZy8HTuPvM47Sb2nvG89gLuku7c8TL3evDM9DT0QPMQ88zxdvAs7jD0kPUe7Y72oObU7Er3JPHy7vLyqvJm80DwVPQq7mb1NPQu847yxvIw9Nr1VPLa8qLsLPZE8krwmuyu9Dr2Ivcw8vrwXvSY9qL01Pf07IjzFvIE9WjzIu4G92buBPFI70TwmvCE8ET1EvZ+817sfPD48nL3BOrM8arsRPSa9hjw0vZO8dT0UPZY877zwPD+9ZT1WvBq9Gj2HvRE9tLtvPUe9B7vbvDs9B7snvRk7Qb0pviE8rrzVOzA8M7yjPIE8Hj1CPfe8/js+vZa9xbwjvRQ9/DxXPSo9Fbu+vAU7O71guZY8Qb3xvBQ9Lr06vVy89zyVPPg6sbtBvYg8iT2aPBU8uTzOvIa8gjxTvN08hbyMvDG8czwFvI688DoOvc45L7zWvA28XTq3PNG8zTxWPdM8dTuOOw89cLvmvCU9AbzyPNo8lLsiPeK8rD2qPFM95jwNPQ49OT0uPUG8Sr1FPVM8jLyWuhI92jt7PUC9jLzHOxs9hz00Pbm8lD2rPea8ED30u2K8wbxYvJ48tzzEPKu88LwCPNg8jj1XvK48mj2dPSa9Srw/vSm9KL2FvAS81LszPSk9C7uZvFY8iby7vGu9crsvPGQ8Mb0gvRo9Lr2DvaM8PT0QvAg9bD0RPK28QDw1PJg8zTzGPA89h7yGPKM7wbxuvIY6HDwxPIi88DtMPTq9nLwGvfM8zjwPPVK9szztvTC7AL2GvG+9Nj1Wuhk9SrxYvIG7HD2wvJY9iLwRvVK9Fz27PFa7rbwEvUa8fL2kO4y8DL0DvE+8hTx1vFW7Az1fvB69MLyMvbK8rb05vJg9szzavKs8tDrKvBW7TrzIPZ68C70HPSq9ibtSPA09ND3tO6G8Ub2kuwe+X7piPTQ9AbzXvCy9/jyOPCI9qLsfvYm8MzvcPKi8MjyjvJS8br2ovLQ9uDxWvM48Cz2rPDm+4zvyPBE9MD0ku1k8CbywvQ89GD0ePOg7aD0LvAm95rxUvci8xrw7PRS9vLz0vbw85rxDPUq9mrzNu488OT3uOTa8zzz/PCc9G72UvEI9N72zPK48tDqbPW+8Yb2Zvbo8jzvcvDI84LzcvAO9d7ykPGQ8oj3NPPU8o7yQvJC93bxIPSg9+DyFvB29Ar6LPG29jr26vX29GLzNvYy9crxbPd08mb3pO1S9y7yavGk9yrwpvMM8mbzDvQI+mT3fvIg817vJveY9JD0LPZ27Pz39Pci8hjyGPXM8y7zWPWg7qLoePSa9c7twvQq7hj2tPIU9srwovMu8YDzrPIc9kTzGPXY9AL0kPSy7rrz5utQ6lDu0vRG+HT2AuzG9trzpPDC8k7wxu049az1vvaC8m72gPHS8r72OPR67gz34PCy8gL0APpe8oLuMvOw8tzwbvaG8NbwZvUM9/jyEvUm9yLp8PaG8m7y9vJ27WT35vIi9kj05PcM8PD3CvNg8p7x4PFQ9ab2vuxU9Wj3vve28PL2ivZy9GD3vPRM9TT6SuzQ8vLyHPU49MzsPPaA7Xb0DPme9X7yvvIq9/Dx0vQa9r7mRvDs9jLwBPUe9g7zjuhO977yGuwK8jbzHu4U9ML0jPcW6/7zwu/W8fT2WvRM8Lr2BPBw95z0IvcK94T2nvQc91jyFvUS8kTwMvYi9yLxDPSs9Db2HPK49/TwnvQs7hr0DvZ88gLxIPSM9Eb0gvQC9jr1XvRy8VL3vvK49I72QvKk8Hz0mvfi8PTyRPaw9bj1pPJg6Pr3Uu/m8dj1YvF099L1ZvY49DTxyvZU9azwCPZo8NLznPJM9e7u8PN67U70SPZK9l70qvZ49Fr1juzs9Wb1hvZW7IT31vBG82TslPKU8GD0QPbu8LT1HPa+7tD0Hvc87jb2yPLa9/bo3vcG9u72Zu529qr0qvfs8cj0KPaE7t7yQPT896ztlvVW7ILxGPOU8+z2curW8tryNPYg9zDxluzc837yWOsU8mj0fvb88tz3+vHI9D70pvBY9aD2Mugm967zUPJ68v73OvHk9RD2iPCw9Ib1WPbi8u7yiPFO8Vj0NPVg6VLvFPCW8Nz27PK46LL0hPSw7Ar2uPRM9Az2RPB28hjwfvPk8pb1rPNO9jrzYu5Y9Lz0IvZc8UT28O6u6JbyAPNs9ZLvBPQA8pLtZvZ+9Gz1ivds9gL2yPRQ9ZDx5uxk9eD1jPXe9Gr3gOwY+6zx9vQu9WTzluyU9pL0avcC9Lr1EPDc9ybyqvbO8sjpdPSa9zLyQu+S7mL36OzM8A76gPIY787zgvAy9H7kfvak9ZD1iuwW9vbyVvHy75jwePg69Er1iPEy9FjxJPcK9c71dPVy7lj0RPLS9VT1evF+9SLybPI082D2puy09sL2Xve+7uLsOvHg9nzx1vXW8Lj20vWo9A70ePDS8hz0Bvlq9ZL0KPd67e7ydPEe9FDulPAm+DjxLPPu88jzau908Fz2rPG09pb0KPDW9xDyDu7a9qjxFPUm94Ty2PWC8C74Xvbs8mDwhPc08tb3XvJk9QT1evbU9w7sSPD491D3APHi9azsvvTE9cboCPWY9KL2jO268e717PUY8QTw7vVI9zD2CPY08uD06vem9O71DPeg7Q7t4vdG8E7zpvJ+9HL0+vUu9qj2EPDQ9kjsFPTg9eTqxvCc9LLwgPJ68B71NveC8i7yGPZs9sLpEPZu9uT2YvYq9hD0pvYI9xTw0PYO9LL0IvWw9iTzivKo9aL2Wuz+8kj28PSQ8M71Fvfc9Ez0BPoc9vr2IPWs89byHvSQ96T1hPDO8njyZvDs93z3GuxK9gzyxvYO8ODy+uhO94DwtvaM8Xz3kPUC8tLt4vSI93r2OPIw8Hj05vBc96Dygvdi74LyRPZ46djtjPcg7k71bve+7njynPYQ7Yz0cvQO7Cz0IPUC9Xj2dPMG9LzwBPHo8Nz1sPLU8bjsnPCC7iD12vL48j7zBPUW9ML3IvaK8ErxxOk290rxgPH49s7wKvIQ96DzRPQU8oDv6PKk8FDztvWY9Az0OPOA7dj0VvEY9AT0IPDE9kr1JvQS9QzwVPZM9Dz0Kvbo9o70gPeU71LwBvTg9mz0dPRi9C72EPXq8bzy4va88YrwuPIi9ET4xPJ49Jr1uPcI7/bqAvYk9drscvq49DL0GPNE80LwmvT+9JbzEPXi9Zj0ePcm86ztXvNq937uMvFa9Fb0QPN66fDzVPKy7KrsoPRu9qjwmPaY6nT2JvPO8BrwXPYy8AL51PPs8cju0PBe86jmFu7A8YDxLvWK8Qb2aPQG+Ir2CPFS9aL1Bvai7Bj60PH28or2WvTy9RTyzPBg7zDxlPeS8Sz2LPQQ9Gr0gPfo7ATsCPQ88aD2fvTE9CDx0PCW8vryJvC48uTzwPKw9I71WO5e8l70kPfm8QL3kvA881Lz+vKQ8UT3jPYc87zyevPM8Pjx2uwy91zt7PKk9g70JvSq9uj3uOgY9Ob0DvWs9uLpNvYe7ar2LvKu7cL2ovZc8ED20vGU9WL2wPaI8QT1UvfQ9P7uNPQw+67qHPIG9jj0bvTa9fb09PUu9ZD3kvJm73DkXvKc9Eb12vBu9fz0jvBg9W71FPSE9qL3GPGC9bj0Tvow9cjwKPRk9GTw9PVS9Gj2tPRw+jjwOPMk7BrvcPAc9p708PbC7sDxSPZy8X7t7PU47Qb0DvWG9Fb0TvKw9sLzGPI49CL4+vQw9pr0UvX880rzdO4Y9SbxEvWC9gzwgO0K9Uz3JvKA8bT0wvaa8nLyzPBA77byKvfI7gjwUPV09mz2KvRi97Dxhvaq7tb2BvVu8Pz2QPfQ89jvfvN88h72IPau9S72tvIA9jToIPhW94j1dvDo8ozt1vYW7Vz2RvVq9/ruGPb69Uj0RvZW9zLzOPM88Dz2evZ28ij0dPZi8+7yxPHG8YT06vSI84rwzPK67dj0Bvbs8Hb25vaE8M70UvW09nj0qPFk6kjtyu8m8Kj3jvIU9mr1dvT09/DuMPII9gr3bPG09xzwDvYI9Tj0OPbE7oD3IPCI9/rrWO3A8XjxTPQk9UD1bvSQ8RT2Wu4y9eTupPWq8+Lp0PZQ71zxJvL89RD36PIc5Kb2HPYq7ijy4PW+9R70kPWi9uDwIPZs8Eb2wPFU8tb2cPeG8U7uePWQ9cL2sPea6Bz2jO2C9Lj2RvXy8K72DPTA89TlYveE8Gj2SPUG8S721Pdy8t7zBvAc9vrlAPMi7v7tRPJi9Qz1euxS7Q71LvTg7hjqQPds837wnPV88j72DvAu9PLuGuo48RD2BPBE8ED3rvHy8Hz0gvvI8F70MvYU8qL0UvNm7Bj21vdE9uL1pvDY8tzyUuoo9urzmPJO9yjxtPPM8BztAvfq8zz0svME9Iz0LPSS9Aj6nO8U9zj1+PKO7rryaPO+9frxNPV89V70OvYe96ryPPX49xrtMvZw8lDw4PdC9qD1HOV293b0SvYc94TxSu5S9mrukPdA8AjxJPKS8wLsrPRE8iT3KvO89Cbw+vbi8bbySvTE9KzzGPSe9hT1NvVE7mL3TvQk9hrzSOwc+zbs6vbu6j7xVvXY9gb11u6g9jbwIPTU93TyNvCi9dDyXvJY8iTyHPJS9JD0UvGa9cT3svCe94zxJPU87H7yQu/I8Iz03vH87O7q9PKW7Xz1xvSs9qz3BO229yL3aPN27grxxPbY8vb2QPby7Gz1QvY88Pr0gPQC9dj3APJ+8w709u5W90LybPV29k7ylPJa9mzxFPem94Lw8Pf48pDyvOea8fT2FOSi9CT3AO727p7xovXC9/Dtju4E9f73NvOy8wz1evZi8z71YO6W9U71EPQM8kDxnPeG8Jj16vSC8pDu6PJO8xDytPQg9mD2Gu2u8Sz1AvRW99zw1vY89nDypumI8l7zaOnO97DoNuqe8sDr8PI68qb0fOyI9MjzgvJA9IT2PvFw8YjuFvZm9sjpPPCC9i7uavJ89Br1SvVw8Pj3BvEq9dD0HPZe9Bj2RvP28ervUvYW8Sj2MPai7+DuKvTQ9sLxOPfe7Gr0ivC88ubqlvdw8Qz1ZPZs87TnbPPM8kj0XPgO9eL0MPcs9pj1RPWs8oD33vCq9T7vBvSm8srzxPRI9Gb3YPYo8irxjvWU9aTzMvSU9FLw+vXe8+bwGvbU9aT2jvGi8hbwhvRI8BTw4u5K9uTwCPau8UL3UvQK8xTxoPdu8bDvwvSO7Az0WvZ4717xbvci8ZjxgvAi9OrzoOqG94j1LvDw9ejyzPOS95r0WvEK98TzxO5i9hL2SvUO9I722PZE97DvLuxU9QD0MvQ0+Fz0FvaY9kDzXPH28vT1Ku+c62rzNvMu7xz17vSA9xLxtPSG9ezskvDi9Y7yNvIG7tr0VPb88uT2CPeg8Mr3bPGm9pr3FvBC9LbuBO0o81LygvM09WLyhvbo9hj0svek8Xb2BPds8mz0XPPQ8k71uO4g8kb2JPG69qL3su5E9hjyvvRW9u724PPE8tjyMPdA9lbzpO449DL33vTk8ujxyvX+8fz3auwA8wLwAO708oryqvbI9rzy0Paa8YzytPEI9x7zvPBC95r2uu487Prw8PRO8rb2/PPW9qzyUPJ28Br2kPew8Sj3Tu8Q8g7tOPOg7NLzqvR49oTsWvQQ9UT24PLE8ojzJvbK83rzGPRC9iz3VPLY8rT0nPdy8lrsbPGM9TT1ZPFW9rD3IPWU94rugO2E9irzkvPw8aD3qvDm91z2tvJi9Aj2gvH68p714vci92Tz6vH68/r3TPWY9AL0MvdU9Ur20O6k9PL3FvBS90jx6vM48jDyyvG+96Tx7vYM8BbyyvBw9JrxpvSi8HzzxPJ698DzAvFe88bxGvQE9pD2HPdI8Jz3gPIo7dzyuvGY8xj1fveE9Uz1wvJm6+jsbPDW9ETwxvZM8RD0AvW290LyOPTY8XTrZO4I95Dsdu9E7GD0PPL+8oD36PRY9VbvyPJW98buEPea8Hz0Mvd888ruLOzw9v7yNvfq9ATyHvFE9KD1VPQa9VL0ZPdM8qTyRvTM9Lj3XPZi8kzr5vA2+pDwHO5W83DwBPfI9tLxrvDW9wzwRvvq9uj1IvJW8yryZPb+9njxYPSc7ND3tPXo8KL1JPZo81bhEPVY7D73pvRQ9PrxmPEI9Mz2fu769NT0yvcc9XT3AvXe9VL0WvfM7Ob26Pdq8hz2UPPQ9Tz3yvOo7Rz2QvU28cDscPWE8mDuyPRM9Gj1qPHA6cj1OPSE8ur2EPA69AjxBOyg9U72wPPy8w7zlPOW8Z73/PCM9iz1HvXC7yTwQPIm8Xz2DPX48lrx9vJm9tjy9PA+9Yz2BvcE9+7sBPTc8x70uPUK98zy3u8K9DDuPPfg7Bjy6PaM9hj04vUi9AbzDvQ49yzxsOwc9Wj26vEW9Q7zSvZg9Kz6xPXg9Nr3Nuz471byQvUQ9sDwWPa28grxEPYI8Ez2WPQ48NL0Dvbq8krvPu8w9kT0JvPS8eLx+O3475DyXvJg9lD2BPHY88zxqPQ89Iz0WPdm8sjvEujI9JL12PKY8V70LvMe8w702vYS9YDwIPXO8eD0XvY+9Zz0APpG8Kr1dPLk9bLzJPPQ8CT2xvBm9Nr1uPAi8Vbu1vVQ9gLwsvZg9vTvcu267oLzhvXS9uzwbO607db2lPHi8lj3ivIc8Eb3KvJy7Gb14vae8XzwEvVM9aLxZPdM8gr32PVk9pj04Pce8y7xMPIA9pj2tPEw90TxVPYW8tjynvb08N72qvYi8yLxBvIM9HbvjuxO7NL0CPAK9aD3/PAu7Br2ZvSC8Zby7vC29pr1rvUe94rzBvK47Fbp0vZ49wz0tvLY9JzylPau9HT0KPb09kr1DOpG8Gj1oPSs9I7yvvYy9yb1fvaM8IT3pOiS96r3uvDI9Vz2UvCE99byHvOA9ebuyu4G9ObuDPU087z0ovTA+pb3KvTO8a72DPV48Tj2GPUi9BL0qPAg9RLzEPfy8vj0APfS8Mb2zvfM8XDzzPCq8EL0OvFA9abtmPRI9TjxKPdm9lD2QPA283rywPd87SD2nPIW9zrwsvUy9ob30PBK+ET0rPRK9PT3euxW+wbwIPdY7lL3Guok7Cr0bvTu82DyfvOs7Jz1dPMw8xz0QPfK7DzwevZG8KT03Pds83rtsPce9nr2rPEc9Fr19PZm9OrxAvZg91DxmPQC9RTsZveI8nDxcPDy9BryWO0295LztO2i8WT3qvGO8pDxGvNe8rD3YO9y9iD3CPBO9EL5pPYi907t8PWS8XT3KPZS9jb20vWi9ajwXvaK9xTunvbM8vbw4PZm9yDx1vQK9LT3/PGC7KTxJPU+9mL3OPKE8SDuPPb+8url1PI49t7yQvW6827xJPGU9Djz6PIo8iTzXPEa9UbsJPP+8vzxvPdI7pTxiPUK9UL3lPJ67qT3qvE69sz2vvCa72bzlvJQ9Prs9PWQ8AL3rOzS9Jbt7vGC7fL0jPaa8Qbz3vLY9IL0ePDy8nT3CPBs9kr3AvJ09ULuOvMY80jpkPcO9p7tavIu6aDwTPYu8Tbskvfi6tL0gPec8uL0kPOS8d73MPb+8D7wIvTq9IL3yuja9l7ykvLq9KDyQvKM9GD2IvKo9hj2bvVE8eT3cvBa8z7yFvD297TuXPYu9brwDPKA9QL2Fu7S9ez0tPYS8ED3lOxY9zL3KPVY9ob1ZvK+8YT0TvUE9j7pevYM80z2ZPCO8LT1IvBg9E72juZq9QTzpu8K9Ez3DvJY9Dz1cvAM9w72SvAA9qbwju6C9ZLvUvBO9/jyautK9OT27vA+9IbsYPVk98rwJvDc98zzgvRK9jrrSOyS9070BPZW9Cz2VvcE8/bxtO827Br2wu+y9Xr0XvTA8qr2lOAs+Q72MPHg9YT2wvem8qbwJuA69rD3zvMy8QLwDPXu9Pr2OPVy8BDyQPA08Gb1XvF69oT2dvUQ9kjzFvTS6S7wRvU09pz06PW09rD2HvYo9ETwzPJI9bb10PSc897qKPUk9gTyauzu8qbxzPdm8P7wLPC68FbwMvYk89zy5PAg9Uzu1PCe8NTxJvVK98D0HPQk9nDz3OjO9p7wuPWa8TTxsvSm9lLyVvdy80Dxgvbm8bj20vXw8KL0OPc08hjw/vZ67Jb05PQg8tz2qPFG92jyUPGM94ry5uv09nLsau8k7XT31vHi8sT3nvA69X71gPRU95bzbvCu73bwavhU9qDuoPI+9l70qPGE8/Lymvfk7xzofPWm87rzpOxW9mD0QPSc9yjtGOz09tLz9vAI8bD2IvQs9vj0BPcE9mD3Svd89Y7y+vC29MT2uuxG9rTu9PDw9Sj1vPCC8Hz21PKS7ar14PHg9YDylvYM9mzyJPXe7mD0cPCA+rL0+PQK9TT0Ou7+8TL3Xu/C8GL2SO0e9lT0UvTu9q73ZvKK8GTsMPQ078T14PZ29Lz3UPYw9ET0zPTa8uTyevPU9HL3HPTw9wDyuvHq7Lb3nvCC9AD25Og69A75sPIa9rr2AvY49Lju5POo7CztJPWi8kjx4vR49xryUvUW9qTyePOq92bzOO2W8CbzXPMc9tTyLvMs9pz3hvO48BL2jvJQ7DTzmPMy8m734PLy8kj2/ux+9hDwkO4S9yLr6PNm8zrw8PKO9fby8Ows9Xz2hPQ+9vLxuPUe9P71UPdC9rrt/PFC9DzzzutI9jb0DvXi9yLqqu5U9YjykvPa8+r03PLI9hLwZvZY82D0yPPk6w7ykPQ+8Hz3aPVU627yRvQ68ezw8vbC8HTswPZq9Eb3ivN085zwWPbe9Iz0EvUG9hz3gvVG8Pb2LvCO7Lj2UOwm9+rt8PRi++LpmvQm9Zr2TvI49ozocvNc82TuJPT89k72hPQU9tr1iPV09Jz2EvQm9G71CvJU9Oz3ovM68grwOvYG8Mz2JO488kT2GvI69WT0iPb+85zyHvHI9fLoLvTc9sb32um+747wMvWe86z2RvZM9Fry7PRy69zx2Ok49NL12PIM9qb1PPba9gjw7PAA9SjwfvVC6M72JvFW7i72avM+8KT1APZE8i72DPKu8/LyQPWm9d7w3PYY7HjzCPaO8nryoPbO9Mb4quyU8Jj2ovNG8WDzmPPk8nbz+vGc8or09PVO9DD3APQE+ljywu5I8ML01PU+94zxFPFK9GD0fPPQ7GLxcvck9Fj0hPeW8Qr3avZa9HD0ivSA9irs6PR69Ib3WPRi9hz0Gu1Y8wbwSPeu8arxEvUu8ZbytOA09dj1yvX+9Gb2nvGI9aj2zPSS9Rj3ivNc8ujzmPMo8vzwJvRC9Bz2cvUc9GLyUvbS9STpnvbI8gj0SPI68O7x+PYA947yHvLo8fLssvd08DDeBPWm8Rbz2PME8EzscvVa9nL3qvYo80j0lvQA+JT3zPGe9gz1kPYI9/z2NveU8tL3dvLM8QLyKuvA8M73zvAS9mr0DPQU8+TxBPdM88LvsvYw8CbwQPsi9Tr29vFE827yzPd88vDzmPV693j2RPLU86TwqvZY8FL1IPLm9uTw9PZI8Yb2yvfM8NL2aO9U9izyGOk+91zsfvV08nr0WvhO9tDxFPZk9Cb3Gu7O8eD3xvUc9Uj0SPnI9Oj2+Oac8HL2kvBm8er1ivAo9p7wJPcy7ADyFvXg9JL2jPMk8Cj0tva+8bz37PAs9PzuKPck9jjyMvbS9LjxyPSu9uj3hO6g99zzQPXC8tL0wvRm9RLxnvTo9Lz10OyA9nb2yPfW83zvRO428nj2kPBG9i7tIvdg7Lb1BvZg7iz1FO/C8Yb1oPes8yz2KvOO8AT0wvVi8brzzPCS9HzzcvUa8j71OPCm8hLvtO8A9rD2qvC28kL1tvbo9PzydPeo8S7zFO/q8hLzJvZy9DD2CPFW9zDyjvRQ9uLzNPZG9nz2yvc68RbyiPfK8uz2iPDs8mD2xPLg8Jr0JPXW91LyovQc9ij1CvSG8fT0xvDS6grw5PIa9qLkrPW+9j72hO+s8Hbz/vI49sL3mPCo9jj1NvZ+8BrxUvao8ez3VPDS92Tw4PO68KDzhPZi86b1DvVq9xTwwvIU9FL4mPY88KTzyPJO82DyfPBc9TzqOPFY9hb2DvN68jj2mPBg6AzuNPa68+rsHO1U8OzvpvCW9DL2ZPLA8RT3dvAe9rzuFvcK8ejw3PH88pby2Pf28Ojx6PJQ8gj23vJM8KD2yPUc9hr3sPPW8xzuMuoW87bzAPHO9Jz0dPWc8qTwuvfk5pLwlvMo9mT0uPTU7VD2KPLa8TL31PIS9Vj2uvJO9Lbt1vc68mj12vNg9lj2lPO45F76IPSq8gjxePJi847yHvXi9Fbz/PDK7mj1UvQK9jTsJPDq9QT0RPGU7DT3mPJw9Br5XPEI8JTxnPU677r13PV69CL29PIs8GjnPvCc9mL0yPdu8FzyhPcw9MT1gPPA8TL3QPLC7lz07Peg8Ur21PEo9+zsNvUK9wbwxPKo85byqOw09WD2uuwq9pT1yvBQ8E73qPGg9yDsgPU29zz3xvJ47Zr2mO6G8xjyyPTu8Tjw+vbi9ybx0PTY8mDyMPWM7jr2Rvew87j0lvIs8Yr0uPKa9njzlPJO8wT0MvDu9tDu2uwi9LbztPOq9Ez0UPZQ83r0nPTw9Pjw9vd+8EL4pvYa8tTy4PTi7t71svH+91z2KPag8bb22PZU9HrxpPQ28xbz0vDW7Xj0vuvm5ojybPS49Mz5vuWS8TLygvCQ98zuKPTG9mT0zvGU9azoCPD49lz0OOyo9dz3tvQO9vLuCvFm8bj29PdI8aT2EvFE9ML36vCo8Wzygux29ibytvQy9b73BvZE9BDzsvHA88bxbvB+8YL1wPCQ7f73PPVQ9qr2FvEW8gb17O4c6+Lwxvau9ILzqO6Q8v7zEvEM9PzzXvH29kr0jPcK9y72OPcQ8CLtQvXw9HL0LvSe9qr2wvK09vru1PYe8jL0iPZk8nr20PGK9vbxEvP28Nr2rPDO90TxHvaw9yzuovPC8Sjyevcm8YL3Evfm71D2ZvMS81Ts0PM49kzx4O/872zuePQk9LL15vd+9l73PPJY92rxTveK6Db23PI28j7tCPTg99bwjvHY8CD6TvEQ9hT2YPXO98byMvUe83b09vTs9n70wvZ49+rzNPNI8l71+vTU9or0QvQ29Tz19vSo8SrzjvQi8ljyIO5+8eL2VPIK9IjvMPIY9Gjz8OM89lTyJvLc9gz1kvSO9+zygvA48EL16PUG9UTwSPGS9i71VPBM+MD0BvQW8a705PaC7gr3jPB892bzRPGc9Gj0dPTE9hT3CPBW87TrovOI6Vz0CvZ+89Ly/O8g8k72LPCO8n70JvZS8NTuTvDI9LDxavca8Eb0FvY09ZLyiPdi6CT1YPQc9pD0wPTm8JL2fvM08Vz2IPGI8hT3EPI29/jyjuyO8rzyRPfW82z3QPCG9FL2jvZS8Fj1UPbm7tryAO4G9JLyGu0u81D1XvKY5hr0/vVK9Nr3pO0M9J73Cvbi8xz1ePUO9gDzzu+I8kzzWvcs8tTyhvWc9Oj6xPR69jbtGPUu9krydvFi8o7xtO9+9pjvvPM888TyqvTu9NbylPEw8Vj0svQ29vzzBPY+7I7ymu1S83TzEO629wbyavU29VDyGPaw9yb0VvXo8ITstPXC7UD1qPI69h7ooPjY9PbqvPJa8qD1hPdy8C708PWQ8tTvHvAC9wju6u7w8q7w0PRA9orwFvam9rLwsvDM9DT0fPbg8Oj0JPlu9qLslvGs9YL1uPZ68N70GvMw7or3mPBc817yavUC7jT2dvXg9tLsSPZ+9G70fvS09eb0kPI29BT3hvEA9az20PLQ98jsxvS09Eb2EPKk9Uj05ve48nTwAPF+9trzYvIs91T1mPZ69gLswPe07Bz3OPHO9azm3u5u9/7yJPWS8aDyRvU29y7xqu6C9tL2tvDY9lrtIPY+9zzzau7e9G73JPHe9yTyYPV486DsGPZI9ATyTvZw9Nrx+PXS84ju7POu9DL1RPCS9QrpFPVC9Gz2nPBg8dj0zPe888LtsPSQ7BTzaPJC9HDu6vLM8LL2IvbW7m7wOvLS9gT3JvDg8jj0QPSg9DTydPbM96LyQPWQ9RLpLPdo7bT3uPEe9p7zFPTK8Sz03vHU8fTzuPCw6qD1aO8w7XT3KvEO8Vj2bPQM91zxbvYq7kj0Bu+49PjwKvEs8ULytvWm8mz22Op+8SLr1vHm6hbwjPXG9nbuWPGe9EDy0vKU8Mr39Ori9fj0gvYs9XzygvFy9j701vDE8Sr2/PB+9/DwmvVI88bxqvZC8NrzWO8u8yr2dvZI8/jwWvbc9Ir2GPSk8xD0ZPYe9vb3fPIm6rz09PBs9AT2OvLc9urrjPJa9ojwavL28TL0dPVe8srwaPGO927wAPbQ9rLx+PIE9lj1jPHU9B70bPcG8E70Pvh+7gD2Bu0y9Yj1VPcg9Hj2GPUK9ODycvVG9hr0OPZI8Hz2sug29yjxFPdu8GTzcu0M8kjycvfe8Az2gPF69eL2AvFC8Aj0xPVe77bzzvD271Dz+PNW7Iz3kvCk90rxGPao8Yz33vG69qbzyN828ezxxPN+9Uz1zvLS9pb0+PZ08m72WvaK8bL1NPYY9Lb2QvTK9mjzovNy8dTzPvJS8RL08PZK8CLxPvBo99DujvRM9krwzPYa8J7uzPQU9G73jPI+8Rr3MvZw8IzxIvLm8mT0jPRO8fL1evT89kjo1vTS8IT1PvV09nTzPPXa9uT2DPBG9TD2MPQU8YTzovL6817x3PLY6a70ounE8Kr2bPBC9lr3XPQK8BTz5PMs8Zz2HPJK8HL2vvVS9Sb29PAU+wr1fPVu8mrvHO3u9KjuPPZQ9orxOPBS9EL00vRy9Fr1OPac9STyNPd+8Cb1TPZM8jL2+PWi8fbxBvDO9sbziO+W7CrzoPcg9Jb1oPba9Dr2kvO48BD1cvJ+7+zwnvYk6+jzTvI89uzwPPS48xjx4vbG9kbyQPcQ7c7yeO/u8bL0LvsS80b0OvcS8f7mTvUs8Fb2YPQe9bjs/vUm9eb0KPrA8ET2qvfg8hj1avbE7Sz3CvNc70jy4vFQ9NDwbPdI9wzyhvcG7Qr1VPYi9JD2gPaU88D1oPWW5RzyQOmC8cb2APE09oL1UvC88ZbuKPZS8sDydvem8X70PPWm8mT0dPZw8srz8PMe7d734PLc6Wz1qvRk9XjtmvKe86L3aPII8Pj2OO9I82jwmvBo8s7wZPUK9dD1ZvHe95zuKvDG9CL0ZvUI9dbxPvD49nz0DPO68Cj33vIU9K70+u8W8UzwbvdK8x73SPJu95LzWPUq92btOvUm9UL3TPT69/LztPD29aT1/PQO9CjyWvb68+byFOjW837vUvQs81js5PNo8qzyqPOE8Gz0kPR28KT2BPa48wL22PUG8Qz0YvPw9Cj4Xu0Q9ZTy3O7K9CDswvZw98z02PYI66bxsvHc8mTrZPJ680jx6vIk93j2RvXe9Az3XPFO84jwxvaw9Sz27vEI8Pr1vPfA99bx/vJW80rzOPQW9Ar5pPJm9o7wtvGG9YjxavFi9fb16vCu7Ur5PvOE90D1+vVG9BT1/vJY9aj0YvQi9Eb3ytou9aDsTvQE8nz2TPQG8cLwPPtc8aL13PMe7T73VPZq9lr3MPbK6ETydvBC9+jzLPDy99zy+vAI9VL0fPYS94zxKPZu9VDyfO0S9yLxIPau8IDwfvag8Lz00vUK9tD0bvW29nTztPKo8jb2Uu4k8iT1ZPW69R7zbu4Q8GTzFPH29gb1APOu9nLxBPR+9Oj0JPem7YLxKOww9TDyIvPM8RL0XPWi88DkPvHG9D72CvMq9w7ySPbG5CD0QvSi94bvQPJQ9Kb1qPXg9Ib0BvQU9FL38u708tL33PNI8hj2+vc29XTxUPIo9PLxyOYM7PL3nvI69BDyjPSe9+LzkvJ89DD3quom93rztvLA8IrwqPXG7QLkVPN47mL2VvOC9mbxVvLK9Aj05PaK8grt+vbY8Q7osu5i7MjzMPL09lj05PSi9zLtmPFG9vzubPR49GL0xvT+8XrwcO1I97DrPu9q837tMu007jLz0vEi8Ebw0vUU8cL2Ousq8Ur3ZPXO9/bwTPHE9GrwLvXU9lrm3OJ27c7y1vEK8Nz2VvSw8CTybvL08Lb0rPWE9MDurvUa9Qzy7PF49ND2uOy68Oz1Nvdc8KT2RvC49M72SPSo9kz0qPdm8hr0JvJ47Jz2LPB67Jz3QvIA9ojzOvSG9Hr0APU87xLyYOxY9R7wMPGK9ZTz2uz29arzUPZE9kL35vHu9u73BPSw9+7yjPGo8uDyRvQi7pLyEPQo8Sb2TvXe86zu7u+I8aL3iuwS7VrxUvfS8NLygvf89+rwkvQg9cL2PveU6nr2nvfe85Ly1vSu8iD19PbU7jzyhvTi9xzz0Pdm9ir1XvcA9kjw8PKI9sr2wu9S9gj1avAG9Uz2rvdE9nb1IPd+8pDvouw69ujwnPOS9Db3EPbg9tLw+PH28hrsyvCO8kz3UvDW93TxJva+8QTxWvTm97T2zvRq9AzwQvZs8kT14PK09db0UO5670LyRPEQ9iL2HvZ+9i71Nvao7RTs2vIi83zycPVM7Dr2Iu6c9hz0Cvrk8ET12PSg9Mb0eO4k8A70BPfI8G71xPUs93jwuvdk8vbxfvZM83DxHPTm9gL0+PUw9rrz7vIw9J72iPEO9QrxgvLM73zuZPck8wrxgO728ST2fvWU8hj0bPdm7Pr2CPQ==", "ffn_branch_bf16": "Y761PKu+yL1vPD68BD/6vTM+jD5PvZI+fz1rvji+eT7IvRa//T6XPci9ZL7yvpY+hD72vSC+uz69PaS9DT0rvn2+uDmuvrE9Db8Yv/Y9BT4Lvcg+qb0lPsu97b2Svbc9ib2GvlW+nz1uvna+V70cPny+Bz6XPGy9eDxHvgy9wTttvim+9L4ZPp++zT5dPjE+gz4EP6S9uD2EPre+Xj53PuY8Sj5Dvum9vT2Kvg8+wrvhvVs+kb1qPrQ9Hr9vPiI+MD45vl28Sj03vj0+0z0SPss93LxqvgK+p72WOxA/qb5Kvg0+tb5DvBA+H77tvsC+jLtYvtY9Lj46vbQ9ET6rPoO9ZD2cPlK+E74lvTe+tL0ZPoO+8D7dPDo7Wz0BP0y+nr72vuC9cr2uveo8mT2BPui+Bz3FvYy8arxyvT8+Yz6lPpM9QT0SPhK+U74svgC+2Tw3PRA+Ub7yvoG+Ab87vve9OD6+PnS+z73MvbW9hr6qPnO8Tb7Svqu8IT0FPcI++T29vae+SDymvLA+970XPsU9Q71kO/A+HD4bvgO+CT09vTC+m71VPqI+1jrNvu++xTy+Pio+xb2wPdS9sT1tvr08Ib5Rvjs+s75Jvn0+ST6GPou+K77jva49mz6ivom+l7wePkW+/zyQviy8oL4avY886L2evH8+tb6lPQo9Ob5dPgI9LT2KPgK9jD5yvNW97r1DPh6+q72UPgk9j77pPfo96j3rPDO+iz4lvTo+er5VPkc+Uz32vjM+jr6CPga8p7ysPei88T4Ev1k+Uz3NPqg9Mz5iPIM9aL6NPiC+Qb6dPX0+mD7qvhS96b3qPby9Wz6gPkY+hz4mvrE9CT78vZE+dL6Xvjw9B73wPQA+T70BPjG+hLuVvbQ9WT5tPZo9uT2Evf490D3QPc08Rz1lPUm+xTxfvhC+gj7svh0/Ez5qPb8+qryAvZc+s76IPq69zL4vPaG7mj0Jvg+/qTwfvv09sj2zvls+4b7Tvai+Lz5CPr09mD7avbA+/70Wvg8+gD6/Pn49yz4FPQg+3b1BPvC+rb0yPqG9DL7IvBC+kj2jvvs8iD3jPB0+gT1zPmG+2b0JPn++8T29PRK9jT7Jvf+9nj73Pl0+bL46vks+Cr5rPmQ+4D2FvQm+uT4JvW6+oT5HPoM9wrzPvQS9Hr6Mvp8+dj7yPVS+NT5fPeu+Ar5Rvhu+WL6vvYQ+973iPTK+ODyvPrq9ij7gPei7+D2SPq+95j6cvgU6ib5oPhG+Nb7iPCS+hL23PcU+wj7jPYY+2D76vj893T3UvXE957xBPpu+br4hPeS9x74NPgm+gb5zvv48i70WPXG9aj2kv1m+Fz6APY0+P76NvoO+hD45viq9jLxwPtm9pj6uvrm95T0cv3S+Ur+tvWe9yj3mOsw9iD1jvpg+tT6jPi6+Qr6mvIg+Wb7dvBq+bT6aviu+qzxLPr6+Hz2BvlQ+lTz3PWi+r76Mvh8+ZjqzPmY+yrwzvgS+CD7IvYu+8ryqvbe+Aj/VvIw+776NPAw/C72Pvqo+Hj7ePdU8cD78vBq+nj2yvUA+ML5hvg4+PL67PpK+kj6QvtU93j3wvEg+qT0EPls+tz31PTm+0jwivha8kb3VvdS+2TzMPjs/6b4wPoQ+ar1nPiY9Mz4RvrC9IL1xvdy+bz0pPT674r21vlq+Kj67vc6+lT4pvQq+4rwPPVw+a71uPkA97b4YvqM9BT3ZPjK+m72nPuY72z4SPki8CL60Phu+Mb5zvtU9CL76vg2+jb4Nvi6+k74TPuG94j0IvWu+oD6rvgo+EL57vDO+GT3XPX0+Fj6uvTa+3D4ePxy+ED7RPiW+yDomvsU+kL6NvdS+izwQPgm+Xb77PY8+oD3pvV094jsaPpK+tr2+vSW+AL7IvmA+jT6rvvw9UL3lvPM8o73vPG6+jT7vvPI9Ib0cvUC+Oj4TPrq9wb7ePYa9Lj46vo49N77KvXc82L1LvoS98j5PPS4+bb22vcK+ij3vvXo+hL5uva8+lr2UvTa+9z5hPkO+JDr3Puu9FL5/PXg+0D6Tvqo+u71FPkE+J7xQPmQ9RD5FvZk9Cb42Pg6+kL5WPr67c7zWPQo+mD2DPSa+vL2VPZK9TT7UOwQ/Tz1VvoC+tD6Pvhe+OD5bPiO+XrwavUk+XLz2PtS+ez3Dvtk+8L2FPso+a74RPo4+5T6BvYc+YD7gvkK88b1cvpg+Jb7QPLa+P71wPY8+5T64Ptu+ib4fPom+NLzjvfe9gT60Pts9h74cvpc9iT5vvZs++zwqviO+uj5KPi++Ib7Ivum+6z2NPKa+Jr4Evvc+i76lvbS9AD4YPo++Zb6cPrk+7Ly0vXu8OL3APvi9QTz6Pve9OD5dPnm9Pr40PPu9hD7PvCO+j76tPZQ9Db5rPvm9ST5Nvnc9o74FPvo8gT3ovZa+kr7aPcC+vTzAPHe+LL1iPts9cr6tPiy+Cz+bPsg8mr5avVg+8b33vSC+BD7+u6U+VD5RPVW+HD3UPl2++b11vjA+oD2yPZQ+SDzmvQO+HT5jPhS8o7zqvOc9v70APi29oLwCPno+wz5GPRQ+Bz9lPS0+qL6bvqS+Sr25PIc+MD00vgc+Vb1JvQ2+X76tvmu+0r5QPiI+lrwLPSa+kL55O2Q+pb7BvYO+B79PvQK/2T46vos9FT5Hviu9jD59vic+1T0svlM+hruFvZ2+Ob6KPjI+kT66PVA+Ab+zvii+lD5nPs++Nj2FvRe+mb3WPms+EDqBO5i9lz54vg2+njz3vYe+pb5zvlI9JL4EPVq9AT4ovD++TT1EPoy9LT4gO+q8XT2fPda97D01v86+pb10Paa9CL6Dvkq9iL0jPm68XjxYvoO8gb5NPds9jLxHPi8+YrwdPiC+Ib64Poi+R77aPbO8NL84Phc+lT1jvrm+gj0iv6s+ML6rPo29Bj6Hvp893jxNvii9Zj2mPpY+KD1FPkm+/D1YPZm8Sb14PqG+EL2XPpc7ub6NvuI8kD12vvu+A774vXy94L6QvoG+Xj7HPgu8xL1SvQA6Ub4qPkU+yD3oPhc+lL7aPAS+c74TvgW+o762Ptq9RT4HP0G+oj2mPEE+8D04PaM9Nz4fPqi+k7yhPXi+zT2rvpC9sb72vWc+pb5Ivja+I77QPVK+77wKvgw6Oz6dPuc+p76WPZa9Iz7vPQw+tT39Pio9Hz5Kuw4/bj7CvgK9+jwWPis+CL5wvgQ+5D1EPiM9D76CvG48CT5Lvle+ir3hvQI+5L6APaU+Xr7bPQU9GT30vSC9s75Yujw+G8BmPus+5r0rPZe+6r0bPKW9Qb4kvlC8372bPm88rL5IOw8+CL7MOg493b20PEG95z6tvWu9GT1nPiS+RD7evT2+z73ZPQ88Kb4gPZc+7b6UPUu9nz4tPpc9Jj7KPgI+0DpYvh8+ir6QPUY9E74QP6M9qL18vrW9y76Yvsg+2j4iPpE+PD6aPkc+qL7sPJg9Bb7pvSC+lT7IvR6+h773vS09W75Avr0+Bzw4PR4+KLyDPXs9Qz40vs69mjz9PaC+iD6avJm+uDvzPhW+SL4dveI92L4ovH6+J72/Pd4+lL0Cvr++jz2JPYG9sj06PqQ91b4wPhQ9ob2zvsO9tj6SvJA7PTxQP8q8fD6nveg9AjzLPtg9L71rPtI9RL4Hvo4+lr7+vb09uL6aPSI+i756Pag+Ub36vrQ+xT7xPRE+YT6lPqw+QL7VPok+yb12vio9aD5kvmu+Xr4QPj6+vD0kvvM7Rz7mvO++tL2fPrS+Lz2BPTE+nz3uvRg9JT7VvTe+Wjzfvg2+ED8PPZG++Lv7PEs9676lPpQ+zr0ZvgE7TL2xPgE+LT4zvgc+ED6dPdK+zb2Tvqa+Vr42vU++sb5YPQi+a72XPq294D2wvgM/Dr+iPYm9pL25PB0+972ivQM/Rj4EP6G+2L3aPbU6Aj6gvqy97b4XvpQ+IL7pvjG+aj5kPqc+Zb7bvkY+bD25PqC8qr7ivRa9SL6Pvb288b05PQw+Jj3JP22+JD6tPgY+hT64voA8vL0mvtQ+s757PtY+fD1mvp89gb0Fvpy9wb5FPjq+Az4FPk6+Cj+gvas+8z31PZi9BL0RvSS+8j0Cvwe+RT6/PYw+hj6TvQQ+R76YPZa+Wr6svYa8ED70vSO9iD7KvCO8xb5QPga+ob30Pa8+Ar/IvQ4+RL6tPtw+pj2zPls+JD6XvrM+Nr6RPo++QD6jvk2+n76OvqC+lj49Pbg9nz1Yvso9rz2QvZy9VT0hPkC+CrzMvXC+kj2GvTU/p725Pv48Q76SPhk96LyKOxQ+hz7YPqo82Tuoukg8Fr0rv4u+HT6tPp++Gb6FPIs+MT4KPui+hb7kPe28db1Gvii65b0WvZS9xDwcPhq+oz5zvoI9Ij7XPp+9RD7nvKS9lr4ePtu8M743PW69Lz6YvrI+4L5aPJ6+0L5MPau+6b0uPri+GL3mvfc9oz42Pp4+Hr6UPVE9lD3sPQY9br3PPYq+hz2fvEO+Vz4HvaG9FL9CPAg9Kj7BPZO+fL7NPlM9Tz4kPoy9Sr4EvJy+9TuhvQ8+5r7Nveo9ij62Pk09iL6mPZs+2z3fPQe+tb4SPmi+zD2xPmW+Ij4OPsK9cT15vvk9Pb0+vsU9dr5kvn49xr0xPqW+Bz8Svga/mz5bvQG/Fr6Gvpg8eb6HPVo8Mb6XPYI9xbzOO6W8A7+kPcG9iT67vVy/2rwDPmS+Uz5FvSW9Ir69vXI9t76VPoy9drvVPly++LwqPlM+BLyPPo+9tL4YvSS+9b1Svhm+PzyaPcc9FD6KPoe8kr3/PUO+nTytPkg+fL4kvcC8Bz5Mvq8+h76xPtC9iz0VvsA9uz0Mvd09lT6APqs+Vz6jvHW+Bb5uvuq8GD4uvBA+Zj4RPu69fj6OPrw8LD5SPc2+mj2zPqu+P74+PX6+kr0/vhi+arwevjk/WD3LvXi+ab3AvYY+e70PPUI+yz42v3S9dj6WvIC+zb7lPQ6+fz4yvkK+T75TPeQ8rD2MPoo+6L1zu1o+pb7vvue+Tz7rPYU+1zxDPvQ9Ar+PPkI+2r0UPvY+vT4FvoS+i75Kvf879r0BPWG+JL57vrQ9gj3DvVU+ub1+vlg+Tj6MPu49cb04voM+Yr5VvlK8oD64viW+gL7LvtA+BD6LveA7J76bPnQ9t7wCPoq9XD6RvkM+kL6UOrq9Zz0ePdC+yj47vno+TDz8vdy8CT1bPi8+qT34PWa8lD2dvpy9Mr6fPVq+Tz2uPBE/PT7zPdU9bb5CPmi+uD5UPgm9/r1OPvO9jb5/Pj4+gj7rvbo+k76LvZ29PL6wvii9jT63vTi+Ar7lvgM9+r3YPUS+Kz6HPUg9q774PcG91bx8vV+74L1pvYI+xb4yPVw7t72AvAG8c767Pgk+Cb6QPbQ+9r0TvvQ9FD4Gvre9tDwTPTc+EL5zvqo9kT1dvqy+Wrwevu0+ab5dvFQ+NT4DPjq9Bj7QvVo+/b1tPny9CL2IPl29HL0vvZ49Lj7uPa6+2z7LPX0+Xr6mPsG9272CvRq8YT6WPgA/Qr0CPiu+CL2JvoY8JD/+vjO+6r65PYs+iD7ePIa+Bj4qP3m+Ab6sPly9o77JPgw+QL7XvgE+j724PbY+Bj/zvc09Ajp8vt68Dr7APoQ+cLzgvQy+Br6cPrA8Yb51Pt29Kb7hvIk+kj6uvhK+Fj6Tvr2+FTyEvoM+8r21vjY+lz6YvfI+5j2zvSa+2r6gPoS+yb2oPuc+Hjo3PWQ+Lb7SPCI9Xr1uPZC+qj3gPqE9bj5XPo2+L75JPdQ+Bj41va29y724vvk8Ar6hPoE+7b2PvZy+bb7ku5k+XD6rvpK+s72BPj+91j4QPqu+T76DPnG+YL7wPig+tz4SPes+tL6ovZs+mj2PPbq+i7xsvtA6LD4UPpm+Pr2CPmG+kD01PmA+DL4Ovru+X723vsO+Fj6TPow+hj7NPZC8RL4GP9S9kT6kvrO8Hj7DvvI+Er4+PZS+mz6RPiq8HL6jvsq9Nb7UPUm+Tb0MPMQ7Ir7ZPoy+Rz6BviO+VL4Mvhs/Ur4UPoI8FD7MPQ++G76oviK8fD4gPgi9asBmOgg/gT4yvvy9KzyBPgQ8kL4ZPp09kzzIPR0+nr70Pfa9I726PGC+9j2+vv67P76Ovn696z1ovmw+pr6TviG9Ub6BvoA9ODr9Ply7Sj5CPCk+7z14PX6+Nz6dPt28RLsUvg6+Vr4SPm09rb6bPda8or7PPZi+DL4ovdg7hr70u7k+Ab1Ivjk+cb3oPGK+7r3iPbu9kz6SO4U+i77EvfI8Y7xgvic+Sz2mPcC+Sz05PVQ+GL6CvDO+gj2qvqG95TyZPXq95r57vqK+5r6Avkw++D6WvZM+1TxRvTc+jr46PkY9Zj7QPc69YD0fvuE9ub6evTY+g732veG8xD1TvjO+gr6KPie+2DycPie+qL3LPRu+dDxavIe+qj4DP7s+nb7avm27k77gPoM+Jz4aPlW+T75RPlW+FT6MvZ6+uz4wvr+8Bj6kvtO9Pj59vCA+br4WPRC+rT7pPsY9Zr4Uvku9Oz7BvAY+Oz5kPUE9Tr5aPbI9CT4wPpe9Ar+pPsa9yr5pPpa+6L3EPio+7D6Avqc7i74sPSS+Cr6BvP88ub0hvs2+CT4lPue9s7yfPZU+qb0PPvI+Ib6bPvQ9Tj5ePiK+sT05vlm+Kz7YPYo+Pz5CPl0+A7+wPPK85z0zvsi9XbtkvY47yz6hPp8+kb4bvl6+/j2GPY89uLvxveU9kT6Nvty9iz6NPQA/Z76yvq4+1T1+PpA+ND+8PSA+bz5XPkK+djzSPaO+aTwaPs6+Pb4vPao8FD5DPsU+fz0hvlO9LL0Lvvk+TT6LPjc+Ej7Ivuw8w75MPA++CL+nu1k9L75VPb894L58PRU9Fz8bvbM8gD2vPsM9ZD6OvdQ9fL4FPpG+rD3OPUM96j28vYe+1z0Zvoa9Pb0zPV6+1T0wPvg41j3YPrq9ej27PQy+LL2VPpA+8jytvcC+V73UvRG+bD6FvcO8MD5avqW9pL6CvQI+b74SPho+nb7IPAg+YL7jPfK9kztFPLQ8sT6VvmC+8T3gOl48EL4FPz4+KDzYPcU9Az+WPaI7yL53PS4+ar3KPvm+GL7JPBE+3z4HPsM+aj4uPrK+8b1WPk69PT6TvFK8DT5Rvd894T3DPaK+DL44Pkm+GLtavbG+ej5BPeU9X72+PgY/170oPsq+FT5pPuC9TL4OvsY+TL3uPV0+Yz0svsO9OD22PQU9Ar0FPm88nD4XPYQ+pb43vqU+Ej6hPkq+Rj0MPqK9yT7qvpY+2L31u6y+g77ePNy8iD4Rvik+uL17Pja+Br5gvqa9Ir5hPju+5r21vjE+gj4Nvie+gj6ivVW+gj0nvoc+RD4dPuA7i74OPp4+OL6XvlW+wr3uvWm+UT6wPYS+P76ZvhS+1b2ZvlY+tL1rvoa9tb0Svq89HL2IvM66GT47vps98T4SPbW+3T68vi++iz2RvuS9Qb6wvfy9iDyAuKk9fDy1vTW+db7Runo+sT77PV69Ej4fPjG+uj4ePkS+lDxivs2+8r2vvqY+pr0KPjG8zT7fva49Rj7jvSQ+n71tPiu+4r3aPhy9lb24PhE+vj3cPrE8+L1IvLc9gj1Nvoi+Qj57vrq9fD7VPT29qT2hvIg+srynPRy+8r6sPSS/ULy5Pq0+qj5+vBi9jz5HPs29oTyJPpK+crwtPu6+RD2LvVA+hr7fu4i+Er3XPjW+Y72yvo89Nr2vuwg+rb2zvay9Db4Vvoy+lL0pvdE9ED6iPVY+rT47vt69gb0/PlU+kz7Evi+/gb4qvCI9Jz6MPgy9wb3YPSQ+nT62vAk/wz6WPe89vD1iPoQ+yD7DPTA+az52vXq+2L5UPkO+lz7fOxI+172HvRc+t74Wvrw+ir5svkK+ar5HPsM+mz00vla+sL04PqE+CT7TPWC+aj4BPdS+XT6pvWK9Yr3wPjK+HT7UPgk+mz7sPeA8Ur5EPck8KT2zvaq7Wj0KPok+iz6+PTC8hD2NPlg+6T5Avji+2r3UPiU9mD78PCc+tT7lPVu+tDpLvna94726PXo+67yAvp68Gb8Qvio+3j1xvok+Bb5Ovmi9Xb6GPVY9ury0vRs9oj2yvV2+JT5avla+Dz66PkA+6j3dPN29mj6pPQ0+oT4Fvia8ibzAvjK+YjxKPpi7az3tvGS9S74uvY+8Fz7DPcE+NjnQPsa+Jj5JPS6+gz0sPim8hL7evCe9vz0nvkY8ID4kPG09vrzfPjY7yz6GvDo+zL1ZvRS+Gb64PjQ+Jr59PU29zjvRPju+vT7GPdU+er3QPfa+Vr7QvkW+rz6FvA8+Gj7Qvlm+zj6NPfg9gr7iPp2+ez4qvs89BT6NvsK+cj6cPeK9gLwwvkA9uj6mPbe9fL2svg8+zzzovaI8hLyAPbO9X7zRPde8uL4hvpK+HT9wvX6+7z3OvVq+mD5RvSo+4z7rPfu+nb0uPb89j71avW8+EL6ePUq+Uj5ovuE8Ub2PvgG+cT4pPfw9kj72PBE9iTu3PW283b1mPbE9xD7AvdQ+ljz6vbE8lb4kPmQ9gL0Gv1++0b0Dv4w+OT6jvrc7br77PU09570OvoK8tb1JPsI9L77ivaK+Ab6/va6++jwLPlc9DD6HPni+zT2lvQ09P77JPTU8mT4yvg6+kz1WPB8+Rj0cPdw+yr5dPfs9+D1NvvK+Y7x4Poo8vD3yvlu9cL58Pqs94b3oPvW9WT3uPQM7aD4hPXK+PD5gvQK9Tr6EPn29Rj40PUw82byhvhE/8r3rvEc9Cz4QvhU+bj2IPMU8Vb1hPai+kD2hvXm9Yj52Pj++VT7rPbU+kb45PRO+gL1sPgg+TD7kPQm+tD69vey+i72Qvrq7Ej5/PsS9Oj6aPh++8L7rPZa+Kb4ivqK+RT7Hvh2/h745vEo+bz2iviE/TT6qPZy9GL+LvpM+wD2KPog9676yvnG+cr4Lvum9wD5HvYa+zz5IPKG9Lz4wPKa+Xr1EO1s+mr6Yvts+8j0TvhG+/TwIPj4+kb3evoq9EL4lvrK9cD0BPpE9o70vPsA+MLx/PiK+172bvq09oL12vno+hL6/vdU+ij6Ovpg9Kz0+PqA8zz0ZPsa9JD9oPYq8OT6Vu7y9Lj7EPsq+Xz7jPQa+d71Fvsq8oD3EvJQ8TsBivZo9lj4SPmK+Fz7Tvdy+MD6rPeC9hL5CPsO9uj5sPoK84z0yvke8oz5Gvu49z72bvrC+4T7LvQE+tT2WPui+mr0oPlc+UT61PS+90b5mvq2+gD6hvtI9hb4vvjI+gruBvnQ+FL12vuY8Mj5aPci+WL4NP4Y8Pz4OvQE+C704Pdw9Ez6GPoM9ZL6xvBk+N7s5PsC+dD38PXI9mr3xvWi9dD5ovQG+fj5LPlI+jz0DvYg+db6OPpo+ID2ovQA/l7zTvQU+Zj5/PpI+qb4UvXq+9L1hvQs+UT7uvcG+L74Mvag+3j5Xvgi88r0Xvg69CL6HPR8+R74MPkY+Lb4aPne+pT7gvQM+tTz2Pcc9zb1HPYq8hD5Qvv09Dj4MvYq+zT1BPgm/Rr0hPVa+nT72vhe+DT0QvWY+cL64vkC9CD6gPsi9pj6FvpS9kT62vRy/lT0pPjC9lb4Wvga+KT42vuE9gj6cvY6+C761PpC9O746Plm+AT2uvT8+lry0vVQ+Nr2nPAY9Yr6tPra+l74ZPkC+rryEPhW+trsqPoO+trwMPh0+xj5VPkm9r70KPgW+FrwvvW6+ubvhvpq8Sr4ovYC9Bj3yPUM9Jj0gvvA8ub7yvZ6967ySPXQ+cb6zPT8+LTwlPve+mb6Xvnq9ub40voO+Dr7hvf2+bbvIPnw+7D0EvZy8UL4fvvk9db13PRu+qzyRPZo9sb5GvT+9Rb3YPci8lL6Wves82j3YPoc+Az57vpW+b75cvRW+oL2TPf49lD5TPdE9VT62vv8+aLwxvvC8ob2LPQi8uTwTPgi+Cz0/vmI8HT65Pi4+Iz5Qvf29kD66PMA97z3tPdU9gL3SPlc+yz1Ivsw+qL6FP4C9uj5qPr29sb2pPDU+lL69PH++bT1ivLg+sz46vkS8ET6avu29Xb0LPu49k70gPmm+Qb20vi6+kr6tvac+Sj6TPbY+Ab5pvq08Jz2OvV0+o74JPro+Fz2uvIg+mb7QvTg+wD0cPdW+JL50PQE+r75hvmi9mL50vjK+jr4BP1g+izw0v3A+BD52OoM9fb5pPtQ9ob2wO529r73TvTs+MT6SPpC8cb5Vvtu9Nj6wvqi+5r0rPtw+Tr0LvuQ9Az6DvhY+ND7SvLs9MD4/vu+9ab1Ivlo+2L3QPnS+hj7EvVY+6LmYPs49QT6NvtG9dT2qvYI+aL5Jvu496j3avNs9Oj2hvlY9fD2lvsG9P75rvQu+iEC+vFo+XT2QvrK+s74svpU+RL7zPcS9rD65vsW9fj2uvd89tb69vqU+FD+JvOW9Sj6HvcG+B72gvYs+bD5dPtK+773nvsa+RT7SPRM+fj49vRe+CL0QPgo8yL0sPrk9oz05Pji8MD5mu2K+Sb7JvlC+HD4/vpE+Xj2wPrm+/T3RvlU+zz2GPkE9gbx9PWo9Cr5TPoA+bj5GPdK9Rj7EvVs+br11PYQ8xL6CPYm+Az2evqE+rr6lvWy9lD0PvqM+8b2IvlO+kLsvPjW+SD5qPZ6+HD4UPpA9rj1wvYo9EL4Uvk0+1T1fvh0+8byNvE88wD5Vvqo9rz7mvl+9+D4SPAi+Gz6wvQS97TzjPtQ9tz4bvhe8Wz0LPF+9uL5iPsi+xT2VvWW+Fr4xPjq+E76FPba9iL64vUO+P75uvY28Gb77PWs+1L7mPIW+nD4APtO87j77PTA+Ur5+Pog5Lj3VvRa+xLuIPve93b6lPWI+7j6pPYq8cr5aPra887zfPRY+hD52Pl48Wj4BPoq+xz0NPS0+uLyYPgC8ej46Ppk+jT7nPRK+lT0APeU9gz29vZe+cT5JPYA9/j2YPtq7Bz5YvJS9tj6+vo4+G76Bvtk+qb11PZQ+5r40vmC9Gj5IPlg8Nb2tPao+aD/rvIU+cz6xvnC9Eb6oPAy+nT0ivt08RT4qPgA+br2jvjo+oD0YvrW9jTymvvy81T7Svoa+R76zvL+9QD0kPZu8wL1HvQG+8D2MPVs+Eb1GvbM9f76QPSA+Ar7NvvK+DD3TPR8+Bj+oPoI9BL70vjC8e74pPQS+hD1Pvhy80D1CPjG8PT4UPhQ/hz76PQk8Yb0gvt29C74ovgq/kT0WPhw9zb6bPYM+HL5AvmI+jT0zPpI+Qz4mPpE+jz6AvQe+Yj5APne+oz2cPne9zr2jvIi+pjwxvmi8RD40PiU9NbuAvZ6+s7wBPWI8yrw3vom9nr3NvRi+Nj5CvRM+Qj0Pvmq+Gb5QvpC+uryOPcC+dD4bvsE+Q74sPxY9Vz0oPv0+b778PXA81b5dvq2++D0aPn08Oj6bPpa+vr6cPqw90b3KvQC+Gj2gPkq+HL4IPsG7Fb7LvdK92D2RPVw9Pz3gvlK9YD0KvWS+170/Pqs9OD0FPoW+pT0KPsK+rb3bvHO+Jj6UPbg8GL41v8Y+bT7/PN2+1by3Phk+d76VPZo+BL5NvqI9Db7aPU2+/j05vkY9Cj+tPvu9lb2LPZc9Jb09Po89lr6/PCK9Fr54vQM/Sz/ovQu/j76Ivtc+6LyNPYW+Xj0vPvU+oL2RPje9x75XPiS/Vj3LPUg6fz3kPQO+mT7ivrs+h74nvVY+Zj2ivrS7PLw1Psi+sT6WO2k++D2Tu8g9y70GPos+C743v8O+UT6EvE0+hz5tvQS/pT72PPU+lL6ZPk49Vbu6vso8nb4oPqc9Tj7XPgI+C77HPYc9QLzBPR0+Pj0Xvm2+WbzTvTG9WryRPh89IL4TPR2+kD6Jvq+8xLokPte8Gb+MviO+Pb8APm+9Dj7uvuq+MT6hvZQ9ez73vZu++L1GPgW+Wr2evpK+uL5gvf+9Pb6uPb++hzyiPRW81D3YOZk9/rsMvFm+Y7yyPCY99L3WviI+R72SvqK+CT/NvO+9UD4yvne+ib4Svky+9735vTA+nT4/PcI7L779PJq+pTsGvma+pT6HPkw+ir6+PS4+oD7vvEs+eD1fPmy9lz0pvkG/2L0qO/m9XT3QPt87qr2zPl2+XT7XvhK+Zz6vvjw+yr3TPi0+aD6Qvvs9CT4JvU49gz5JPpc+1b7ivWW8kD7PvLe+zL23vYY+zL5yvn6+J72bPlc7gb5BvtY9dz7tPP29PD2OvlY90r16vhQ9uT5GPJO95L0Qvn6+Ar4hPu69CDwTPg++Tb6bPUk+3T0svjE+Eb7tPa89Pz5lvHS+Fj7RvEU8qrwEPuM9gzyHPfw8ML2WPu0+gzwtPgm+i74mvWe9Dz6IPkg9Rb2TvlK+vj6EvYI+V76nviu+xL3fPYs+Cjt8vo49DT5Cvke+sr7JPiW+qLwxPhQ+KT6MPgW+yzy2vky95byCPuq+KT4Ev7q9zz6evDY+BT5EPuy7qD5RvjW+Tb6Cvb49hzu3vE89XD6Cvm2+cD5Hvoi+hT6XvXE+TT6KPpW+2DyqPsW9UD6JPns9fr4vvoG+Bz9SPj69k76kviy9gT5uvnS+xT7ePSU+1T04vmI+PD7Cvlo+KT1Lvuq+Jr3CPg6+vb2wvRG84j20Pqc9jT7wvLC+rr6WPQY8ob2MvSo+pj64PoS+2r04Pde8Bj1zvY0+ij1uvRc+mL0SPlA9N74Hvsw9tb1sPkQ+E74Uvge97D18Pfg9az07Prm9ND2cvb48lb7UPcY9FL9xvS2+qL7jPVg+4zxGPg8+bj6dPnY8DD7Hvpi+iL1HvZM9hb4sPqU+M75XvXm9M73RPW8+Oz+Ovf+9FD20Omg93r2Qvm6+5z5rvqK+nLwLPdQ9gD40Pwk9a76iPcO+zL5SPqs+kL5yvFe+cz7Ovpo9mD53vru8pj5+Pgo+qj4LPtI++L2nPcw9hL5uPUW9OD3evZM+G74DvsO+Rz0/vpO92D4GvT++Ojw2PFG+ij5Jvc69hL6Dvka+AbsgvjI+Kb6Uve295D1BPUa9vr1Pvri+qr0Fv8I+ED3ZPQi/n77KPfA9+r2Vvc69xD1kPni+cD7PPR8+67wxPqo+mb61vTK7j70LPku93L0avQ0+ez5TPRE83j2UPQe/hL5Nvr28AD2cvTg+SL62vqQ8kT3oul++tD63vo49ar7AveM+4b5vvQq+HL58vRO+pr6FPhy/BT4qPAu+Bz5Ivo4+Db3HPoe8rz27vZU+Vj6wvvA+BT6Hva6+Mb6xvSC+9r0qvkG+zL2cvtM9or6Pvp29eb2KvR2+iz0lvPq+hbyDPkO+Oj5VPh8+cz3APnO+0j4oP7M8nb6cvbi+qr6Tvce7t77DPZY+Ir4kPqm+GrxiPDs9tb44PlM+7r2xPrO9Uz4bPhI9rj4WPXi9A76lvrs+Ar9Uvru8A7wYvhE98r1hvk++/7wTPF+9wD5wvcE9Qr6Cvpu9DT7dPqO9q722vP69hT2pPY6+Rj6uvFs+tT4BPmq+QT6Xvh4+kb0Gvko9NT6VvSs9BL5vPjq+8D3qPpA+6b4qPiy+7b7PvkS+wr6Avg2+3j7hPv+9I7z2vfM9Er1MO9e9I728PlA+9T13Pg==", "ffn_scale_f32": "Bhz3PJHrlTwcmx89", "ffn_base_f32": "BPIhQBPEM8C59XLA1KI8wOgPZL/y+Wq/EUWpv5Qsgr8WokZAcqJHwNkEW8AT7C/AhaNTwFOsIEAZ8zLAl2YewLxjMMBQY1DAtu03QIyWSMDG8zDArWIiwK8lfMBUUCNA", "provenance": "Retained mHC stage inputs, no expected outputs. eight_rows_bf16: the eight teacher token embeddings (4x5120 bf16 rows) from ordinary-rms-eight-embeddings-fixture.json. ffn_stream_bf16/ffn_branch_bf16/ffn_scale_f32/ffn_base_f32: token-1 layer FFN hyperconnection stream, branch and real checkpoint hc_ffn scale/base from hc-residual-token1-ffn-proof.json. Base64 payloads are little-endian BF16 or FP32. SHA256: eight=6942a581c6b9d84d stream=9409aee30d5836ff branch=74d00910644e0bd8"} \ No newline at end of file diff --git a/zig/src/families/deepseek_v41/index_precision_test.zig b/zig/src/families/deepseek_v41/index_precision_test.zig new file mode 100644 index 0000000000..1822f7ad0f --- /dev/null +++ b/zig/src/families/deepseek_v41/index_precision_test.zig @@ -0,0 +1,57 @@ +//! BF16 tensor boundaries are observable through which compressed positions enter sparse attention. +const std = @import("std"); +const mtl = @import("metal"); +const index = @import("csa2_index.zig"); +const kv = @import("kv.zig"); + +// The last key's extra 2^-9 dot term disappears at the source BF16 boundary; all 513 scores must tie. +test "index BF16 dot product and reduction boundaries preserve top-512 tie selection" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var ip = try index.Pipelines.load(device); + defer ip.deinit(); + var kp = try kv.Pipelines.load(device); + defer kp.deinit(); + var scratch = try index.Scratch.init(device, 1); + defer scratch.deinit(); + const q = try device.buffer(4096 * 2, mtl.ResourceOptions.shared); + defer q.deinit(); + const w = try device.buffer(32 * 2, mtl.ResourceOptions.shared); + defer w.deinit(); + const raw = try device.buffer(513 * 128 * 2, mtl.ResourceOptions.shared); + defer raw.deinit(); + const storage = try device.buffer(513 * 68, mtl.ResourceOptions.shared); + defer storage.deinit(); + @memset(q.slice(u16, 4096), 0); + @memset(w.slice(u16, 32), 0); + @memset(raw.slice(u16, 513 * 128), 0); + q.slice(u16, 4096)[0] = 0x3f80; // 1 + q.slice(u16, 4096)[32] = 0x3b00; // 2^-9, in a separate FP4 scale block + q.slice(u16, 4096)[128] = 0x3f80; + w.slice(u16, 32)[0] = 0x3f40; // 3/4 + w.slice(u16, 32)[1] = 0x3b00; + for (0..513) |row| raw.slice(u16, 513 * 128)[row * 128] = 0x3f80; + raw.slice(u16, 513 * 128)[512 * 128 + 32] = 0x3f80; + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try kp.encodePack(enc, .{ .buffer = raw }, .{ .buffer = storage }, .{ .rows = 513, .width = 128, .stride = 128, .packed_stride = 68, .block = 32, .kind = .index_fp4 }); + try ip.encode(enc, .{ .buffer = q }, .{ .buffer = w }, .{ .buffer = storage }, scratch, .{ .rows = 1, .q_stride = 4096, .weight_stride = 32, .position = 512, .ratio = 1, .keys = 513 }); + enc.end(); + cb.commit(); + cb.wait(); + if (cb.failure()) |message| { + std.log.err("index precision qualification: {s}", .{message}); + return error.GpuExecutionFailed; + } + const positions = scratch.positions(); + const ids: [*]const i32 = @ptrCast(@alignCast(positions.buffer.contents() + positions.offset)); + for (0..512) |i| try std.testing.expectEqual(@as(i32, @intCast(i)), ids[i]); + const scores = scratch.scores(); + const values: [*]const f32 = @ptrCast(@alignCast(scores.buffer.contents() + scores.offset)); + try std.testing.expectEqual(@as(f32, 0.01171875), values[0]); + try std.testing.expectEqual(values[0], values[512]); +} diff --git a/zig/src/families/deepseek_v41/kv.zig b/zig/src/families/deepseek_v41/kv.zig new file mode 100644 index 0000000000..ac9c4839b9 --- /dev/null +++ b/zig/src/families/deepseek_v41/kv.zig @@ -0,0 +1,162 @@ +//! Cache operators accept projected inputs; no source projection or hidden allocations live here. +const std = @import("std"); +const mtl = @import("metal"); +const sources = @import("kernel_sources"); +const View = @import("view.zig").View; +pub const cache = @import("kv_state.zig"); +pub const Kind = enum(u32) { window_fp8 = 0, main_fp4 = 1, index_fp4 = 2 }; +pub const Pack = extern struct { + rows: u32, + width: u32, + stride: u32, + packed_stride: u32, + block: u32, + kind: Kind, + first: u32 = 0, + ring: u32 = 0, + inplace: u32 = 0, + pub fn validate(p: Pack) !void { + const block: u32 = if (p.kind == .main_fp4) 16 else 32; + const bytes = if (p.kind == .window_fp8) p.width else p.width / 2; + if (p.rows == 0 or p.width == 0 or p.width % block != 0 or p.block != block or + p.stride < p.width or @as(u64, p.packed_stride) < @as(u64, bytes) + p.width / block or + p.inplace > 1 or @as(u64, p.first) + p.rows > std.math.maxInt(u32) or + @as(u64, p.rows -| 1) * p.stride + p.width > std.math.maxInt(u32) or + @as(u64, if (p.ring != 0) p.ring else p.first +| p.rows) * p.packed_stride > + std.math.maxInt(u32) or (p.ring != 0 and p.rows > p.ring)) + return error.InvalidCachePackShape; + } +}; +pub const Pool = extern struct { + rows: u32, + width: u32 = 512, + ratio: u32, + position: u32, + stride: u32, + input_f32: u32, + eps: f32 = 1e-20, + pub fn completed(p: Pool) u32 { + return (p.position + p.rows) / p.ratio - p.position / p.ratio; + } + pub fn validate(p: Pool) !void { + if (p.rows == 0 or p.width == 0 or p.width > 512 or (p.ratio != 1 and p.ratio != 2) or + p.stride < p.width or p.input_f32 != @intFromBool(p.ratio > 1) or + !std.math.isFinite(p.eps) or p.eps <= 0 or + @as(u64, p.position) + p.rows > cache.max_context or + @as(u64, p.rows -| 1) * p.stride + p.width > std.math.maxInt(u32)) + return error.InvalidCompressorShape; + } +}; +pub const Pipelines = struct { + pack: mtl.Pipeline, + pool: mtl.Pipeline, + tail: mtl.Pipeline, + copy: mtl.Pipeline, + publish: mtl.Pipeline, + unpack: mtl.Pipeline, + pub fn load(device: mtl.Device) !Pipelines { + var options = mtl.CompileOptions.mlx(); + options.math_functions = .precise; + const lib = try mtl.Library.fromSource(device, sources.deepseek_v41_kv, options); + defer lib.deinit(); + var p: Pipelines = undefined; + p.pack = try mtl.Pipeline.init(device, lib, "dsv41_pack", false); + errdefer p.pack.deinit(); + p.pool = try mtl.Pipeline.init(device, lib, "dsv41_pool", false); + errdefer p.pool.deinit(); + p.tail = try mtl.Pipeline.init(device, lib, "dsv41_pool_tail", false); + errdefer p.tail.deinit(); + p.copy = try mtl.Pipeline.init(device, lib, "dsv41_copy", false); + errdefer p.copy.deinit(); + p.publish = try mtl.Pipeline.init(device, lib, "dsv41_publish_window", false); + errdefer p.publish.deinit(); + p.unpack = try mtl.Pipeline.init(device, lib, "dsv41_unpack_rows", false); + return p; + } + pub fn deinit(p: *Pipelines) void { + p.pack.deinit(); + p.pool.deinit(); + p.tail.deinit(); + p.copy.deinit(); + p.publish.deinit(); + p.unpack.deinit(); + } + pub fn encodePack(p: *const Pipelines, enc: mtl.ComputeEncoder, x: View, storage: View, args: Pack) !void { + try args.validate(); + try x.requireRows(args.rows, args.stride, args.width, 2); + try storage.requireRows(if (args.ring != 0) args.ring else args.first + args.rows, args.packed_stride, args.width / (if (args.kind == .window_fp8) @as(u32, 1) else 2) + + args.width / args.block, 1); + enc.setPipeline(p.pack); + enc.setBuffer(x.buffer, x.offset, 0); + enc.setBuffer(storage.buffer, storage.offset, 1); + enc.setValue(args, 2); + enc.dispatchGroups(mtl.Size.of(args.width / args.block, args.rows, 1), mtl.Size.of(32, 1, 1)); + enc.barrier(); + } + pub fn encodeUnpack(p: *const Pipelines, enc: mtl.ComputeEncoder, storage: View, out: View, args: Pack) !void { + try args.validate(); + try out.requireRows(args.rows, args.stride, args.width, 2); + try storage.requireRows(if (args.ring != 0) args.ring else args.first + args.rows, args.packed_stride, args.width / (if (args.kind == .window_fp8) @as(u32, 1) else 2) + + args.width / args.block, 1); + enc.setPipeline(p.unpack); + enc.setBuffer(storage.buffer, storage.offset, 0); + enc.setBuffer(out.buffer, out.offset, 1); + enc.setValue(args, 2); + enc.dispatchThreads(mtl.Size.of(args.width, args.rows, 1), mtl.Size.of(32, 1, 1)); + enc.barrier(); + } + /// ratio2 values/gates are FP32; ratio1 values and RMS weight are BF16. Latent output is BF16. + pub fn encodePool(p: *const Pipelines, enc: mtl.ComputeEncoder, values: View, gates: View, partial: View, norm_weight: View, latent: View, args: Pool) !void { + try args.validate(); + try values.requireRows(args.rows, args.stride, args.width, if (args.input_f32 != 0) 4 else 2); + if (args.ratio > 1) { + try gates.requireRows(args.rows, args.stride, args.width, 4); + try partial.require(@as(usize, 2) * args.ratio * args.width * 4, 4); + } + if (args.completed() != 0) { + try norm_weight.require(@as(usize, args.width) * 2, 2); + try latent.requireRows(args.completed(), args.width, args.width, 2); + enc.setPipeline(p.pool); + for ([_]View{ values, gates, partial, norm_weight, latent }, 0..) |v, i| + enc.setBuffer(v.buffer, v.offset, i); + enc.setValue(args, 5); + enc.dispatchGroups(mtl.Size.of(args.completed(), 1, 1), mtl.Size.of(128, 1, 1)); + enc.barrier(); + } + if (args.ratio > 1) { + enc.setPipeline(p.tail); + for ([_]View{ values, gates, partial }, 0..) |v, i| enc.setBuffer(v.buffer, v.offset, i); + enc.setValue(args, 3); + enc.dispatchThreads(mtl.Size.of(args.width, 1, 1), mtl.Size.of(128, 1, 1)); + enc.barrier(); + } + } + pub fn encodeCopy(p: *const Pipelines, enc: mtl.ComputeEncoder, src: View, dst: View, bytes: u32) void { + enc.setPipeline(p.copy); + enc.setBuffer(src.buffer, src.offset, 0); + enc.setBuffer(dst.buffer, dst.offset, 1); + enc.setValue(bytes, 2); + enc.dispatchThreads(mtl.Size.of(bytes, 1, 1), mtl.Size.of(256, 1, 1)); + enc.barrier(); + } + /// Copy only the final window; each destination slot has one writer even for large prefill chunks. + pub fn publishWindow(p: *const Pipelines, enc: mtl.ComputeEncoder, chunk: View, ring: View, rows: u32, position: u32) !void { + if (rows == 0 or @as(u64, position) + rows > cache.max_context) return error.NoCacheRows; + try chunk.requireRows(rows, 528, 528, 1); + try ring.require(128 * 528, 1); + enc.setPipeline(p.publish); + enc.setBuffer(chunk.buffer, chunk.offset, 0); + enc.setBuffer(ring.buffer, ring.offset, 1); + enc.setValue([4]u32{ rows, position, 528, 528 }, 2); + enc.dispatchThreads(mtl.Size.of(528, @min(rows, 128), 1), mtl.Size.of(32, 1, 1)); + enc.barrier(); + } + pub fn snapshot(p: *const Pipelines, enc: mtl.ComputeEncoder, state: *cache.State) !void { + try state.begin(); + p.encodeCopy(enc, .{ .buffer = state.live }, .{ .buffer = state.backup }, @intCast(state.plan.mutableBytes())); + } + pub fn rollback(p: *const Pipelines, enc: mtl.ComputeEncoder, state: *cache.State) !void { + try state.rollback(); + p.encodeCopy(enc, .{ .buffer = state.backup }, .{ .buffer = state.live }, @intCast(state.plan.mutableBytes())); + } +}; diff --git a/zig/src/families/deepseek_v41/kv_state.zig b/zig/src/families/deepseek_v41/kv_state.zig index 708257cf8d..f2b70175a7 100644 --- a/zig/src/families/deepseek_v41/kv_state.zig +++ b/zig/src/families/deepseek_v41/kv_state.zig @@ -1,61 +1,111 @@ -// Shared logical KV transactions. Backends supply copies and explicit physical formats. +//! Shared cache ownership and exact transaction snapshots; appended rows are cursor-delimited. const std = @import("std"); -pub const Encoding = enum { bf16, f32, cuda_fp8_584, cuda_index_fp8_132, metal_fp4_288, metal_index_fp4_68 }; -pub const Region = struct { - live: u64, - backup: u64, - row_bytes: usize, - rows: u32, - kind: enum { ring, carry }, - encoding: Encoding = .bf16, - pub fn bytes(r: Region) usize { - return r.row_bytes * r.rows; +const mtl = @import("metal"); +const View = @import("view.zig").View; +pub const owners = [_]u32{ 2, 8, 14, 20 }; +pub const ratios = [_]u32{ 2, 2, 2, 1 }; +pub const index_sources = [_]u32{ 2, 8, 14, 20, 24, 28, 32, 36 }; +pub const main_stride = 288; +pub const index_stride = 68; +pub const ring_stride = 528; +pub const window = 128; +pub const max_context = 1048576; + +pub fn ownerFor(layer: u32, ratio: u32) !usize { + if (ratio == 0) return error.NoCompressedCache; + if (layer >= 40) return error.InvalidCacheLayer; + if (layer < 2 or ratio != (if (layer < 20) @as(u32, 2) else 1)) return error.InvalidCacheOwner; + var found: ?usize = null; + for (owners, 0..) |owner, i| { + if (owner <= layer and ratios[i] == ratio) found = i; } -}; -pub const Copy = struct { - ctx: *anyopaque, - run: *const fn (*anyopaque, u64, u64, usize) anyerror!void, - pub fn copy(c: Copy, dst: u64, src: u64, bytes: usize) !void { - try c.run(c.ctx, dst, src, bytes); + return found orelse error.InvalidCacheOwner; +} +pub fn isIndexSource(layer: u32) bool { + return std.mem.indexOfScalar(u32, &index_sources, layer) != null; +} +pub const Source = struct { owner: u32, ratio: u32, rows: u32, main: View, index: View, partial: View }; +pub const Plan = struct { + capacity: u32, + layers: u32, + pub fn validate(p: Plan) !void { + if (p.capacity == 0 or p.capacity > max_context or p.layers != 40) + return error.InvalidCacheCapacity; + } + pub fn packedBytes(p: Plan) usize { + var bytes: usize = 0; + for (ratios) |r| bytes += @as(usize, p.capacity / r) * (main_stride + index_stride); + return bytes; + } + pub fn mutableBytes(p: Plan) usize { + return @as(usize, p.layers) * window * ring_stride + 7 * 2 * 512 * 4; + } + pub fn residentBytes(p: Plan) usize { + return p.packedBytes() + 2 * p.mutableBytes(); } }; pub const State = struct { - position: u64 = 0, - pending: ?u32 = null, - // A failed copy leaves the transaction active. The owner must retry or retain its buffers. - pub fn begin(s: *State, regions: []const Region, c: Copy, n: u32) !void { - if (s.pending != null) return error.CacheTransactionActive; - if (n == 0 or n > 64 or s.position > std.math.maxInt(u64) - @as(u64, n)) return error.BadWindow; - for (regions) |r| { - _ = std.math.mul(usize, r.row_bytes, r.rows) catch return error.BadRegion; - if (r.row_bytes == 0 or r.rows == 0 or (r.kind == .ring and n > r.rows)) return error.BadRegion; - } - for (regions) |r| try c.copy(r.backup, r.live, r.bytes()); - s.pending = n; - } - // Carries are selected from the accepted projection before keep. A full rollback restores them here. - pub fn keep(s: *State, regions: []const Region, c: Copy, count: u32) !void { - const n = s.pending orelse return error.NoCacheTransaction; - if (count > n) return error.BadWindow; - for (regions) |r| { - if (count == 0) { - try c.copy(r.live, r.backup, r.bytes()); - } else if (r.kind == .ring) { - // Only rejected positions were overwritten. Accepted rows stay in their physical ring slots. - for (count..n) |i| { - const off = ((s.position + i) % r.rows) * r.row_bytes; - try c.copy(r.live + off, r.backup + off, r.row_bytes); - } - } + plan: Plan, + storage: mtl.Buffer, + live: mtl.Buffer, + backup: mtl.Buffer, + position: u32 = 0, + saved_position: ?u32 = null, + + pub fn init(device: mtl.Device, plan: Plan) !State { + try plan.validate(); + const storage = try device.buffer(plan.packedBytes(), mtl.ResourceOptions.shared); + errdefer storage.deinit(); + const live = try device.buffer(plan.mutableBytes(), mtl.ResourceOptions.shared); + errdefer live.deinit(); + const backup = try device.buffer(plan.mutableBytes(), mtl.ResourceOptions.shared); + @memset(live.contents()[0..plan.mutableBytes()], 0); + return .{ .plan = plan, .storage = storage, .live = live, .backup = backup }; + } + pub fn deinit(s: *State) void { + s.storage.deinit(); + s.live.deinit(); + s.backup.deinit(); + } + pub fn residentBytes(s: *const State) usize { + return s.plan.residentBytes(); + } + pub fn registerResidency(s: *const State, set: mtl.ResidencySet) void { + set.add(s.storage); + set.add(s.live); + set.add(s.backup); + } + pub fn ring(s: *const State, layer: u32) !View { + if (layer >= s.plan.layers) return error.InvalidCacheLayer; + return .{ .buffer = s.live, .offset = @as(usize, layer) * window * ring_stride }; + } + pub fn source(s: *const State, layer: u32, ratio: u32) !Source { + if (layer >= s.plan.layers) return error.InvalidCacheLayer; + if (s.position > s.plan.capacity) return error.CacheCapacityExceeded; + const id = try ownerFor(layer, ratio); + var packed_off: usize = 0; + var partial_off: usize = @as(usize, s.plan.layers) * window * ring_stride; + for (ratios[0..id]) |r| { + packed_off += @as(usize, s.plan.capacity / r) * (main_stride + index_stride); + partial_off += 2 * r * 512 * 4; } - s.position += count; - s.pending = null; + return .{ .owner = owners[id], .ratio = ratio, .rows = s.position / ratio, .main = .{ .buffer = s.storage, .offset = packed_off }, .index = .{ .buffer = s.storage, .offset = packed_off + @as(usize, s.plan.capacity / ratio) * main_stride }, .partial = .{ .buffer = s.live, .offset = partial_off } }; + } + pub fn append(s: *State, rows: u32) !void { + if (s.position > s.plan.capacity or rows == 0 or rows > s.plan.capacity - s.position) + return error.CacheCapacityExceeded; + s.position += rows; } - pub fn rollback(s: *State, regions: []const Region, c: Copy) !void { - try s.keep(regions, c, 0); + pub fn begin(s: *State) !void { + if (s.saved_position != null) return error.CacheTransactionActive; + s.saved_position = s.position; + } + pub fn commit(s: *State) !void { + if (s.saved_position == null) return error.NoCacheTransaction; + s.saved_position = null; + } + pub fn rollback(s: *State) !void { + s.position = s.saved_position orelse return error.NoCacheTransaction; + s.saved_position = null; } }; - -test { - _ = @import("kv_state_test.zig"); -} diff --git a/zig/src/families/deepseek_v41/kv_test.zig b/zig/src/families/deepseek_v41/kv_test.zig new file mode 100644 index 0000000000..0d1ae738bc --- /dev/null +++ b/zig/src/families/deepseek_v41/kv_test.zig @@ -0,0 +1,234 @@ +const std = @import("std"); +const mtl = @import("metal"); +const kv = @import("kv.zig"); +const cache = @import("kv_state.zig"); +const View = @import("view.zig").View; +fn bf(x: f32) u16 { + const bits: u32 = @bitCast(x); + return @intCast((bits +% 0x7fff +% ((bits >> 16) & 1)) >> 16); +} +fn view(b: mtl.Buffer) View { + return .{ .buffer = b }; +} +fn submit(cb: mtl.CommandBuffer, enc: mtl.ComputeEncoder) !void { + enc.end(); + cb.commit(); + cb.wait(); + try std.testing.expect(cb.failure() == null); +} + +test "cache owners and one million token storage match the released layout" { + try std.testing.expectEqual(@as(usize, 0), try cache.ownerFor(7, 2)); + try std.testing.expectEqual(@as(usize, 1), try cache.ownerFor(13, 2)); + try std.testing.expectEqual(@as(usize, 2), try cache.ownerFor(19, 2)); + try std.testing.expectEqual(@as(usize, 3), try cache.ownerFor(39, 1)); + try std.testing.expectError(error.NoCompressedCache, cache.ownerFor(0, 0)); + const plan = cache.Plan{ .capacity = 1048576, .layers = 40 }; + try std.testing.expectEqual(@as(usize, 1048576 * 890), plan.packedBytes()); + try std.testing.expectEqual(@as(usize, 2732032), plan.mutableBytes()); + try plan.validate(); + try std.testing.expectError(error.InvalidCacheCapacity, (cache.Plan{ .capacity = 1, .layers = 41 }).validate()); + try std.testing.expectError(error.InvalidCacheLayer, cache.ownerFor(40, 1)); + try std.testing.expectError(error.InvalidCacheOwner, cache.ownerFor(39, 2)); +} + +test "FP4 E2M1 ties, signs, zero scales and BF16 in-place values" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var pipes = try kv.Pipelines.load(device); + defer pipes.deinit(); + const x = try device.buffer(64, mtl.ResourceOptions.shared); + defer x.deinit(); + const storage = try device.buffer(36, mtl.ResourceOptions.shared); + defer storage.deinit(); + const values = [_]f32{ 0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.5, -1, -1.5, -2, 0.25, 0.75, 1.25, 1.75 }; + for (values, 0..) |v, i| x.slice(u16, 32)[i] = bf(v); + @memset(x.slice(u16, 32)[16..32], 0); + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try pipes.encodePack(enc, view(x), view(storage), .{ .rows = 2, .width = 16, .stride = 16, .packed_stride = 9, .block = 16, .kind = .main_fp4, .inplace = 1 }); + try submit(cb, enc); + const expected = [_]u8{ 0x10, 0x32, 0x54, 0x76, 0xa9, 0xcb, 0x20, 0x42, 0x38 }; + try std.testing.expectEqualSlices(u8, &expected, storage.contents()[0..9]); + try std.testing.expectEqual(@as(u8, 1), storage.contents()[17]); + const rounded = [_]f32{ 0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.5, -1, -1.5, -2, 0, 1, 1, 2 }; + for (rounded, 0..) |v, i| try std.testing.expectEqual(bf(v), x.slice(u16, 32)[i]); +} + +test "compressor carries a partial group across chunks without changing the pooled latent" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var pipes = try kv.Pipelines.load(device); + defer pipes.deinit(); + const values = try device.buffer(3 * 512 * 4, mtl.ResourceOptions.shared); + defer values.deinit(); + const gates = try device.buffer(3 * 512 * 4, mtl.ResourceOptions.shared); + defer gates.deinit(); + const partial = try device.buffer(4 * 512 * 4, mtl.ResourceOptions.shared); + defer partial.deinit(); + const one_shot = try device.buffer(4 * 512 * 4, mtl.ResourceOptions.shared); + defer one_shot.deinit(); + const weight = try device.buffer(512 * 2, mtl.ResourceOptions.shared); + defer weight.deinit(); + const out = try device.buffer(2 * 512 * 2, mtl.ResourceOptions.shared); + defer out.deinit(); + @memset(partial.contents()[0 .. 4 * 512 * 4], 0); + @memset(one_shot.contents()[0 .. 4 * 512 * 4], 0); + @memset(weight.slice(u16, 512), bf(1)); + for (0..3 * 512) |i| { + values.slice(f32, 3 * 512)[i] = @as(f32, @floatFromInt((i % 512) % 11)) - 5 + @as(f32, @floatFromInt(i / 512)); + gates.slice(f32, 3 * 512)[i] = @as(f32, @floatFromInt(i % 7)) * 0.25; + } + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try pipes.encodePool(enc, view(values), view(gates), view(partial), view(weight), view(out), .{ .rows = 1, .ratio = 2, .position = 0, .stride = 512, .input_f32 = 1 }); + try pipes.encodePool(enc, view(values).shifted(512 * 4), view(gates).shifted(512 * 4), view(partial), view(weight), view(out), .{ .rows = 2, .ratio = 2, .position = 1, .stride = 512, .input_f32 = 1 }); + try pipes.encodePool(enc, view(values), view(gates), view(one_shot), view(weight), view(out).shifted(512 * 2), .{ .rows = 3, .ratio = 2, .position = 0, .stride = 512, .input_f32 = 1 }); + try submit(cb, enc); + try std.testing.expectEqualSlices(u16, out.slice(u16, 1024)[0..512], out.slice(u16, 1024)[512..1024]); + try std.testing.expectEqualSlices(f32, partial.slice(f32, 2048)[0..512], one_shot.slice(f32, 2048)[0..512]); + try std.testing.expectEqualSlices(f32, partial.slice(f32, 2048)[1024..1536], one_shot.slice(f32, 2048)[1024..1536]); +} + +test "compressor rounds the pooled FP32 latent to BF16 before RMS normalization" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var pipes = try kv.Pipelines.load(device); + defer pipes.deinit(); + const values = try device.buffer(4 * 4, mtl.ResourceOptions.shared); + defer values.deinit(); + const gates = try device.buffer(4 * 4, mtl.ResourceOptions.shared); + defer gates.deinit(); + const partial = try device.buffer(4 * 2 * 4, mtl.ResourceOptions.shared); + defer partial.deinit(); + const weight = try device.buffer(2 * 2, mtl.ResourceOptions.shared); + defer weight.deinit(); + const out = try device.buffer(2 * 2, mtl.ResourceOptions.shared); + defer out.deinit(); + @memcpy(values.slice(f32, 4), &[_]f32{ 1.00390625, 0.5, 1.00390625, 0.5 }); + @memset(gates.slice(f32, 4), 0); + @memset(partial.contents()[0..32], 0); + @memset(weight.slice(u16, 2), bf(1)); + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try pipes.encodePool(enc, view(values), view(gates), view(partial), view(weight), view(out), .{ .rows = 2, .width = 2, .ratio = 2, .position = 0, .stride = 2, .input_f32 = 1 }); + try submit(cb, enc); + // 1.00390625 ties to 1 in BF16; normalizing the unrounded value changes the second result. + try std.testing.expectEqual(bf(1.265625), out.slice(u16, 2)[0]); + try std.testing.expectEqual(bf(0.6328125), out.slice(u16, 2)[1]); +} + +test "ring publish handles chunks beyond the window and transaction rollback restores every mutable byte" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var pipes = try kv.Pipelines.load(device); + defer pipes.deinit(); + var state = try cache.State.init(device, .{ .capacity = 1024, .layers = 40 }); + defer state.deinit(); + const chunk = try device.buffer(131 * 528, mtl.ResourceOptions.shared); + defer chunk.deinit(); + for (0..131) |r| @memset(chunk.contents()[r * 528 .. (r + 1) * 528], @intCast(r)); + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try pipes.publishWindow(enc, view(chunk), try state.ring(0), 131, 0); + try state.append(131); + try pipes.snapshot(enc, &state); + try pipes.publishWindow(enc, view(chunk), try state.ring(0), 131, 131); + try state.append(131); + try pipes.rollback(enc, &state); + try submit(cb, enc); + try std.testing.expectEqual(@as(u32, 131), state.position); + try std.testing.expectEqualSlices(u8, state.backup.contents()[0..state.plan.mutableBytes()], state.live.contents()[0..state.plan.mutableBytes()]); + for (3..131) |r| { + const slot = r % 128; + for (state.live.contents()[slot * 528 .. (slot + 1) * 528]) |byte| + try std.testing.expectEqual(@as(u8, @intCast(r)), byte); + } + const cb2 = queue.commandBuffer(); + const enc2 = cb2.compute(.serial); + try pipes.snapshot(enc2, &state); + try pipes.publishWindow(enc2, view(chunk), try state.ring(0), 131, 131); + try state.append(131); + try state.commit(); + try submit(cb2, enc2); + try std.testing.expectEqual(@as(u32, 262), state.position); + try std.testing.expect(state.saved_position == null); +} + +test "window FP8 quantization includes the RoPE tail and index FP4 uses E8M0 scale bytes" { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const device = try mtl.Device.init(); + defer device.deinit(); + const queue = try device.queue(); + defer queue.deinit(); + var pipes = try kv.Pipelines.load(device); + defer pipes.deinit(); + const x = try device.buffer(64 * 2, mtl.ResourceOptions.shared); + defer x.deinit(); + const storage = try device.buffer(66, mtl.ResourceOptions.shared); + defer storage.deinit(); + const decoded = try device.buffer(64 * 2, mtl.ResourceOptions.shared); + defer decoded.deinit(); + @memset(x.slice(u16, 64), 0); + @memset(x.slice(u16, 64)[32..64], bf(0.3)); + x.slice(u16, 64)[63] = bf(448); + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + const args = kv.Pack{ .rows = 1, .width = 64, .stride = 64, .packed_stride = 66, .block = 32, .kind = .window_fp8, .inplace = 1 }; + try pipes.encodePack(enc, view(x), view(storage), args); + try pipes.encodeUnpack(enc, view(storage), view(decoded), args); + try submit(cb, enc); + try std.testing.expectEqual(@as(u8, 127), storage.contents()[65]); + try std.testing.expectEqual(@as(u8, 126), storage.contents()[63]); + try std.testing.expectEqual(bf(0.3125), decoded.slice(u16, 64)[62]); + try std.testing.expectEqualSlices(u16, x.slice(u16, 64), decoded.slice(u16, 64)); + @memset(x.slice(u16, 64)[0..32], bf(6)); + const cb2 = queue.commandBuffer(); + const enc2 = cb2.compute(.serial); + try pipes.encodePack(enc2, view(x), view(storage), .{ .rows = 1, .width = 32, .stride = 32, .packed_stride = 17, .block = 32, .kind = .index_fp4, .inplace = 1 }); + try submit(cb2, enc2); + for (storage.contents()[0..16]) |byte| try std.testing.expectEqual(@as(u8, 0x77), byte); + try std.testing.expectEqual(@as(u8, 127), storage.contents()[16]); +} + +test { + _ = @import("csa2_test.zig"); +} + +test "cache pack rejects undersized borrowed input and output before encoding" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const small = try device.buffer(16, mtl.ResourceOptions.shared); + defer small.deinit(); + const input = try device.buffer(64, mtl.ResourceOptions.shared); + defer input.deinit(); + const args = kv.Pack{ .rows = 1, .width = 32, .stride = 32, .packed_stride = 33, .block = 32, .kind = .window_fp8 }; + const pipelines: *const kv.Pipelines = undefined; + try std.testing.expectError(error.InvalidTensorView, pipelines.encodePack(undefined, view(small), view(input), args)); + try std.testing.expectError(error.InvalidTensorView, pipelines.encodePack(undefined, view(input), view(small), args)); + try std.testing.expectError(error.InvalidTensorView, pipelines.encodePack(undefined, view(input).shifted(1), view(input), args)); +} + +test "cache shapes reject overflow and nonfinite compressor epsilon" { + try std.testing.expectError(error.InvalidCachePackShape, (kv.Pack{ .rows = 2, .width = 32, .stride = 32, .packed_stride = 33, .block = 32, .kind = .window_fp8, .first = std.math.maxInt(u32) }).validate()); + try std.testing.expectError(error.InvalidCompressorShape, (kv.Pool{ .rows = 1, .ratio = 1, .position = 0, .stride = 512, .input_f32 = 0, .eps = std.math.nan(f32) }).validate()); +} diff --git a/zig/src/families/deepseek_v41/load.zig b/zig/src/families/deepseek_v41/load.zig index 6d9476e97b..337e178ec0 100644 --- a/zig/src/families/deepseek_v41/load.zig +++ b/zig/src/families/deepseek_v41/load.zig @@ -1,89 +1,259 @@ -//! Shared checkpoint admission and host images. Backend resources belong in adapters. -pub const Pack = @import("pack.zig").Pack; -pub const Info = @import("pack.zig").Info; -pub const plan = @import("plan.zig"); -pub const stage = @import("stage.zig"); -pub const named = @import("named.zig"); -pub const prepared = @import("prepared.zig"); +//! Header-first resident loading: no mapped weights, deferred disk lookup or duplicate Engram projections. const std = @import("std"); -/// Strict production checkpoint admission. Pack itself also supports partial header inspection. -pub fn open(a: std.mem.Allocator, io: std.Io, dir: []const u8) !Pack { - var p = try Pack.open(a, io, dir); - errdefer p.deinit(); - if (p.missing != 0) return error.CheckpointIndexMismatch; - var owners = p.owner.iterator(); - while (owners.next()) |e| { - const info = p.get(e.key_ptr.*) orelse return error.CheckpointIndexMismatch; - if (!std.mem.eql(u8, p.files.items[info.file], e.value_ptr.*)) return error.CheckpointIndexMismatch; +const mtl = @import("metal"); +const st = @import("../../core/safetensors.zig"); +const cp = @import("../../core/checkpoint_metal.zig"); +const paced = @import("../../core/paced_read.zig"); + +pub const Reference = struct { shard: usize, entry: st.Entry }; +pub const Shard = struct { + path: [:0]const u8, + file_bytes: usize, + data_start: usize, + begin: usize, + end: usize, + names: st.Header, +}; + +pub const Plan = struct { + arena: std.heap.ArenaAllocator, + shards: std.ArrayList(Shard) = .empty, + tensors: std.StringHashMapUnmanaged(Reference) = .empty, + bytes: usize = 0, + + pub fn open(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, table_dir: []const u8) !Plan { + var self = Plan{ .arena = std.heap.ArenaAllocator.init(gpa) }; + errdefer self.deinit(); + const a = self.arena.allocator(); + const index_path = try std.fs.path.join(a, &.{ dir, "model.safetensors.index.json" }); + const index_bytes = try std.Io.Dir.cwd().readFileAlloc(io, index_path, a, .limited(64 << 20)); + const index = try std.json.parseFromSliceLeaky(struct { weight_map: std.json.Value }, a, index_bytes, .{ + .ignore_unknown_fields = true, + }); + if (index.weight_map != .object or index.weight_map.object.count() == 0) return error.BadCheckpointIndex; + var files: std.StringHashMapUnmanaged(void) = .empty; + var entries = index.weight_map.object.iterator(); + while (entries.next()) |item| { + if (item.value_ptr.* != .string) return error.BadCheckpointIndex; + const file = item.value_ptr.string; + if (!safeBasename(file)) return error.BadCheckpointIndex; + try files.put(a, file, {}); + } + const ordered = try a.alloc([]const u8, files.count()); + var file_it = files.keyIterator(); + var n: usize = 0; + while (file_it.next()) |file| : (n += 1) ordered[n] = file.*; + std.mem.sort([]const u8, ordered, {}, less); + var owned: std.StringHashMapUnmanaged([]const u8) = .empty; + entries.reset(); + while (entries.next()) |item| try owned.put(a, item.key_ptr.*, item.value_ptr.string); + for (ordered) |file| try self.add(try std.fs.path.join(a, &.{ dir, file }), null, &owned); + if (self.tensors.count() != index.weight_map.object.count()) return error.CheckpointIndexMismatch; + entries = index.weight_map.object.iterator(); + while (entries.next()) |item| { + const ref = self.tensors.get(item.key_ptr.*) orelse return error.CheckpointIndexMismatch; + if (!std.mem.eql(u8, std.fs.path.basename(self.shards.items[ref.shard].path), item.value_ptr.string)) + return error.CheckpointIndexMismatch; + } + for ([_]struct { layer: u32, file: []const u8, rows: usize }{ + .{ .layer = 1, .file = "model-00047-of-00048.safetensors", .rows = 384006168 }, + .{ .layer = 14, .file = "model-00048-of-00048.safetensors", .rows = 384016682 }, + }) |table| { + const prefix = try std.fmt.allocPrint(a, "layers.{d}.engram.embed.", .{table.layer}); + try self.add(try std.fs.path.join(a, &.{ table_dir, table.file }), prefix, null); + const weight = try std.fmt.allocPrint(a, "{s}weight", .{prefix}); + const scale = try std.fmt.allocPrint(a, "{s}scale", .{prefix}); + try self.require(weight, .f8_e4m3, &.{ table.rows, 256 }); + try self.require(scale, .f8_e8m0, &.{ table.rows, 8 }); + if (self.shards.items[self.shards.items.len - 1].names.count() != 2) return error.UnexpectedEngramTensor; + } + return self; + } + + pub fn deinit(self: *Plan) void { + self.arena.deinit(); + self.* = undefined; + } + + pub fn get(self: *const Plan, name: []const u8) !st.Entry { + return (self.tensors.get(name) orelse return error.MissingTensor).entry; + } + + pub fn has(self: *const Plan, name: []const u8) bool { + return self.tensors.contains(name); + } + + pub fn require(self: *const Plan, name: []const u8, dtype: st.DType, shape: []const usize) !void { + const e = try self.get(name); + if (e.dtype != dtype or e.rank != shape.len or !std.mem.eql(usize, e.shape[0..e.rank], shape)) { + std.log.err("unsupported tensor {s}: dtype {s}, shape {any}", .{ name, @tagName(e.dtype), e.shape[0..e.rank] }); + return error.UnsupportedTensor; + } } - return p; + + fn add(self: *Plan, path: []const u8, prefix: ?[]const u8, owned: ?*const std.StringHashMapUnmanaged([]const u8)) !void { + const a = self.arena.allocator(); + const own_path = try a.dupeSentinel(u8, path, 0); + const fd = std.c.open(own_path, .{ .ACCMODE = .RDONLY }); + if (fd < 0) return error.OpenFailed; + defer _ = std.c.close(fd); + const length = std.c.lseek(fd, 0, std.c.SEEK.END); + if (length < 8) return error.BadSafetensors; + const file_bytes: usize = @intCast(length); + var head: [8]u8 = undefined; + try readAll(fd, &head, 0); + const header_bytes = std.mem.readInt(u64, &head, .little); + if (header_bytes > @min(file_bytes - 8, 64 << 20)) return error.BadSafetensors; + const json = try a.alloc(u8, @intCast(header_bytes)); + try readAll(fd, json, 8); + const data_start: usize = 8 + @as(usize, @intCast(header_bytes)); + const names = try st.parseHeaderPrefix(a, json, file_bytes - data_start, prefix); + if (names.count() == 0) return error.MissingTensor; + var begin: usize = if (prefix == null) 0 else file_bytes - data_start; + var end: usize = if (prefix == null) file_bytes - data_start else 0; + for (names.keys(), names.values()) |name, entry| { + // A rank-split pack duplicates whole-load tensors beside the shards; the index decides the owner file. + if (owned) |map| { + const owner = map.get(name) orelse continue; + if (!std.mem.eql(u8, owner, std.fs.path.basename(path))) continue; + } + begin = @min(begin, entry.begin); + end = @max(end, entry.end); + const slot = try self.tensors.getOrPut(a, name); + if (slot.found_existing) { + return error.DuplicateTensor; + } + slot.value_ptr.* = .{ .shard = self.shards.items.len, .entry = entry }; + } + if (prefix != null) { + var selected_bytes: usize = 0; + for (names.values()) |entry| selected_bytes = try std.math.add(usize, selected_bytes, entry.end - entry.begin); + if (selected_bytes != end - begin) return error.NoncontiguousEngramTables; + } + self.bytes = try std.math.add(usize, self.bytes, end - begin); + try self.shards.append(a, .{ .path = own_path, .file_bytes = file_bytes, .data_start = data_start, .begin = begin, .end = end, .names = names }); + } + + /// Only called after architecture validation and the complete memory admission calculation. + pub fn materialize(self: *const Plan, gpa: std.mem.Allocator, device: mtl.Device, residency: mtl.ResidencySet, queue: mtl.Queue) !cp.Checkpoint { + var out = cp.Checkpoint.init(gpa); + errdefer out.deinit(); + const stage = try paced.stage(gpa, 32 << 20); + defer gpa.free(stage); + var warm = mtl.keepalive.Target{ .queue = queue, .sets = @as(*const [1]mtl.ResidencySet, &residency) }; + var last_tick: f64 = -std.math.inf(f64); + var short: paced.Memory = undefined; + for (self.shards.items, 0..) |shard, i| { + const fd = std.c.open(shard.path, .{ .ACCMODE = .RDONLY }); + if (fd < 0) return error.OpenFailed; + defer _ = std.c.close(fd); + if (std.c.fcntl(fd, 48, @as(c_int, 1)) != 0) return error.UncachedReadUnavailable; + if (std.c.lseek(fd, 0, std.c.SEEK.END) != shard.file_bytes) return error.CheckpointChanged; + const bytes = shard.end - shard.begin; + const buffer = try device.buffer(@max(bytes, 16), mtl.ResourceOptions.shared | mtl.ResourceOptions.untracked); + var owned = false; + errdefer if (!owned) buffer.deinit(); + try out.own(buffer, bytes); + owned = true; + residency.add(buffer); + residency.commit(); + residency.requestResidency(); + var copied: usize = 0; + while (copied < bytes) { + const count = @min(bytes - copied, stage.len - paced.alignment); + paced.read(fd, buffer.contents()[copied..][0..count], shard.data_start + shard.begin + copied, stage, &short) catch |err| { + if (err == error.MemoryShort) { + var msg: [512]u8 = undefined; + std.log.err("deepseek-v41: {s}", .{paced.refusal(&msg, short, paced.floor())}); + } + return err; + }; + copied += count; + const now = mtl.clock.seconds(); + if (now - last_tick >= 0.4) { + mtl.keepalive.Target.tick(&warm); + last_tick = now; + } + } + for (shard.names.keys(), shard.names.values()) |name, e| { + try out.set(name, .{ .buffer = buffer, .offset = e.begin - shard.begin, .bytes = e.end - e.begin, .dtype = e.dtype, .shape = e.shape, .rank = e.rank }); + } + std.log.info("resident checkpoint shard {d}/{d}: {d} bytes, residency set {d} bytes", .{ i + 1, self.shards.items.len, bytes, residency.allocatedSize() }); + } + residency.commit(); + residency.requestResidency(); + return out; + } +}; + +fn readAll(fd: std.c.fd_t, dest: []u8, at: usize) !void { + var done: usize = 0; + while (done < dest.len) { + const n = std.c.pread(fd, dest.ptr + done, dest.len - done, @intCast(at + done)); + if (n <= 0) return error.ShortRead; + done += @intCast(n); + } +} + +fn safeBasename(s: []const u8) bool { + return s.len > 0 and !std.mem.eql(u8, s, ".") and !std.mem.eql(u8, s, "..") and + std.mem.indexOfAny(u8, s, "/\\") == null and std.mem.indexOfScalar(u8, s, 0) == null and + std.mem.endsWith(u8, s, ".safetensors"); } -pub const Projection = Pack.Projection; -pub const projection = @import("projection.zig").read; -test "shared projection requires dense rotation scales and refuses half-bit groups" { - var p = Pack.init(std.testing.allocator); - defer p.deinit(); - try p.addHeader("dense", "{\"p.weight\":{\"dtype\":\"BF16\",\"shape\":[128,128],\"data_offsets\":[0,32768]}}", 32768); - try std.testing.expectError(error.MissingTensor, projection(&p, "p")); - try std.testing.expectError(error.BadTrellis, @import("core").exl3_layout.Group.fromShape(&.{ 8, 8, 8 })); +fn less(_: void, a: []const u8, b: []const u8) bool { + return std.mem.lessThan(u8, a, b); } -test "shared projection admits packed sign owners and gives dense siblings precedence" { + +test "selected table plan excludes projections and preserves tensor byte offsets" { const a = std.testing.allocator; - const image = try @import("pack.zig").image(a, &.{ - .{ .name = "p.trellis", .dtype = "I16", .shape = &.{ 8, 8, 32 } }, - .{ .name = "p.su", .dtype = "I16", .shape = &.{8} }, - .{ .name = "p.sv", .dtype = "I16", .shape = &.{8} }, - }); - defer a.free(image); - var p = Pack.init(a); - defer p.deinit(); - try p.addHeaderDump("packed", image); - var packed_plan = try @import("model.zig").Plan.init(a, &p, &.{"p"}); - defer packed_plan.deinit(); - const packed_storage = packed_plan.projections[0].storage; - try std.testing.expect(packed_storage == .exl3); - try std.testing.expectEqual(p.get("p.su").?, packed_storage.exl3.scales.input); - try std.testing.expectEqual(p.get("p.sv").?, packed_storage.exl3.scales.output); - try std.testing.expectError(error.UnsupportedPackedScales, packed_storage.cudaGroup()); - const dense = try @import("pack.zig").image(a, &.{ - .{ .name = "p.weight", .dtype = "BF16", .shape = &.{ 128, 256 } }, - .{ .name = "p.suh", .dtype = "F16", .shape = &.{256} }, - .{ .name = "p.svh", .dtype = "F16", .shape = &.{128} }, - }); - defer a.free(dense); - var both = Pack.init(a); - defer both.deinit(); - try both.addHeaderDump("dense", dense); - const trellis = try @import("pack.zig").image(a, &.{.{ .name = "p.trellis", .dtype = "I16", .shape = &.{ 16, 8, 32 } }}); - defer a.free(trellis); - try both.addHeaderDump("trellis", trellis); - var dense_plan = try @import("model.zig").Plan.init(a, &both, &.{"p"}); - defer dense_plan.deinit(); - const dense_storage = dense_plan.projections[0].storage; - try std.testing.expect(dense_storage == .dense); - try std.testing.expectEqual(@as(usize, 256), dense_storage.dense.k); - try std.testing.expectEqual(@as(usize, 128), dense_storage.dense.n); - try std.testing.expectEqual(both.get("p.weight").?, dense_storage.dense.weight); - try std.testing.expectEqual(both.get("p.suh").?, dense_storage.dense.scales.input); - try std.testing.expectEqual(both.get("p.svh").?, dense_storage.dense.scales.output); - try std.testing.expectError(error.UnsupportedDenseProjection, dense_storage.cudaGroup()); + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const header = + \\{"padding":{"dtype":"U8","shape":[3],"data_offsets":[0,3]}, + \\ "engram.embed.weight":{"dtype":"U8","shape":[4],"data_offsets":[3,7]}, + \\ "engram.embed.scale":{"dtype":"U8","shape":[2],"data_offsets":[7,9]}, + \\ "engram.wkv":{"dtype":"U8","shape":[1],"data_offsets":[9,10]}} + ; + var data: [8 + header.len + 10]u8 = undefined; + std.mem.writeInt(u64, data[0..8], header.len, .little); + @memcpy(data[8..][0..header.len], header); + @memset(data[8 + header.len ..], 0); + try tmp.dir.writeFile(io, .{ .sub_path = "table.safetensors", .data = &data }); + const path = try std.fmt.allocPrint(a, ".zig-cache/tmp/{s}/table.safetensors", .{tmp.sub_path}); + defer a.free(path); + var plan = Plan{ .arena = std.heap.ArenaAllocator.init(a) }; + defer plan.deinit(); + try plan.add(path, "engram.embed.", null); + try std.testing.expectEqual(@as(usize, 6), plan.bytes); + try std.testing.expectEqual(@as(usize, 3), plan.shards.items[0].begin); + try std.testing.expectEqual(@as(usize, 9), plan.shards.items[0].end); + try std.testing.expectEqual(@as(usize, 7), (try plan.get("engram.embed.scale")).begin); + try std.testing.expectError(error.MissingTensor, plan.get("engram.wkv")); + try std.testing.expectError(error.DuplicateTensor, plan.add(path, "engram.embed.", null)); } -test "odd half-widths require a mul1 marker and scale pairs cannot be ambiguous" { + +test "selected table payload must be contiguous and indexed files cannot escape checkpoint directory" { const a = std.testing.allocator; - const data = try @import("pack.zig").image(a, &.{ - .{ .name = "p.trellis", .dtype = "I16", .shape = &.{ 8, 8, 24 } }, - .{ .name = "p.suh", .dtype = "F16", .shape = &.{128} }, - .{ .name = "p.svh", .dtype = "F16", .shape = &.{128} }, - }); - defer a.free(data); - var p = Pack.init(a); - defer p.deinit(); - try p.addHeaderDump("odd", data); - try std.testing.expectError(error.BadMarker, p.codebook(std.testing.io, "p")); - try std.testing.expectError(error.BadMarker, projection(&p, "p")); - const signs = try @import("pack.zig").image(a, &.{.{ .name = "p.su", .dtype = "I16", .shape = &.{8} }}); - defer a.free(signs); - try p.addHeaderDump("signs", signs); - try std.testing.expectError(error.AmbiguousScales, projection(&p, "p")); + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const header = + \\{"engram.embed.weight":{"dtype":"U8","shape":[4],"data_offsets":[0,4]}, + \\ "projection":{"dtype":"U8","shape":[3],"data_offsets":[4,7]}, + \\ "engram.embed.scale":{"dtype":"U8","shape":[2],"data_offsets":[7,9]}} + ; + var data: [8 + header.len + 9]u8 = undefined; + std.mem.writeInt(u64, data[0..8], header.len, .little); + @memcpy(data[8..][0..header.len], header); + @memset(data[8 + header.len ..], 0); + try tmp.dir.writeFile(io, .{ .sub_path = "gap.safetensors", .data = &data }); + const path = try std.fmt.allocPrint(a, ".zig-cache/tmp/{s}/gap.safetensors", .{tmp.sub_path}); + defer a.free(path); + var plan = Plan{ .arena = std.heap.ArenaAllocator.init(a) }; + defer plan.deinit(); + try std.testing.expectError(error.NoncontiguousEngramTables, plan.add(path, "engram.embed.", null)); + for ([_][]const u8{ "../x.safetensors", "/x.safetensors", "a\\x.safetensors", ".", "..", "x.safetensors\x00", "x.json", "" }) |name| try std.testing.expect(!safeBasename(name)); + try std.testing.expect(safeBasename("model-00001-of-00039.safetensors")); } diff --git a/zig/src/families/deepseek_v41/mhc.zig b/zig/src/families/deepseek_v41/mhc.zig new file mode 100644 index 0000000000..3ac499423f --- /dev/null +++ b/zig/src/families/deepseek_v41/mhc.zig @@ -0,0 +1,95 @@ +//! Four-copy mHC; callers validate shapes/storage and own previous-pre coefficients. +const mtl = @import("metal"); +const sources = @import("kernel_sources"); +pub const View = @import("view.zig").View; +pub const Pipelines = struct { + rms: mtl.Pipeline, + split: mtl.Pipeline, + collapse: mtl.Pipeline, + post: mtl.Pipeline, + expand: mtl.Pipeline, + pub fn load(device: mtl.Device) !Pipelines { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + var options = mtl.CompileOptions.mlx(); + options.math_functions = .precise; + const lib = try mtl.Library.fromSource(device, sources.deepseek_v41_mhc, options); + defer lib.deinit(); + const rms = try mtl.Pipeline.init(device, lib, "dsv41_hc_rms", false); + errdefer rms.deinit(); + const split = try mtl.Pipeline.init(device, lib, "dsv41_hc_split", false); + errdefer split.deinit(); + const collapse = try mtl.Pipeline.init(device, lib, "dsv41_hc_collapse", false); + errdefer collapse.deinit(); + const post = try mtl.Pipeline.init(device, lib, "dsv41_hc_post", false); + errdefer post.deinit(); + const expand = try mtl.Pipeline.init(device, lib, "dsv41_hc_expand", false); + return .{ .rms = rms, .split = split, .collapse = collapse, .post = post, .expand = expand }; + } + pub fn deinit(p: *Pipelines) void { + p.rms.deinit(); + p.split.deinit(); + p.collapse.deinit(); + p.post.deinit(); + p.expand.deinit(); + } +}; +fn bind(e: mtl.ComputeEncoder, refs: anytype) void { + inline for (refs, 0..) |r, i| e.setBuffer(r.buffer, r.offset, i); +} +pub const Rms = extern struct { rows: u32, width: u32, stream_stride: u32, eps: f32 }; +/// BF16 flattened [rows,4*width] -> FP32 inv[rows]; fixed CUDA single-row tree, not every batched CUDA grouping. +pub fn encodeRms(e: mtl.ComputeEncoder, p: *const Pipelines, stream: View, inv: View, d: Rms) void { + e.setPipeline(p.rms); + bind(e, .{ stream, inv }); + e.setValue(d, 2); + e.dispatchGroups(mtl.Size.of(d.rows, 1, 1), mtl.Size.of(512, 1, 1)); +} +pub const Split = extern struct { + rows: u32, + mix_stride: u32 = 24, + coefficient_stride: u32 = 4, + comb_stride: u32 = 16, + eps: f32 = 1e-6, +}; +/// FP32 mixes[rows,24], inv, scale[3], base[24] -> pre/post[rows,4], comb[rows,4,4]; 20 iterations. +pub fn encodeSplit(e: mtl.ComputeEncoder, p: *const Pipelines, mixes: View, inv: View, scale: View, base: View, pre: View, post: View, comb: View, d: Split) void { + e.setPipeline(p.split); + bind(e, .{ mixes, inv, scale, base, pre, post, comb }); + e.setValue(d, 7); + e.dispatchThreads(mtl.Size.of(d.rows, 1, 1), mtl.Size.of(32, 1, 1)); +} +pub const Collapse = extern struct { rows: u32, width: u32, stream_stride: u32, y_stride: u32, pre_stride: u32 = 4 }; +/// Collapse BF16 [rows,4,width] with previous sublayer pre[rows,4]; head uses final FFN pre, initially one-hot. +pub fn encodeCollapse(e: mtl.ComputeEncoder, p: *const Pipelines, stream: View, previous_pre: View, y: View, d: Collapse) void { + e.setPipeline(p.collapse); + bind(e, .{ stream, previous_pre, y }); + e.setValue(d, 3); + e.dispatchThreads(mtl.Size.of(d.width, d.rows, 1), mtl.Size.of(256, 1, 1)); +} +pub const Post = extern struct { + rows: u32, + width: u32, + branch_stride: u32, + residual_stride: u32, + y_stride: u32, + coefficient_stride: u32 = 4, + comb_stride: u32 = 16, +}; +/// y[k,d]=post[k]*branch[d]+sum_j comb[j,k]*residual[j,d]; BF16 inputs/output, FP32 coefficients/math. + +// Element pitches; exact residual alias is safe, but output must not alias branch/post/comb. +pub fn encodePost(e: mtl.ComputeEncoder, p: *const Pipelines, branch: View, residual: View, post: View, comb: View, y: View, d: Post) void { + e.setPipeline(p.post); + bind(e, .{ branch, residual, post, comb, y }); + e.setValue(d, 5); + e.dispatchThreads(mtl.Size.of(d.width, d.rows, 1), mtl.Size.of(256, 1, 1)); +} +pub const Expand = extern struct { rows: u32, width: u32, x_stride: u32, stream_stride: u32, pre_stride: u32 = 4 }; +/// BF16 x[rows,width] copied into four streams and FP32 initial pre[rows,4] set to [1,0,0,0]. +pub fn encodeExpand(e: mtl.ComputeEncoder, p: *const Pipelines, x: View, stream: View, pre: View, d: Expand) void { + e.setPipeline(p.expand); + bind(e, .{ x, stream, pre }); + e.setValue(d, 3); + e.dispatchThreads(mtl.Size.of(d.width, d.rows, 1), mtl.Size.of(256, 1, 1)); +} diff --git a/zig/src/families/deepseek_v41/mhc_test.zig b/zig/src/families/deepseek_v41/mhc_test.zig new file mode 100644 index 0000000000..ee0d888e28 --- /dev/null +++ b/zig/src/families/deepseek_v41/mhc_test.zig @@ -0,0 +1,209 @@ +//! Consumer-visible mHC coefficient, reduction and BF16 boundary regressions. +const std = @import("std"); +const mtl = @import("metal"); +const mhc = @import("mhc.zig"); +const support = @import("test_support.zig"); +const bf = support.bf; +const fp = support.fp; +const view = support.view; +const Buffers = support.Buffers; +const done = support.done; +const close = support.close; +fn splitReference(mixes: []const f32, inv: f64, scales: []const f32, base: []const f32) [24]f64 { + var out: [24]f64 = undefined; + for (0..4) |j| { + out[j] = 1 / (1 + @exp(-(@as(f64, mixes[j]) * inv * scales[0] + base[j]))) + 1e-6; + out[4 + j] = 2 / (1 + @exp(-(@as(f64, mixes[4 + j]) * inv * scales[1] + base[4 + j]))); + } + for (0..4) |j| { + var max: f64 = -std.math.inf(f64); + for (0..4) |k| { + const q = 8 + 4 * j + k; + out[q] = @as(f64, mixes[q]) * inv * scales[2] + base[q]; + max = @max(max, out[q]); + } + var sum: f64 = 0; + for (0..4) |k| { + const q = 8 + 4 * j + k; + out[q] = @exp(out[q] - max); + sum += out[q]; + } + for (0..4) |k| out[8 + 4 * j + k] = out[8 + 4 * j + k] / sum + 1e-6; + } + for (0..20) |it| { + if (it > 0) for (0..4) |j| { + var s: f64 = 0; + for (0..4) |k| s += out[8 + 4 * j + k]; + for (0..4) |k| out[8 + 4 * j + k] /= s + 1e-6; + }; + for (0..4) |k| { + var s: f64 = 0; + for (0..4) |j| s += out[8 + 4 * j + k]; + for (0..4) |j| out[8 + 4 * j + k] /= s + 1e-6; + } + } + return out; +} + +test "V4.1 flattened mHC RMS, 20 Sinkhorn iterations, previous collapse, source-destination post and identity expand" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try mhc.Pipelines.load(device); + defer p.deinit(); + var b = Buffers{ .device = device }; + defer b.deinit(); + const stream = try b.get(u16, 16); + for (stream.slice(u16, 16), 0..) |*v, i| v.* = bf(@as(f32, @floatFromInt(i + 1)) * 0.25); + const inv = try b.get(f32, 1); + const mixes = try b.get(f32, 24); + for (mixes.slice(f32, 24), 0..) |*v, i| v.* = (@as(f32, @floatFromInt(i % 9)) - 4) * 3; + const scale = try b.get(f32, 3); + @memcpy(scale.slice(f32, 3), &[_]f32{ 0.3, -0.7, 1.2 }); + const base = try b.get(f32, 24); + for (base.slice(f32, 24), 0..) |*v, i| v.* = (@as(f32, @floatFromInt(i % 5)) - 2) * 0.3; + const pre = try b.get(f32, 4); + const post = try b.get(f32, 4); + const comb = try b.get(f32, 16); + const previous = try b.get(f32, 4); + @memcpy(previous.slice(f32, 4), &[_]f32{ 0.1, 0.2, 0.3, 0.4 }); + const collapse = try b.get(u16, 4); + const branch = try b.get(u16, 4); + @memset(branch.slice(u16, 4), bf(2)); + const y = try b.get(u16, 16); + const expanded = try b.get(u16, 16); + const initial = try b.get(f32, 4); + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + mhc.encodeRms(e, &p, view(stream), view(inv), .{ .rows = 1, .width = 4, .stream_stride = 16, .eps = 1e-20 }); + mhc.encodeSplit(e, &p, view(mixes), view(inv), view(scale), view(base), view(pre), view(post), view(comb), .{ .rows = 1 }); + mhc.encodeCollapse(e, &p, view(stream), view(previous), view(collapse), .{ .rows = 1, .width = 4, .stream_stride = 16, .y_stride = 4 }); + mhc.encodePost(e, &p, view(branch), view(stream), view(post), view(comb), view(y), .{ .rows = 1, .width = 4, .branch_stride = 4, .residual_stride = 16, .y_stride = 16 }); + mhc.encodeExpand(e, &p, view(branch), view(expanded), view(initial), .{ .rows = 1, .width = 4, .x_stride = 4, .stream_stride = 16 }); + try done(cb, e); + var squares: f64 = 0; + for (stream.slice(u16, 16)) |v| squares += @as(f64, fp(v)) * fp(v); + const ir = 1 / @sqrt(squares / 16 + 1e-20); + try close(ir, inv.slice(f32, 1)[0], 1e-6); + const ref = splitReference(mixes.slice(f32, 24), ir, scale.slice(f32, 3), base.slice(f32, 24)); + for (0..4) |j| { + try close(ref[j], pre.slice(f32, 4)[j], 2e-6); + try close(ref[4 + j], post.slice(f32, 4)[j], 2e-6); + } + for (0..16) |q| try close(ref[8 + q], comb.slice(f32, 16)[q], 3e-6); + for (0..4) |i| { + var collapsed: f64 = 0; + for (0..4) |j| collapsed += @as(f64, previous.slice(f32, 4)[j]) * fp(stream.slice(u16, 16)[j * 4 + i]); + try close(collapsed, fp(collapse.slice(u16, 4)[i]), 0.02); + for (0..4) |k| { + var expected: f64 = 2 * ref[4 + k]; + for (0..4) |j| expected += ref[8 + 4 * j + k] * fp(stream.slice(u16, 16)[j * 4 + i]); + try close(expected, fp(y.slice(u16, 16)[k * 4 + i]), 0.04); + try std.testing.expectEqual(bf(2), expanded.slice(u16, 16)[k * 4 + i]); + } + try std.testing.expectEqual(@as(f32, if (i == 0) 1 else 0), initial.slice(f32, 4)[i]); + } +} + +test "V4.1 source single-row HC RMS reaches the retained attention BF16 consumer" { + const Fixture = struct { + embedding_bf16: []const u8, + mixes_f32: []const u8, + scale_f32: []const u8, + base_f32: []const u8, + branch_bf16_bits: [2]u16, + residual_bf16_bits: [8]u16, + expected_source_tree_target_bf16_bits: [2]u16, + }; + const fixture = try std.json.parseFromSlice(Fixture, std.testing.allocator, @embedFile("fixtures/hc_attention_rms.json"), .{ .ignore_unknown_fields = true }); + defer fixture.deinit(); + const f = fixture.value; + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try mhc.Pipelines.load(device); + defer p.deinit(); + var b = Buffers{ .device = device }; + defer b.deinit(); + const width = 5120; + const stream = try b.get(u16, 4 * width); + const xs = stream.slice(u16, 4 * width); + try std.base64.standard.Decoder.decode(std.mem.sliceAsBytes(xs[0..width]), f.embedding_bf16); + for (1..4) |copy| @memcpy(xs[copy * width ..][0..width], xs[0..width]); + const inv = try b.get(f32, 1); + const mixes = try b.get(f32, 24); + const scale = try b.get(f32, 3); + const base = try b.get(f32, 24); + try std.base64.standard.Decoder.decode(std.mem.sliceAsBytes(mixes.slice(f32, 24)), f.mixes_f32); + try std.base64.standard.Decoder.decode(std.mem.sliceAsBytes(scale.slice(f32, 3)), f.scale_f32); + try std.base64.standard.Decoder.decode(std.mem.sliceAsBytes(base.slice(f32, 24)), f.base_f32); + const pre = try b.get(f32, 4); + const post = try b.get(f32, 4); + const comb = try b.get(f32, 16); + const branch = try b.get(u16, 2); + const residual = try b.get(u16, 8); + const output = try b.get(u16, 8); + @memcpy(branch.slice(u16, 2), &f.branch_bf16_bits); + @memcpy(residual.slice(u16, 8), &f.residual_bf16_bits); + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + // Recompute statistic, coefficients and post products from the retained projection; never inject an oracle inverse. + mhc.encodeRms(e, &p, view(stream), view(inv), .{ .rows = 1, .width = width, .stream_stride = 4 * width, .eps = 1e-20 }); + mhc.encodeSplit(e, &p, view(mixes), view(inv), view(scale), view(base), view(pre), view(post), view(comb), .{ .rows = 1 }); + mhc.encodePost(e, &p, view(branch), view(residual), view(post), view(comb), view(output), .{ .rows = 1, .width = 2, .branch_stride = 2, .residual_stride = 8, .y_stride = 8 }); + try done(cb, e); + // The former reduction produces [14091,46835], despite unchanged branch BF16 bits. + for (0..2) |i| try std.testing.expectEqual(f.expected_source_tree_target_bf16_bits[i], output.slice(u16, 8)[6 + i]); +} + +test "V4.1 hyperconnections preserve FP32 products and residual boundaries under cancellation" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try mhc.Pipelines.load(device); + defer p.deinit(); + var b = Buffers{ .device = device }; + defer b.deinit(); + const stream = try b.get(u16, 8); + @memcpy(stream.slice(u16, 8), &[_]u16{ bf(3), bf(3), bf(0), bf(0), bf(1), bf(1), bf(1), bf(1) }); + const previous = try b.get(f32, 8); + @memcpy(previous.slice(f32, 8), &[_]f32{ -1, 1 + 0x1p-23, 0, 0, 1, 0x1p-24, 0x1p-24, -1 }); + const collapsed = try b.get(u16, 2); + const branch = try b.get(u16, 2); + @memcpy(branch.slice(u16, 2), &[_]u16{ bf(0), bf(1) }); + const post = try b.get(f32, 8); + post.slice(f32, 8)[4] = -1; + const comb = try b.get(f32, 32); + comb.slice(f32, 32)[0] = -1; + comb.slice(f32, 32)[4] = 1 + 0x1p-23; + comb.slice(f32, 32)[16] = 1; + comb.slice(f32, 32)[20] = 0x1p-24; + comb.slice(f32, 32)[24] = 0x1p-24; + const output = try b.get(u16, 8); + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + mhc.encodeCollapse(e, &p, view(stream), view(previous), view(collapsed), .{ .rows = 2, .width = 1, .stream_stride = 4, .y_stride = 1 }); + mhc.encodePost(e, &p, view(branch), view(stream), view(post), view(comb), view(output), .{ .rows = 2, .width = 1, .branch_stride = 1, .residual_stride = 4, .y_stride = 4 }); + try done(cb, e); + try std.testing.expectEqual(bf(0x1p-21), collapsed.slice(u16, 2)[0]); + try std.testing.expectEqual(bf(0), collapsed.slice(u16, 2)[1]); + try std.testing.expectEqual(bf(0x1p-21), output.slice(u16, 8)[0]); + try std.testing.expectEqual(bf(0), output.slice(u16, 8)[4]); + branch.slice(u16, 2)[0] = bf(3); + post.slice(f32, 8)[0] = 1 + 0x1p-23; + comb.slice(f32, 32)[4] = 0; + const branch_cb = queue.commandBuffer(); + const branch_enc = branch_cb.compute(.serial); + mhc.encodePost(branch_enc, &p, view(branch), view(stream), view(post), view(comb), view(output), .{ .rows = 1, .width = 1, .branch_stride = 1, .residual_stride = 4, .y_stride = 4 }); + try done(branch_cb, branch_enc); + try std.testing.expectEqual(bf(0x1p-21), output.slice(u16, 8)[0]); +} diff --git a/zig/src/families/deepseek_v41/model.zig b/zig/src/families/deepseek_v41/model.zig index 130f288657..443b3b2747 100644 --- a/zig/src/families/deepseek_v41/model.zig +++ b/zig/src/families/deepseek_v41/model.zig @@ -1,7 +1,377 @@ -//! Backend-neutral checkpoint owners. CUDA rank slicing is an explicit capability adapter. -pub const Config = @import("config.zig").Config; -pub const Plan = @import("plan.zig").CheckpointPlan; -pub const CudaRankPlan = @import("plan.zig").Plan; -pub const Image = @import("named.zig").Named; +//! Plain V4.1 execution over resident EXL3 weights and source-owned packed caches. +const std = @import("std"); +const mtl = @import("metal"); +const checkpoint = @import("../../core/checkpoint_metal.zig"); +const exl3 = @import("../../core/exl3_projection.zig"); +const config = @import("config.zig"); +const load = @import("load.zig"); +const weights = @import("weights.zig"); +const work = @import("workspace.zig"); +pub const attention = @import("attention.zig"); +pub const kv = @import("kv.zig"); +pub const ops = @import("ops.zig"); +pub const mhc = @import("mhc.zig"); +pub const csa2 = @import("csa2.zig"); +pub const csa2_index = @import("csa2_index.zig"); +pub const rope = @import("rope.zig"); +const route = @import("route.zig"); +const engram = @import("engram.zig"); +const View = @import("view.zig").View; +const gpu_counters = @import("profiler.zig"); +const expert_groups = @import("../../core/expert_groups.zig"); -pub const KvState = @import("kv_state.zig").State; +pub const prefill_step = 256; +pub const Logits = enum { none, last, all }; +pub const Options = struct { + dir: []const u8, + table_dir: ?[]const u8 = null, + token_map: []const u8, + context: u32 = 0, + chunk_rows: u32 = prefill_step, +}; + +const Pipelines = struct { + projection: exl3.Pipelines, + ops: ops.Pipelines, + mhc: mhc.Pipelines, + route: route.Pipelines, + engram: engram.Pipelines, + attention: attention.Pipelines, + groups: expert_groups.Pipelines, + + fn init(device: mtl.Device, capacity: u32, c: config.Text) !Pipelines { + var ep = try exl3.Pipelines.load(device); + errdefer ep.deinit(); + var op = try ops.Pipelines.load(device); + errdefer op.deinit(); + var hp = try mhc.Pipelines.load(device); + errdefer hp.deinit(); + var rp = try route.Pipelines.load(device); + errdefer rp.deinit(); + var gp = try engram.Pipelines.load(device); + errdefer gp.deinit(); + var groups = try expert_groups.Pipelines.load(device); + errdefer groups.deinit(); + const ap = try attention.Pipelines.load(device, capacity, c); + return .{ .projection = ep, .ops = op, .mhc = hp, .route = rp, .engram = gp, .attention = ap, .groups = groups }; + } + + fn deinit(p: *Pipelines) void { + p.attention.deinit(); + p.engram.deinit(); + p.route.deinit(); + p.groups.deinit(); + p.mhc.deinit(); + p.ops.deinit(); + p.projection.deinit(); + } +}; + +/// Serialized unique owner; submissions finish before return/release, cancellation is between bounded chunks. +pub const Model = struct { + gpa: std.mem.Allocator, + device: mtl.Device, + queue: mtl.Queue, + residency: mtl.ResidencySet, + checkpoint: checkpoint.Checkpoint, + weights: weights.Model, + cfg: config.Config, + cfg_arena: std.heap.ArenaAllocator, + cache: kv.cache.State, + workspace: work.Workspace, + attention_workspace: attention.Workspace, + pipelines: Pipelines, + admitted_bytes: usize, + last_gpu_seconds: f64 = 0, + logits_rows: usize = 0, + valid: bool = true, + profile: ?*gpu_counters.Recorder = null, + grouped_prefill: bool = true, + observer: ?Observer = null, + pub const Observer = struct { + ctx: *anyopaque, + encode: *const fn (*anyopaque, mtl.ComputeEncoder, u32, gpu_counters.Stage, *const Model) void, + }; + + pub fn open(gpa: std.mem.Allocator, io: std.Io, o: Options) !Model { + var arena = std.heap.ArenaAllocator.init(gpa); + errdefer arena.deinit(); + const c = try config.Config.load(arena.allocator(), io, o.dir); + const capacity = try c.context(o.context); + const table_dir = o.table_dir orelse c.engram_table_dir orelse return error.MissingEngramTables; + var plan = try load.Plan.open(gpa, io, o.dir, table_dir); + defer plan.deinit(); + try weights.validate(&plan); + const map_blob = try std.Io.Dir.cwd().readFileAlloc(io, o.token_map, arena.allocator(), .limited(2 << 20)); + const map = try engram.hash.TokenMap.parse(map_blob); + const wp = try work.Plan.init(o.chunk_rows); + const kp = kv.cache.Plan{ .capacity = capacity, .layers = 40 }; + const workspace_bytes = try std.math.add(usize, wp.bytes, attention.Workspace.bytes(o.chunk_rows)); + const cache_bytes = try std.math.add(usize, kp.residentBytes(), weights.sizeofDescriptors()); + const execution_bytes = try std.math.add(usize, workspace_bytes, cache_bytes); + const rotary_bytes = attention.Pipelines.residentBytes(capacity, c.text_config); + const needed = try std.math.add(usize, plan.bytes, try std.math.add(usize, execution_bytes, rotary_bytes)); + const device = try mtl.Device.init(); + errdefer device.deinit(); + const margin: u64 = 8 << 30; + const available = device.maxWorkingSet() -| device.allocated() -| margin; + if (needed > available) { + std.log.err("full residency needs {d} bytes plus {d} margin; Metal admission allows {d}", .{ needed, margin, available }); + return error.InsufficientResidentMemory; + } + const max_buffer = mtl.objc.msg(u64, device.id, "maxBufferLength", .{}); + for (plan.shards.items) |shard| if (shard.end - shard.begin > max_buffer) return error.ShardExceedsMetalBufferLimit; + std.log.info("full resident admission: weights={d}, total={d}, context={d}, headroom={d}", .{ plan.bytes, needed, capacity, available - needed }); + var pipelines = try Pipelines.init(device, capacity, c.text_config); + errdefer pipelines.deinit(); + const queue = try device.queue(); + errdefer queue.deinit(); + const residency = try device.residencySet(plan.shards.items.len + 128); + errdefer residency.deinit(); + var cp = try plan.materialize(gpa, device, residency, queue); + errdefer cp.deinit(); + var bound = try weights.bind(device, &cp); + errdefer bound.deinit(); + var cache = try kv.cache.State.init(device, kp); + errdefer cache.deinit(); + var workspace = try work.Workspace.init(device, wp, map.bytes); + errdefer workspace.deinit(); + var aw = try attention.Workspace.init(device, o.chunk_rows); + errdefer aw.deinit(); + bound.registerResidency(residency); + cache.registerResidency(residency); + residency.add(workspace.buffer); + aw.registerResidency(residency); + pipelines.attention.registerResidency(residency); + residency.commit(); + residency.requestResidency(); + queue.addResidencySet(residency); + return .{ .gpa = gpa, .device = device, .queue = queue, .residency = residency, .checkpoint = cp, .weights = bound, .cfg = c, .cfg_arena = arena, .cache = cache, .workspace = workspace, .attention_workspace = aw, .pipelines = pipelines, .admitted_bytes = needed }; + } + + pub fn deinit(m: *Model) void { + m.queue.removeResidencySet(m.residency); + m.residency.deinit(); + m.attention_workspace.deinit(); + m.workspace.deinit(); + m.cache.deinit(); + m.weights.deinit(); + m.checkpoint.deinit(); + m.pipelines.deinit(); + m.queue.deinit(); + m.device.deinit(); + m.cfg_arena.deinit(); + } + + pub fn reset(m: *Model) !void { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + m.valid = false; + m.logits_rows = 0; + m.cache.position = 0; + m.attention_workspace.selection_layer = null; + m.attention_workspace.selection_rows = 0; + m.attention_workspace.candidate_rows = 0; + m.last_gpu_seconds = 0; + m.cache.saved_position = null; + @memset(m.cache.live.contents()[0..m.cache.plan.mutableBytes()], 0); + const cb = m.queue.commandBufferUnretained(); + const enc = cb.compute(.serial); + engram.encodeReset(enc, &m.pipelines.engram, m.workspace.view(.history)); + enc.end(); + cb.commit(); + cb.wait(); + try completed(cb); + m.valid = true; + m.logits_rows = 0; + } + + /// One command buffer per chunk; no operator waits or reads routes back to the CPU. + pub fn forward(m: *Model, ids: []const u32, logits: Logits) ![]const f32 { + if (!m.valid) return error.InvalidModelState; + if (ids.len == 0 or ids.len > m.workspace.plan.rows) return error.InvalidPrefillChunk; + if (m.cache.position > m.cache.plan.capacity or + ids.len > m.cache.plan.capacity - m.cache.position) return error.ContextFull; + for (ids) |id| if (id >= 129280) return error.UnsupportedToken; + m.logits_rows = 0; + @memcpy(m.workspace.tokenIds(ids.len), ids); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const rows: u32 = @intCast(ids.len); + const e = m.queue.commandBufferUnretained(); + var enc = e.compute(.serial); + for (m.checkpoint.shards.items) |shard| enc.useResource(shard.buffer, mtl.ResourceUsage.read); + var encoding = true; + var record_started = false; + errdefer { + if (m.profile) |recorder| { + if (record_started) { + if (encoding) recorder.cancelEncoding(&enc) catch {} else recorder.discardAfterCompletion() catch {}; + } else if (encoding) enc.end(); + } else if (encoding) enc.end(); + } + if (m.profile) |recorder| { + try recorder.beginCommand(e); + record_started = true; + } + try m.profileSpan(&enc, .other); + const p = &m.pipelines; + const ws = &m.workspace; + for (engram.hash.layers, 0..) |meta, index| { + try engram.encodeHash(enc, &p.engram, meta, ws.view(.ids), ws.view(.token_map), null, ws.view(.history), ws.view(if (index == 0) .engram_ids_0 else .engram_ids_1), rows); + } + try engram.encodeCommit(enc, &p.engram, ws.view(.ids), ws.view(.token_map), null, ws.view(.history), rows); + ops.encodeEmbedding(enc, &p.ops, ws.view(.ids), tensor(m.weights.embed), ws.view(.collapsed), .{ .rows = rows, .width = 5120, .table_stride = 5120, .y_stride = 5120, .copies = 1 }); + enc.barrier(); + mhc.encodeExpand(enc, &p.mhc, ws.view(.collapsed), ws.view(.stream), ws.view(.previous_pre), .{ .rows = rows, .width = 5120, .x_stride = 5120, .stream_stride = 20480 }); + enc.barrier(); + for (&m.weights.layers, 0..) |*layer, index| { + if (layer.engram) |g| { + try m.profileSpan(&enc, .engram); + try engram.encodeGather(enc, &p.engram, .{ .weight = tensor(g.table), .scale = tensor(g.scales), .rows = @intCast(g.table.shape[0]) }, ws.view(if (index == 1) .engram_ids_0 else .engram_ids_1), ws.view(.engram_gather), rows, 6144); + try m.project(enc, g.wkv, ws.view(.engram_gather), ws.view(.engram_kv), rows, 6144, 25600, .bf16); + try engram.encodeGate(enc, &p.engram, .{ .q_weight = tensor(g.q_weight), .k_weight = tensor(g.k_weight), .dim = 5120, .copies = 4, .weight_dtype = switch (g.q_weight.dtype) { + .bf16 => .bf16, + else => .f32, + }, .hidden_stride = 20480, .kv_stride = 25600, .output_stride = 20480, .eps = m.cfg.text_config.rms_norm_eps }, ws.view(.stream), ws.view(.engram_kv), null, ws.view(.stream), rows); + } + try m.profileSpan(&enc, .mhc); + m.mixes(enc, layer.hc_attn, ws.view(.attention_pre), rows); + mhc.encodeCollapse(enc, &p.mhc, ws.view(.stream), ws.view(.previous_pre), ws.view(.collapsed), .{ .rows = rows, .width = 5120, .stream_stride = 20480, .y_stride = 5120 }); + enc.barrier(); + m.normalize(enc, ws.view(.collapsed), layer.attn_norm, ws.view(.normalized), rows); + try m.profileSpan(&enc, .attention); + try attention.encode(&p.attention, enc, &layer.attn, &m.cache, &m.attention_workspace, ws.view(.normalized), ws.view(.branch), &p.projection, ws.projection(), @intCast(index), rows, m.cfg.text_config); + enc.barrier(); + try m.profileSpan(&enc, .mhc); + m.post(enc, rows); + if (m.observer) |observer| observer.encode(observer.ctx, enc, @intCast(index), .attention, m); + m.mixes(enc, layer.hc_ffn, ws.view(.previous_pre), rows); + mhc.encodeCollapse(enc, &p.mhc, ws.view(.stream), ws.view(.attention_pre), ws.view(.collapsed), .{ .rows = rows, .width = 5120, .stream_stride = 20480, .y_stride = 5120 }); + enc.barrier(); + m.normalize(enc, ws.view(.collapsed), layer.ffn_norm, ws.view(.normalized), rows); + try m.profileSpan(&enc, .moe); + try m.moe(enc, &layer.moe, rows); + try m.profileSpan(&enc, .mhc); + m.post(enc, rows); + if (m.observer) |observer| observer.encode(observer.ctx, enc, @intCast(index), .moe, m); + } + if (logits != .none) { + try m.profileSpan(&enc, .head); + const output_rows: u32 = if (logits == .all) rows else 1; + const first: usize = if (logits == .all) 0 else rows - 1; + mhc.encodeCollapse(enc, &p.mhc, ws.view(.stream).shifted(first * 20480 * 2), ws.view(.previous_pre).shifted(first * 4 * 4), ws.view(.collapsed), .{ .rows = output_rows, .width = 5120, .stream_stride = 20480, .y_stride = 5120 }); + enc.barrier(); + m.normalize(enc, ws.view(.collapsed), m.weights.norm, ws.view(.normalized), output_rows); + try m.project(enc, m.weights.head, ws.view(.normalized), ws.view(.logits), output_rows, 5120, 129280, .f32); + } + if (m.profile) |recorder| { + if (recorder.activeSpan()) try recorder.endSpan(&enc); + try recorder.finishEncoding(&enc); + } else enc.end(); + encoding = false; + m.valid = false; + e.commit(); + e.wait(); + try completed(e); + if (m.profile) |recorder| _ = try recorder.readAfterCompletion(); + try m.cache.append(rows); + m.last_gpu_seconds = e.gpuSeconds(); + m.valid = true; + m.logits_rows = if (logits == .all) rows else if (logits == .last) 1 else 0; + return ws.logits(m.logits_rows); + } + + fn profileSpan(m: *Model, encoder: *mtl.ComputeEncoder, stage: gpu_counters.Stage) !void { + const recorder = m.profile orelse return; + if (!recorder.enabled()) return; + if (recorder.activeSpan()) try recorder.endSpan(encoder); + try recorder.beginSpan(encoder, stage); + for (m.checkpoint.shards.items) |shard| encoder.useResource(shard.buffer, mtl.ResourceUsage.read); + } + + fn normalize(m: *Model, enc: mtl.ComputeEncoder, x: View, w: checkpoint.Tensor, y: View, rows: u32) void { + ops.encodeRms(enc, &m.pipelines.ops, x, tensor(w), y, .{ .rows = rows, .width = 5120, .x_stride = 5120, .y_stride = 5120, .eps = m.cfg.text_config.rms_norm_eps }); + enc.barrier(); + } + + fn mixes(m: *Model, enc: mtl.ComputeEncoder, w: weights.Hc, pre: View, rows: u32) void { + const ws = &m.workspace; + mhc.encodeRms(enc, &m.pipelines.mhc, ws.view(.stream), ws.view(.inv), .{ .rows = rows, .width = 5120, .stream_stride = 20480, .eps = m.cfg.text_config.rms_norm_eps }); + ops.encodeDense(enc, &m.pipelines.ops, ws.view(.stream), tensor(w.projection), ws.view(.mixes), .{ .rows = rows, .k = 20480, .n = 24, .x_stride = 20480, .w_stride = 20480, .y_stride = 24, .w_dtype = .f32, .y_dtype = .f32 }); + enc.barrier(); + mhc.encodeSplit(enc, &m.pipelines.mhc, ws.view(.mixes), ws.view(.inv), tensor(w.scale), tensor(w.base), pre, ws.view(.post), ws.view(.comb), .{ .rows = rows, .eps = m.cfg.text_config.hc_eps }); + enc.barrier(); + } + + fn post(m: *Model, enc: mtl.ComputeEncoder, rows: u32) void { + const ws = &m.workspace; + mhc.encodePost(enc, &m.pipelines.mhc, ws.view(.branch), ws.view(.stream), ws.view(.post), ws.view(.comb), ws.view(.stream), .{ .rows = rows, .width = 5120, .branch_stride = 5120, .residual_stride = 20480, .y_stride = 20480 }); + enc.barrier(); + } + + fn project(m: *Model, enc: mtl.ComputeEncoder, matrix: exl3.Matrix, x: View, y: View, rows: u32, xs: usize, ys: usize, output: exl3.Output) !void { + try exl3.encode(&m.pipelines.projection, enc, matrix, convert(x), convert(y), m.workspace.projection(), .{ .count = rows, .x_stride = xs, .y_stride = ys, .output = output }); + enc.barrier(); + } + + fn moe(m: *Model, enc: mtl.ComputeEncoder, w: *const weights.Moe, rows: u32) !void { + const ws = &m.workspace; + const p = &m.pipelines; + ops.encodeDense(enc, &p.ops, ws.view(.normalized), tensor(w.gate), ws.view(.router_logits), .{ .rows = rows, .k = 5120, .n = 384, .x_stride = 5120, .w_stride = 5120, .y_stride = 384, .w_dtype = switch (w.gate.dtype) { + .bf16 => .bf16, + else => .f16, + }, .y_dtype = .f32 }); + enc.barrier(); + route.encode(enc, &p.route, ws.view(.router_logits), tensor(w.bias), ws.view(.route_ids), ws.view(.route_weights), .{ .rows = rows, .bias_dtype = switch (w.bias.dtype) { + .f32 => .f32, + .f16 => .f16, + .bf16 => .bf16, + else => .f16, + } }); + enc.barrier(); + const grouped: ?expert_groups.Scratch = if (m.grouped_prefill and rows >= 8) try ws.grouping(rows) else null; + var routed_rows = exl3.Rows{ .count = @as(usize, rows) * 6, .x_stride = 5120, .y_stride = 2304, .input_rows = rows, .output_rows = @as(usize, rows) * 6, .routes = .{ .ids = convert(ws.view(.route_ids)), .repeat = 6 } }; + if (grouped) |g| { + try expert_groups.encode(enc, &p.groups, convert(ws.view(.route_ids)), g, 6); + routed_rows.count = g.plan.padded; + routed_rows.input_ids = g.view(.token_ids); + routed_rows.output_ids = g.view(.slot_ids); + routed_rows.routes = .{ .ids = g.view(.expert_ids) }; + routed_rows.grouped = true; + } + for (0..2) |i| { + try exl3.encodeIndexed(&p.projection, enc, .{ .descriptors = .{ .buffer = w.descriptors[i] }, .count = 384, .k = 5120, .n = 2304 }, convert(ws.view(.normalized)), convert(ws.view(if (i == 0) .gate else .up)), ws.projection(), routed_rows); + enc.barrier(); + } + ops.encodeSwiGLU(enc, &p.ops, ws.view(.gate), ws.view(.up), ws.view(.route_weights), ws.view(.activation), .{ .rows = rows * 6, .width = 2304, .input_stride = 2304, .output_stride = 2304, .weighted = true }); + enc.barrier(); + routed_rows.x_stride = 2304; + routed_rows.y_stride = 5120; + routed_rows.routes.?.repeat = 1; + routed_rows.input_rows = @as(usize, rows) * 6; + if (grouped) |g| routed_rows.input_ids = g.view(.slot_ids); + try exl3.encodeIndexed(&p.projection, enc, .{ .descriptors = .{ .buffer = w.descriptors[2] }, .count = 384, .k = 2304, .n = 5120 }, convert(ws.view(.activation)), convert(ws.view(.routed)), ws.projection(), routed_rows); + enc.barrier(); + try m.project(enc, w.shared[0], ws.view(.normalized), ws.view(.shared_gate), rows, 5120, 2304, .bf16); + try m.project(enc, w.shared[1], ws.view(.normalized), ws.view(.shared_up), rows, 5120, 2304, .bf16); + ops.encodeSwiGLU(enc, &p.ops, ws.view(.shared_gate), ws.view(.shared_up), ws.view(.route_weights), ws.view(.shared_activation), .{ .rows = rows, .width = 2304, .input_stride = 2304, .output_stride = 2304 }); + enc.barrier(); + try m.project(enc, w.shared[2], ws.view(.shared_activation), ws.view(.shared_down), rows, 2304, 5120, .bf16); + ops.encodeCombine(enc, &p.ops, ws.view(.routed), ws.view(.shared_down), ws.view(.route_ids), ws.view(.branch), .{ .rows = rows, .width = 5120, .routed_row_stride = 6 * 5120, .slot_stride = 5120, .shared_stride = 5120, .y_stride = 5120 }); + enc.barrier(); + } +}; + +fn tensor(t: checkpoint.Tensor) View { + return .{ .buffer = t.buffer, .offset = t.offset }; +} +fn convert(v: View) exl3.View { + return .{ .buffer = v.buffer, .offset = v.offset }; +} +fn completed(cb: mtl.CommandBuffer) !void { + if (cb.failure()) |message| { + std.log.err("DeepSeek V4.1 GPU execution failed: {s}", .{message}); + return error.GpuExecutionFailed; + } +} diff --git a/zig/src/families/deepseek_v41/ops.zig b/zig/src/families/deepseek_v41/ops.zig new file mode 100644 index 0000000000..68fd169857 --- /dev/null +++ b/zig/src/families/deepseek_v41/ops.zig @@ -0,0 +1,110 @@ +//! Trusted plaintext encoders: callers validate nonzero shapes/storage; strides count elements, EXL3 is separate. +const mtl = @import("metal"); +const sources = @import("kernel_sources"); +pub const View = @import("view.zig").View; + +pub const Pipelines = struct { + rms: mtl.Pipeline, + dense: mtl.Pipeline, + embedding: mtl.Pipeline, + swiglu: mtl.Pipeline, + combine: mtl.Pipeline, + + pub fn load(device: mtl.Device) !Pipelines { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + var options = mtl.CompileOptions.mlx(); + options.math_functions = .precise; + const lib = try mtl.Library.fromSource(device, sources.deepseek_v41_ops, options); + defer lib.deinit(); + const rms = try mtl.Pipeline.init(device, lib, "dsv41_rms", false); + errdefer rms.deinit(); + const dense = try mtl.Pipeline.init(device, lib, "dsv41_dense", false); + errdefer dense.deinit(); + const embedding = try mtl.Pipeline.init(device, lib, "dsv41_embedding", false); + errdefer embedding.deinit(); + const swiglu = try mtl.Pipeline.init(device, lib, "dsv41_swiglu", false); + errdefer swiglu.deinit(); + const combine = try mtl.Pipeline.init(device, lib, "dsv41_combine", false); + return .{ .rms = rms, .dense = dense, .embedding = embedding, .swiglu = swiglu, .combine = combine }; + } + pub fn deinit(p: *Pipelines) void { + p.rms.deinit(); + p.dense.deinit(); + p.embedding.deinit(); + p.swiglu.deinit(); + p.combine.deinit(); + } +}; +fn bind(e: mtl.ComputeEncoder, refs: anytype) void { + inline for (refs, 0..) |r, i| e.setBuffer(r.buffer, r.offset, i); +} +pub const Rms = struct { rows: u32, width: u32, x_stride: u32, y_stride: u32, eps: f32, weight_f32: bool = false }; +/// BF16 x/y [rows,width], BF16 or FP32 weight [width]; sums, mean and affine product are FP32. +pub fn encodeRms(e: mtl.ComputeEncoder, p: *const Pipelines, x: View, weight: View, y: View, d: Rms) void { + e.setPipeline(p.rms); + bind(e, .{ x, weight, y }); + const D = extern struct { rows: u32, width: u32, x_stride: u32, y_stride: u32, eps: f32, weight_f32: u32 }; + e.setValue(D{ .rows = d.rows, .width = d.width, .x_stride = d.x_stride, .y_stride = d.y_stride, .eps = d.eps, .weight_f32 = @intFromBool(d.weight_f32) }, 3); + e.dispatchGroups(mtl.Size.of(d.rows, 1, 1), mtl.Size.of(256, 1, 1)); +} +pub const Dtype = enum(u32) { bf16 = 0, f32 = 1, f16 = 2 }; +pub const Dense = struct { + rows: u32, + k: u32, + n: u32, + groups: u32 = 1, + x_stride: u32, + w_stride: u32, + y_stride: u32, + x_dtype: Dtype = .bf16, + w_dtype: Dtype = .bf16, + y_dtype: Dtype = .bf16, +}; +/// x[rows,groups,k], w[groups,n,k], y[rows,groups,n]; packed groups, padded token pitches, FP32 sums, final BF16. +pub fn encodeDense(e: mtl.ComputeEncoder, p: *const Pipelines, x: View, w: View, y: View, d: Dense) void { + e.setPipeline(p.dense); + bind(e, .{ x, w, y }); + e.setValue([10]u32{ d.rows, d.k, d.n, d.groups, d.x_stride, d.w_stride, d.y_stride, @backingInt(d.x_dtype), @backingInt(d.w_dtype), @backingInt(d.y_dtype) }, 3); + e.dispatchGroups(mtl.Size.of((d.n + 15) / 16, d.rows, d.groups), mtl.Size.of(128, 1, 1)); +} +pub const Embedding = extern struct { rows: u32, width: u32, table_stride: u32, y_stride: u32, copies: u32 = 4 }; +/// ids[rows] u32 must be in vocabulary; table[vocab,width] BF16 -> y[rows,copies,width] BF16. +pub fn encodeEmbedding(e: mtl.ComputeEncoder, p: *const Pipelines, ids: View, table: View, y: View, d: Embedding) void { + e.setPipeline(p.embedding); + bind(e, .{ ids, table, y }); + e.setValue(d, 3); + e.dispatchThreads(mtl.Size.of(d.width, d.rows, 1), mtl.Size.of(256, 1, 1)); +} +pub const Activation = struct { + rows: u32, + width: u32, + input_stride: u32, + output_stride: u32, + weighted: bool = false, + input_f32: bool = false, +}; +/// Separate gate/up; apply FP32 weights before BF16 cast/down. Unweighted calls borrow but never read weights. +pub fn encodeSwiGLU(e: mtl.ComputeEncoder, p: *const Pipelines, gate: View, up: View, weights: View, y: View, d: Activation) void { + e.setPipeline(p.swiglu); + bind(e, .{ gate, up, weights, y }); + e.setValue([6]u32{ d.rows, d.width, d.input_stride, d.output_stride, @intFromBool(d.weighted), @intFromBool(d.input_f32) }, 4); + e.dispatchThreads(mtl.Size.of(d.width, d.rows, 1), mtl.Size.of(256, 1, 1)); +} +pub const Combine = struct { + rows: u32, + width: u32, + routed_row_stride: u32, + slot_stride: u32, + shared_stride: u32, + y_stride: u32, + input_f32: bool = false, + id_stride: u32 = 6, +}; +/// Sum six weighted down slots by ascending expert ID in FP32, then shared; final BF16, invalid IDs yield NaN. +pub fn encodeCombine(e: mtl.ComputeEncoder, p: *const Pipelines, routed: View, shared: View, ids: View, y: View, d: Combine) void { + e.setPipeline(p.combine); + bind(e, .{ routed, shared, ids, y }); + e.setValue([8]u32{ d.rows, d.width, d.routed_row_stride, d.slot_stride, d.shared_stride, d.y_stride, @intFromBool(d.input_f32), d.id_stride }, 4); + e.dispatchThreads(mtl.Size.of(d.width, d.rows, 1), mtl.Size.of(256, 1, 1)); +} diff --git a/zig/src/families/deepseek_v41/ops_check.zig b/zig/src/families/deepseek_v41/ops_check.zig new file mode 100644 index 0000000000..bfe08f4dd8 --- /dev/null +++ b/zig/src/families/deepseek_v41/ops_check.zig @@ -0,0 +1,191 @@ +//! Independent GPU consumer fixtures; pinned official inference equations and provenance in THIRD_PARTY_NOTICES.md. +const std = @import("std"); +const mtl = @import("metal"); +const ops = @import("ops.zig"); +const support = @import("test_support.zig"); +const bf = support.bf; +const fp = support.fp; +const view = support.view; +const Buffers = support.Buffers; +const done = support.done; +const close = support.close; + +test "V4.1 BF16 RMS, F16 dense row independence, grouped wo_a, embedding, clamped weighted SwiGLU and shared combine" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try ops.Pipelines.load(device); + defer p.deinit(); + var b = Buffers{ .device = device }; + defer b.deinit(); + const x = try b.get(u16, 16); + const w = try b.get(u16, 4); + const norm = try b.get(u16, 16); + const xs = x.slice(u16, 16); + const ws = w.slice(u16, 4); + for (0..2) |r| for (0..4) |i| { + xs[r * 8 + i] = bf(@as(f32, @floatFromInt(1 + i + r))); + }; + for (ws, 0..) |*v, i| v.* = bf(@as(f32, @floatFromInt(i + 1)) * 0.5); + const f16w = try b.get(f16, 8); + const fws = f16w.slice(f16, 8); + for (fws, 0..) |*v, i| v.* = @floatCast(@as(f32, @floatFromInt(i + 1)) * 0.125 + 0.0009765625); + const dense = try b.get(f32, 8); + const dense_one = try b.get(f32, 2); + const grouped = try b.get(u16, 8); + const gw = try b.get(u16, 8); + for (gw.slice(u16, 8), 0..) |*v, i| v.* = bf(@as(f32, @floatFromInt(i + 1)) * 0.25); + const ids = try b.get(u32, 2); + ids.slice(u32, 2)[0] = 1; + ids.slice(u32, 2)[1] = 0; + const embed = try b.get(u16, 32); + const gate = try b.get(f32, 8); + const up = try b.get(f32, 8); + @memcpy(gate.slice(f32, 8), &[_]f32{ -100, -12, 1, 100, -2, 0, 8, 12 }); + @memcpy(up.slice(f32, 8), &[_]f32{ -100, -12, 1, 100, 12, -20, 0.5, 1 }); + const weights = try b.get(f32, 2); + @memcpy(weights.slice(f32, 2), &[_]f32{ 0.25, 1.25 }); + const act = try b.get(u16, 8); + const routed = try b.get(u16, 48); + const combine_ids = try b.get(u32, 12); + @memcpy(combine_ids.slice(u32, 12), &[_]u32{ 5, 3, 1, 4, 2, 0, 5, 3, 1, 4, 2, 0 }); + const shared = try b.get(u16, 8); + const combined = try b.get(u16, 8); + const values = [_]f32{ 1, 33554432, -33554432, 2, 4, 8 }; + for (routed.slice(u16, 48), 0..) |*v, i| v.* = bf(values[(i % 24) / 4]); + @memset(shared.slice(u16, 8), bf(2.5)); + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + ops.encodeRms(e, &p, view(x), view(w), view(norm), .{ .rows = 2, .width = 4, .x_stride = 8, .y_stride = 8, .eps = 1e-20 }); + ops.encodeDense(e, &p, view(x), view(f16w), view(dense), .{ .rows = 2, .k = 4, .n = 2, .x_stride = 8, .w_stride = 4, .y_stride = 4, .w_dtype = .f16, .y_dtype = .f32 }); + ops.encodeDense(e, &p, view(x).shifted(16), view(f16w), view(dense_one), .{ .rows = 1, .k = 4, .n = 2, .x_stride = 8, .w_stride = 4, .y_stride = 2, .w_dtype = .f16, .y_dtype = .f32 }); + ops.encodeDense(e, &p, view(x), view(gw), view(grouped), .{ .rows = 2, .k = 2, .n = 2, .groups = 2, .x_stride = 8, .w_stride = 2, .y_stride = 4 }); + ops.encodeEmbedding(e, &p, view(ids), view(x), view(embed), .{ .rows = 2, .width = 4, .table_stride = 8, .y_stride = 16 }); + ops.encodeSwiGLU(e, &p, view(gate), view(up), view(weights), view(act), .{ .rows = 2, .width = 4, .input_stride = 4, .output_stride = 4, .weighted = true, .input_f32 = true }); + ops.encodeCombine(e, &p, view(routed), view(shared), view(combine_ids), view(combined), .{ .rows = 2, .width = 4, .routed_row_stride = 24, .slot_stride = 4, .shared_stride = 4, .y_stride = 4 }); + try done(cb, e); + for (0..2) |r| { + var sum: f64 = 0; + for (0..4) |i| sum += @as(f64, fp(xs[r * 8 + i])) * fp(xs[r * 8 + i]); + for (0..4) |i| try close(@as(f64, fp(xs[r * 8 + i])) / @sqrt(sum / 4 + 1e-20) * fp(ws[i]), fp(norm.slice(u16, 16)[r * 8 + i]), 0.04); + for (0..2) |j| { + var ref: f64 = 0; + for (0..4) |k| ref += @as(f64, fp(xs[r * 8 + k])) * @as(f64, fws[j * 4 + k]); + try close(ref, dense.slice(f32, 8)[r * 4 + j], 1e-6); + } + for (0..2) |g| for (0..2) |j| { + var ref: f32 = 0; + for (0..2) |k| ref += fp(xs[r * 8 + g * 2 + k]) * fp(gw.slice(u16, 8)[g * 4 + j * 2 + k]); + try std.testing.expectEqual(bf(ref), grouped.slice(u16, 8)[r * 4 + g * 2 + j]); + }; + for (0..4) |copy| for (0..4) |i| try std.testing.expectEqual(xs[(1 - r) * 8 + i], embed.slice(u16, 32)[r * 16 + copy * 4 + i]); + for (0..4) |i| { + const k = r * 4 + i; + const a: f64 = @min(gate.slice(f32, 8)[k], 10); + const u: f64 = @max(-10, @min(up.slice(f32, 8)[k], 10)); + const activated = a / (1 + @exp(-a)) * u * weights.slice(f32, 2)[r]; + try close(activated, fp(act.slice(u16, 8)[k]), @abs(activated) * 0.004 + 1e-35); + var ref: f32 = 0; + for ([_]usize{ 5, 2, 4, 1, 3, 0 }) |slot| ref += fp(routed.slice(u16, 48)[r * 24 + slot * 4 + i]); + try std.testing.expectEqual(bf(ref + 2.5), combined.slice(u16, 8)[k]); + } + } + for (0..2) |j| try std.testing.expectEqual(dense.slice(f32, 8)[4 + j], dense_one.slice(f32, 2)[j]); +} + +test "V4.1 CUDA RMS mean projection preserves the BF16 affine tie" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try ops.Pipelines.load(device); + defer p.deinit(); + var buffers = Buffers{ .device = device }; + defer buffers.deinit(); + const width = 5120; + const x = try buffers.get(u16, width); + const weight = try buffers.get(f32, width); + const output = try buffers.get(u16, width); + x.slice(u16, width)[0] = bf(3); + @memset(weight.slice(f32, width), @bitCast(@as(u32, 0x3c65de29))); + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + ops.encodeRms(e, &p, view(x), view(weight), view(output), .{ .rows = 1, .width = width, .x_stride = width, .y_stride = width, .eps = 1e-20, .weight_f32 = true }); + try done(cb, e); + // Pinned CUDA RMSNorm.forward returns 1; sum / width crosses to the next BF16 value. + try std.testing.expectEqual(bf(1), output.slice(u16, width)[0]); + for (output.slice(u16, width)[1..]) |v| try std.testing.expectEqual(@as(u16, 0), v); +} + +test "V4.1 FP32 dense retains product and reduction cancellation" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try ops.Pipelines.load(device); + defer p.deinit(); + var buffers = Buffers{ .device = device }; + defer buffers.deinit(); + const k = 96; + const n = 3; + const x = try buffers.get(u16, k); + const w = try buffers.get(f32, k * n); + const y = try buffers.get(f32, n); + @memset(x.slice(u16, k), bf(3)); + const weights = w.slice(f32, k * n); + weights[0] = 0x1p25; + weights[32] = 1; + weights[64] = -0x1p25; + weights[k] = 1 + 0x1p-23; + weights[k + 32] = -1; + weights[2 * k] = 0x1p25; + weights[2 * k + 1] = 1; + weights[2 * k + 2] = -0x1p25; + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + ops.encodeDense(enc, &p, view(x), view(w), view(y), .{ .rows = 1, .k = k, .n = n, .x_stride = k, .w_stride = k, .y_stride = n, .w_dtype = .f32, .y_dtype = .f32 }); + try done(cb, enc); + for (0..n) |column| { + var expected: f64 = 0; + for (0..k) |index| expected += @as(f64, fp(x.slice(u16, k)[index])) * weights[column * k + index]; + try std.testing.expectEqual(@as(f32, @floatCast(expected)), y.slice(f32, n)[column]); + } +} + +test "V4.1 weighted SiLU preserves division rounding before BF16 activation" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try ops.Pipelines.load(device); + defer p.deinit(); + var buffers = Buffers{ .device = device }; + defer buffers.deinit(); + const gate = try buffers.get(u16, 1); + const up = try buffers.get(u16, 1); + const weights = try buffers.get(f32, 1); + const output = try buffers.get(u16, 1); + gate.slice(u16, 1)[0] = bf(0.78515625); + up.slice(u16, 1)[0] = bf(-6.96875); + weights.slice(f32, 1)[0] = 0.25; + const cb = queue.commandBuffer(); + const encoder = cb.compute(.serial); + ops.encodeSwiGLU(encoder, &p, view(gate), view(up), view(weights), view(output), .{ .rows = 1, .width = 1, .input_stride = 1, .output_stride = 1, .weighted = true }); + try done(cb, encoder); + // Source SiLU divides before up/route multiplication; reciprocal multiplication crosses this BF16 tie. + try std.testing.expectEqual(bf(-0.94140625), output.slice(u16, 1)[0]); +} + +test { + _ = @import("mhc_test.zig"); + _ = @import("route_test.zig"); +} diff --git a/zig/src/families/deepseek_v41/profiler.zig b/zig/src/families/deepseek_v41/profiler.zig new file mode 100644 index 0000000000..fe3a85b5fc --- /dev/null +++ b/zig/src/families/deepseek_v41/profiler.zig @@ -0,0 +1,93 @@ +//! Family stage labels over the generic same-command-buffer counter recorder; no extra storage or GPU waits. +const std = @import("std"); +const mtl = @import("metal"); +const counters = @import("../../core/gpu_counters.zig"); + +pub const Stage = enum { engram, mhc, attention, moe, head, other }; +pub const stage_count = @typeInfo(Stage).@"enum".field_names.len; +pub const Options = counters.Options; +pub const Aggregate = counters.Aggregate; +pub const Span = counters.Span; + +pub fn class(stage: Stage) counters.Class { + return switch (stage) { + .engram => .embedding, + .mhc => .norm, + .attention => .attention, + .moe, .head => .dense, + .other => .copy, + }; +} +pub fn stageOf(span: Span) !Stage { + if (span.metadata >= stage_count) return error.InvalidProfileStage; + return @fromBackingInt(@intCast(span.metadata)); +} + +/// Unique owner of the recorder; stage metadata and class totals share the recorder's bounded span storage. +pub const Recorder = struct { + recorder: counters.Recorder, + totals: [stage_count]Aggregate = @splat(.{}), + + pub fn init(allocator: std.mem.Allocator, device: mtl.Device, options: Options) !Recorder { + return .{ .recorder = try counters.Recorder.init(allocator, device, options) }; + } + pub fn enabled(self: *const Recorder) bool { + return self.recorder.enabled(); + } + pub fn capability(self: *const Recorder) mtl.counters.Capability { + return self.recorder.capability(); + } + pub fn activeSpan(self: *const Recorder) bool { + return self.recorder.active_span != null; + } + pub fn beginCommand(self: *Recorder, cb: mtl.CommandBuffer) !void { + try self.recorder.beginCommand(cb); + self.totals = @splat(.{}); + } + pub fn beginSpan(self: *Recorder, encoder: *mtl.ComputeEncoder, stage: Stage) !void { + try self.recorder.beginSpan(encoder, class(stage), @intCast(@backingInt(stage))); + } + pub fn endSpan(self: *Recorder, encoder: *mtl.ComputeEncoder) !void { + try self.recorder.endSpan(encoder); + } + pub fn finishEncoding(self: *Recorder, encoder: *mtl.ComputeEncoder) !void { + try self.recorder.finishEncoding(encoder); + } + pub fn cancelEncoding(self: *Recorder, encoder: *mtl.ComputeEncoder) !void { + try self.recorder.cancelEncoding(encoder); + self.totals = @splat(.{}); + } + pub fn discardAfterCompletion(self: *Recorder) !void { + try self.recorder.discardAfterCompletion(); + self.totals = @splat(.{}); + } + pub fn readAfterCompletion(self: *Recorder) ![]const Span { + const spans = try self.recorder.readAfterCompletion(); + var totals: [stage_count]Aggregate = @splat(.{}); + for (spans) |span| { + const total = &totals[@backingInt(try stageOf(span))]; + total.calls += 1; + total.gpu_seconds += span.gpu_seconds; + } + self.totals = totals; + return spans; + } + pub fn aggregates(self: *const Recorder) [stage_count]Aggregate { + return self.totals; + } + pub fn deinit(self: *Recorder) !void { + try self.recorder.deinit(); + self.totals = @splat(.{}); + } +}; + +test "disabled family profiling allocates nothing and leaves no stage receipts" { + var recorder = try Recorder.init(std.testing.failing_allocator, undefined, .{}); + defer recorder.deinit() catch unreachable; + var encoder: mtl.ComputeEncoder = undefined; + try recorder.beginSpan(&encoder, .mhc); + try recorder.endSpan(&encoder); + try std.testing.expect(!recorder.activeSpan()); + try std.testing.expectEqual(@as(usize, 0), (try recorder.readAfterCompletion()).len); + for (recorder.aggregates()) |total| try std.testing.expectEqual(@as(u64, 0), total.calls); +} diff --git a/zig/src/families/deepseek_v41/rope.zig b/zig/src/families/deepseek_v41/rope.zig new file mode 100644 index 0000000000..cc3419f7c3 --- /dev/null +++ b/zig/src/families/deepseek_v41/rope.zig @@ -0,0 +1,134 @@ +//! Adjacent-pair BF16 RoPE using resident, source-style complex rotation factors. +const std = @import("std"); +const mtl = @import("metal"); +const sources = @import("kernel_sources"); +const View = @import("view.zig").View; + +pub const Spec = struct { + theta: f32, + factor: f32 = 16, + original: u32 = 0, + beta_fast: f32 = 32, + beta_slow: f32 = 1, + + fn validate(s: Spec) !void { + if (!std.math.isFinite(s.theta) or s.theta <= 1 or + !std.math.isFinite(s.factor) or s.factor <= 0 or + !std.math.isFinite(s.beta_fast) or s.beta_fast <= 0 or + !std.math.isFinite(s.beta_slow) or s.beta_slow <= 0) + return error.InvalidRotarySpec; + } +}; +pub const Spectrum = enum(u32) { normal, compressed }; +pub const Params = extern struct { + rows: u32, + heads: u32, + width: u32, + rotary: u32, + stride: u32, + position: u32, + step: u32 = 1, + inverse: u32 = 0, + spectrum: Spectrum = .normal, + + pub fn validate(p: Params) !void { + if (p.rows == 0 or p.heads == 0 or p.rotary == 0 or p.rotary % 2 != 0 or + p.rotary > p.width or @as(u64, p.stride) < @as(u64, p.heads) * p.width or + p.step == 0 or p.inverse > 1 or + @as(u64, p.rows - 1) * p.stride + @as(u64, p.heads) * p.width > std.math.maxInt(u32)) + return error.InvalidRotaryShape; + } +}; + +// Keep FP32 tensor/angle boundaries, FP64 corrected_dim logs, and FP32-rounded polar unaries; never per-head recompute. +fn fillFactors(out: []f32, capacity: u32, rotary: u32, spec: Spec) void { + @setFloatMode(.strict); + const dim: f32 = @floatFromInt(rotary); + const pairs: usize = rotary / 2; + var low: f32 = 0; + var high: f32 = 0; + if (spec.original > 0) { + const original: f64 = @floatFromInt(spec.original); + const dim64: f64 = @floatFromInt(rotary); + const denominator = 2 * @log(@as(f64, spec.theta)); + low = @floatCast(@max(@floor(dim64 * @log(original / + (@as(f64, spec.beta_fast) * 2 * std.math.pi)) / denominator), 0)); + high = @floatCast(@min(@ceil(dim64 * @log(original / + (@as(f64, spec.beta_slow) * 2 * std.math.pi)) / denominator), dim64 - 1)); + } + for (0..pairs) |pair| { + const exponent: f32 = @as(f32, @floatFromInt(pair * 2)) / dim; + const power: f32 = @floatCast(std.math.pow(f64, spec.theta, exponent)); + var frequency: f32 = 1 / power; + if (spec.original > 0) { + const ramp: f32 = std.math.clamp((@as(f32, @floatFromInt(pair)) - low) / + @max(high - low, @as(f32, 1e-3)), 0, 1); + const smooth: f32 = 1 - ramp; + const scaled: f32 = frequency / spec.factor; + const slow: f32 = scaled * (1 - smooth); + const fast: f32 = frequency * smooth; + frequency = slow + fast; + } + for (0..capacity) |position| { + const angle: f32 = @as(f32, @floatFromInt(position)) * frequency; + const at = (position * pairs + pair) * 2; + out[at] = @floatCast(@cos(@as(f64, angle))); + out[at + 1] = @floatCast(@sin(@as(f64, angle))); + } + } +} + +pub const Pipelines = struct { + rotate: mtl.Pipeline, + factors: mtl.Buffer, + capacity: u32, + rotary: u32, + + pub fn residentBytes(capacity: u32, rotary: u32) usize { + return 2 * @as(usize, capacity) * (rotary / 2) * 2 * @sizeOf(f32); + } + + pub fn load(device: mtl.Device, capacity: u32, rotary: u32, normal: Spec, compressed: Spec) !Pipelines { + if (capacity == 0 or rotary == 0 or rotary % 2 != 0 or + @as(u64, capacity) * rotary > std.math.maxInt(usize) / (2 * @sizeOf(f32))) + return error.InvalidRotaryShape; + try normal.validate(); + try compressed.validate(); + var options = mtl.CompileOptions.mlx(); + options.math_functions = .precise; + const lib = try mtl.Library.fromSource(device, sources.deepseek_v41_rope, options); + defer lib.deinit(); + const rotate = try mtl.Pipeline.init(device, lib, "dsv41_rope", false); + errdefer rotate.deinit(); + const factors = try device.buffer(residentBytes(capacity, rotary), mtl.ResourceOptions.shared | mtl.ResourceOptions.untracked); + errdefer factors.deinit(); + const count = @as(usize, capacity) * rotary; + const values = factors.slice(f32, 2 * count); + fillFactors(values[0..count], capacity, rotary, normal); + fillFactors(values[count..], capacity, rotary, compressed); + return .{ .rotate = rotate, .factors = factors, .capacity = capacity, .rotary = rotary }; + } + pub fn deinit(p: *Pipelines) void { + p.factors.deinit(); + p.rotate.deinit(); + } + pub fn registerResidency(p: *const Pipelines, set: mtl.ResidencySet) void { + set.add(p.factors); + } + pub fn encode(p: *const Pipelines, enc: mtl.ComputeEncoder, x: View, args: Params) !void { + try args.validate(); + const last = @as(u64, args.position) + @as(u64, args.rows - 1) * args.step; + if (args.rotary != p.rotary or last >= p.capacity) return error.InvalidRotaryShape; + try x.requireRows(args.rows, args.stride, @as(usize, args.heads) * args.width, 2); + const offset = switch (args.spectrum) { + .normal => 0, + .compressed => @as(usize, p.capacity) * p.rotary * @sizeOf(f32), + }; + enc.setPipeline(p.rotate); + enc.setBuffer(x.buffer, x.offset, 0); + enc.setValue(args, 1); + enc.setBuffer(p.factors, offset, 2); + enc.dispatchThreads(mtl.Size.of(args.rotary / 2, args.heads, args.rows), mtl.Size.of(32, 1, 1)); + enc.barrier(); + } +}; diff --git a/zig/src/families/deepseek_v41/route.zig b/zig/src/families/deepseek_v41/route.zig new file mode 100644 index 0000000000..00c1a1ff10 --- /dev/null +++ b/zig/src/families/deepseek_v41/route.zig @@ -0,0 +1,39 @@ +//! Exhaustive 384-expert sqrt-softplus top-six routing. Bias affects selection, never returned weights. +const mtl = @import("metal"); +const sources = @import("kernel_sources"); +pub const View = @import("view.zig").View; +pub const Dtype = @import("ops.zig").Dtype; +pub const Pipelines = struct { + select: mtl.Pipeline, + pub fn load(device: mtl.Device) !Pipelines { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + var options = mtl.CompileOptions.mlx(); + options.math_functions = .precise; + const lib = try mtl.Library.fromSource(device, sources.deepseek_v41_route, options); + defer lib.deinit(); + return .{ .select = try mtl.Pipeline.init(device, lib, "dsv41_route", false) }; + } + pub fn deinit(p: *Pipelines) void { + p.select.deinit(); + } +}; +pub const Shape = struct { + rows: u32, + logit_stride: u32 = 384, + output_stride: u32 = 6, + temperature: f32 = 1, + bias_dtype: Dtype = .f32, +}; +/// FP32 logits[rows,384], bias F16/BF16/FP32 -> u32 IDs and FP32 weights[rows,6]; lower ID wins biased-score ties. + +// Nonzero rows, logit/output pitches >=384/6, finite positive temperature, complete storage are caller invariants. + +// All six execute; weights use unbiased scores / (sum+1e-20) * 1.5. Invalid comparisons emit invalid IDs/NaNs. +pub fn encode(e: mtl.ComputeEncoder, p: *const Pipelines, logits: View, bias: View, ids: View, weights: View, d: Shape) void { + e.setPipeline(p.select); + inline for (.{ logits, bias, ids, weights }, 0..) |r, i| e.setBuffer(r.buffer, r.offset, i); + const D = extern struct { rows: u32, logit_stride: u32, output_stride: u32, temperature: f32, bias_dtype: u32 }; + e.setValue(D{ .rows = d.rows, .logit_stride = d.logit_stride, .output_stride = d.output_stride, .temperature = d.temperature, .bias_dtype = @backingInt(d.bias_dtype) }, 4); + e.dispatchGroups(mtl.Size.of(d.rows, 1, 1), mtl.Size.of(128, 1, 1)); +} diff --git a/zig/src/families/deepseek_v41/route_test.zig b/zig/src/families/deepseek_v41/route_test.zig new file mode 100644 index 0000000000..f653fa9241 --- /dev/null +++ b/zig/src/families/deepseek_v41/route_test.zig @@ -0,0 +1,152 @@ +//! Consumer-visible expert IDs, weights and nonfinite-input routing boundaries. +const std = @import("std"); +const mtl = @import("metal"); +const route = @import("route.zig"); +const ops = @import("ops.zig"); +const support = @import("test_support.zig"); +const view = support.view; +const Buffers = support.Buffers; +const done = support.done; +const close = support.close; +fn scoreReference(x: f64) f64 { + return @sqrt(if (x > 20) x else std.math.log1p(@exp(x))); +} +test "V4.1 exhaustive router F16 correction bias, unbiased normalization, extreme logits and row independence" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try route.Pipelines.load(device); + defer p.deinit(); + var b = Buffers{ .device = device }; + defer b.deinit(); + const logits = try b.get(f32, 2 * 389); + const bias = try b.get(f16, 384); + for (0..384) |i| { + logits.slice(f32, 778)[i] = (@as(f32, @floatFromInt(i)) - 190) * 0.5; + logits.slice(f32, 778)[389 + i] = 0; + bias.slice(f16, 384)[i] = if (i >= 17 and i < 23) 100 else 0; + } + const ids = try b.get(u32, 16); + const weights = try b.get(f32, 16); + const ids_one = try b.get(u32, 6); + const weights_one = try b.get(f32, 6); + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + route.encode(e, &p, view(logits), view(bias), view(ids), view(weights), .{ .rows = 2, .logit_stride = 389, .output_stride = 8, .bias_dtype = .f16 }); + route.encode(e, &p, view(logits).shifted(389 * 4), view(bias), view(ids_one), view(weights_one), .{ .rows = 1, .bias_dtype = .f16 }); + try done(cb, e); + for (0..2) |r| { + var used: [384]bool = @splat(false); + var picks: [6]usize = undefined; + var raw: [6]f64 = undefined; + var sum: f64 = 0; + for (0..6) |slot| { + var best: f64 = -std.math.inf(f64); + var pick: usize = 0; + for (0..384) |i| { + const sc = scoreReference(logits.slice(f32, 778)[r * 389 + i]) + @as(f64, bias.slice(f16, 384)[i]); + if (!used[i] and sc > best) { + best = sc; + pick = i; + } + } + used[pick] = true; + picks[slot] = pick; + raw[slot] = scoreReference(logits.slice(f32, 778)[r * 389 + pick]); + sum += raw[slot]; + } + for (0..6) |slot| { + try std.testing.expectEqual(@as(u32, @intCast(picks[slot])), ids.slice(u32, 16)[r * 8 + slot]); + try close(raw[slot] / (sum + 1e-20) * 1.5, weights.slice(f32, 16)[r * 8 + slot], 3e-6); + } + } + for (0..6) |i| { + try std.testing.expectEqual(ids.slice(u32, 16)[8 + i], ids_one.slice(u32, 6)[i]); + try std.testing.expectEqual(weights.slice(f32, 16)[8 + i], weights_one.slice(f32, 6)[i]); + } +} + +test "V4.1 router normalization matches source CUDA six-score FP32 reduction boundaries" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try route.Pipelines.load(device); + defer p.deinit(); + var b = Buffers{ .device = device }; + defer b.deinit(); + // Power-of-two sqrt(softplus(score^2)) with logits >20 eliminates unary-rounding ambiguity. + const scores = [2][6]f32{ + .{ 268435456, 16, 16, 16, 16, 16 }, + .{ 2097152, 8, 16, 134217728, 8388608, 8192 }, + }; + // Independent pinned Torch CUDA Gate expected bits; the second row also distinguishes wide normalization. + const expected_bits = [2][6]u32{ + .{ 1069547517, 868220925, 868220925, 868220925, 868220925, 868220925 }, + .{ 1018303405, 867308461, 875697069, 1068635053, 1035080621, 951194541 }, + }; + const logits = try b.get(f32, scores.len * 384); + const bias = try b.get(f32, 384); + const ids = try b.get(u32, scores.len * 6); + const weights = try b.get(f32, scores.len * 6); + @memset(logits.slice(f32, scores.len * 384), -100); + for (scores, 0..) |row, r| { + for (row, 0..) |s, slot| logits.slice(f32, scores.len * 384)[r * 384 + slot] = s * s; + } + // Distinct biases force slot order without topk ties; weights still use unbiased scores. + for (0..6) |slot| bias.slice(f32, 384)[slot] = 1099511627776 - @as(f32, @floatFromInt(slot)) * 68719476736; + const cb = queue.commandBuffer(); + const encoder = cb.compute(.serial); + route.encode(encoder, &p, view(logits), view(bias), view(ids), view(weights), .{ .rows = scores.len, .bias_dtype = .f32 }); + try done(cb, encoder); + for (expected_bits, 0..) |row, r| { + for (row, 0..) |expected, slot| { + try std.testing.expectEqual(@as(u32, @intCast(slot)), ids.slice(u32, scores.len * 6)[r * 6 + slot]); + try std.testing.expectEqual(expected, @as(u32, @bitCast(weights.slice(f32, scores.len * 6)[r * 6 + slot]))); + } + } +} + +test "nonfinite router inputs propagate invalid routes without reading outside the expert arrays" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try route.Pipelines.load(device); + defer p.deinit(); + var buffers = Buffers{ .device = device }; + defer buffers.deinit(); + const logits = try buffers.get(f32, 384); + const bias = try buffers.get(f32, 384); + const ids = try buffers.get(u32, 6); + const weights = try buffers.get(f32, 6); + @memset(logits.slice(f32, 384), std.math.nan(f32)); + const cb = queue.commandBuffer(); + const encoder = cb.compute(.serial); + route.encode(encoder, &p, view(logits), view(bias), view(ids), view(weights), .{ .rows = 1 }); + try done(cb, encoder); + for (ids.slice(u32, 6), weights.slice(f32, 6)) |id, weight| { + try std.testing.expectEqual(std.math.maxInt(u32), id); + try std.testing.expect(std.math.isNan(weight)); + } + var combine = try ops.Pipelines.load(device); + defer combine.deinit(); + const routed = try buffers.get(u16, 6 * 4); + const shared = try buffers.get(u16, 4); + const output = try buffers.get(u16, 4); + // Invalid routes skip expert writes; finite slots from a prior chunk must not hide that failure. + @memset(routed.slice(u16, 24), support.bf(2)); + @memset(shared.slice(u16, 4), support.bf(3)); + const combine_cb = queue.commandBuffer(); + const combine_encoder = combine_cb.compute(.serial); + ops.encodeCombine(combine_encoder, &combine, view(routed), view(shared), view(ids), view(output), .{ .rows = 1, .width = 4, .routed_row_stride = 24, .slot_stride = 4, .shared_stride = 4, .y_stride = 4 }); + try done(combine_cb, combine_encoder); + for (output.slice(u16, 4)) |value| try std.testing.expect(std.math.isNan(support.fp(value))); +} diff --git a/zig/src/families/deepseek_v41/test_support.zig b/zig/src/families/deepseek_v41/test_support.zig new file mode 100644 index 0000000000..80bf87e8d2 --- /dev/null +++ b/zig/src/families/deepseek_v41/test_support.zig @@ -0,0 +1,40 @@ +const std = @import("std"); +const mtl = @import("metal"); +const View = @import("view.zig").View; + +pub fn bf(x: f32) u16 { + const u: u32 = @bitCast(x); + return @intCast((u +% 0x7fff +% ((u >> 16) & 1)) >> 16); +} +pub fn fp(x: u16) f32 { + return @bitCast(@as(u32, x) << 16); +} +pub fn view(b: mtl.Buffer) View { + return .{ .buffer = b }; +} +pub const Buffers = struct { + device: mtl.Device, + items: [32]mtl.Buffer = undefined, + count: usize = 0, + + pub fn get(b: *Buffers, comptime T: type, n: usize) !mtl.Buffer { + if (b.count == b.items.len) return error.TooManyTestBuffers; + const x = try b.device.buffer(try std.math.mul(usize, @sizeOf(T), n), mtl.ResourceOptions.shared); + b.items[b.count] = x; + b.count += 1; + @memset(x.slice(T, n), 0); + return x; + } + pub fn deinit(b: *Buffers) void { + for (b.items[0..b.count]) |x| x.deinit(); + } +}; +pub fn done(cb: mtl.CommandBuffer, e: mtl.ComputeEncoder) !void { + e.end(); + cb.commit(); + cb.wait(); + try std.testing.expect(cb.failure() == null); +} +pub fn close(expected: f64, actual: f32, tolerance: f64) !void { + try std.testing.expect(@abs(expected - @as(f64, actual)) <= tolerance); +} diff --git a/zig/src/families/deepseek_v41/view.zig b/zig/src/families/deepseek_v41/view.zig new file mode 100644 index 0000000000..fbaa08ce39 --- /dev/null +++ b/zig/src/families/deepseek_v41/view.zig @@ -0,0 +1,25 @@ +//! A borrowed Metal buffer region; element precision is carried by each operator's explicit descriptor. +const std = @import("std"); +const mtl = @import("metal"); + +pub const View = struct { + buffer: mtl.Buffer, + offset: usize = 0, + + pub fn shifted(self: View, bytes: usize) View { + return .{ .buffer = self.buffer, .offset = self.offset + bytes }; + } + + pub fn require(self: View, bytes: usize, alignment: usize) !void { + if (self.offset % alignment != 0 or self.offset > self.buffer.length() or + bytes > self.buffer.length() - self.offset) return error.InvalidTensorView; + } + + /// Rows are strided elements; only the final row's used width is required. + pub fn requireRows(self: View, rows: usize, stride: usize, width: usize, element_bytes: usize) !void { + if (rows == 0 or width == 0 or stride < width) return error.InvalidTensorView; + const preceding = try std.math.mul(usize, rows - 1, stride); + const elements = try std.math.add(usize, preceding, width); + try self.require(try std.math.mul(usize, elements, element_bytes), element_bytes); + } +}; diff --git a/zig/src/families/deepseek_v41/weights.zig b/zig/src/families/deepseek_v41/weights.zig new file mode 100644 index 0000000000..0795e0db8e --- /dev/null +++ b/zig/src/families/deepseek_v41/weights.zig @@ -0,0 +1,348 @@ +//! Strict pinned backbone schema and borrowed resident bindings; only expert descriptor tables are owned. +const std = @import("std"); +const mtl = @import("metal"); +const cp = @import("../../core/checkpoint_metal.zig"); +const st = @import("../../core/safetensors.zig"); +const exl3 = @import("../../core/exl3_projection.zig"); +const load = @import("load.zig"); + +pub const Tensor = cp.Tensor; +pub const Matrix = exl3.Matrix; + +/// A kept-dense repack stores small matrices as a bf16 weight beside the rotation scales. +pub const Dense = struct { weight: Tensor, suh: Tensor, svh: Tensor, k: usize, n: usize }; +pub const AnyMatrix = union(enum) { trellis: Matrix, dense: Dense }; +pub const Hc = struct { projection: Tensor, scale: Tensor, base: Tensor }; +pub const Compressor = struct { wkv: AnyMatrix, wgate: ?AnyMatrix, norm: Tensor }; +pub const Indexer = struct { wq_b: AnyMatrix, wk: ?AnyMatrix, weights_proj: Tensor, k_norm: ?Tensor }; +pub const Attention = struct { + wq_a: Matrix, + wq_b: Matrix, + wkv: Matrix, + wo_b: Matrix, + wo_a: [8]Matrix, + sink: Tensor, + q_norm: Tensor, + kv_norm: Tensor, + compressor: ?Compressor, + indexer: ?Indexer, +}; +/// Projection order is w1 (gate), w3 (up), w2 (down), for shared and routed experts alike. +pub const Moe = struct { + gate: Tensor, + bias: Tensor, + shared: [3]Matrix, + descriptors: [3]mtl.Buffer, +}; +pub const Engram = struct { table: Tensor, scales: Tensor, q_weight: Tensor, k_weight: Tensor, wkv: Matrix }; +pub const Layer = struct { + attn_norm: Tensor, + ffn_norm: Tensor, + hc_attn: Hc, + hc_ffn: Hc, + attn: Attention, + moe: Moe, + engram: ?Engram, +}; +pub const Model = struct { + embed: Tensor, + head: Matrix, + norm: Tensor, + layers: [40]Layer, + + /// Checkpoint buffers must outlive this borrowed model and every submitted command using it. + pub fn deinit(self: *Model) void { + for (&self.layers) |*layer| for (layer.moe.descriptors) |buffer| buffer.deinit(); + self.* = undefined; + } + pub fn registerResidency(self: *const Model, set: mtl.ResidencySet) void { + for (&self.layers) |*layer| for (layer.moe.descriptors) |buffer| set.add(buffer); + } + pub fn residentBytes(_: *const Model) usize { + return sizeofDescriptors(); + } +}; + +/// Admission adds these bytes to load.Plan.bytes; no dequantized weights or duplicate projections are allocated. +pub fn sizeofDescriptors() usize { + return 40 * 3 * 384 * @sizeOf(exl3.ExpertDescriptor); +} + +/// Every executed tensor is checked using headers only, before checkpoint materialization or model allocation. +pub fn validate(plan: *const load.Plan) !void { + try populate(false, plan, null); +} + +/// Bind resident tensors and marker payloads; route descriptors replace redundant host matrix metadata. +pub fn bind(device: mtl.Device, checkpoint: *const cp.Checkpoint) !Model { + var model: Model = undefined; + try populate(true, checkpoint, &model); + var owned: usize = 0; + errdefer for (0..owned) |i| model.layers[i / 3].moe.descriptors[i % 3].deinit(); + var prefix: [128]u8 = undefined; + for (&model.layers, 0..) |*layer, layer_id| { + inline for (.{ "w1", "w3", "w2" }, 0..) |part, projection| { + const bytes = 384 * @sizeOf(exl3.ExpertDescriptor); + const buffer = try device.buffer(bytes, mtl.ResourceOptions.shared | mtl.ResourceOptions.untracked); + layer.moe.descriptors[projection] = buffer; + owned += 1; + const out: [*]exl3.ExpertDescriptor = @ptrCast(@alignCast(buffer.contents())); + for (0..384) |expert| { + var m: Matrix = undefined; + const k: usize = if (projection == 2) 2304 else 5120; + const n: usize = if (projection == 2) 5120 else 2304; + try matrix(true, checkpoint, try named(&prefix, "layers.{d}.ffn.experts.{d}.{s}", .{ layer_id, expert, part }), k, n, &m); + out[expert] = try exl3.descriptor(m); + } + } + } + return model; +} + +/// A borrowed CPU/GPU matrix view for checkpoint qualification, using the same strict storage contract. +pub fn lookup(checkpoint: *const cp.Checkpoint, prefix: []const u8, k: usize, n: usize) !Matrix { + var out: Matrix = undefined; + try matrix(true, checkpoint, prefix, k, n, &out); + return out; +} + +fn named(buffer: []u8, comptime fmt: []const u8, args: anytype) ![]const u8 { + return std.fmt.bufPrint(buffer, fmt, args); +} + +/// A plain tensor the encoders decode as f32, f16 or bf16; the shape must match exactly. +fn plainEither(comptime binding: bool, source: anytype, name: []const u8, shape: []const usize, destination: ?*Tensor) !void { + const tensor = try source.get(name); + if ((tensor.dtype != .f32 and tensor.dtype != .f16 and tensor.dtype != .bf16) or tensor.rank != shape.len or + !std.mem.eql(usize, tensor.shape[0..tensor.rank], shape)) + { + std.log.err("unsupported tensor {s}: {s} {any}", .{ name, @tagName(tensor.dtype), tensor.shape[0..tensor.rank] }); + return error.UnsupportedTensor; + } + if (binding) { + var count: usize = 1; + for (shape) |dim| count = try std.math.mul(usize, count, dim); + if (tensor.bytes != try std.math.mul(usize, count, tensor.dtype.size())) return error.UnsupportedTensor; + if (tensor.offset % tensor.dtype.size() != 0 or tensor.offset > tensor.buffer.length() or + tensor.bytes > tensor.buffer.length() - tensor.offset) return error.InvalidTensorView; + destination.?.* = tensor; + } else if (tensor.begin % 4 != 0) return error.InvalidTensorView; +} + +fn plain(comptime binding: bool, source: anytype, name: []const u8, dtype: st.DType, shape: []const usize, destination: ?*Tensor) !void { + const tensor = try source.get(name); + if (tensor.dtype != dtype or tensor.rank != shape.len or + !std.mem.eql(usize, tensor.shape[0..tensor.rank], shape)) + { + std.log.err("unsupported backbone tensor {s}: {s} {any}", .{ name, @tagName(tensor.dtype), tensor.shape[0..tensor.rank] }); + return error.UnsupportedTensor; + } + const offset = if (binding) tensor.offset else tensor.begin; + if (offset % dtype.size() != 0) return error.InvalidTensorView; + if (binding) { + var count: usize = 1; + for (shape) |dim| count = try std.math.mul(usize, count, dim); + if (tensor.bytes != try std.math.mul(usize, count, dtype.size())) return error.UnsupportedTensor; + if (tensor.offset % dtype.size() != 0 or tensor.offset > tensor.buffer.length() or + tensor.bytes > tensor.buffer.length() - tensor.offset) return error.InvalidTensorView; + destination.?.* = tensor; + } +} + +fn has(source: anytype, name: []const u8) bool { + return source.has(name); +} + +const Scale = struct { name: []const u8, format: exl3.ScaleFormat }; +fn scale(comptime binding: bool, source: anytype, prefix: []const u8, size: usize, comptime fp: []const u8, comptime signs: []const u8, storage: []u8) !Scale { + var alternative: [160]u8 = undefined; + const f = try named(storage, "{s}.{s}", .{ prefix, fp }); + const s = try named(&alternative, "{s}.{s}", .{ prefix, signs }); + const have_fp = has(source, f); + if (have_fp == has(source, s)) return error.MissingOrAmbiguousExl3Scale; + const name = if (have_fp) f else try named(storage, "{s}.{s}", .{ prefix, signs }); + const t = try source.get(name); + if (t.rank != 1 or t.shape[0] != (if (have_fp) size else size / 16) or + t.dtype != (if (have_fp) st.DType.f16 else st.DType.i16)) return error.UnsupportedExl3Scale; + const offset = if (binding) t.offset else t.begin; + if (offset % 2 != 0) return error.InvalidTensorView; + if (binding) { + const bytes = (if (have_fp) size else size / 16) * 2; + if (t.bytes != bytes or t.offset % 2 != 0 or t.offset > t.buffer.length() or + bytes > t.buffer.length() - t.offset) return error.InvalidTensorView; + } + return .{ .name = name, .format = if (have_fp) .f16 else .packed_signs }; +} + +/// Compressor/indexer matrices ride either storage: packs quantize; rank-split repacks stay dense with rotation scales. +fn matrixAny(comptime binding: bool, source: anytype, prefix: []const u8, k: usize, n: usize, destination: ?*AnyMatrix) !void { + var name_buf: [160]u8 = undefined; + if (has(source, try named(&name_buf, "{s}.weight", .{prefix}))) { + const weight_name = try named(&name_buf, "{s}.weight", .{prefix}); + const w = try source.get(weight_name); + if (w.dtype != .bf16 or w.rank != 2 or w.shape[0] != n or w.shape[1] != k) { + std.log.err("unsupported dense tensor {s}: {s} {any}", .{ weight_name, @tagName(w.dtype), w.shape[0..w.rank] }); + return error.UnsupportedTensor; + } + const su = try source.get(try named(&name_buf, "{s}.suh", .{prefix})); + const sv = try source.get(try named(&name_buf, "{s}.svh", .{prefix})); + if (su.dtype != .f16 or su.rank != 1 or su.shape[0] != k or + sv.dtype != .f16 or sv.rank != 1 or sv.shape[0] != n) return error.UnsupportedExl3Scale; + if (binding) { + if (w.offset % 2 != 0) return error.InvalidTensorView; + destination.?.* = .{ .dense = .{ .weight = w, .suh = su, .svh = sv, .k = k, .n = n } }; + } + return; + } + var m: Matrix = undefined; + try matrix(binding, source, prefix, k, n, if (binding) &m else null); + if (binding) destination.?.* = .{ .trellis = m }; +} + +fn matrix(comptime binding: bool, source: anytype, prefix: []const u8, k: usize, n: usize, destination: ?*Matrix) !void { + var name: [160]u8 = undefined; + const trellis = try source.get(try named(&name, "{s}.trellis", .{prefix})); + if (trellis.dtype != .i16 or trellis.rank != 3 or trellis.shape[0] != k / 16 or + trellis.shape[1] != n / 16 or trellis.shape[2] % 8 != 0) return error.UnsupportedExl3Shape; + if ((if (binding) trellis.offset else trellis.begin) % 2 != 0) return error.InvalidTensorView; + const halves = trellis.shape[2] / 8; + if (halves < 2 or halves > 16 or (halves % 2 != 0 and halves > 7)) return error.UnsupportedExl3Width; + // Pinned checkpoint uses mul1 throughout; missing markers cannot silently select the legacy codebook. + const marker = try source.get(try named(&name, "{s}.mul1", .{prefix})); + if (marker.dtype != .i32 or marker.rank != 0) return error.UnsupportedExl3Marker; + if ((if (binding) marker.offset else marker.begin) % 4 != 0) return error.InvalidTensorView; + if (has(source, try named(&name, "{s}.mcg", .{prefix})) or + has(source, try named(&name, "{s}.3inst", .{prefix})) or + has(source, try named(&name, "{s}.bias", .{prefix}))) + { + std.log.err("unexpected exl3 parts beside {s}", .{prefix}); + return error.UnsupportedExl3Parts; + } + var su_name: [160]u8 = undefined; + var sv_name: [160]u8 = undefined; + const su = try scale(binding, source, prefix, k, "suh", "su", &su_name); + const sv = try scale(binding, source, prefix, n, "svh", "sv", &sv_name); + if (binding) { + const expected = k / 16 * (n / 16) * halves * 16; + if (trellis.bytes != expected or trellis.offset % 2 != 0 or + trellis.offset > trellis.buffer.length() or expected > trellis.buffer.length() - trellis.offset) + return error.InvalidTensorView; + if (marker.bytes != 4 or marker.offset % 4 != 0 or marker.offset > marker.buffer.length() or + 4 > marker.buffer.length() - marker.offset) return error.InvalidTensorView; + const marker_bytes = marker.buffer.contents()[marker.offset..][0..4]; + if (std.mem.readInt(u32, marker_bytes, .little) != exl3.format.Mul1Mul) return error.UnsupportedExl3Marker; + const suh = try source.get(su.name); + const svh = try source.get(sv.name); + destination.?.* = .{ + .trellis = .{ .buffer = trellis.buffer, .offset = trellis.offset }, + .suh = .{ .buffer = suh.buffer, .offset = suh.offset }, + .svh = .{ .buffer = svh.buffer, .offset = svh.offset }, + .k = k, + .n = n, + .width = .{ .halves = @intCast(halves) }, + .codebook = .mul1, + .suh_format = su.format, + .svh_format = sv.format, + }; + try destination.?.validate(); + } +} + +fn populate(comptime binding: bool, source: anytype, model: ?*Model) !void { + try plain(binding, source, "embed.weight", .bf16, &.{ 129280, 5120 }, if (binding) &model.?.embed else null); + try plain(binding, source, "norm.weight", .bf16, &.{5120}, if (binding) &model.?.norm else null); + try matrix(binding, source, "head", 5120, 129280, if (binding) &model.?.head else null); + var name: [160]u8 = undefined; + var prefix: [128]u8 = undefined; + for (0..40) |layer| { + const out: ?*Layer = if (binding) &model.?.layers[layer] else null; + try plain(binding, source, try named(&name, "layers.{d}.attn_norm.weight", .{layer}), .bf16, &.{5120}, if (binding) &out.?.attn_norm else null); + try plain(binding, source, try named(&name, "layers.{d}.ffn_norm.weight", .{layer}), .bf16, &.{5120}, if (binding) &out.?.ffn_norm else null); + inline for (.{ "attn", "ffn" }) |kind| { + const hc: ?*Hc = if (binding) (if (std.mem.eql(u8, kind, "attn")) &out.?.hc_attn else &out.?.hc_ffn) else null; + try plain(binding, source, try named(&name, "layers.{d}.hc_{s}_fn", .{ layer, kind }), .f32, &.{ 24, 20480 }, if (binding) &hc.?.projection else null); + try plain(binding, source, try named(&name, "layers.{d}.hc_{s}_scale", .{ layer, kind }), .f32, &.{3}, if (binding) &hc.?.scale else null); + try plain(binding, source, try named(&name, "layers.{d}.hc_{s}_base", .{ layer, kind }), .f32, &.{24}, if (binding) &hc.?.base else null); + } + const attn: ?*Attention = if (binding) &out.?.attn else null; + inline for (.{ "wq_a", "wq_b", "wkv", "wo_b" }, .{ 5120, 1280, 5120, 8192 }, .{ 1280, 32768, 512, 5120 }) |part, k, n| { + try matrix(binding, source, try named(&prefix, "layers.{d}.attn.{s}", .{ layer, part }), k, n, if (binding) &@field(attn.?, part) else null); + } + for (0..8) |slice| try matrix(binding, source, try named(&prefix, "layers.{d}.attn.wo_a.slice.{d}", .{ layer, slice }), 4096, 1024, if (binding) &attn.?.wo_a[slice] else null); + try plain(binding, source, try named(&name, "layers.{d}.attn.attn_sink", .{layer}), .f32, &.{64}, if (binding) &attn.?.sink else null); + try plain(binding, source, try named(&name, "layers.{d}.attn.q_norm.weight", .{layer}), .bf16, &.{1280}, if (binding) &attn.?.q_norm else null); + try plain(binding, source, try named(&name, "layers.{d}.attn.kv_norm.weight", .{layer}), .bf16, &.{512}, if (binding) &attn.?.kv_norm else null); + const owner = layer == 2 or layer == 8 or layer == 14 or layer == 20; + if (binding) { + attn.?.compressor = if (owner) @as(Compressor, undefined) else null; + attn.?.indexer = if (owner or layer == 24 or layer == 28 or layer == 32 or layer == 36) + @as(Indexer, undefined) + else + null; + } + if (owner) { + const compressor: ?*Compressor = if (binding) &attn.?.compressor.? else null; + try matrixAny(binding, source, try named(&prefix, "layers.{d}.attn.compressor.wkv", .{layer}), 5120, 512, if (binding) &compressor.?.wkv else null); + try plain(binding, source, try named(&name, "layers.{d}.attn.compressor.norm.weight", .{layer}), .bf16, &.{512}, if (binding) &compressor.?.norm else null); + if (binding) compressor.?.wgate = if (layer < 20) @as(AnyMatrix, undefined) else null; + if (layer < 20) try matrixAny(binding, source, try named(&prefix, "layers.{d}.attn.compressor.wgate", .{layer}), 5120, 512, if (binding) &compressor.?.wgate.? else null); + } + if (owner or layer == 24 or layer == 28 or layer == 32 or layer == 36) { + const indexer: ?*Indexer = if (binding) &attn.?.indexer.? else null; + try matrixAny(binding, source, try named(&prefix, "layers.{d}.attn.indexer.wq_b", .{layer}), 1280, 4096, if (binding) &indexer.?.wq_b else null); + try plainEither(binding, source, try named(&name, "layers.{d}.attn.indexer.weights_proj.weight", .{layer}), &.{ 32, 5120 }, if (binding) &indexer.?.weights_proj else null); + if (binding) { + indexer.?.wk = if (owner) @as(AnyMatrix, undefined) else null; + indexer.?.k_norm = if (owner) @as(Tensor, undefined) else null; + } + if (owner) { + try matrixAny(binding, source, try named(&prefix, "layers.{d}.attn.indexer.wk", .{layer}), 512, 128, if (binding) &indexer.?.wk.? else null); + try plain(binding, source, try named(&name, "layers.{d}.attn.indexer.k_norm.weight", .{layer}), .bf16, &.{128}, if (binding) &indexer.?.k_norm.? else null); + } + } + const moe: ?*Moe = if (binding) &out.?.moe else null; + // The released checkpoints store the router gate f16; rank-split repacks store it bf16. + if (binding) { + const tensor = try source.get(try named(&name, "layers.{d}.ffn.gate.weight", .{layer})); + if ((tensor.dtype != .f16 and tensor.dtype != .bf16) or tensor.rank != 2 or + tensor.shape[0] != 384 or tensor.shape[1] != 5120) + { + std.log.err("unsupported router gate {s}: {s} {any}", .{ name, @tagName(tensor.dtype), tensor.shape[0..tensor.rank] }); + return error.UnsupportedTensor; + } + moe.?.gate = tensor; + } else { + const tensor = try source.get(try named(&name, "layers.{d}.ffn.gate.weight", .{layer})); + if ((tensor.dtype != .f16 and tensor.dtype != .bf16) or tensor.rank != 2 or + tensor.shape[0] != 384 or tensor.shape[1] != 5120) return error.UnsupportedTensor; + } + // The bias rides the same three dtypes the route kernel decodes (f32, f16, bf16). + if (binding) { + const tensor = try source.get(try named(&name, "layers.{d}.ffn.gate.bias", .{layer})); + if ((tensor.dtype != .f16 and tensor.dtype != .bf16 and tensor.dtype != .f32) or tensor.rank != 1 or tensor.shape[0] != 384) + return error.UnsupportedTensor; + moe.?.bias = tensor; + } else { + const tensor = try source.get(try named(&name, "layers.{d}.ffn.gate.bias", .{layer})); + if ((tensor.dtype != .f16 and tensor.dtype != .bf16 and tensor.dtype != .f32) or tensor.rank != 1 or tensor.shape[0] != 384) + return error.UnsupportedTensor; + } + inline for (.{ "w1", "w3", "w2" }, 0..) |part, projection| { + const k: usize = if (projection == 2) 2304 else 5120; + const n: usize = if (projection == 2) 5120 else 2304; + try matrix(binding, source, try named(&prefix, "layers.{d}.ffn.shared_experts.{s}", .{ layer, part }), k, n, if (binding) &moe.?.shared[projection] else null); + if (!binding) for (0..384) |expert| try matrix(false, source, try named(&prefix, "layers.{d}.ffn.experts.{d}.{s}", .{ layer, expert, part }), k, n, null); + } + const engram = layer == 1 or layer == 14; + if (binding) out.?.engram = if (engram) @as(Engram, undefined) else null; + if (engram) { + const e: ?*Engram = if (binding) &out.?.engram.? else null; + const rows: usize = if (layer == 1) 384006168 else 384016682; + try plain(binding, source, try named(&name, "layers.{d}.engram.embed.weight", .{layer}), .f8_e4m3, &.{ rows, 256 }, if (binding) &e.?.table else null); + try plain(binding, source, try named(&name, "layers.{d}.engram.embed.scale", .{layer}), .f8_e8m0, &.{ rows, 8 }, if (binding) &e.?.scales else null); + try plainEither(binding, source, try named(&name, "layers.{d}.engram.q_weight", .{layer}), &.{ 4, 5120 }, if (binding) &e.?.q_weight else null); + try plainEither(binding, source, try named(&name, "layers.{d}.engram.k_weight", .{layer}), &.{ 4, 5120 }, if (binding) &e.?.k_weight else null); + try matrix(binding, source, try named(&prefix, "layers.{d}.engram.wkv", .{layer}), 6144, 25600, if (binding) &e.?.wkv else null); + } + } +} diff --git a/zig/src/families/deepseek_v41/workspace.zig b/zig/src/families/deepseek_v41/workspace.zig new file mode 100644 index 0000000000..fcef57cd6e --- /dev/null +++ b/zig/src/families/deepseek_v41/workspace.zig @@ -0,0 +1,107 @@ +//! Fixed chunk-sized activations: no layer history or context-by-query score matrices. +const std = @import("std"); +const mtl = @import("metal"); +const exl3 = @import("../../core/exl3_projection.zig"); +const View = @import("view.zig").View; +const hash = @import("engram_hash.zig"); +const groups = @import("../../core/expert_groups.zig"); + +pub const Field = enum { + ids, + token_map, + history, + stream, + collapsed, + normalized, + branch, + previous_pre, + attention_pre, + post, + comb, + mixes, + inv, + engram_ids_0, + engram_ids_1, + engram_gather, + engram_kv, + router_logits, + route_ids, + route_weights, + gate, + up, + activation, + routed, + shared_gate, + shared_up, + shared_activation, + shared_down, + logits, + projection, + grouping, +}; +const count = std.enums.values(Field).len; + +pub const Plan = struct { + rows: u32, + offsets: [count]usize, + bytes: usize, + + pub fn init(rows: u32) !Plan { + if (rows == 0 or rows > 4096) return error.InvalidPrefillChunk; + const t: usize = rows; + const grouped = if (rows > 1) try groups.Plan.init(t * 6, 384) else null; + const projection_bytes = @max(try exl3.scratchBytes(5120, 129280, t), if (grouped) |g| @max(try exl3.scratchBytes(5120, 2304, g.padded), try exl3.scratchBytes(2304, 5120, g.padded)) else try exl3.scratchBytes(2304, 5120, t * 6)); + const sizes = [_]usize{ + t * 4, hash.raw_vocab * 4, 16, t * 20480 * 2, t * 5120 * 2, t * 5120 * 2, t * 5120 * 2, + t * 4 * 4, t * 4 * 4, t * 4 * 4, t * 16 * 4, t * 24 * 4, t * 4, t * 24 * 4, + t * 24 * 4, t * 6144 * 2, t * 25600 * 2, t * 384 * 4, t * 6 * 4, t * 6 * 4, t * 6 * 2304 * 2, + t * 6 * 2304 * 2, t * 6 * 2304 * 2, t * 6 * 5120 * 2, t * 2304 * 2, t * 2304 * 2, t * 2304 * 2, t * 5120 * 2, + t * 129280 * 4, projection_bytes, if (grouped) |g| g.bytes else 0, + }; + var out = Plan{ .rows = rows, .offsets = undefined, .bytes = 0 }; + for (sizes, 0..) |size, i| { + out.offsets[i] = std.mem.alignForward(usize, out.bytes, 16); + out.bytes = try std.math.add(usize, out.offsets[i], size); + } + out.bytes = std.mem.alignForward(usize, out.bytes, 16384); + return out; + } +}; + +pub const Workspace = struct { + buffer: mtl.Buffer, + plan: Plan, + + pub fn init(device: mtl.Device, plan: Plan, token_map: []const u8) !Workspace { + if (token_map.len != hash.raw_vocab * 4) return error.InvalidTokenMap; + const buffer = try device.buffer(plan.bytes, mtl.ResourceOptions.shared | mtl.ResourceOptions.untracked); + const w = Workspace{ .buffer = buffer, .plan = plan }; + @memcpy(buffer.contents()[plan.offsets[@backingInt(Field.token_map)]..][0..token_map.len], token_map); + @memset(buffer.contents()[plan.offsets[@backingInt(Field.history)]..][0..16], 0); + return w; + } + + pub fn deinit(w: *Workspace) void { + w.buffer.deinit(); + } + pub fn view(w: *const Workspace, field: Field) View { + return .{ .buffer = w.buffer, .offset = w.plan.offsets[@backingInt(field)] }; + } + pub fn projection(w: *const Workspace) exl3.View { + const v = w.view(.projection); + return .{ .buffer = v.buffer, .offset = v.offset }; + } + pub fn grouping(w: *const Workspace, rows: u32) !groups.Scratch { + if (rows < 2 or rows > w.plan.rows) return error.InvalidPrefillChunk; + const v = w.view(.grouping); + return .{ .buffer = v.buffer, .offset = v.offset, .plan = try groups.Plan.init(@as(usize, rows) * 6, 384) }; + } + pub fn tokenIds(w: *const Workspace, rows: usize) []u32 { + const v = w.view(.ids); + return @as([*]u32, @ptrCast(@alignCast(v.buffer.contents() + v.offset)))[0..rows]; + } + pub fn logits(w: *const Workspace, rows: usize) []const f32 { + const v = w.view(.logits); + return @as([*]const f32, @ptrCast(@alignCast(v.buffer.contents() + v.offset)))[0 .. rows * 129280]; + } +}; diff --git a/zig/src/metal/counters.zig b/zig/src/metal/counters.zig index 610e22086b..0bbd65f186 100644 --- a/zig/src/metal/counters.zig +++ b/zig/src/metal/counters.zig @@ -1,4 +1,4 @@ -//! Metal counter sampling; owned resources must outlive every command buffer using them. +//! Metal counter sampling; owned descriptors, samples and fences must outlive their command buffers. const std = @import("std"); const objc = @import("objc.zig"); const types = @import("types.zig"); @@ -107,7 +107,7 @@ pub const SampleBuffer = struct { objc.msg(void, encoder.id, "sampleCountersInBuffer:atSampleIndex:withBarrier:", .{ self.id, index, true }); } - /// Caller must ensure GPU completion; borrows a bounded NSData result without a GPU blit or second copy. + /// Borrow bounded NSData bytes after completion; same-command blit resolution is unsupported, no result copy. pub fn resolveCompleted(self: SampleBuffer, count: usize) !Resolved { if (count == 0 or count > self.count) return error.InvalidSampleCount; const data = objc.msg(?objc.Id, self.id, "resolveCounterRange:", .{ diff --git a/zig/src/metal_counter_consumer.zig b/zig/src/metal_counter_consumer.zig index 3ea67c6991..e098e75b16 100644 --- a/zig/src/metal_counter_consumer.zig +++ b/zig/src/metal_counter_consumer.zig @@ -81,7 +81,7 @@ pub const Consumer = struct { const scales = [_]f32{ 2, 1, 0.5 }; const biases = [_]f32{ 1, -3, 2 }; for (classes, 0..) |kind, i| { - try recorder.beginSpan(&encoder, kind); + try recorder.beginSpan(&encoder, kind, @intCast(i)); const input = if (i % 2 == 0) self.a else self.b; const output = if (i % 2 == 0) self.b else self.a; encoder.setPipeline(self.pipeline); @@ -207,11 +207,11 @@ test "bounded recorder rejects nesting, overflow, open spans and premature reuse try std.testing.expectError(error.CounterEncoderStillOpen, recorder.discardAfterCompletion()); try std.testing.expectError(error.CounterRecordingInFlight, recorder.beginCommand(cb)); try std.testing.expectError(error.NoActiveCounterSpan, recorder.endSpan(&encoder)); - try recorder.beginSpan(&encoder, .dense); - try std.testing.expectError(error.NestedCounterSpan, recorder.beginSpan(&encoder, .copy)); + try recorder.beginSpan(&encoder, .dense, 0); + try std.testing.expectError(error.NestedCounterSpan, recorder.beginSpan(&encoder, .copy, 0)); try std.testing.expectError(error.UnclosedCounterSpan, recorder.finishEncoding(&encoder)); try recorder.endSpan(&encoder); - try std.testing.expectError(error.CounterSpanCapacityExceeded, recorder.beginSpan(&encoder, .copy)); + try std.testing.expectError(error.CounterSpanCapacityExceeded, recorder.beginSpan(&encoder, .copy, 0)); try recorder.finishEncoding(&encoder); try std.testing.expectError(error.CounterCommandNotCompleted, recorder.readAfterCompletion()); try recorder.cancelEncoding(&encoder); diff --git a/zig/src/tensorfold.zig b/zig/src/tensorfold.zig index 2ed6d0a1af..adaeb80099 100644 --- a/zig/src/tensorfold.zig +++ b/zig/src/tensorfold.zig @@ -22,6 +22,15 @@ pub const row_projection = @import("core/row_projection.zig"); pub const affine_mm = @import("core/affine_mm.zig"); /// The checkpoint formats (MLX affine and its fp64 reader for the device checks, exact widen). pub const quant = @import("core/quant/quant.zig"); +pub const deepseek_v41_config = @import("families/deepseek_v41/config.zig"); +pub const deepseek_v41_load = @import("families/deepseek_v41/load.zig"); +pub const deepseek_v41_weights = @import("families/deepseek_v41/weights.zig"); +pub const deepseek_v41_view = @import("families/deepseek_v41/view.zig"); +pub const deepseek_v41 = @import("families/deepseek_v41/model.zig"); +pub const deepseek_v41_profiler = @import("families/deepseek_v41/profiler.zig"); +pub const gpu_counters = @import("core/gpu_counters.zig"); +pub const expert_groups = @import("core/expert_groups.zig"); +pub const deepseek_v41_engram_hash = @import("families/deepseek_v41/engram_hash.zig"); pub const exl3_format = @import("core/exl3_format.zig"); pub const exl3_projection = @import("core/exl3_projection.zig"); pub const exlr8 = @import("core/exlr8.zig"); diff --git a/zig/tests/deepseek_v41_reference/attention_math.zig b/zig/tests/deepseek_v41_reference/attention_math.zig new file mode 100644 index 0000000000..09a4b1155d --- /dev/null +++ b/zig/tests/deepseek_v41_reference/attention_math.zig @@ -0,0 +1,446 @@ +//! CPU CSA2 from DeepSeek AI inference/{model,kernel}.py, rev 2cba9e42aa026125f3ed06c6d98c1db82f7ca027. +const std = @import("std"); +const tf = @import("tensorfold"); +const m = @import("math.zig"); +const projection = @import("projection.zig"); +const Config = tf.deepseek_v41_config.Text; +const Attention = tf.deepseek_v41_weights.Attention; + +pub const max_capacity = 4096; +/// Kernel-shaped key addressing: window rows at physical ring slots, then compressed rows. +pub const KeyTable = struct { + window: []const u16, + window_stride: usize = 512, + compressed: []const u16 = &[_]u16{}, + compressed_stride: usize = 512, + pub fn keyRow(t: KeyTable, at: usize) []const u16 { + if (at < 128) return t.window[at * t.window_stride ..][0..512]; + const row = at - 128; + return t.compressed[row * t.compressed_stride ..][0..512]; + } +}; +fn narrow(comptime T: type, x: f64) T { + return if (T == f32) @as(f32, @floatCast(x)) else x; +} +/// Compressor softmax pooling at reference precision T: FP32 is the source contract, FP64 the ideal. +pub fn poolOf(comptime T: type, kv: [2][]const T, gate: [2][]const T, out: []u16) void { + @setFloatMode(.strict); + for (out, 0..) |*o, d| { + const high = @max(gate[0][d], gate[1][d]); + const p0 = @exp(gate[0][d] - high); + const p1 = @exp(gate[1][d] - high); + const denominator = p0 + p1; + const pooled = kv[0][d] * (p0 / denominator) + kv[1][d] * (p1 / denominator); + o.* = if (T == f32) m.b(pooled) else m.b64(pooled); + } +} +pub const State = struct { + gpa: std.mem.Allocator, + capacity: usize, + window: []u16, + sources: []u16, + scores: []f32, + candidates: []bool, + blocks: []f32, + active_owner: usize = 0, + selected: [512]u32 = undefined, + selected_len: usize = 0, + partial_kv: [4][2][512]f32 = undefined, + partial_gate: [4][2][512]f32 = undefined, + qr: [1280]u16 = undefined, + q: [32768]u16 = undefined, + index_q: [4096]u16 = undefined, + index_weights: [32]f32 = undefined, + latent: [512]u16 = undefined, + index_key: [128]u16 = undefined, + kv: [512]u16 = undefined, + o: [32768]u16 = undefined, + grouped: [8192]u16 = undefined, + observer: ?Observer = null, + sparse_snapshot: ?[]u16 = null, + /// The indexer-selection probe: per scored window, the contract and FP64-ideal margins, and the pick flips. + score_probe: ?ScoreProbe = null, + score_copy: []f32 = &.{}, + ideal_scores: []f32 = &.{}, + ideal_copy: []f32 = &.{}, + pick_work: []f32 = &.{}, + selected_ideal: []u32 = &.{}, + pub const ScoreProbe = struct { + ctx: *anyopaque, + record: *const fn (*anyopaque, layer: usize, position: usize, count: usize, selected_len: usize, margin: f32, margin_ideal: f32, flips: usize, lowest: f32, highest_unselected: f32) void, + }; + pub const Observer = struct { + ctx: *anyopaque, + capture: *const fn (*anyopaque, tf.deepseek_v41.attention.DiagnosticStage, []const u16) void, + }; + fn observe(self: *State, stage: tf.deepseek_v41.attention.DiagnosticStage, values: []const u16) void { + if (self.observer) |observer| observer.capture(observer.ctx, stage, values); + } + + pub fn init(gpa: std.mem.Allocator, capacity: usize) !State { + if (capacity == 0 or capacity > max_capacity) return error.InvalidAttentionReferenceCapacity; + const window = try gpa.alloc(u16, 40 * 128 * 512); + errdefer gpa.free(window); + const sources = try gpa.alloc(u16, 4 * capacity * 640); + errdefer gpa.free(sources); + const scores = try gpa.alloc(f32, capacity); + errdefer gpa.free(scores); + const candidates = try gpa.alloc(bool, capacity); + errdefer gpa.free(candidates); + const blocks = try gpa.alloc(f32, (capacity + 7) / 8); + errdefer gpa.free(blocks); + const score_copy = try gpa.alloc(f32, capacity); + errdefer gpa.free(score_copy); + const ideal_scores = try gpa.alloc(f32, capacity); + errdefer gpa.free(ideal_scores); + const ideal_copy = try gpa.alloc(f32, capacity); + errdefer gpa.free(ideal_copy); + const pick_work = try gpa.alloc(f32, capacity); + errdefer gpa.free(pick_work); + const selected_ideal = try gpa.alloc(u32, capacity); + errdefer gpa.free(selected_ideal); + @memset(window, 0); + return .{ .gpa = gpa, .capacity = capacity, .window = window, .sources = sources, .scores = scores, .candidates = candidates, .blocks = blocks, .score_copy = score_copy, .ideal_scores = ideal_scores, .ideal_copy = ideal_copy, .pick_work = pick_work, .selected_ideal = selected_ideal }; + } + pub fn deinit(self: *State) void { + self.gpa.free(self.window); + self.gpa.free(self.sources); + self.gpa.free(self.scores); + self.gpa.free(self.candidates); + self.gpa.free(self.blocks); + self.gpa.free(self.score_copy); + self.gpa.free(self.ideal_scores); + self.gpa.free(self.ideal_copy); + self.gpa.free(self.pick_work); + self.gpa.free(self.selected_ideal); + self.* = undefined; + } + fn mainRow(self: *State, owner: usize, row: usize) []u16 { + return self.sources[owner * self.capacity * 640 + row * 640 ..][0..512]; + } + fn indexRow(self: *State, owner: usize, row: usize) []u16 { + return self.sources[owner * self.capacity * 640 + row * 640 + 512 ..][0..128]; + } + fn publish(self: *State, a: Attention, layer: usize, x: []const u16, position: usize, c: Config, scratch: *projection.Scratch) !bool { + @setFloatMode(.strict); + const compressor = a.compressor orelse return false; + const owner = (layer - 2) / 6; + self.active_owner = owner; + const ratio: usize = c.compress_ratios[layer]; + if (ratio == 1) { + try projection.apply(compressor.wkv, x, &self.latent, scratch); + } else { + const slot = position % ratio; + try projection.applyF32(compressor.wkv, x, &self.partial_kv[owner][slot], scratch); + try projection.applyF32(compressor.wgate.?, x, &self.partial_gate[owner][slot], scratch); + if ((position + 1) % ratio != 0) return false; + poolOf(f32, .{ &self.partial_kv[owner][0], &self.partial_kv[owner][1] }, .{ &self.partial_gate[owner][0], &self.partial_gate[owner][1] }, &self.latent); + } + m.rms(&self.latent, compressor.norm, c.rms_norm_eps, &self.latent); + const indexer = a.indexer.?; + try projection.apply(indexer.wk.?, &self.latent, &self.index_key, scratch); + m.rms(&self.index_key, indexer.k_norm.?, c.rms_norm_eps, &self.index_key); + const group_position = position + 1 - ratio; + m.rope(&self.index_key, group_position, true, false, c); + m.quantize(&self.index_key, .index); + const row = position / ratio; + @memcpy(self.indexRow(owner, row), &self.index_key); + m.rope(&self.latent, group_position, true, false, c); + m.quantize(&self.latent, .main); + @memcpy(self.mainRow(owner, row), &self.latent); + return true; + } + fn chooseCandidates(self: *State, count: usize, c: Config) void { + @memset(self.candidates[0..count], false); + const nblocks = (count + 7) / 8; + for (0..nblocks) |block| { + var score: f32 = -std.math.inf(f32); + for (block * 8..@min(block * 8 + 8, count)) |row| score = @max(score, self.scores[row]); + self.blocks[block] = score; + } + if (nblocks == 0) return; + self.blocks[nblocks - 1] = std.math.inf(f32); + for (0..@min(nblocks, c.candidate_topk_blocks)) |_| { + var best: usize = 0; + for (0..nblocks) |i| if (self.blocks[i] > self.blocks[best]) { + best = i; + }; + if (self.blocks[best] == -std.math.inf(f32)) break; + for (best * 8..@min(best * 8 + 8, count)) |row| self.candidates[row] = true; + self.blocks[best] = -std.math.inf(f32); + } + } + fn selectPositions(self: *State, a: Attention, layer: usize, x: []const u16, position: usize, count: usize, c: Config, scratch: *projection.Scratch) !void { + const indexer = a.indexer orelse return; + self.selected_len = @min(count, c.index_topk); + if (count == 0) return; + try projection.apply(indexer.wq_b, &self.qr, &self.index_q, scratch); + for (0..32) |head| { + const q = self.index_q[head * 128 ..][0..128]; + m.rope(q, position, true, false, c); + m.quantize(q, .index); + } + m.dense(indexer.weights_proj, x, &self.index_weights); + const scale: f32 = 1 / @sqrt(@as(f32, 128 * 32)); + // Python scalar multiplication preserves the source weights projection's BF16 dtype. + for (&self.index_weights) |*v| v.* = m.f(m.b(m.f(m.b(v.*)) * scale)); + for (0..count) |row| { + const key = self.indexRow(self.active_owner, row); + var score: f64 = 0; + for (0..32) |head| { + var dot: f64 = 0; + for (key, 0..) |v, d| dot += @as(f64, m.f(v)) * m.f(self.index_q[head * 128 + d]); + // einsum produces BF16, then BF16 multiply and head sum, in the pinned source. + const rectified = @max(m.f(m.b(@floatCast(dot))), 0); + const weighted = m.f(m.b(rectified * self.index_weights[head])); + score += weighted; + } + self.scores[row] = m.f(m.b(@floatCast(score))); + } + if (layer == c.candidate_source_layer_id) self.chooseCandidates(count, c); + if (layer > c.candidate_source_layer_id) for (0..count) |row| { + if (!self.candidates[row]) self.scores[row] = -std.math.inf(f32); + }; + @memcpy(self.score_copy[0..count], self.scores[0..count]); + self.idealScores(a, count); + for (0..self.selected_len) |pick| { + var best: usize = 0; + for (0..count) |i| if (self.scores[i] > self.scores[best]) { + best = i; + }; + self.selected[pick] = @intCast(best); + self.scores[best] = -std.math.inf(f32); + } + std.mem.sort(u32, self.selected[0..self.selected_len], {}, std.sort.asc(u32)); + if (self.score_probe) |probe| self.reportSelection(probe, layer, position, count); + } + /// The same selection on FP64-ideal scores: same BF16 inputs, no intermediate roundings; pick flips are the signal. + fn idealScores(self: *State, a: Attention, count: usize) void { + @setFloatMode(.strict); + if (a.indexer == null) return; + for (0..count) |row| { + if (self.score_copy[row] == -std.math.inf(f32)) { + self.ideal_scores[row] = -std.math.inf(f32); + continue; + } + const key = self.indexRow(self.active_owner, row); + var score: f64 = 0; + for (0..32) |head| { + var dot: f64 = 0; + for (key, 0..) |v, d| dot += @as(f64, m.f(v)) * m.f(self.index_q[head * 128 + d]); + score += @max(dot, 0) * @as(f64, self.index_weights[head]); + } + self.ideal_scores[row] = @floatCast(score); + } + } + fn pickTop(self: *State, scores: []const f32, into: []u32) void { + @memcpy(self.pick_work[0..scores.len], scores); + const work = self.pick_work[0..scores.len]; + for (into) |*pick| { + var best: usize = 0; + for (work, 0..) |v, i| if (v > work[best]) { + best = i; + }; + pick.* = @intCast(best); + work[best] = -std.math.inf(f32); + } + std.mem.sort(u32, into, {}, std.sort.asc(u32)); + } + fn pickedAt(selection: []const u32, row: usize) bool { + var lo: usize = 0; + var hi: usize = selection.len; + while (lo < hi) { + const mid = (lo + hi) / 2; + if (selection[mid] == row) return true; + if (selection[mid] < row) lo = mid + 1 else hi = mid; + } + return false; + } + fn reportSelection(self: *State, probe: ScoreProbe, layer: usize, position: usize, count: usize) void { + var lowest: f32 = std.math.inf(f32); + var highest_unselected: f32 = -std.math.inf(f32); + for (self.selected[0..self.selected_len]) |row| { + lowest = @min(lowest, self.score_copy[row]); + } + for (0..count) |row| { + if (!pickedAt(self.selected[0..self.selected_len], row) and self.score_copy[row] > -std.math.inf(f32)) + highest_unselected = @max(highest_unselected, self.score_copy[row]); + } + const margin: f32 = if (self.selected_len == 0 or lowest == std.math.inf(f32)) 0 else lowest - highest_unselected; + // The ideal picks, on a copy so the probe's ideal scores survive for callers. + @memcpy(self.ideal_copy[0..count], self.ideal_scores[0..count]); + const ideal_len = @min(self.selected_len, self.selected_ideal.len); + self.pickTop(self.ideal_copy[0..count], self.selected_ideal[0..ideal_len]); + var flips: usize = 0; + if (self.selected_len == ideal_len) { + for (self.selected[0..self.selected_len], self.selected_ideal[0..ideal_len]) |s, i| { + if (s != i) flips += 1; + } + } else flips = self.selected_len; + var ideal_lowest: f32 = std.math.inf(f32); + var ideal_highest_unselected: f32 = -std.math.inf(f32); + for (self.selected_ideal[0..ideal_len]) |row| { + ideal_lowest = @min(ideal_lowest, self.ideal_scores[row]); + } + for (0..count) |row| { + if (!pickedAt(self.selected_ideal[0..ideal_len], row) and self.ideal_scores[row] > -std.math.inf(f32)) + ideal_highest_unselected = @max(ideal_highest_unselected, self.ideal_scores[row]); + } + const margin_ideal: f32 = if (ideal_len == 0 or ideal_lowest == std.math.inf(f32)) 0 else ideal_lowest - ideal_highest_unselected; + probe.record(probe.ctx, layer, position, count, self.selected_len, margin, margin_ideal, flips, lowest, highest_unselected); + } + pub fn apply(self: *State, a: Attention, layer: usize, x: []const u16, position: usize, c: Config, out: []u16, scratch: *projection.Scratch) !void { + const compressed = c.compress_ratios[layer] != 0; + try projection.apply(.{ .trellis = a.wq_a }, x, &self.qr, scratch); + self.observe(.qr_projection, &self.qr); + m.rms(&self.qr, a.q_norm, c.rms_norm_eps, &self.qr); + self.observe(.qr_norm, &self.qr); + try projection.apply(.{ .trellis = a.wq_b }, &self.qr, &self.q, scratch); + self.observe(.q_projection, &self.q); + for (0..64) |head| m.rope(self.q[head * 512 ..][0..512], position, compressed, false, c); + self.observe(.q, &self.q); + try projection.apply(.{ .trellis = a.wkv }, x, &self.kv, scratch); + self.observe(.kv_projection, &self.kv); + m.rms(&self.kv, a.kv_norm, c.rms_norm_eps, &self.kv); + self.observe(.kv_norm, &self.kv); + m.rope(&self.kv, position, compressed, false, c); + self.observe(.kv_rope, &self.kv); + m.quantize(&self.kv, .window); + self.observe(.kv_quantized, &self.kv); + const window = self.window[layer * 128 * 512 ..][0 .. 128 * 512]; + @memcpy(window[(position % 128) * 512 ..][0..512], &self.kv); + var count: usize = 0; + if (compressed) { + _ = try self.publish(a, layer, x, position, c, scratch); + count = (position + 1) / c.compress_ratios[layer]; + try self.selectPositions(a, layer, x, position, count, c, scratch); + } + var indices: [640]i32 = undefined; + var slots: usize = 0; + if (position == 0) { + indices[0] = 0; + slots = 1; + } else { + const oldest = position % 128 + 1; + for (0..128) |i| { + const at = (oldest + i) % 128; + indices[i] = if (at > position) -1 else @intCast(at); + } + slots = 128; + } + if (compressed) for (self.selected[0..self.selected_len]) |row| { + indices[slots] = @intCast(128 + row); + slots += 1; + }; + for (0..64) |head| { + self.sparseHead(self.q[head * 512 ..][0..512], window, indices[0..slots], m.value(a.sink, head), self.o[head * 512 ..][0..512]); + self.observeHead(head); + m.rope(self.o[head * 512 ..][0..512], position, compressed, true, c); + } + if (self.sparse_snapshot) |snapshot| self.observe(.sparse_o, snapshot); + self.observe(.o, &self.o); + for (0..8) |group| try projection.apply(.{ .trellis = a.wo_a[group] }, self.o[group * 4096 ..][0..4096], self.grouped[group * 1024 ..][0..1024], scratch); + self.observe(.grouped, &self.grouped); + try projection.apply(.{ .trellis = a.wo_b }, &self.grouped, out, scratch); + self.observe(.wo_b, out); + } + fn observeHead(self: *State, head: usize) void { + if (self.sparse_snapshot) |snapshot| + @memcpy(snapshot[head * 512 ..][0..512], self.o[head * 512 ..][0..512]); + } + pub fn sparseHead(self: *State, q: []const u16, window: []const u16, indices: []const i32, sink: f32, out: []u16) void { + // The source-contract FP32 instance; mainRow addresses compressed rows at the 640 stride. + sparseHeadOf(f32, q, .{ + .window = window, + .compressed = self.sources[self.active_owner * self.capacity * 640 ..], + .compressed_stride = 640, + }, indices, sink, out); + } +}; + +/// Sparse online-softmax head: f32 reproduces the pinned source contract bit-for-bit; f64 makes every boundary ideal. +pub fn sparseHeadOf(comptime T: type, q: []const u16, keys: KeyTable, indices: []const i32, sink: f32, out: []u16) void { + return sparseHeadPre(T, q, keys, indices, sink, out, null); +} + +/// Same head; `pre` captures acc/denominator pre-rounding, so FP32 vs FP64 ULP is the true rounding level, not BF16's. +pub fn sparseHeadPre( + comptime T: type, + q: []const u16, + keys: KeyTable, + indices: []const i32, + sink: f32, + out: []u16, + pre: ?[]T, +) void { + @setFloatMode(.strict); + const scale: T = if (T == f32) 1 / @sqrt(@as(f32, 512)) else 1 / @sqrt(@as(f64, 512)); + var acc: [512]T = @splat(0); + var high: T = m.wide(T, -1e30); + var denominator: T = 0; + var base: usize = 0; + while (base < indices.len) : (base += 64) { + var probabilities: [64]T = undefined; + const width = @min(64, indices.len - base); + var next_high = high; + for (0..width) |j| { + const index = indices[base + j]; + var dot: f64 = 0; + if (index >= 0) { + const key = keys.keyRow(@intCast(index)); + for (q, key) |a, v| dot += m.d64(a) * m.d64(v); + } + probabilities[j] = if (index < 0) -std.math.inf(T) else narrow(T, dot) * scale; + next_high = @max(next_high, probabilities[j]); + } + const correction = @exp(high - next_high); + var sum: f64 = 0; + for (0..width) |j| { + probabilities[j] = @exp(probabilities[j] - next_high); + sum += probabilities[j]; + } + denominator = denominator * correction + narrow(T, sum); + for (0..512) |d| { + var accumulated: f64 = acc[d] * correction; + for (0..width) |j| if (indices[base + j] >= 0) { + // Online 64-column source kernel rounds unnormalized probabilities to BF16 before P*V. + const p: f64 = if (T == f32) @as(f64, m.f(m.b(probabilities[j]))) else m.d64(m.b64(probabilities[j])); + accumulated += p * m.d64(keys.keyRow(@intCast(indices[base + j]))[d]); + }; + acc[d] = @floatCast(accumulated); + } + high = next_high; + } + denominator += @exp(m.wide(T, sink) - high); + for (out, acc, 0..) |*v, a, i| { + const quotient = a / denominator; + if (pre) |cells| cells[i] = quotient; + v.* = if (T == f32) m.b(quotient) else m.b64(quotient); + } +} + +test "online 64-column sparse attention rounds probabilities before value GEMM" { + var state = try State.init(std.testing.allocator, 2); + defer state.deinit(); + var q: [512]u16 = @splat(0); + q[0] = m.b(8); + state.window[1] = m.b(1); + state.window[512] = m.b(1); + var indices: [65]i32 = @splat(-1); + indices[0] = 0; + indices[1] = 1; + indices[64] = 0; + var out: [512]u16 = undefined; + state.sparseHead(&q, state.window[0..1024], &indices, 0, &out); + const probability = @exp(-@as(f32, 8) * (1 / @sqrt(@as(f32, 512)))); + var denominator = @as(f32, @floatCast(@as(f64, probability) + 1)); + denominator += probability; + denominator += probability; + const expected = m.b((2 * m.f(m.b(probability))) / denominator); + try std.testing.expectEqual(expected, out[1]); +} + +test "attention reference capacity rejects unbounded history before allocation" { + try std.testing.expectError(error.InvalidAttentionReferenceCapacity, State.init(std.testing.allocator, 0)); + try std.testing.expectError(error.InvalidAttentionReferenceCapacity, State.init(std.testing.allocator, max_capacity + 1)); +} diff --git a/zig/tests/deepseek_v41_reference/engram_math.zig b/zig/tests/deepseek_v41_reference/engram_math.zig new file mode 100644 index 0000000000..88772898de --- /dev/null +++ b/zig/tests/deepseek_v41_reference/engram_math.zig @@ -0,0 +1,106 @@ +//! CPU Engram from DeepSeek AI inference/engram.py and model.py, rev 2cba9e42aa026125f3ed06c6d98c1db82f7ca027. +const std = @import("std"); +const tf = @import("tensorfold"); +const m = @import("math.zig"); +const projection = @import("projection.zig"); +const Engram = tf.deepseek_v41_weights.Engram; +const Config = tf.deepseek_v41_config.Text; +const multipliers = [2][4]u64{ + .{ 76632096046245, 4839876093313, 35959672319349, 73987337458391 }, + .{ 67716810739261, 51510806800915, 30921347202721, 82619226485591 }, +}; +const primes = [2][24]u32{ + .{ 16000057, 16000079, 16000081, 16000097, 16000121, 16000129, 16000133, 16000183, 16000189, 16000207, 16000211, 16000253, 16000277, 16000289, 16000307, 16000321, 16000339, 16000381, 16000393, 16000399, 16000403, 16000409, 16000447, 16000463 }, + .{ 16000477, 16000487, 16000499, 16000507, 16000511, 16000573, 16000609, 16000627, 16000667, 16000669, 16000693, 16000697, 16000711, 16000729, 16000759, 16000769, 16000781, 16000799, 16000813, 16000819, 16000841, 16000877, 16000879, 16000889 }, +}; +fn word(raw: []const u8, at: usize) u32 { + return std.mem.readInt(u32, raw[at..][0..4], .little); +} +pub const State = struct { + map: []const u8, + history: [3]u32 = .{ 2, 2, 2 }, + gathered: [6144]u16 = undefined, + kv: [25600]u16 = undefined, + + pub fn init(blob: []const u8) !State { + const at = 96 + 2 * 232; + if (blob.len != at + 129280 * 4 or !std.mem.eql(u8, blob[0..8], "DSENGR01")) + return error.InvalidEngramMetadata; + for ([_]u32{ 129280, 99092, 2, 4, 8, 2 }, 0..) |v, i| + if (word(blob, 8 + i * 4) != v) return error.InvalidEngramMetadata; + for (0..2) |layer| { + const base = 96 + layer * 232; + if (word(blob, base) != (if (layer == 0) @as(u32, 1) else @as(u32, 14))) + return error.InvalidEngramMetadata; + var offset: u32 = 0; + for (multipliers[layer], 0..) |v, j| + if (std.mem.readInt(u64, blob[base + 8 + 8 * j ..][0..8], .little) != v) + return error.InvalidEngramMetadata; + for (primes[layer], 0..) |v, j| { + if (word(blob, base + 40 + 4 * j) != v or word(blob, base + 136 + 4 * j) != offset) + return error.InvalidEngramMetadata; + offset += v; + } + if (word(blob, base + 4) != offset) return error.InvalidEngramMetadata; + } + const map = blob[at..]; + const expected = [_]u8{ 0xc6, 0x0a, 0x86, 0x32, 0x2e, 0xc1, 0x7b, 0x41, 0x42, 0xbf, 0xef, 0x3c, 0x57, 0xa8, 0xd8, 0x1f, 0xb4, 0x28, 0x55, 0x0c, 0xdf, 0x48, 0x7f, 0x88, 0xa4, 0x32, 0x0c, 0xb5, 0x9f, 0xe4, 0x64, 0x81 }; + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(map, &digest, .{}); + if (!std.mem.eql(u8, &expected, &digest)) return error.InvalidEngramTokenMap; + return .{ .map = map }; + } + pub fn compressed(self: *const State, id: u32) !u32 { + if (id >= 129280) return error.InvalidToken; + return word(self.map, @as(usize, id) * 4); + } + pub fn advance(self: *State, id: u32) void { + self.history[2] = self.history[1]; + self.history[1] = self.history[0]; + self.history[0] = id; + } + pub fn apply(self: *State, e: Engram, layer: usize, id: u32, h: []u16, c: Config, scratch: *projection.Scratch) !void { + @setFloatMode(.strict); + const tokens = [4]u32{ id, self.history[0], self.history[1], self.history[2] }; + var rolling = @as(u64, tokens[0]) * multipliers[layer][0]; + var offset: usize = 0; + for (1..4) |lookback| { + rolling ^= @as(u64, tokens[lookback]) * multipliers[layer][lookback]; + for (0..8) |head| { + const column = (lookback - 1) * 8 + head; + const row = offset + @as(usize, @intCast(rolling % primes[layer][column])); + for (0..256) |d| self.gathered[column * 256 + d] = + m.b(m.value(e.table, row * 256 + d) * m.value(e.scales, row * 8 + d / 32)); + offset += primes[layer][column]; + } + } + try projection.apply(.{ .trellis = e.wkv }, &self.gathered, &self.kv, scratch); + for (0..4) |copy| { + var hh: f64 = 0; + var kk: f64 = 0; + var dot: f64 = 0; + for (0..5120) |d| { + const at = copy * 5120 + d; + const x = m.f(h[at]); + const k = m.f(self.kv[at]); + const weight = m.value(e.q_weight, at) * m.value(e.k_weight, at); + hh += @as(f64, x) * x; + kk += @as(f64, k) * k; + const weighted: f32 = (x * weight) * k; + dot += weighted; + } + const h_mean = m.meanFromWide(hh, 5120); + const k_mean = m.meanFromWide(kk, 5120); + const h_inverse = m.idealRsqrt(h_mean + c.rms_norm_eps); + const k_inverse = m.idealRsqrt(k_mean + c.rms_norm_eps); + const rstd: f32 = h_inverse * k_inverse; + const dot_f32: f32 = @floatCast(dot); + const normalized: f32 = (dot_f32 * rstd) * (1 / @sqrt(@as(f32, 5120))); + const gate = m.sigmoid(std.math.copysign(@sqrt(@max(@abs(normalized), 1e-6)), normalized)); + for (0..5120) |d| { + const at = copy * 5120 + d; + h[at] = m.b(m.f(h[at]) + gate * m.f(self.kv[20480 + d])); + } + } + } +}; diff --git a/zig/tests/deepseek_v41_reference/margins.zig b/zig/tests/deepseek_v41_reference/margins.zig new file mode 100644 index 0000000000..2267b17de5 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/margins.zig @@ -0,0 +1,94 @@ +//! Indexer-margin capture: a long prefill, one token, per-layer selection scratch copied verbatim for offline analysis. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const Model = tf.deepseek_v41.Model; +const index = tf.deepseek_v41.csa2_index; +const gpu_counters = tf.deepseek_v41_profiler; + +const copy_source = + \\kernel void margins_copy(const device uint* x [[buffer(0)]], device uint* y [[buffer(1)]], constant uint& n [[buffer(2)]], uint i [[thread_position_in_grid]]) { if(i 0 and self.slots[self.count - 1].layer == layer) return; // the hook fires twice per layer + const work = &model.attention_workspace; + if (work.selection_layer == null or work.selection_layer.? != layer) return; + if (self.count >= self.slots.len) return; + const scratch_bytes = index.Scratch.bytes(work.selection.rows); + const slot = &self.slots[self.count]; + slot.* = .{ .layer = layer, .position = work.selection_position, .rows = work.selection_rows, .offset = @as(u64, self.count) * @as(u64, self.per_layer) }; + self.count += 1; + const uints: u32 = @intCast(scratch_bytes / 4); + enc.setPipeline(self.pipeline); + enc.setBuffer(work.selection.buffer, 0, 0); + enc.setBuffer(self.buffer, @intCast(slot.offset), 1); + enc.setValue(uints, 2); + enc.dispatchThreads(.{ .width = uints }, .{ .width = 1024 }); + } +}; + +pub fn run(gpa: std.mem.Allocator, io: std.Io, model: *Model, ids: []const u32, dir: []const u8, deep: usize) !void { + if (ids.len <= deep + 1) return error.InvalidMarginsPrompt; + var indexer_layers: usize = 0; + for (model.weights.layers) |layer| { + if (layer.attn.indexer != null) indexer_layers += 1; + } + if (indexer_layers == 0) return error.MissingAttentionIndexer; + var capture = try Capture.init(gpa, model, indexer_layers); + defer capture.deinit(); + + try model.reset(); + var at: usize = 0; + while (at < deep) { + const end = @min(at + model.workspace.plan.rows, deep); + _ = try model.forward(ids[at..end], .last); + at = end; + } + const previous = model.observer; + model.observer = .{ .ctx = &capture, .encode = Capture.hook }; + defer model.observer = previous; + _ = try model.forward(&.{ids[deep]}, .last); + + const blob_path = try std.fs.path.join(gpa, &.{ dir, "margins-scratch.bin" }); + defer gpa.free(blob_path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = blob_path, .data = capture.buffer.contents()[0 .. capture.per_layer * capture.count] }); + var meta: std.ArrayList(u8) = .empty; + defer meta.deinit(gpa); + for (capture.slots[0..capture.count]) |slot| { + const line = try std.fmt.allocPrint(gpa, "{{\"layer\":{d},\"position\":{d},\"rows\":{d},\"offset\":{d},\"bytes\":{d}}}\n", .{ slot.layer, slot.position, slot.rows, slot.offset, index.Scratch.bytes(model.attention_workspace.max_rows) }); + defer gpa.free(line); + try meta.appendSlice(gpa, line); + } + const meta_path = try std.fs.path.join(gpa, &.{ dir, "margins-meta.jsonl" }); + defer gpa.free(meta_path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = meta_path, .data = meta.items }); + std.debug.print("margins: {d} layers captured at position {d}\n", .{ capture.count, deep }); +} diff --git a/zig/tests/deepseek_v41_reference/math.zig b/zig/tests/deepseek_v41_reference/math.zig new file mode 100644 index 0000000000..6ba08d1d72 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/math.zig @@ -0,0 +1,400 @@ +//! Independent CPU equations: DeepSeek AI inference/{model,kernel}.py, rev 2cba9e42aa026125f3ed06c6d98c1db82f7ca027. +const std = @import("std"); +const tf = @import("tensorfold"); +const Tensor = tf.checkpoint_metal.Tensor; +const Hc = tf.deepseek_v41_weights.Hc; +const Config = tf.deepseek_v41_config.Text; + +pub fn f(x: u16) f32 { + return @bitCast(@as(u32, x) << 16); +} +pub fn b(x: f32) u16 { + const bits: u32 = @bitCast(x); + if ((bits & 0x7fffffff) > 0x7f800000) return @truncate((bits >> 16) | 0x40); + return @truncate((bits +% 0x7fff +% ((bits >> 16) & 1)) >> 16); +} +pub fn bytes(t: Tensor) []const u8 { + return t.buffer.contents()[t.offset..][0..t.bytes]; +} +pub fn value(t: Tensor, index: usize) f32 { + const raw = bytes(t); + return switch (t.dtype) { + .bf16 => f(std.mem.readInt(u16, raw[index * 2 ..][0..2], .little)), + .f16 => @floatCast(@as(f16, @bitCast(std.mem.readInt(u16, raw[index * 2 ..][0..2], .little)))), + .f32 => @bitCast(std.mem.readInt(u32, raw[index * 4 ..][0..4], .little)), + .f8_e4m3 => e4(raw[index]), + .f8_e8m0 => e8(raw[index]), + else => unreachable, + }; +} +pub fn e4(code: u8) f32 { + const magnitude = code & 127; + if (magnitude == 127) return std.math.nan(f32); + const exponent = magnitude >> 3; + const mantissa = magnitude & 7; + const power: f32 = @bitCast((@as(u32, exponent) + 120) << 23); + const v: f32 = if (exponent == 0) @as(f32, @floatFromInt(mantissa)) / 512 else power * (1 + @as(f32, @floatFromInt(mantissa)) / 8); + return if ((code & 128) != 0) -v else v; +} +pub fn e8(code: u8) f32 { + if (code == 255) return std.math.nan(f32); + if (code == 0) return 0x1p-127; + return @bitCast(@as(u32, code) << 23); +} +pub fn roundE4(x: f32) f32 { + const a = @min(@abs(x), 448); + var low: u8 = 0; + var high: u8 = 126; + while (low < high) { + const mid = low + (high - low + 1) / 2; + if (e4(mid) <= a) low = mid else high = mid - 1; + } + var best = low; + if (low < 126) { + const below = a - e4(low); + const above = e4(low + 1) - a; + if (above < below or (above == below and (low + 1) % 2 == 0)) best += 1; + } + return std.math.copysign(e4(best), x); +} +pub fn roundE2(x: f32) f32 { + const levels = [_]f32{ 0, 0.5, 1, 1.5, 2, 3, 4, 6 }; + var best: usize = 0; + var distance: f32 = std.math.inf(f32); + for (levels, 0..) |v, i| { + const d = @abs(@abs(x) - v); + if (d < distance or (d == distance and i % 2 == 0)) { + best = i; + distance = d; + } + } + return std.math.copysign(levels[best], x); +} +fn powerScale(x: f32) f32 { + const bits: u32 = @bitCast(x); + const exponent = (bits >> 23) & 255; + const rounded = exponent + @as(u32, @intFromBool((bits & 0x7fffff) != 0)); + return @bitCast(rounded << 23); +} +pub const Quant = enum { window, main, index }; +pub fn quantize(x: []u16, kind: Quant) void { + const group: usize = if (kind == .main) 16 else 32; + var start: usize = 0; + while (start < x.len) : (start += group) { + var a: f32 = if (kind == .window) 1e-4 else if (kind == .main) 6 * 0x1p-9 else 6 * 0x1p-126; + for (x[start..][0..group]) |v| a = @max(a, @abs(f(v))); + const scale = if (kind == .main) roundE4(a / 6) else powerScale(a * (if (kind == .window) @as(f32, 1.0 / 448.0) else @as(f32, 1.0 / 6.0))); + for (x[start..][0..group]) |*v| { + const z = f(v.*) / scale; + v.* = b((if (kind == .window) roundE4(z) else roundE2(@max(-6, @min(6, z)))) * scale); + } + } +} +pub fn dense(t: Tensor, x: []const u16, out: []f32) void { + for (out, 0..) |*y, row| { + var sum: f64 = 0; + for (x, 0..) |v, k| sum += @as(f64, f(v)) * value(t, row * x.len + k); + y.* = @floatCast(sum); + } +} +/// CUDA MeanOps scalar projection; the internal reduction remains independently wide. +pub fn meanFromWide(sum: f64, count: usize) f32 { + @setFloatMode(.strict); + const accumulator: f32 = @floatCast(sum); + const factor: f32 = 1 / @as(f32, @floatFromInt(count)); + return accumulator * factor; +} +/// Independent ideal FP32 unary result, not an emulation of CUDA's rsqrt approximation. +pub fn idealRsqrt(x: f32) f32 { + @setFloatMode(.strict); + return @floatCast(1 / @sqrt(@as(f64, x))); +} +pub fn rms(x: []const u16, weight: Tensor, eps: f32, out: []u16) void { + rmsOf(f32, x, weight, eps, out); +} +/// FP64 decode of a BF16 cell, exact at both reference precisions; FP32 is the source contract, FP64 the ideal. +pub fn d64(x: u16) f64 { + return @as(f64, f(x)); +} +/// FP64 to BF16 once (RNE), no FP32 intermediate; RMS at T: contract FP32 mean for f32, wide FP64 mean+rsqrt for f64. +pub fn rmsOf(comptime T: type, x: []const u16, weight: Tensor, eps: f32, out: []u16) void { + @setFloatMode(.strict); + var sum: f64 = 0; + for (x) |v| sum += @as(f64, f(v)) * f(v); + if (T == f32) { + const r = idealRsqrt(meanFromWide(sum, x.len) + eps); + for (x, out, 0..) |v, *y, i| y.* = b(f(v) * r * value(weight, i)); + return; + } + const r = 1 / @sqrt(sum / @as(f64, @floatFromInt(x.len)) + @as(f64, eps)); + for (x, out, 0..) |v, *y, i| y.* = b64(@as(f64, f(v)) * r * @as(f64, value(weight, i))); +} +/// Same boundary with weight cells explicit (decoded BF16 or native FP32), so a harness needs no checkpoint tensor. +pub fn rmsOfValues(comptime T: type, x: []const u16, weight: []const f32, eps: f32, out: []u16) void { + @setFloatMode(.strict); + var sum: f64 = 0; + for (x) |v| sum += @as(f64, f(v)) * f(v); + if (T == f32) { + const r = idealRsqrt(meanFromWide(sum, x.len) + eps); + for (x, weight, 0..) |v, w, i| out[i] = b(f(v) * r * w); + return; + } + const r = 1 / @sqrt(sum / @as(f64, @floatFromInt(x.len)) + @as(f64, eps)); + for (x, weight, 0..) |v, w, i| out[i] = b64(@as(f64, f(v)) * r * @as(f64, w)); +} +pub fn sigmoid(x: f32) f32 { + return 1 / (1 + @exp(-x)); +} +/// Element type widened to the reference precision; FP32 is the source contract, FP64 the ideal. +pub fn wide(comptime T: type, v: f32) T { + return if (T == f32) v else @as(f64, v); +} + +/// Round an FP64 value once to BF16 (round-to-nearest-even) with no intermediate FP32 rounding. +pub fn b64(x: f64) u16 { + const bits: u64 = @bitCast(x); + const sign: u16 = @intCast((bits >> 63) << 15); + const magnitude = bits & 0x7fffffffffffffff; + if (magnitude > 0x7ff0000000000000) return sign | 0x7fc0; // NaN keeps the quiet bit. + if (magnitude >= 0x7ff0000000000000) return sign | 0x7f80; + if (magnitude == 0) return sign; + const exponent: i32 = @intCast(magnitude >> 52); + if (exponent == 0) return sign; // FP64 subnormals sit far below any BF16 subnormal. + const unbiased: i32 = exponent - 1023; + const significand: u64 = (magnitude & 0x000fffffffffffff) | (1 << 52); + if (unbiased > 127) return sign | 0x7f80; + if (unbiased >= -126) { + // Normal BF16 keeps seven fraction bits; RNE the remaining 45. + var kept = significand >> 45; + const round_bit: u64 = (significand >> 44) & 1; + const sticky: u64 = significand & ((@as(u64, 1) << 44) - 1); + if (round_bit == 1 and (sticky != 0 or (kept & 1) == 1)) kept += 1; + if (kept == (1 << 8)) return sign | (@as(u16, @intCast(unbiased + 128)) << 7); + return sign | (@as(u16, @intCast(unbiased + 127)) << 7) | @as(u16, @intCast(kept & 0x7f)); + } + // Subnormal BF16: value = significand * 2^(unbiased-52); representable steps of 2^-133. + const drop: i32 = -81 - unbiased; + if (drop >= 54) return sign; // Below half of the smallest subnormal, including the 2^-134 tie. + var k: u64 = significand >> @intCast(drop); + const round_bit: u64 = (significand >> @intCast(drop - 1)) & 1; + const sticky: u64 = significand & ((@as(u64, 1) << @intCast(drop - 1)) - 1); + if (round_bit == 1 and (sticky != 0 or (k & 1) == 1)) k += 1; + if (k >= (1 << 7)) return sign | (1 << 7); // Rounded up into the minimum normal. + return sign | @as(u16, @intCast(k)); +} + +pub fn MixOf(comptime T: type) type { + return struct { pre: [4]T, post: [4]T, comb: [16]T }; +} +pub const Mix = MixOf(f32); +/// The compensated dense dot keeps FP64 accumulation for both reference precisions. +pub fn denseOf(comptime T: type, t: Tensor, x: []const u16, out: []T) void { + for (out, 0..) |*y, row| { + var sum: f64 = 0; + for (x, 0..) |v, k| sum += @as(f64, f(v)) * value(t, row * x.len + k); + y.* = if (T == f32) @floatCast(sum) else sum; + } +} +fn logistic(comptime T: type, x: T) T { + return 1 / (1 + @exp(-x)); +} +/// The HC RMS statistic inverse: the source-contract FP32 projection for f32, the ideal FP64 for f64. +pub fn inverseOf(comptime T: type, x: []const u16, eps: f32) T { + @setFloatMode(.strict); + var squares: f64 = 0; + for (x) |v| squares += @as(f64, f(v)) * f(v); + if (T == f32) return idealRsqrt(meanFromWide(squares, x.len) + eps); + return 1 / @sqrt(squares / @as(f64, @floatFromInt(x.len)) + eps); +} +pub fn mixes(x: []const u16, hc: Hc, c: Config) Mix { + return mixesOf(f32, x, hc, c); +} +pub fn mixesOf(comptime T: type, x: []const u16, hc: Hc, c: Config) MixOf(T) { + var projected: [24]T = undefined; + denseOf(T, hc.projection, x, &projected); + const r = inverseOf(T, x, c.rms_norm_eps); + return splitOf(T, &projected, r, hc, c); +} +/// The affine, softmax and Sinkhorn coefficient stage at reference precision T. +pub fn splitOf(comptime T: type, projected: []const T, r: T, hc: Hc, c: Config) MixOf(T) { + @setFloatMode(.strict); + var m: MixOf(T) = undefined; + for (0..4) |i| { + m.pre[i] = logistic(T, @mulAdd(T, projected[i] * r, wide(T, value(hc.scale, 0)), wide(T, value(hc.base, i)))) + wide(T, c.hc_eps); + m.post[i] = 2 * logistic(T, @mulAdd(T, projected[i + 4] * r, wide(T, value(hc.scale, 1)), wide(T, value(hc.base, i + 4)))); + } + for (0..4) |i| { + var high: T = -std.math.inf(T); + for (0..4) |j| { + const at = i * 4 + j; + m.comb[at] = @mulAdd(T, projected[at + 8] * r, wide(T, value(hc.scale, 2)), wide(T, value(hc.base, at + 8))); + high = @max(high, m.comb[at]); + } + for (0..4) |j| { + m.comb[i * 4 + j] = @exp(m.comb[i * 4 + j] - high); + } + // TileLang 0.1.8 emits FP32 XOR reductions with offsets 2, then 1. + const denominator = (m.comb[i * 4] + m.comb[i * 4 + 2]) + + (m.comb[i * 4 + 1] + m.comb[i * 4 + 3]); + for (0..4) |j| m.comb[i * 4 + j] = m.comb[i * 4 + j] / denominator + wide(T, c.hc_eps); + } + for (0..c.hc_sinkhorn_iters) |iteration| { + if (iteration != 0) for (0..4) |i| { + const denominator = ((m.comb[i * 4] + m.comb[i * 4 + 2]) + + (m.comb[i * 4 + 1] + m.comb[i * 4 + 3])) + wide(T, c.hc_eps); + for (0..4) |j| m.comb[i * 4 + j] /= denominator; + }; + for (0..4) |j| { + const denominator = ((m.comb[j] + m.comb[8 + j]) + + (m.comb[4 + j] + m.comb[12 + j])) + wide(T, c.hc_eps); + for (0..4) |i| m.comb[i * 4 + j] /= denominator; + } + } + return m; +} +pub fn collapse(h: []const u16, pre: [4]f32, out: []u16) void { + collapseOf(f32, h, pre, out); +} +pub fn collapseOf(comptime T: type, h: []const u16, pre: [4]T, out: []u16) void { + @setFloatMode(.strict); + for (out, 0..) |*v, d| { + var sum: T = 0; + for (0..4) |i| { + const product: T = pre[i] * wide(T, f(h[i * out.len + d])); + sum += product; + } + v.* = if (T == f32) b(sum) else b64(sum); + } +} +pub fn expand(h: []u16, x: []const u16, m: Mix) void { + expandOf(f32, h, x, m); +} +pub fn expandOf(comptime T: type, h: []u16, x: []const u16, m: MixOf(T)) void { + @setFloatMode(.strict); + for (0..x.len) |d| { + var old: [4]T = undefined; + for (0..4) |i| old[i] = wide(T, f(h[i * x.len + d])); + for (0..4) |i| { + var residual: T = 0; + for (0..4) |j| { + const product: T = m.comb[j * 4 + i] * old[j]; + residual += product; + } + h[i * x.len + d] = if (T == f32) b(m.post[i] * wide(T, f(x[d])) + residual) else b64(m.post[i] * wide(T, f(x[d])) + residual); + } + } +} +pub fn ropeFrequency(index: usize, theta: f32, original: u32, factor: f32, beta_fast: f32, beta_slow: f32) f32 { + @setFloatMode(.strict); + const exponent: f32 = @as(f32, @floatFromInt(index * 2)) / 64; + const power: f32 = @floatCast(std.math.pow(f64, theta, exponent)); + var frequency: f32 = 1 / power; + if (original > 0) { + const original_f64: f64 = @floatFromInt(original); + const theta_f64: f64 = theta; + const low: f32 = @floatCast(@max(@floor(64 * @log(original_f64 / + (@as(f64, beta_fast) * 2 * std.math.pi)) / (2 * @log(theta_f64))), 0)); + const high: f32 = @floatCast(@min(@ceil(64 * @log(original_f64 / + (@as(f64, beta_slow) * 2 * std.math.pi)) / (2 * @log(theta_f64))), 63)); + const ramp: f32 = @max(0, @min(1, (@as(f32, @floatFromInt(index)) - low) / @max(high - low, 1e-3))); + const smooth: f32 = 1 - ramp; + frequency = frequency / factor * (1 - smooth) + frequency * smooth; + } + return frequency; +} +pub fn rope(x: []u16, position: usize, compressed: bool, inverse: bool, c: Config) void { + for (0..32) |i| { + const frequency = ropeFrequency(i, if (compressed) c.compress_rope_theta else c.rope_theta, if (compressed) c.rope_scaling.original_max_position_embeddings else 0, c.rope_scaling.factor, c.rope_scaling.beta_fast, c.rope_scaling.beta_slow); + const angle: f32 = @as(f32, @floatFromInt(position)) * frequency; + const cosine = @cos(angle); + const sine = @sin(angle) * (if (inverse) @as(f32, -1) else @as(f32, 1)); + const at = x.len - 64 + 2 * i; + const a = f(x[at]); + const z = f(x[at + 1]); + x[at] = b(a * cosine - z * sine); + x[at + 1] = b(a * sine + z * cosine); + } +} + +test "FP64 to BF16 rounds once and agrees with the FP32 boundary on FP32 values" { + // Every FP32 value rounds identically through either path. + const samples = [_]f32{ 0, 1, -1, 3, 0x1p-21, 0x1p-126, 3.4e38, 0.0371947, 26.5, 26.625, 1.4393715858459473 }; + for (samples) |v| try std.testing.expectEqual(b(v), b64(v)); + // Exact tie rounds to even mantissa. + try std.testing.expectEqual(@as(u16, 0x3f80), b64(1.00390625)); + try std.testing.expectEqual(@as(u16, 0x3f81), b64(1.00390625 + 0x1p-30)); + // The double-rounding trap: FP64 -> FP32 -> BF16 would keep 0x3f80 here. + try std.testing.expectEqual(@as(u16, 0x3f80), b(@floatCast(1.00390625 + 0x1p-30))); + // Subnormal BF16 range and the 2^-134 tie. + try std.testing.expectEqual(@as(u16, 0), b64(0x1p-134)); + try std.testing.expectEqual(@as(u16, 2), b64(1.5 * 0x1p-133)); + try std.testing.expectEqual(@as(u16, 0x7f80), b64(std.math.inf(f64))); + try std.testing.expectEqual(@as(u16, 0xff80), b64(-std.math.inf(f64))); +} + +test "BF16 probability ties and FP4 ties retain source boundaries" { + try std.testing.expectEqual(@as(u16, 0x3f80), b(@bitCast(@as(u32, 0x3f808000)))); + try std.testing.expectEqual(@as(f32, 2), roundE2(2.5)); + try std.testing.expectEqual(@as(f32, 4), roundE2(5)); + try std.testing.expectEqual(@as(f32, 448), roundE4(500)); +} + +test "delayed hyperconnection residual uses comb columns, not rows" { + var h = [_]u16{ b(1), b(2), b(3), b(4) }; + const mix = Mix{ .pre = .{ 1, 0, 0, 0 }, .post = .{ 0, 0, 0, 0 }, .comb = .{ 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0 } }; + expand(&h, &.{b(0)}, mix); + try std.testing.expectEqualSlices(u16, &.{ b(4), b(1), b(2), b(3) }, &h); +} + +test "hyperconnection FP32 accumulation preserves collapse and branch cancellation" { + var h = [_]u16{ b(1), b(1), b(1), b(1) }; + var collapsed: [1]u16 = undefined; + collapse(&h, .{ 1, 0x1p-24, 0x1p-24, -1 }, &collapsed); + try std.testing.expectEqual(@as(u16, 0), collapsed[0]); + const mix = Mix{ .pre = .{ 1, 0, 0, 0 }, .post = .{ -1, 0, 0, 0 }, .comb = .{ 1, 0, 0, 0, 0x1p-24, 0, 0, 0, 0x1p-24, 0, 0, 0, 0, 0, 0, 0 } }; + expand(&h, &.{b(1)}, mix); + try std.testing.expectEqual(@as(u16, 0), h[0]); +} + +test "hyperconnection rounds elementwise products before cancelling reductions" { + var h = [_]u16{ b(3), b(3), b(0), b(0) }; + const coefficient: f32 = 1 + 0x1p-23; + var collapsed: [1]u16 = undefined; + collapse(&h, .{ -1, coefficient, 0, 0 }, &collapsed); + try std.testing.expectEqual(b(0x1p-21), collapsed[0]); + const mix = Mix{ .pre = .{ 1, 0, 0, 0 }, .post = .{ 0, 0, 0, 0 }, .comb = .{ -1, 0, 0, 0, coefficient, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; + expand(&h, &.{b(0)}, mix); + try std.testing.expectEqual(b(0x1p-21), h[0]); + h = .{ b(3), b(0), b(0), b(0) }; + const branch_mix = Mix{ .pre = .{ 1, 0, 0, 0 }, .post = .{ coefficient, 0, 0, 0 }, .comb = .{ -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; + expand(&h, &.{b(3)}, branch_mix); + try std.testing.expectEqual(b(0x1p-21), h[0]); +} + +test "RoPE and YaRN frequency vectors retain official FP32 elementwise boundaries" { + const parsed = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, @embedFile("rope_frequencies.json"), .{}); + defer parsed.deinit(); + for (parsed.value.object.get("vectors").?.array.items) |vector| { + const theta: f32 = @floatFromInt(vector.object.get("theta").?.integer); + const original: u32 = @intCast(vector.object.get("original").?.integer); + for (vector.object.get("bits").?.array.items, 0..) |expected, index| { + const actual = ropeFrequency(index, theta, original, 16, 32, 1); + const bits: u32 = @intCast(expected.integer); + const source: f32 = @bitCast(bits); + try std.testing.expectApproxEqAbs(source, actual, source * std.math.floatEps(f32)); + } + } +} + +test "ideal FP32 inverse square root avoids an intermediate square-root rounding" { + // Actual CUDA agrees on this HC RMS argument; CPU/MPS's two-round result is one ULP higher. + try std.testing.expectEqual(@as(f32, 0.3651483356952667), idealRsqrt(7.500000953674316)); +} + +test "CUDA mean projection preserves the BF16 RMS affine tie" { + @setFloatMode(.strict); + // Constructed exact-sum row isolates canonical CUDA RMSNorm mean projection through the BF16 affine tie. + const gamma: f32 = @bitCast(@as(u32, 0x3c65de29)); + const inverse = idealRsqrt(meanFromWide(9, 5120) + @as(f32, 1e-20)); + try std.testing.expectEqual(b(1), b((@as(f32, 3) * inverse) * gamma)); +} diff --git a/zig/tests/deepseek_v41_reference/operator_attention.zig b/zig/tests/deepseek_v41_reference/operator_attention.zig new file mode 100644 index 0000000000..73e94641c3 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_attention.zig @@ -0,0 +1,155 @@ +//! Single-position retained-input attention isolation. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const projection = @import("projection.zig"); +const math = @import("math.zig"); +const attention = tf.deepseek_v41.attention; +const Model = tf.deepseek_v41.Model; +const diagnostic = @import("operator_diagnostic.zig"); +const diagnosticRow = diagnostic.diagnosticRow; +const attentionMetric = diagnostic.attentionMetric; +const attention_math = @import("attention_math.zig"); +const AttentionStage = diagnostic.AttentionStage; +const AttentionCapture = diagnostic.AttentionCapture; +const attention_elements = diagnostic.attention_elements; +const attentionProjection = diagnostic.attentionProjection; +const hyperconnections = @import("operator_hc.zig").hyperconnections; + +/// Local isolation receipt, not a replacement for the full eight-token oracle or its 1% gate. +pub fn runAttention(gpa: std.mem.Allocator, io: std.Io, model: *Model, snapshot: []const u8, dir: []const u8, layer: usize) !void { + const blob = try std.Io.Dir.cwd().readFileAlloc(io, snapshot, gpa, .limited(8 << 20)); + defer gpa.free(blob); + const cfg = model.cfg.text_config; + if (layer >= model.weights.layers.len) return error.InvalidAttentionLayer; + const snapshot_offset = layer * 2 * 61488; + if (blob.len < snapshot_offset + 40960 + 10240) return error.InvalidAttentionSnapshot; + try model.reset(); + const input = model.workspace.view(.normalized); + @memcpy(input.buffer.contents()[input.offset..][0..10240], blob[snapshot_offset + 40960 ..][0..10240]); + const input_row = @as([*]const u16, @ptrCast(@alignCast(input.buffer.contents() + input.offset)))[0..5120]; + const native = try model.device.buffer(attention_elements * 2, mtl.ResourceOptions.shared); + defer native.deinit(); + const reference_rows = try gpa.alloc(u16, attention_elements); + defer gpa.free(reference_rows); + var capture = AttentionCapture{ .model = model, .native = native, .reference = reference_rows }; + model.residency.add(native); + model.residency.commit(); + model.residency.requestResidency(); + defer { + model.residency.remove(native); + model.residency.commit(); + } + var scratch = try projection.Scratch.init(gpa, 8192, 32768); + defer scratch.deinit(); + var state = try attention_math.State.init(gpa, 1); + defer state.deinit(); + var output: [5120]u16 = undefined; + // Replay prerequisite normalized layers; complete each GPU command before reusing the CPU input buffer. + for (0..layer) |prior| { + const prior_offset = prior * 2 * 61488 + 40960; + @memcpy(input.buffer.contents()[input.offset..][0..10240], blob[prior_offset..][0..10240]); + const warm_cb = model.queue.commandBufferUnretained(); + const warm_enc = warm_cb.compute(.serial); + var warm_encoding = true; + errdefer if (warm_encoding) warm_enc.end(); + for (model.checkpoint.shards.items) |shard| warm_enc.useResource(shard.buffer, mtl.ResourceUsage.read); + try attention.encode(&model.pipelines.attention, warm_enc, &model.weights.layers[prior].attn, &model.cache, &model.attention_workspace, input, model.workspace.view(.branch), &model.pipelines.projection, model.workspace.projection(), @intCast(prior), 1, cfg); + warm_enc.end(); + warm_encoding = false; + warm_cb.commit(); + warm_cb.wait(); + if (warm_cb.failure() != null) return error.OperatorGpuFailure; + try state.apply(model.weights.layers[prior].attn, prior, input_row, 0, cfg, &output, &scratch); + } + @memcpy(input.buffer.contents()[input.offset..][0..10240], blob[snapshot_offset + 40960 ..][0..10240]); + const weights = &model.weights.layers[layer].attn; + const cb = model.queue.commandBufferUnretained(); + const enc = cb.compute(.serial); + var encoding = true; + errdefer if (encoding) enc.end(); + for (model.checkpoint.shards.items) |shard| enc.useResource(shard.buffer, mtl.ResourceUsage.read); + try attention.encodeDiagnostic(&model.pipelines.attention, enc, weights, &model.cache, &model.attention_workspace, input, model.workspace.view(.branch), &model.pipelines.projection, model.workspace.projection(), @intCast(layer), 1, cfg, true, .{ .ctx = &capture, .capture = AttentionCapture.gpu }); + enc.end(); + encoding = false; + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.OperatorGpuFailure; + const sparse_snapshot = try gpa.alloc(u16, 32768); + defer gpa.free(sparse_snapshot); + state.sparse_snapshot = sparse_snapshot; + state.observer = .{ .ctx = &capture, .capture = AttentionCapture.cpu }; + try state.apply(weights.*, layer, input_row, 0, cfg, &output, &scratch); + state.observer = null; + state.sparse_snapshot = null; + + var report: std.ArrayList(u8) = .empty; + defer report.deinit(gpa); + var count: usize = 0; + var header: [512]u8 = undefined; + try report.appendSlice(gpa, try std.fmt.bufPrint(&header, "{{\"mode\":\"attention-sameinput\",\"token\":0,\"layer\":{d},\"input_offset_bytes\":{d},\"input_length_bytes\":10240,\"identical_normalized_input\":true,\"retained_normalized_prefix_layers\":{d},\"compression_ratio\":{d},\"reference_dot\":\"independent_EXL3_FP64\",\"qualification_gate_applied\":false,\"cases\":[\n", .{ layer, snapshot_offset + 40960, layer, cfg.compress_ratios[layer] })); + inline for (std.enums.values(AttentionStage)) |stage| + try attentionMetric(u16, gpa, &report, &count, @tagName(stage), "same_normalized_input_chain", capture.row(stage), capture.cpuRow(stage)); + + const expected = try gpa.alloc(f32, 32768); + defer gpa.free(expected); + try attentionProjection(gpa, model, weights.wq_a, input, "wq_a", &scratch, expected, &report, &count); + try attentionProjection(gpa, model, weights.wq_b, capture.view(.qr_norm), "wq_b", &scratch, expected, &report, &count); + try attentionProjection(gpa, model, weights.wkv, input, "wkv", &scratch, expected, &report, &count); + for (weights.wo_a, 0..) |matrix, group| { + var name: [32]u8 = undefined; + try attentionProjection(gpa, model, matrix, capture.view(.o).shifted(group * 4096 * 2), try std.fmt.bufPrint(&name, "wo_a_group{d}", .{group}), &scratch, expected, &report, &count); + } + try attentionProjection(gpa, model, weights.wo_b, capture.view(.grouped), "wo_b", &scratch, expected, &report, &count); + + var same_input: [32768]u16 = undefined; + math.rms(capture.row(.qr_projection), weights.q_norm, cfg.rms_norm_eps, same_input[0..1280]); + try attentionMetric(u16, gpa, &report, &count, "qr_norm", "same_native_input_reference_rms", capture.row(.qr_norm), same_input[0..1280]); + math.rms(capture.row(.kv_projection), weights.kv_norm, cfg.rms_norm_eps, same_input[0..512]); + try attentionMetric(u16, gpa, &report, &count, "kv_norm", "same_native_input_reference_rms", capture.row(.kv_norm), same_input[0..512]); + @memcpy(same_input[0..512], capture.row(.kv_rope)); + math.quantize(same_input[0..512], .window); + try attentionMetric(u16, gpa, &report, &count, "kv_quantized", "same_native_input_reference_quantize", capture.row(.kv_quantized), same_input[0..512]); + if (cfg.compress_ratios[layer] == 1) { + const source = try model.cache.source(@intCast(layer), 1); + const compressed = model.workspace.view(.collapsed); + const unpack_cb = model.queue.commandBufferUnretained(); + const unpack_enc = unpack_cb.compute(.serial); + var unpack_encoding = true; + errdefer if (unpack_encoding) unpack_enc.end(); + try model.pipelines.attention.cache.encodeUnpack(unpack_enc, source.main, compressed, .{ .rows = 1, .width = 512, .stride = 512, .packed_stride = 288, .block = 16, .kind = .main_fp4 }); + unpack_enc.end(); + unpack_encoding = false; + unpack_cb.commit(); + unpack_cb.wait(); + if (unpack_cb.failure() != null) return error.OperatorGpuFailure; + const actual_compressed = diagnosticRow(u16, compressed, 512); + const reference_compressed = state.sources[state.active_owner * state.capacity * 640 ..][0..512]; + try attentionMetric(u16, gpa, &report, &count, "compressed_kv", "same_retained_normalized_prefix", actual_compressed, reference_compressed); + var sinks: [64]f32 = undefined; + for (&sinks, 0..) |*v, head| v.* = math.value(weights.sink, head); + inline for (.{ "attention-compressed-native.bf16", "attention-compressed-reference.bf16", "attention-sinks.f32" }, .{ std.mem.sliceAsBytes(actual_compressed), std.mem.sliceAsBytes(reference_compressed), std.mem.sliceAsBytes(&sinks) }) |name, data| { + const path = try std.fs.path.join(gpa, &.{ dir, name }); + defer gpa.free(path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = data }); + } + @memcpy(reference_compressed, actual_compressed); + const indices = [_]i32{ 0, 128 }; + for (0..64) |head| + state.sparseHead(capture.row(.q)[head * 512 ..][0..512], capture.row(.kv_quantized), &indices, sinks[head], same_input[head * 512 ..][0..512]); + try attentionMetric(u16, gpa, &report, &count, "sparse_o", "same_native_q_window_and_compressed_kv_reference_sparse", capture.row(.sparse_o), &same_input); + } + if (cfg.compress_ratios[layer] == 0) { + const indices = [_]i32{0}; + for (0..64) |head| + state.sparseHead(capture.row(.q)[head * 512 ..][0..512], capture.row(.kv_quantized), &indices, math.value(weights.sink, head), same_input[head * 512 ..][0..512]); + try attentionMetric(u16, gpa, &report, &count, "sparse_o", "same_native_q_and_quantized_kv_reference_sparse", capture.row(.sparse_o), &same_input); + } + try hyperconnections(gpa, model, blob[snapshot_offset..], capture.view(.wo_b), &report, &count, layer); + try report.appendSlice(gpa, "\n]}\n"); + inline for (.{ "attention-sameinput.json", "attention-native.bf16", "attention-reference.bf16", "attention-input.bf16" }, .{ report.items, std.mem.sliceAsBytes(native.slice(u16, attention_elements)), std.mem.sliceAsBytes(reference_rows), std.mem.sliceAsBytes(input_row) }) |name, data| { + const path = try std.fs.path.join(gpa, &.{ dir, name }); + defer gpa.free(path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = data }); + } +} diff --git a/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig b/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig new file mode 100644 index 0000000000..113e602c2e --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_csa2_fp64.zig @@ -0,0 +1,298 @@ +//! Shared CSA2 FP64 ideal-reference harness; receipts print "CSA64" (DeepSeek-V4.1-Flash inference, MIT). +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const math = @import("math.zig"); +const attention_math = @import("attention_math.zig"); + +const kv = tf.deepseek_v41.kv; +const csa2 = tf.deepseek_v41.csa2; +pub const View = tf.deepseek_v41.attention.View; + +pub const width: usize = 512; +const ratio: usize = 2; +const eps: f32 = 0.000001; + +/// The local FP32 spacing at the FP64 reference value: the ULP unit for kernel deltas. +fn nextUpF32(x: f32) f32 { + if (std.math.isNan(x) or std.math.isInf(x)) return x; + const bits: u32 = @bitCast(x); + if (bits == 0 or bits == 0x80000000) return @bitCast(@as(u32, 1)); + return @bitCast(if (bits >> 31 == 0) bits + 1 else bits - 1); +} + +fn ulp32At(v: f64) f64 { + const f: f32 = @floatCast(v); + if (std.math.isNan(f) or std.math.isInf(f)) return @abs(v); + const a = @abs(f); + if (a == 0) return 0x1p-149; + const up = nextUpF32(a); + if (std.math.isInf(up)) return @as(f64, a) - @as(f64, -nextUpF32(-a)); + return @as(f64, up) - @as(f64, a); +} + +/// A BF16-cell comparison: NRMS against the ideal, and the count of cells the two sides disagree on. +pub const BfDelta = struct { + count: usize = 0, + nrms: f64 = 0, + unequal: usize = 0, + worst: usize = 0, + worst_relative: f64 = 0, +}; + +pub fn deltaBf16(actual: []const u16, ideal: []const f64) BfDelta { + var d: BfDelta = .{ .count = actual.len }; + var squared: f64 = 0; + var magnitude: f64 = 0; + for (actual, ideal, 0..) |cell, r, i| { + const a: f64 = math.f(cell); + const diff = a - r; + squared += diff * diff; + magnitude += r * r; + const rel = @abs(diff) / @max(@abs(r), 0x1p-126); + if (cell != math.b64(r)) { + d.unequal += 1; + if (rel > d.worst_relative) { + d.worst_relative = rel; + d.worst = i; + } + } + } + d.nrms = if (magnitude == 0) @sqrt(squared) else @sqrt(squared / magnitude); + return d; +} + +/// An FP32-cell comparison, used for pre-rounding stage boundaries that stay in FP32. +pub const Delta = struct { + count: usize = 0, + nrms: f64 = 0, + max_abs: f64 = 0, + max_ulp: f64 = 0, + worst: usize = 0, +}; + +pub fn deltaF32VsF64(actual: []const f32, ideal: []const f64) Delta { + var d: Delta = .{ .count = actual.len }; + var squared: f64 = 0; + var magnitude: f64 = 0; + for (actual, ideal, 0..) |a, r, i| { + const diff = @as(f64, a) - r; + squared += diff * diff; + magnitude += r * r; + const ad = @abs(diff); + if (ad > d.max_abs) { + d.max_abs = ad; + d.worst = i; + } + const u = ad / ulp32At(r); + if (u > d.max_ulp) d.max_ulp = u; + } + d.nrms = if (magnitude == 0) @sqrt(squared) else @sqrt(squared / magnitude); + return d; +} + +pub fn printBf(case_name: []const u8, stage: []const u8, label: []const u8, d: BfDelta) void { + std.debug.print( + \\CSA64 {{"case":"{s}","stage":"{s}","side":"{s}","cells":{d},"nrms_vs_ideal":{d},"bf16_unequal_vs_ideal":{d},"worst_index":{d},"worst_relative":{d}}} + , .{ case_name, stage, label, d.count, d.nrms, d.unequal, d.worst, d.worst_relative }); +} + +pub fn printF32(case_name: []const u8, stage: []const u8, label: []const u8, d: Delta) void { + std.debug.print("CSA64 {{\"case\":\"{s}\",\"stage\":\"{s}\",\"side\":\"{s}\",\"cells\":{d}," ++ + "\"nrms_vs_ideal\":{d},\"max_abs_vs_ideal\":{d},\"max_ulp_vs_ideal\":{d:.3}," ++ + "\"worst_index\":{d}}}\n", .{ case_name, stage, label, d.count, d.nrms, d.max_abs, d.max_ulp, d.worst }); +} + +pub const Lcg = struct { + state: u64 = 0x853c49e6748fea9b, + pub fn next(s: *Lcg) u64 { + s.state = s.state *% 6364136223846793005 +% 1442695040888963407; + return s.state >> 11; + } + /// Uniform [-1, 1) with 24-bit granularity, exactly representable in FP32. + pub fn uniform(s: *Lcg) f32 { + const v: f64 = @as(f64, @floatFromInt(s.next() & 0xffffff)) / 16777216.0; + return @floatCast(v * 2 - 1); + } + pub fn scale(s: *Lcg, amplitude: f32) f32 { + return s.uniform() * amplitude; + } +}; + +pub const Buffers = struct { + device: mtl.Device, + items: [16]mtl.Buffer = undefined, + count: usize = 0, + pub fn get(b: *Buffers, comptime T: type, n: usize) !mtl.Buffer { + const count = @max(n, 1); + const x = try b.device.buffer(try std.math.mul(usize, @sizeOf(T), count), mtl.ResourceOptions.shared); + b.items[b.count] = x; + b.count += 1; + @memset(x.slice(T, n), 0); + return x; + } + pub fn deinit(b: *Buffers) void { + for (b.items[0..b.count]) |x| x.deinit(); + } +}; + +pub fn view(buffer: mtl.Buffer) View { + return .{ .buffer = buffer }; +} + +/// One compressor-pool group: two projected ratio-2 positions, pooled then RMS-normed from FP32 wkv/wgate outputs. +fn poolCase(device: mtl.Device, queue: mtl.Queue, p: *const kv.Pipelines, case_name: []const u8, values: []const f32, gates: []const f32, weight: []const u16) !void { + var b = Buffers{ .device = device }; + defer b.deinit(); + const values_gpu = try b.get(f32, 2 * width); + const gates_gpu = try b.get(f32, 2 * width); + const partial_gpu = try b.get(f32, 2 * ratio * width); + const weight_gpu = try b.get(u16, width); + const latent_gpu = try b.get(u16, width); + @memcpy(values_gpu.slice(f32, 2 * width), values); + @memcpy(gates_gpu.slice(f32, 2 * width), gates); + @memcpy(weight_gpu.slice(u16, width), weight); + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encodePool(enc, view(values_gpu), view(gates_gpu), view(partial_gpu), view(weight_gpu), view(latent_gpu), .{ .rows = 2, .width = @intCast(width), .ratio = @intCast(ratio), .position = 2, .stride = @intCast(width), .input_f32 = 1, .eps = eps }); + enc.end(); + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.Csa2Fp64GpuFailure; + const metal = latent_gpu.slice(u16, width); + + // Both references: the same pool, then the same RMS boundary, each at its own precision. + var ref32: [width]u16 = undefined; + var ref64: [width]u16 = undefined; + var pooled32: [width]u16 = undefined; + var pooled64: [width]u16 = undefined; + attention_math.poolOf(f32, .{ values[0..width], values[width .. 2 * width] }, .{ gates[0..width], gates[width .. 2 * width] }, &pooled32); + var wide_values: [2][width]f64 = undefined; + var wide_gates: [2][width]f64 = undefined; + for (0..2) |slot| wide_values[slot] = widen(values[slot * width ..][0..width]); + for (0..2) |slot| wide_gates[slot] = widen(gates[slot * width ..][0..width]); + attention_math.poolOf(f64, .{ &wide_values[0], &wide_values[1] }, .{ &wide_gates[0], &wide_gates[1] }, &pooled64); + var cells: [width]f32 = undefined; + for (weight, 0..) |cell, i| cells[i] = math.f(cell); + math.rmsOfValues(f32, &pooled32, &cells, eps, &ref32); + math.rmsOfValues(f64, &pooled64, &cells, eps, &ref64); + + const ideal = widenCells(&ref64); + var from_f32 = widenCells(&ref32); + var magnitude: f64 = 0; + for (ideal) |v| magnitude += v * v; + std.debug.print( + \\CSA64 {{"case":"{s}","stage":"pool","side":"scale","ideal_rms":{d}}} + , .{ case_name, @sqrt(magnitude / @as(f64, width)) }); + printBf(case_name, "pool", "metal", deltaBf16(metal, &ideal)); + printBf(case_name, "pool", "f32_reference", deltaBf16(&ref32, &ideal)); + printBf(case_name, "pool", "metal_vs_f32_reference", deltaBf16(metal, &from_f32)); +} + +fn widen(cells: []const f32) [width]f64 { + var out: [width]f64 = undefined; + for (cells, 0..) |v, i| out[i] = @as(f64, v); + return out; +} + +fn widenCells(cells: *const [width]u16) [width]f64 { + var out: [width]f64 = undefined; + for (cells, 0..) |cell, i| out[i] = math.f(cell); + return out; +} + +test "compressor pool stage sits at rounding level against the FP64 ideal" { + var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_instance.deinit(); + const gpa = arena_instance.allocator(); + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try kv.Pipelines.load(device); + defer p.deinit(); + + var rng = Lcg{ .state = 0x9e3779b97f4a7c15 }; + // Case 1: typical magnitudes; gates straddle each other so the pool weights split. + { + const values = try gpa.alloc(f32, 2 * width); + const gates = try gpa.alloc(f32, 2 * width); + for (0..2 * width) |i| values[i] = rng.scale(1.0); + for (0..2 * width) |i| gates[i] = rng.scale(8.0); + try poolCase(device, queue, &p, "typical", values, gates, try weightCells(gpa, &rng)); + } + // Case 2: cancellation - the two pooled terms cancel to a few ULPs of the larger one. + { + const values = try gpa.alloc(f32, 2 * width); + const gates = try gpa.alloc(f32, 2 * width); + for (0..width) |d| { + const large = rng.scale(64.0); + const small = rng.scale(0.001); + values[d] = large + small; + values[width + d] = -large + rng.scale(0.001); + } + for (0..2 * width) |i| gates[i] = rng.scale(0.5); + try poolCase(device, queue, &p, "cancellation", values, gates, try weightCells(gpa, &rng)); + } + // Case 3: wide dynamic range, so one slot dominates and the other is far below it. + { + const values = try gpa.alloc(f32, 2 * width); + const gates = try gpa.alloc(f32, 2 * width); + for (0..2 * width) |i| { + const exponent = @as(f32, @floatFromInt(rng.next() % 60)) - 30; + values[i] = std.math.pow(f32, 2, exponent) * rng.uniform(); + gates[i] = rng.scale(3.0); + } + try poolCase(device, queue, &p, "wide_range", values, gates, try weightCells(gpa, &rng)); + } + // Case 4: deep cancellation, 1e-7 of the terms it cancels: FP32 error is a visible fraction. + { + const values = try gpa.alloc(f32, 2 * width); + const gates = try gpa.alloc(f32, 2 * width); + for (0..width) |d| { + const large = rng.scale(64.0); + values[d] = large * (1 + rng.scale(1e-7)); + values[width + d] = -large * (1 + rng.scale(1e-7)); + } + for (0..2 * width) |i| gates[i] = rng.scale(0.5); + try poolCase(device, queue, &p, "cancel_1e7", values, gates, try weightCells(gpa, &rng)); + } + // Case 5: gate skew that pushes one softmax weight into FP32 subnormal territory. + { + const values = try gpa.alloc(f32, 2 * width); + const gates = try gpa.alloc(f32, 2 * width); + for (0..2 * width) |i| values[i] = rng.scale(1.0); + for (0..width) |d| { + const base = rng.scale(4.0); + gates[d] = base; + gates[width + d] = base - 85 - @as(f32, @floatFromInt(rng.next() % 8)); + } + try poolCase(device, queue, &p, "gate_skew", values, gates, try weightCells(gpa, &rng)); + } + // Case 6: near-equal gates so the weights are both near one half, with large cancelling values. + { + const values = try gpa.alloc(f32, 2 * width); + const gates = try gpa.alloc(f32, 2 * width); + const shared = try gpa.alloc(f32, width); + for (0..width) |d| shared[d] = rng.scale(32.0); + for (0..width) |d| { + values[d] = shared[d] + rng.scale(1e-4); + values[width + d] = -shared[d] + rng.scale(1e-4); + gates[d] = rng.scale(0.01); + gates[width + d] = rng.scale(0.01); + } + try poolCase(device, queue, &p, "balanced_cancel", values, gates, try weightCells(gpa, &rng)); + } +} + +// Weight cells for the compressor-pool stage: near-unit magnitudes with no exponent extremes. +pub fn weightCells(gpa: std.mem.Allocator, rng: *Lcg) ![]u16 { + const out = try gpa.alloc(u16, width); + for (out) |*cell| { + const amplitude: f32 = 0.5 + 0.25 * @as(f32, @floatFromInt(rng.next() % 1000)) / 1000; + cell.* = math.b(amplitude); + } + return out; +} diff --git a/zig/tests/deepseek_v41_reference/operator_csa2_index_fp64.zig b/zig/tests/deepseek_v41_reference/operator_csa2_index_fp64.zig new file mode 100644 index 0000000000..3fa6e1b299 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_csa2_index_fp64.zig @@ -0,0 +1,252 @@ +//! CSA2 indexer-scoring FP64 ideal-reference test (DeepSeek-V4.1-Flash inference, MIT). +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const math = @import("math.zig"); +const h = @import("operator_csa2_fp64.zig"); +const pack = @import("operator_csa2_pack_fp64.zig"); + +/// One indexer-score row set: 32 heads of 128 FP4-index columns each, scored against packed keys. +const IndexSpec = struct { name: []const u8, rows: usize, keys: usize, topk: usize }; + +fn packIndexRow(values: []const f32, out: []u8) void { + @memset(out[0 .. 128 / 2 + 128 / 32], 0); + for (0..128 / 32) |block| { + var a: f32 = 6.0 * 0x1p-126; + for (values[block * 32 ..][0..32]) |v| a = @max(a, @abs(v)); + const bits: u32 = @bitCast(a / 6.0); + const unbiased: i32 = @as(i32, @intCast((bits >> 23) & 255)) - 127 + @intFromBool((bits & 0x7fffff) != 0); + const scale = std.math.ldexp(@as(f32, 1.0), unbiased); + out[64 + block] = @intCast(unbiased + 127); + for (0..32) |j| { + const code = pack.e2Code(values[block * 32 + j] / scale); + const at = block * 16 + j / 2; + if ((j & 1) != 0) out[at] |= code << 4 else out[at] |= code; + } + } +} + +fn decodeIndexRow(bytes: []const u8) [128]u16 { + var row: [128]u16 = undefined; + for (0..128) |d| { + const nibble: u8 = if ((d & 1) != 0) bytes[d / 2] >> 4 else bytes[d / 2] & 15; + const scale: f32 = @bitCast(@as(u32, bytes[64 + d / 32]) << 23); + row[d] = math.b(pack.e2(nibble) * scale); + } + return row; +} + +fn indexCase( + device: mtl.Device, + queue: mtl.Queue, + p: *const tf.deepseek_v41.csa2_index.Pipelines, + spec: IndexSpec, + q: []const u16, + weights: []const u16, + key_values: []const []const f32, +) !void { + var b = h.Buffers{ .device = device }; + defer b.deinit(); + var scratch = try tf.deepseek_v41.csa2_index.Scratch.init(device, @intCast(spec.rows)); + defer scratch.deinit(); + const q_gpu = try b.get(u16, spec.rows * 32 * 128); + const weights_gpu = try b.get(u16, spec.rows * 32); + const keys_gpu = try b.get(u8, spec.keys * 68); + @memcpy(q_gpu.slice(u16, spec.rows * 32 * 128), q); + @memcpy(weights_gpu.slice(u16, spec.rows * 32), weights); + + // Both sides read the same decoded keys; the references differ only in their arithmetic. + const decoded = try std.testing.allocator.alloc(u16, spec.keys * 128); + defer std.testing.allocator.free(decoded); + for (key_values, 0..) |values, row| { + const bytes = try std.testing.allocator.alloc(u8, 68); + defer std.testing.allocator.free(bytes); + packIndexRow(values, bytes); + @memcpy(keys_gpu.slice(u8, spec.keys * 68)[row * 68 ..][0..68], bytes); + const decoded_row = decodeIndexRow(bytes); + @memcpy(decoded[row * 128 ..][0..128], &decoded_row); + } + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encode(enc, h.view(q_gpu), h.view(weights_gpu), h.view(keys_gpu), scratch, .{ + .rows = @intCast(spec.rows), + .keys = @intCast(spec.keys), + .q_stride = 32 * 128, + .weight_stride = 32, + .position = @intCast(spec.keys), + .ratio = 1, + }); + enc.end(); + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.Csa2Fp64GpuFailure; + + // Positions live behind the scores/heap/candidates regions; read them through the view offset. + const kernel_scores = scratch.scores().buffer.slice(f32, spec.rows * 4096); + const positions_offset = scratch.positions().offset / @sizeOf(i32); + const kernel_positions = scratch.positions().buffer.slice(i32, positions_offset + spec.rows * 512); + var ref32 = try std.testing.allocator.alloc(f32, spec.keys); + defer std.testing.allocator.free(ref32); + var ref64 = try std.testing.allocator.alloc(f64, spec.keys); + defer std.testing.allocator.free(ref64); + // The same scores with only the documented quantizations below the final one: the BF16 quantum stays. + var exact = try std.testing.allocator.alloc(f64, spec.keys); + defer std.testing.allocator.free(exact); + const w_scale: f32 = 1 / @sqrt(@as(f32, 128 * 32)); + for (0..spec.keys) |row| { + const key = decoded[row * 128 ..][0..128]; + var ideal: f64 = 0; + var narrow32: f32 = 0; + for (0..32) |head| { + var dot: f64 = 0; + for (0..128) |d| dot += @as(f64, math.f(key[d])) * @as(f64, math.f(q[head * 128 + d])); + // The source head einsum lands in BF16, so the score is a quantized boundary like the cache codes. + const rectified: f64 = @max(math.f(math.b(@min(@as(f32, @floatCast(dot)), 1e38))), 0); + const weight: f32 = math.f(weights[head]); + const scaled_weight = math.f(math.b(weight * w_scale)); + const weighted = math.f(math.b(@as(f32, @floatCast(rectified)) * scaled_weight)); + ideal += weighted; + narrow32 += @as(f32, @floatCast(weighted)); + } + exact[row] = ideal; + // The kernel and the source contract write the score back through BF16, so the ideal carries it too. + ref64[row] = math.f(math.b64(ideal)); + ref32[row] = math.f(math.b(narrow32)); + } + var ideal_cells: [1024]f64 = undefined; + const shown = @min(spec.keys, ideal_cells.len); + for (0..shown) |row| ideal_cells[row] = ref64[row]; + h.printF32(spec.name, "index_score", "bf16_score_boundary", h.deltaF32VsF64(ref32[0..shown], exact[0..shown])); + h.printF32(spec.name, "index_score", "metal_vs_ideal", h.deltaF32VsF64(kernel_scores[0..shown], ideal_cells[0..shown])); + h.printF32(spec.name, "index_score", "f32_vs_ideal", h.deltaF32VsF64(ref32[0..shown], ideal_cells[0..shown])); + // The kernel against the source-contract FP32 reference on identical keys: the delta selection can see. + var contract: [1024]f64 = undefined; + for (0..shown) |row| contract[row] = @as(f64, ref32[row]); + h.printF32(spec.name, "index_score", "metal_vs_f32_reference", h.deltaF32VsF64(kernel_scores[0..shown], contract[0..shown])); + + // Selection: which keys land in the top-k; the kernel heap and strict-greater scan break ties by position. + var chosen = try std.testing.allocator.alloc(bool, spec.keys); + defer std.testing.allocator.free(chosen); + @memset(chosen, false); + var scores_copy = try std.testing.allocator.alloc(f32, spec.keys); + defer std.testing.allocator.free(scores_copy); + @memcpy(scores_copy, ref32[0..spec.keys]); + var reference_picks: [512]u32 = undefined; + const picks = @min(spec.topk, spec.keys); + for (0..picks) |pick| { + var best: usize = 0; + for (0..spec.keys) |i| { + if (scores_copy[i] > scores_copy[best]) best = i; + } + reference_picks[pick] = @intCast(best); + scores_copy[best] = -std.math.inf(f32); + } + std.mem.sort(u32, reference_picks[0..picks], {}, std.sort.asc(u32)); + var overlap: usize = 0; + for (reference_picks[0..picks]) |at| chosen[at] = true; + var min_selected: f32 = std.math.inf(f32); + var max_unselected: f32 = -std.math.inf(f32); + for (0..spec.keys) |row| { + if (chosen[row]) { + overlap += 1; + min_selected = @min(min_selected, ref32[row]); + } else max_unselected = @max(max_unselected, ref32[row]); + } + var kernel_only: usize = 0; + var out_of_range: usize = 0; + for (0..picks) |i| { + const slot = kernel_positions[positions_offset + i]; + if (slot < 0 or slot >= @as(i32, @intCast(spec.keys))) { + out_of_range += 1; + continue; + } + if (!chosen[@intCast(slot)]) kernel_only += 1; + } + // Arithmetic noise selection absorbs: a pick is knife-edged at one score quantum of margin; else no 1-ULP flip. + var exact_picks: [512]u32 = undefined; + var exact_scores = try std.testing.allocator.alloc(f64, spec.keys); + defer std.testing.allocator.free(exact_scores); + @memcpy(exact_scores, exact[0..spec.keys]); + for (0..picks) |pick| { + var best: usize = 0; + for (0..spec.keys) |i| { + if (exact_scores[i] > exact_scores[best]) best = i; + } + exact_picks[pick] = @intCast(best); + exact_scores[best] = -std.math.inf(f64); + } + var flips: usize = 0; + var exact_chosen = try std.testing.allocator.alloc(bool, spec.keys); + defer std.testing.allocator.free(exact_chosen); + @memset(exact_chosen, false); + for (exact_picks[0..picks]) |at| exact_chosen[at] = true; + for (0..spec.keys) |at| { + if (chosen[at] != exact_chosen[at]) flips += 1; + } + var quantum: f32 = 0; + var margin_in_quanta: f64 = 0; + if (min_selected != std.math.inf(f32) and max_unselected != -std.math.inf(f32)) { + const boundary = min_selected * 0.5 + max_unselected * 0.5; + const exponent: i32 = if (@abs(boundary) > 0) std.math.ilogb(@abs(boundary)) - 7 else -149; + quantum = std.math.pow(f32, 2.0, @as(f32, @floatFromInt(exponent))); + margin_in_quanta = @as(f64, min_selected - max_unselected) / @as(f64, quantum); + } + std.debug.print( + \\CSA64 {{"case":"{s}","stage":"index_select","side":"noise_tolerance","keys":{d},"picks":{d},"precision_flips":{d},"selection_margin":{d},"score_quantum":{d},"margin_in_quanta":{d:.1}}} + , .{ spec.name, spec.keys, picks, flips, min_selected - max_unselected, quantum, margin_in_quanta }); + std.debug.print( + \\CSA64 {{"case":"{s}","stage":"index_select","side":"metal","keys":{d},"picks":{d},"reference_only":{d},"kernel_only":{d},"invalid_positions":{d},"lowest_selected":{d},"highest_unselected":{d}}} + , .{ spec.name, spec.keys, picks, picks - overlap, kernel_only, out_of_range, min_selected, max_unselected }); +} + +test "indexer scoring and selection sit at rounding level against the FP64 ideal" { + var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_instance.deinit(); + const gpa = arena_instance.allocator(); + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try tf.deepseek_v41.csa2_index.Pipelines.load(device); + defer p.deinit(); + + var rng = h.Lcg{ .state = 0x1234_5678_9abc_def0 }; + var amplitude: f32 = 1; + for ([_]IndexSpec{ + .{ .name = "index_random", .rows = 1, .keys = 1024, .topk = 512 }, + .{ .name = "index_random_4k", .rows = 1, .keys = 4096, .topk = 512 }, + .{ .name = "index_ties", .rows = 1, .keys = 1024, .topk = 512 }, + .{ .name = "index_boundary", .rows = 1, .keys = 1024, .topk = 512 }, + }, 0..) |spec, case_index| { + _ = case_index; + const q = try gpa.alloc(u16, 32 * 128); + defer gpa.free(q); + const weights = try gpa.alloc(u16, 32); + defer gpa.free(weights); + const cells = try gpa.alloc(f32, spec.keys * 128); + defer gpa.free(cells); + const keys = try gpa.alloc([]const f32, spec.keys); + defer gpa.free(keys); + for (q) |*cell| cell.* = math.b(rng.scale(1.0)); + for (weights) |*cell| cell.* = math.b(rng.scale(1.0)); + for (cells, 0..) |*v, i| { + v.* = rng.scale(1.0); + keys[i / 128] = cells[i / 128 * 128 ..][0..128]; + } + // Identical keys: every score is the same value, so the whole selection is tie-driven. + if (std.mem.eql(u8, spec.name, "index_ties")) { + // Broadcast row 0's 128 cells into every later row, so each key's slice is a copy of the first key's. + for (cells[128..], 0..) |*dst, i| dst.* = cells[i % 128]; + } + // A score cliff just inside the selection boundary: half the keys sit on the boundary. + if (std.mem.eql(u8, spec.name, "index_boundary")) { + for (cells, 0..) |*v, i| { + if ((i % 128) == 0) amplitude = (if (i / 128 < 512) @as(f32, 1.000001) else @as(f32, 1.0)); + v.* = rng.scale(amplitude); + } + } + try indexCase(device, queue, &p, spec, q, weights, keys); + } +} diff --git a/zig/tests/deepseek_v41_reference/operator_csa2_pack_fp64.zig b/zig/tests/deepseek_v41_reference/operator_csa2_pack_fp64.zig new file mode 100644 index 0000000000..9aed886414 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_csa2_pack_fp64.zig @@ -0,0 +1,103 @@ +//! CSA2 packed-cache code tables for the FP64 ideal-reference stages (DeepSeek-V4.1-Flash inference, MIT). +const std = @import("std"); +const math = @import("math.zig"); +const h = @import("operator_csa2_fp64.zig"); + +/// Packed-cache tables: generated codes give both sides bit-identical BF16 keys, so only attention arithmetic differs. +fn e4(code: u8) f32 { + const magnitude = code & 127; + if (magnitude == 127) return std.math.nan(f32); + const exponent: i32 = @intCast(magnitude >> 3); + const mantissa = magnitude & 7; + const base: f32 = if (exponent == 0) @as(f32, @floatFromInt(mantissa)) / 512 else std.math.ldexp(1 + @as(f32, @floatFromInt(mantissa)) / 8, exponent - 7); + return if ((code & 128) != 0) -base else base; +} + +pub fn e2(code: u8) f32 { + const levels = [_]f32{ 0, 0.5, 1, 1.5, 2, 3, 4, 6 }; + const v = levels[code & 7]; + return if ((code & 8) != 0) -v else v; +} + +/// One ring row: 512 FP8 codes plus the FP32 scale of each 32-column block, exactly as dsv41_unpack reads it. +pub fn decodeWindowRow(bytes: []const u8) [h.width]u16 { + var row: [h.width]u16 = undefined; + for (0..h.width) |d| { + const scale: f32 = @bitCast(@as(u32, bytes[h.width + d / 32]) << 23); + row[d] = math.b(e4(bytes[d]) * scale); + } + return row; +} + +/// One compressed row: 256 FP4 nibble bytes plus the FP8 scale of each 16-column block. +pub fn decodeMainRow(bytes: []const u8) [h.width]u16 { + var row: [h.width]u16 = undefined; + for (0..h.width) |d| { + const nibble: u8 = if ((d & 1) != 0) bytes[d / 2] >> 4 else bytes[d / 2] & 15; + row[d] = math.b(e2(nibble) * e4(bytes[h.width / 2 + d / 16])); + } + return row; +} + +/// FP8 code under the scale the pack kernel picks for the block: Metal rint rounds ties to even. +fn roundHalfEven(x: f32) f32 { + const toward: f32 = @trunc(x); + const magnitude: f32 = if (@abs(x - toward) > 0.5) toward + std.math.sign(x) else toward; + if (@abs(x - magnitude) != 0.5) return magnitude; + const as_int: i64 = @intFromFloat(magnitude); + if (@mod(as_int, 2) == 0) return magnitude; + return if (x < 0) magnitude + 1 else magnitude - 1; +} + +fn e4Code(x: f32) u8 { + const a = @min(@abs(x), 448.0); + if (a == 0) return if (std.math.signbit(x)) 128 else 0; + const bits: u32 = @bitCast(a); + const unbiased: i32 = @as(i32, @intCast((bits >> 23) & 255)) - 127; + const e = @max(unbiased, -6); + const q: i32 = @intFromFloat(roundHalfEven(std.math.ldexp(a, 3 - @as(i32, @intCast(e))))); + const code: u32 = if (e == -6 and q < 8) @intCast(q) else @intCast((e + 7) * 8 + q - 8); + return @as(u8, @intCast(@min(code, 126))) | @as(u8, if (std.math.signbit(x)) 128 else 0); +} + +fn e4Scale(x: f32) u8 { + const bits: u32 = @bitCast(x); + const unbiased: i32 = @as(i32, @intCast((bits >> 23) & 255)) - 127 + @intFromBool((bits & 0x7fffff) != 0); + return @intCast(unbiased + 127); +} + +pub fn e2Code(x: f32) u8 { + const a = @min(@abs(x), 6.0); + const pick: u8 = if (a <= 0.25) 0 else if (a < 0.75) 1 else if (a <= 1.25) 2 else if (a < 1.75) 3 else if (a <= 2.5) 4 else if (a < 3.5) 5 else if (a <= 5.0) 6 else 7; + return pick | @as(u8, if (std.math.signbit(x)) 8 else 0); +} + +/// Pack a window row as dsv41_pack kind 0: per 32-column block, a power-of-two FP8 scale and FP8 codes. +pub fn packWindowRow(values: []const f32, out: []u8) void { + @memset(out[0 .. h.width + h.width / 32], 0); + for (0..h.width / 32) |block| { + var a: f32 = 1e-4; + for (values[block * 32 ..][0..32]) |v| a = @max(a, @abs(v)); + const z = a / 448; + const zbits: u32 = @bitCast(z); + const scale = std.math.ldexp(@as(f32, 1.0), @as(i32, @intCast((zbits >> 23) & 255)) - 127 + @intFromBool((zbits & 0x7fffff) != 0)); + out[h.width + block] = e4Scale(scale); + for (0..32) |j| out[block * 32 + j] = e4Code(values[block * 32 + j] / scale); + } +} + +/// Pack a compressed row as dsv41_pack kind 1: per 16-column block, an FP8 scale and FP4 codes. +pub fn packMainRow(values: []const f32, out: []u8) void { + @memset(out[0 .. h.width / 2 + h.width / 16], 0); + for (0..h.width / 16) |block| { + var a: f32 = 6.0 * 0x1p-9; + for (values[block * 16 ..][0..16]) |v| a = @max(a, @abs(v)); + const scale = e4(e4Code(@max(a, 6.0 * 0x1p-9) / 6.0)); + out[h.width / 2 + block] = e4Code(@max(a, 6.0 * 0x1p-9) / 6.0); + for (0..16) |j| { + const code = e2Code(values[block * 16 + j] / scale); + const at = block * 8 + j / 2; + if ((j & 1) != 0) out[at] |= code << 4 else out[at] |= code; + } + } +} diff --git a/zig/tests/deepseek_v41_reference/operator_csa2_rope_fp64.zig b/zig/tests/deepseek_v41_reference/operator_csa2_rope_fp64.zig new file mode 100644 index 0000000000..5f78bd425b --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_csa2_rope_fp64.zig @@ -0,0 +1,136 @@ +//! CSA2 rotary-stage FP64 ideal-reference test (DeepSeek-V4.1-Flash inference, MIT). +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const math = @import("math.zig"); +const h = @import("operator_csa2_fp64.zig"); + +/// Both sides of one rotary call: the source's FP32 contract against the FP64 ideal at every step. +fn ropeFrequency64( + index: usize, + theta: f64, + original: f64, + factor: f64, + beta_fast: f64, + beta_slow: f64, +) f64 { + const frequency: f64 = 1 / std.math.pow(f64, theta, @as(f64, @floatFromInt(index * 2)) / 64); + if (original <= 0) return frequency; + const denominator = 2 * @log(theta); + const low = @max(@floor(64 * @log(original / (beta_fast * 2 * std.math.pi)) / denominator), 0); + const high = @min(@ceil(64 * @log(original / (beta_slow * 2 * std.math.pi)) / denominator), 63); + const ramp = std.math.clamp((@as(f64, @floatFromInt(index)) - low) / @max(high - low, 1e-3), 0, 1); + return frequency / factor * ramp + frequency * (1 - ramp); +} + +fn ropeCase( + device: mtl.Device, + queue: mtl.Queue, + p: *const tf.deepseek_v41.rope.Pipelines, + name: []const u8, + rng: *h.Lcg, + position: usize, + rows: usize, + heads: usize, + spectrum: tf.deepseek_v41.rope.Spectrum, + inverse: bool, + theta: f32, + original: u32, + factor: f32, +) !void { + const stride = heads * h.width; + const cells = rows * stride; + var b = h.Buffers{ .device = device }; + defer b.deinit(); + const x_gpu = try b.get(u16, cells); + const x = x_gpu.slice(u16, cells); + const before = try std.testing.allocator.alloc(u16, cells); + defer std.testing.allocator.free(before); + for (x, 0..) |*cell, i| { + cell.* = math.b(rng.scale(1.0)); + before[i] = cell.*; + } + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encode(enc, h.view(x_gpu), .{ .rows = @intCast(rows), .heads = @intCast(heads), .width = h.width, .rotary = 64, .stride = @intCast(stride), .position = @intCast(position), .step = 1, .inverse = @intFromBool(inverse), .spectrum = spectrum }); + enc.end(); + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.Csa2Fp64GpuFailure; + + const theta64: f64 = @as(f64, theta); + const original64: f64 = if (original == 0) 0 else @floatFromInt(original); + const ideal_cells = try std.testing.allocator.alloc(u16, cells); + defer std.testing.allocator.free(ideal_cells); + const contract_cells = try std.testing.allocator.alloc(u16, cells); + defer std.testing.allocator.free(contract_cells); + for (0..rows) |row| for (0..heads) |head| { + const base = row * stride + head * h.width; + // Everything outside the rotary window is copied through, so both sides must agree there. + for (0..h.width - 64) |d| { + ideal_cells[base + d] = before[base + d]; + contract_cells[base + d] = before[base + d]; + } + for (0..32) |i| { + const at = base + h.width - 64 + 2 * i; + const a = math.f(before[at]); + const z = math.f(before[at + 1]); + const frequency = math.ropeFrequency(i, theta, original, factor, 32, 1); + const angle: f32 = @as(f32, @floatFromInt(position + row)) * frequency; + const cosine32 = @cos(angle); + const sine32 = @sin(angle) * (if (inverse) @as(f32, -1) else 1); + contract_cells[at] = math.b(a * cosine32 - z * sine32); + contract_cells[at + 1] = math.b(a * sine32 + z * cosine32); + const frequency64 = ropeFrequency64(i, theta64, original64, @as(f64, factor), 32, 1); + const angle64: f64 = @as(f64, @floatFromInt(position + row)) * frequency64; + const cosine64 = @cos(angle64); + const sine64 = @sin(angle64) * (if (inverse) @as(f64, -1) else @as(f64, 1)); + ideal_cells[at] = math.b64(@as(f64, a) * cosine64 - @as(f64, z) * sine64); + ideal_cells[at + 1] = math.b64(@as(f64, a) * sine64 + @as(f64, z) * cosine64); + } + }; + + var stage_buffer: [32]u8 = undefined; + for (0..rows) |row| for (0..heads) |head| { + const stage = std.fmt.bufPrint(&stage_buffer, "row{d}_head{d}", .{ row, head }) catch unreachable; + const base = row * stride + head * h.width; + var ideal: [h.width]f64 = undefined; + var contract: [h.width]f64 = undefined; + for (0..h.width) |d| { + ideal[d] = math.f(ideal_cells[base + d]); + contract[d] = math.f(contract_cells[base + d]); + } + h.printBf(name, stage, "metal", h.deltaBf16(x[base..][0..h.width], &ideal)); + h.printBf(name, stage, "f32_reference", h.deltaBf16(contract_cells[base..][0..h.width], &ideal)); + h.printBf(name, stage, "metal_vs_f32_reference", h.deltaBf16(x[base..][0..h.width], &contract)); + }; +} + +test "the rotary stage sits at rounding level for both spectra and at long positions" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + // The production shape: 64 rotary columns, YaRN on the compressed spectrum only, positions to capacity. + var p = try tf.deepseek_v41.rope.Pipelines.load(device, 16384, 64, .{ .theta = 10000, .original = 0 }, .{ .theta = 10000, .factor = 16, .original = 65536, .beta_fast = 32, .beta_slow = 1 }); + defer p.deinit(); + + const positions = [_]usize{ 1, 7, 63, 64, 4095, 8191, 16383 }; + for (positions) |position| { + var name_buf: [32]u8 = undefined; + var scoped = h.Lcg{ .state = 0x9e3779b97f4a7c15 ^ @as(u64, position) }; + const label = std.fmt.bufPrint(&name_buf, "rope_normal_p{d}", .{position}) catch unreachable; + try ropeCase(device, queue, &p, label, &scoped, position, 1, 2, .normal, false, 10000, 0, 16); + var scoped2 = h.Lcg{ .state = 0x85ebca77c0b98731 ^ @as(u64, position) }; + const label2 = std.fmt.bufPrint(&name_buf, "rope_compressed_p{d}", .{position}) catch unreachable; + try ropeCase(device, queue, &p, label2, &scoped2, position, 1, 2, .compressed, false, 10000, 65536, 16); + var scoped3 = h.Lcg{ .state = 0xc2b2ae3d27d4eb4f ^ @as(u64, position) }; + const label3 = std.fmt.bufPrint(&name_buf, "rope_inverse_p{d}", .{position}) catch unreachable; + try ropeCase(device, queue, &p, label3, &scoped3, position, 1, 1, .compressed, true, 10000, 65536, 16); + } + // A prefill row set: the kernel indexes the factor table by absolute position per row. + var multi = h.Lcg{ .state = 0x2545f4914f6cdd1d }; + try ropeCase(device, queue, &p, "rope_prefill_rows", &multi, 8100, 4, 2, .compressed, false, 10000, 65536, 16); +} diff --git a/zig/tests/deepseek_v41_reference/operator_csa2_sparse_fp64.zig b/zig/tests/deepseek_v41_reference/operator_csa2_sparse_fp64.zig new file mode 100644 index 0000000000..dbc7618646 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_csa2_sparse_fp64.zig @@ -0,0 +1,514 @@ +//! CSA2 sparse-attention FP64 ideal-reference tests (DeepSeek-V4.1-Flash inference, MIT). +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const math = @import("math.zig"); +const attention_math = @import("attention_math.zig"); +const h = @import("operator_csa2_fp64.zig"); +const pack = @import("operator_csa2_pack_fp64.zig"); +const csa2 = tf.deepseek_v41.csa2; + +const SparseSpec = struct { + name: []const u8, + heads: usize, + rows: usize, + position: usize, + ratio: usize, + compressed_rows: usize, + topk: usize, +}; + +/// The kernel's own column addressing, so the reference reads the identical key per column slot. +fn columnKey(spec: SparseSpec, row: usize, slot: usize, indices: []const i32) ?KeySource { + const query = spec.position + row; + const window_columns: usize = if (spec.position + row == 0) 1 else 128; + if (slot < window_columns) { + const pos: usize = if (query == 0) 0 else if (query + 1 < 128) (query + 1 + slot) % 128 else blk: { + const begin = if (query + 1 > 128) query + 1 - 128 else 0; + break :blk begin + slot; + }; + if (pos > query) return null; + return .{ .row = pos % 128 }; + } + if (spec.ratio == 0) return null; + const pos_i = indices[row * spec.topk + slot - window_columns]; + const limit = @min(spec.compressed_rows, (query + 1) / spec.ratio); + if (pos_i < 0 or @as(usize, @intCast(pos_i)) >= limit) return null; + return .{ .compressed = true, .row = @intCast(pos_i) }; +} + +const KeySource = struct { compressed: bool = false, row: usize }; + +fn sparseCase( + device: mtl.Device, + queue: mtl.Queue, + p: *const csa2.Pipelines, + spec: SparseSpec, + q: []const u16, + window_rows: []const []const f32, + compressed_rows: []const []const f32, + indices: []const i32, + sinks: []const f32, +) !void { + var b = h.Buffers{ .device = device }; + defer b.deinit(); + const q_stride = spec.heads * h.width; + const q_gpu = try b.get(u16, spec.rows * q_stride); + const ring_gpu = try b.get(u8, 128 * 528); + const chunk_gpu = try b.get(u8, @max(spec.rows, 1) * 528); + const main_gpu = try b.get(u8, @max(spec.compressed_rows, 1) * 288); + const positions_gpu = try b.get(i32, spec.rows * spec.topk); + const sink_gpu = try b.get(f32, spec.heads); + const out_gpu = try b.get(u16, spec.rows * q_stride); + @memcpy(q_gpu.slice(u16, spec.rows * q_stride), q); + @memcpy(positions_gpu.slice(i32, spec.rows * spec.topk), indices); + @memcpy(sink_gpu.slice(f32, spec.heads), sinks); + + // The reference key table is the kernel's decoded cache: window columns by slot, then selected compressed rows. + var window_table = try std.testing.allocator.alloc(u16, 128 * h.width); + defer std.testing.allocator.free(window_table); + @memset(window_table, 0); + // The reference table is indexed 128 + row, so it needs one row per compressed row. + const compressed_cells = @max(compressed_rows.len, 1); + var compressed_table = try std.testing.allocator.alloc(u16, compressed_cells * h.width); + defer std.testing.allocator.free(compressed_table); + @memset(compressed_table, 0); + // Rows are ring slots: a logical position reads ring[pos % 128] unless this call publishes it. + for (window_rows, 0..) |values, logical| { + const bytes = try std.testing.allocator.alloc(u8, 528); + defer std.testing.allocator.free(bytes); + pack.packWindowRow(values, bytes); + const slot = logical % 128; + @memcpy(ring_gpu.slice(u8, 128 * 528)[slot * 528 ..][0..528], bytes); + const decoded = pack.decodeWindowRow(bytes); + @memcpy(window_table[slot * h.width ..][0..h.width], decoded[0..h.width]); + if (logical >= spec.position and logical < spec.position + spec.rows) + @memcpy(chunk_gpu.slice(u8, @max(spec.rows, 1) * 528)[(logical - spec.position) * 528 ..][0..528], bytes); + } + for (compressed_rows, 0..) |values, row| { + const bytes = try std.testing.allocator.alloc(u8, 288); + defer std.testing.allocator.free(bytes); + pack.packMainRow(values, bytes); + @memcpy(main_gpu.slice(u8, @max(spec.compressed_rows, 1) * 288)[row * 288 ..][0..288], bytes); + const decoded = pack.decodeMainRow(bytes); + @memcpy(compressed_table[row * h.width ..][0..h.width], decoded[0..h.width]); + } + + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + try p.encode(enc, h.view(q_gpu), h.view(ring_gpu), h.view(chunk_gpu), h.view(main_gpu), h.view(positions_gpu), h.view(sink_gpu), h.view(out_gpu), .{ + .rows = @intCast(spec.rows), + .heads = @intCast(spec.heads), + .q_stride = @intCast(q_stride), + .position = @intCast(spec.position), + .chunk_rows = @intCast(spec.rows), + .ratio = @intCast(spec.ratio), + .compressed_rows = @intCast(spec.compressed_rows), + .topk = @intCast(spec.topk), + .index_stride = @intCast(spec.topk), + }); + enc.end(); + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.Csa2Fp64GpuFailure; + const metal = out_gpu.slice(u16, spec.rows * q_stride); + + var column_indices = try std.testing.allocator.alloc(i32, 640); + defer std.testing.allocator.free(column_indices); + var wide32: [h.width]u16 = undefined; + var wide64: [h.width]u16 = undefined; + for (0..spec.rows) |row| { + const window_columns: usize = if (spec.position + row == 0) 1 else 128; + const compressed_columns: usize = if (spec.ratio == 0) 0 else @min(spec.topk, @min(spec.compressed_rows, (spec.position + row + 1) / spec.ratio)); + const count = window_columns + compressed_columns; + for (0..count) |slot| { + if (columnKey(spec, row, slot, indices)) |key| { + column_indices[slot] = if (key.compressed) @intCast(128 + key.row) else @intCast(key.row); + } else column_indices[slot] = -1; + } + const keys = attention_math.KeyTable{ + .window = window_table, + .window_stride = h.width, + .compressed = compressed_table, + .compressed_stride = h.width, + }; + for (0..spec.heads) |head| { + const head_q = q[row * q_stride + head * h.width ..][0..h.width]; + var pre32: [h.width]f32 = undefined; + var pre64: [h.width]f64 = undefined; + attention_math.sparseHeadPre(f32, head_q, keys, column_indices[0..count], sinks[head], wide32[0..h.width], &pre32); + attention_math.sparseHeadPre(f64, head_q, keys, column_indices[0..count], sinks[head], wide64[0..h.width], &pre64); + var ideal: [h.width]f64 = undefined; + for (wide64, 0..) |cell, d| ideal[d] = math.f(cell); + var energy: f64 = 0; + var peak: f64 = 0; + var live: usize = 0; + for (column_indices[0..count]) |at| live += @intFromBool(at >= 0); + for (ideal) |v| { + energy += v * v; + peak = @max(peak, @abs(v)); + } + std.debug.print( + \\CSA64 {{"case":"{s}","stage":"scale","side":"{s}","columns":{d},"live":{d},"ideal_rms":{d},"ideal_peak":{d}}} + , .{ spec.name, "sparse", count, live, @sqrt(energy / @as(f64, h.width)), peak }); + var stage_buffer: [32]u8 = undefined; + const stage = std.fmt.bufPrint(&stage_buffer, "row{d}_head{d}", .{ row, head }) catch unreachable; + // The intrinsic FP32-versus-FP64 level of the stage, before the BF16 quantum hides it; verdicts use it. + h.printF32(spec.name, stage, "f32_vs_ideal_pre_rounding", h.deltaF32VsF64(&pre32, &pre64)); + h.printBf(spec.name, stage, "metal", h.deltaBf16(metal[row * q_stride + head * h.width ..][0..h.width], &ideal)); + h.printBf(spec.name, stage, "f32_reference", h.deltaBf16(wide32[0..h.width], &ideal)); + // Decisive for the verdict: kernel and source-contract reference on identical decoded rows. + var contract_cells: [h.width]f64 = undefined; + for (wide32, 0..) |cell, d| contract_cells[d] = math.f(cell); + h.printBf(spec.name, stage, "metal_vs_f32_reference", h.deltaBf16(metal[row * q_stride + head * h.width ..][0..h.width], &contract_cells)); + } + } +} + +const KeyPattern = enum { random, cancelling, near_tie, one_sided }; + +fn sparseCaseFrom( + gpa: std.mem.Allocator, + device: mtl.Device, + queue: mtl.Queue, + p: *const csa2.Pipelines, + spec: SparseSpec, + rng: *h.Lcg, + key_amplitude: f32, + scale_exponent: i32, + indices_mode: usize, + sink: f32, + pattern: KeyPattern, +) !void { + try sparseCaseScaled(gpa, device, queue, p, spec, rng, key_amplitude, scale_exponent, scale_exponent, indices_mode, sink, pattern); +} + +/// Rows with window and compressed cache on independent scale grids: one accumulator mixes large and tiny terms. +fn sparseCaseScaled( + gpa: std.mem.Allocator, + device: mtl.Device, + queue: mtl.Queue, + p: *const csa2.Pipelines, + spec: SparseSpec, + rng: *h.Lcg, + key_amplitude: f32, + window_exponent: i32, + compressed_exponent: i32, + indices_mode: usize, + sink: f32, + pattern: KeyPattern, +) !void { + const q_stride = spec.heads * h.width; + const q = try gpa.alloc(u16, spec.rows * q_stride); + defer gpa.free(q); + const q_scale = std.math.pow(f32, 2, @as(f32, @floatFromInt(window_exponent))); + for (q) |*cell| cell.* = math.b(rng.uniform() * q_scale); + const logical_window = spec.position + spec.rows; + const window_rows = try gpa.alloc([]const f32, logical_window); + defer gpa.free(window_rows); + for (window_rows, 0..) |*row, i| { + const cells = try gpa.alloc(f32, h.width); + fillPatternRow(cells, rng, i, key_amplitude, window_exponent, pattern); + row.* = cells; + } + const compressed_rows = try gpa.alloc([]const f32, spec.compressed_rows); + defer gpa.free(compressed_rows); + for (compressed_rows, 0..) |*row, i| { + const cells = try gpa.alloc(f32, h.width); + fillPatternRow(cells, rng, i, key_amplitude, compressed_exponent, pattern); + row.* = cells; + } + const indices = try gpa.alloc(i32, spec.rows * spec.topk); + defer gpa.free(indices); + const limit = @max(spec.compressed_rows, 1); + for (0..spec.rows) |r| for (0..spec.topk) |k| { + indices[r * spec.topk + k] = switch (indices_mode) { + 0 => @intCast(k * limit / @max(spec.topk, 1)), // spread and ascending + 1 => @intCast((limit - 1) - k), // descending (the reference sorts; the kernel does not) + else => @as(i32, @intCast((r *% 7 +% 13 * k) % limit)), // arbitrary order, unsorted + }; + }; + const sinks = try gpa.alloc(f32, spec.heads); + defer gpa.free(sinks); + for (sinks) |*v| v.* = sink; + try sparseCase(device, queue, p, spec, q, window_rows, compressed_rows, indices, sinks); +} + +fn fillPatternRow(row: []f32, rng: *h.Lcg, index: usize, amplitude: f32, exponent: i32, pattern: KeyPattern) void { + const scale = std.math.pow(f32, 2, @as(f32, @floatFromInt(exponent))); + if (pattern == .random) { + for (row) |*v| v.* = rng.uniform() * amplitude * scale; + return; + } + for (row, 0..) |*v, d| { + if (pattern == .cancelling) { + // Same key shape: every score ties and the value sum cancels to the ULPs of the terms. + const flip: f32 = if ((d & 1) == 0) 1 else -1; + v.* = flip * amplitude * scale + rng.uniform() * amplitude * scale * 1e-3; + } else if (pattern == .near_tie) { + // Keys differ only far above the score rounding boundary: probabilities tie at the top. + v.* = @as(f32, @floatFromInt(index)) * amplitude * scale + rng.uniform() * amplitude * scale * 1e-7; + } else { + // One dominant key against keys orders of magnitude smaller. + v.* = rng.uniform() * amplitude * scale * @as(f32, if (index == 0) @as(f32, 1e4) else @as(f32, 1e-4)); + } + } +} + +test "sparse attention stages sit at rounding level against the FP64 ideal" { + var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_instance.deinit(); + const gpa = arena_instance.allocator(); + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try csa2.Pipelines.load(device); + defer p.deinit(); + + var rng = h.Lcg{ .state = 0x2545f4914f6cdd1d }; + + // Window only, the decode-step boundary at a wrapped ring. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "window_decode", .heads = 4, .rows = 1, .position = 255, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); + // Compressed rows with a small selection, ascending indices. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "compressed_asc", .heads = 4, .rows = 1, .position = 255, .ratio = 2, .compressed_rows = 128, .topk = 16 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); + // Unsorted indices: the kernel consumes the array in the given order; the reference must too. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "compressed_unsorted", .heads = 2, .rows = 1, .position = 9, .ratio = 2, .compressed_rows = 40, .topk = 12 }, &rng, 1.0, 0, 2, 0, KeyPattern.random); + // Prefill rows, multi-row chunk with a wrapped ring. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "prefill_rows", .heads = 2, .rows = 4, .position = 250, .ratio = 2, .compressed_rows = 100, .topk = 8 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); + // Wide dynamic range keys: the score sum runs across many orders of magnitude. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "wide_range", .heads = 1, .rows = 1, .position = 200, .ratio = 2, .compressed_rows = 32, .topk = 8 }, &rng, 1.0, -20, 0, 0, KeyPattern.random); + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "wide_range_hi", .heads = 1, .rows = 1, .position = 200, .ratio = 2, .compressed_rows = 32, .topk = 8 }, &rng, 1.0, 20, 0, 0, KeyPattern.random); + // Denormal-scale sink: the sink mass underflows FP32 while it is exact in FP64. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "sink_denormal", .heads = 1, .rows = 1, .position = 100, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, -16.0, KeyPattern.random); + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "sink_large", .heads = 1, .rows = 1, .position = 100, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, 12.0, KeyPattern.random); + // Value-product cancellation: identical key shapes tie every score and the P.V sum cancels to the ULPs. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "cancel_values", .heads = 1, .rows = 1, .position = 300, .ratio = 2, .compressed_rows = 64, .topk = 64 }, &rng, 32.0, 0, 0, 0, KeyPattern.cancelling); + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "cancel_wide", .heads = 1, .rows = 1, .position = 300, .ratio = 2, .compressed_rows = 64, .topk = 64 }, &rng, 32.0, 6, 0, 0, KeyPattern.cancelling); + // Tied scores: the denominator sums equal exponentials; accumulation order picks each probability's side. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "score_tie", .heads = 1, .rows = 1, .position = 300, .ratio = 2, .compressed_rows = 32, .topk = 32 }, &rng, 1.0, 0, 0, 0, KeyPattern.near_tie); + // One dominant key against keys orders of magnitude smaller: denominator and accumulator at the ratio. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "one_dominant", .heads = 1, .rows = 1, .position = 300, .ratio = 2, .compressed_rows = 16, .topk = 16 }, &rng, 1.0, 0, 0, 0, KeyPattern.one_sided); + // Many tiles, so the online correction multiplies the accumulator once per 64 columns. + try sparseCaseFrom(gpa, device, queue, &p, .{ .name = "many_tiles", .heads = 1, .rows = 1, .position = 511, .ratio = 2, .compressed_rows = 512, .topk = 384 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); + // Cancellation in the value product: alternating-sign keys make the P.V sum small next to its terms. + { + const spec = SparseSpec{ .name = "pv_cancellation", .heads = 1, .rows = 1, .position = 64, .ratio = 0, .compressed_rows = 0, .topk = 0 }; + const q = try gpa.alloc(u16, h.width); + defer gpa.free(q); + for (q) |*cell| cell.* = math.b(rng.scale(1.0)); + const window_rows = try gpa.alloc([]const f32, spec.position + spec.rows); + defer gpa.free(window_rows); + for (window_rows) |*row| { + const cells = try gpa.alloc(f32, h.width); + for (cells, 0..) |*v, d| v.* = if ((d & 1) == 0) rng.scale(64.0) else rng.scale(64.0) * 0.987654321; + row.* = cells; + } + const sinks = try gpa.alloc(f32, 1); + defer gpa.free(sinks); + sinks[0] = 0; + const empty = try gpa.alloc([]const f32, 0); + defer gpa.free(empty); + const indices = try gpa.alloc(i32, 0); + defer gpa.free(indices); + try sparseCase(device, queue, &p, spec, q, window_rows, empty, indices, sinks); + } + // Large cancelling keys beside small ones: score dot products run through near-cancelling terms. + { + const spec = SparseSpec{ .name = "score_cancellation", .heads = 1, .rows = 1, .position = 96, .ratio = 0, .compressed_rows = 0, .topk = 0 }; + const q = try gpa.alloc(u16, h.width); + defer gpa.free(q); + for (q, 0..) |*cell, d| cell.* = if ((d & 1) == 0) math.b(1.0) else math.b(-1.0); + const window_rows = try gpa.alloc([]const f32, spec.position + spec.rows); + defer gpa.free(window_rows); + for (window_rows, 0..) |*row, i| { + const cells = try gpa.alloc(f32, h.width); + const bias: f32 = @as(f32, @floatFromInt(@rem(@as(i32, @intCast(i)), 8))) * 4.0; + for (cells, 0..) |*v, d| v.* = bias + (if ((d & 1) == 0) rng.scale(0.001) else rng.scale(0.001) * 0.99999); + row.* = cells; + } + const sinks = try gpa.alloc(f32, 1); + defer gpa.free(sinks); + sinks[0] = 0; + const empty = try gpa.alloc([]const f32, 0); + defer gpa.free(empty); + const indices = try gpa.alloc(i32, 0); + defer gpa.free(indices); + try sparseCase(device, queue, &p, spec, q, window_rows, empty, indices, sinks); + } +} + +/// One live column pins the maximum, so sink - max is the whole normalization across the FP32 exp limits. +fn sinkCase( + gpa: std.mem.Allocator, + device: mtl.Device, + queue: mtl.Queue, + p: *const csa2.Pipelines, + name: []const u8, + rng: *h.Lcg, + sink: f32, +) !void { + const spec = SparseSpec{ .name = name, .heads = 1, .rows = 1, .position = 0, .ratio = 0, .compressed_rows = 0, .topk = 0 }; + const q = try gpa.alloc(u16, h.width); + defer gpa.free(q); + for (q) |*cell| cell.* = math.b(rng.scale(1.0)); + const window_rows = try gpa.alloc([]const f32, 1); + defer gpa.free(window_rows); + const cells = try gpa.alloc(f32, h.width); + defer gpa.free(cells); + for (cells) |*v| v.* = rng.scale(1.0); + window_rows[0] = cells; + const sinks = try gpa.alloc(f32, 1); + defer gpa.free(sinks); + sinks[0] = sink; + const empty = try gpa.alloc([]const f32, 0); + defer gpa.free(empty); + const indices = try gpa.alloc(i32, 0); + defer gpa.free(indices); + try sparseCase(device, queue, p, spec, q, window_rows, empty, indices, sinks); +} + +test "sink normalization sits at rounding level across the exp overflow boundaries" { + var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_instance.deinit(); + const gpa = arena_instance.allocator(); + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var pipelines = try csa2.Pipelines.load(device); + defer pipelines.deinit(); + var rng = h.Lcg{ .state = 0x51c0de51c0de51c0 }; + + // One case per FP32 exp() boundary: underflow, overflow edge and the balanced middle where the sink decides. + const sweep = [_]struct { sink: f32, name: []const u8 }{ + .{ .sink = -104, .name = "sink_underflow" }, + .{ .sink = -90, .name = "sink_near_underflow" }, + .{ .sink = -60, .name = "sink_tiny" }, + .{ .sink = -12, .name = "sink_small" }, + .{ .sink = -1, .name = "sink_minus_one" }, + .{ .sink = 0, .name = "sink_zero" }, + .{ .sink = 1, .name = "sink_one" }, + .{ .sink = 12, .name = "sink_medium" }, + .{ .sink = 60, .name = "sink_large" }, + .{ .sink = 80, .name = "sink_edge80" }, + .{ .sink = 83, .name = "sink_edge83" }, + .{ .sink = 84, .name = "sink_edge84" }, + .{ .sink = 85, .name = "sink_edge85" }, + .{ .sink = 86, .name = "sink_over_edge" }, + .{ .sink = 90, .name = "sink_overflow" }, + }; + for (sweep) |entry| try sinkCase(gpa, device, queue, &pipelines, entry.name, &rng, entry.sink); + + // Per-head sinks: the kernel indexes the sink array by head, so heads orders apart check the indexing. + { + const spec = SparseSpec{ .name = "sink_per_head", .heads = 4, .rows = 1, .position = 0, .ratio = 0, .compressed_rows = 0, .topk = 0 }; + var head_rng = h.Lcg{ .state = 0xabcdef0123456789 }; + const q = try gpa.alloc(u16, spec.heads * h.width); + defer gpa.free(q); + for (q) |*cell| cell.* = math.b(head_rng.scale(1.0)); + const window_rows = try gpa.alloc([]const f32, 1); + defer gpa.free(window_rows); + const cells = try gpa.alloc(f32, h.width); + defer gpa.free(cells); + for (cells) |*v| v.* = head_rng.scale(1.0); + window_rows[0] = cells; + const sinks = try gpa.alloc(f32, spec.heads); + defer gpa.free(sinks); + for (0..spec.heads) |head| sinks[head] = @as(f32, @floatFromInt(@as(i32, @intCast(head)) * 24 - 36)); + const empty = try gpa.alloc([]const f32, 0); + defer gpa.free(empty); + const indices = try gpa.alloc(i32, 0); + defer gpa.free(indices); + try sparseCase(device, queue, &pipelines, spec, q, window_rows, empty, indices, sinks); + } +} + +// Tile denominators on flat, ramped and descending scores: the ramp drives the correction through FP32 underflow. +test "tile denominators sit at rounding level for every tile shape" { + var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_instance.deinit(); + const gpa = arena_instance.allocator(); + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var pipelines = try csa2.Pipelines.load(device); + defer pipelines.deinit(); + var rng = h.Lcg{ .state = 0x1a2b3c4d5e6f7081 }; + + // Ten tiles: 639 exponentials and 639 products; identical keys tie every score and cancel the sum. + try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = "tiles_equal_scores", .heads = 1, .rows = 1, .position = 1022, .ratio = 2, .compressed_rows = 512, .topk = 512 }, &rng, 1.0, 0, 0, 0, KeyPattern.cancelling); + // Ascending scores by tile: the correction falls one power of two per step until FP32 flushes it. + for ([_]i32{ -2, 0, 1, 2 }) |step| { + var name_buf: [24]u8 = undefined; + const label = std.fmt.bufPrint(&name_buf, "tiles_ramp{d}", .{step}) catch unreachable; + try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = label, .heads = 1, .rows = 1, .position = 1000, .ratio = 2, .compressed_rows = 512, .topk = 512 }, &rng, 1.0, step, 0, 0, KeyPattern.near_tie); + } + // Descending scores: the running maximum never moves, so every tile lands at full weight. + try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = "tiles_descending", .heads = 1, .rows = 1, .position = 1000, .ratio = 2, .compressed_rows = 512, .topk = 512 }, &rng, 1.0, 0, 1, 0, KeyPattern.near_tie); + // One tile only, the boundary case of the online loop: the correction is applied zero times. + try sparseCaseFrom(gpa, device, queue, &pipelines, .{ .name = "tiles_single", .heads = 1, .rows = 1, .position = 20, .ratio = 0, .compressed_rows = 0, .topk = 0 }, &rng, 1.0, 0, 0, 0, KeyPattern.random); +} + +// Output mix: window and compressed rows on different scale grids, and mirror rows whose shared shape cancels. +test "the output mix sits at rounding level when window and compressed rows cancel" { + var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_instance.deinit(); + const gpa = arena_instance.allocator(); + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var pipelines = try csa2.Pipelines.load(device); + defer pipelines.deinit(); + var rng = h.Lcg{ .state = 0x77a1c0de_5b5e1a2b }; + + // Large window rows against tiny compressed rows, then the reverse: one accumulator six orders apart. + try sparseCaseScaled(gpa, device, queue, &pipelines, .{ .name = "mix_window_dominates", .heads = 1, .rows = 1, .position = 100, .ratio = 2, .compressed_rows = 100, .topk = 64 }, &rng, 1.0, 14, -12, 0, 0, KeyPattern.random); + try sparseCaseScaled(gpa, device, queue, &pipelines, .{ .name = "mix_compressed_dominates", .heads = 1, .rows = 1, .position = 100, .ratio = 2, .compressed_rows = 100, .topk = 64 }, &rng, 1.0, -12, 14, 0, 0, KeyPattern.random); + try sparseCaseScaled(gpa, device, queue, &pipelines, .{ .name = "mix_balanced_magnitudes", .heads = 1, .rows = 1, .position = 100, .ratio = 2, .compressed_rows = 100, .topk = 64 }, &rng, 1.0, 6, 6, 0, 0, KeyPattern.cancelling); + + // Amplification probe: 128 window rows of a large alternating shape that cancels from scores and sum alike. + for ([_]i32{ 8, 16, 23 }) |shape_exponent| { + var name_buf: [24]u8 = undefined; + const label = std.fmt.bufPrint(&name_buf, "mirror_shape_e{d}", .{shape_exponent}) catch unreachable; + const spec = SparseSpec{ .name = label, .heads = 1, .rows = 1, .position = 127, .ratio = 0, .compressed_rows = 0, .topk = 0 }; + const q = try gpa.alloc(u16, h.width); + defer gpa.free(q); + // q dot shape == 0: a uniform query against a sign-alternating shape, so the large part drops out. + for (q) |*cell| cell.* = math.b(1.0); + const window_rows = try gpa.alloc([]const f32, 128); + defer gpa.free(window_rows); + const shape = std.math.pow(f32, 2, @floatFromInt(shape_exponent)); + const shared = try gpa.alloc(f32, h.width); + defer gpa.free(shared); + // One 64th of the cancelling shape near the FP8 window floor: partial sums run 2^16 times the final. + for (shared) |*v| v.* = rng.scale(shape * 0x1p-16); + for (window_rows, 0..) |*row, i| { + const cells = try gpa.alloc(f32, h.width); + const side: f32 = if ((i & 1) == 0) 1 else -1; + for (0..h.width) |d| { + const flip: f32 = if ((d & 1) == 0) 1 else -1; + cells[d] = side * flip * shape + shared[d]; + } + row.* = cells; + } + const sinks = try gpa.alloc(f32, 1); + defer gpa.free(sinks); + sinks[0] = 0; + const empty = try gpa.alloc([]const f32, 0); + defer gpa.free(empty); + const indices = try gpa.alloc(i32, 0); + defer gpa.free(indices); + try sparseCase(device, queue, &pipelines, spec, q, window_rows, empty, indices, sinks); + } +} diff --git a/zig/tests/deepseek_v41_reference/operator_diagnostic.zig b/zig/tests/deepseek_v41_reference/operator_diagnostic.zig new file mode 100644 index 0000000000..66dc039bcf --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_diagnostic.zig @@ -0,0 +1,124 @@ +//! Shared retained-input diagnostic capture and metrics. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const projection = @import("projection.zig"); +const math = @import("math.zig"); +const attention = tf.deepseek_v41.attention; +const Model = tf.deepseek_v41.Model; +const ex = tf.exl3_projection; + +pub const AttentionStage = attention.DiagnosticStage; +pub const attention_widths = [_]usize{ 1280, 1280, 32768, 32768, 512, 512, 512, 512, 32768, 32768, 8192, 5120 }; +pub const attention_elements = blk: { + var total: usize = 0; + for (attention_widths) |width| total += width; + break :blk total; +}; +pub fn attentionOffset(stage: AttentionStage) usize { + var offset: usize = 0; + for (attention_widths[0..@backingInt(stage)]) |width| offset += width; + return offset; +} +pub const AttentionCapture = struct { + model: *Model, + native: mtl.Buffer, + reference: []u16, + + pub fn view(self: *const AttentionCapture, stage: AttentionStage) attention.View { + return .{ .buffer = self.native, .offset = attentionOffset(stage) * 2 }; + } + pub fn row(self: *const AttentionCapture, stage: AttentionStage) []u16 { + return self.native.slice(u16, attention_elements)[attentionOffset(stage)..][0..attention_widths[@backingInt(stage)]]; + } + pub fn cpuRow(self: *const AttentionCapture, stage: AttentionStage) []u16 { + return self.reference[attentionOffset(stage)..][0..attention_widths[@backingInt(stage)]]; + } + pub fn gpu(ctx: *anyopaque, enc: mtl.ComputeEncoder, stage: AttentionStage, source: attention.View) !void { + const self: *AttentionCapture = @ptrCast(@alignCast(ctx)); + const cache = &self.model.pipelines.attention.cache; + if (stage == .kv_quantized) { + try cache.encodeUnpack(enc, source, self.view(stage), .{ .rows = 1, .width = 512, .stride = 512, .packed_stride = 528, .block = 32, .kind = .window_fp8 }); + } else { + cache.encodeCopy(enc, source, self.view(stage), @intCast(attention_widths[@backingInt(stage)] * 2)); + } + } + pub fn cpu(ctx: *anyopaque, stage: AttentionStage, values: []const u16) void { + const self: *AttentionCapture = @ptrCast(@alignCast(ctx)); + @memcpy(self.cpuRow(stage), values); + } +}; + +pub fn attentionMetric(comptime T: type, gpa: std.mem.Allocator, report: *std.ArrayList(u8), count: *usize, name: []const u8, comparison: []const u8, actual: []const T, expected: []const T) !void { + if (actual.len == 0 or actual.len != expected.len) return error.InvalidAttentionMetricShape; + var squared: f64 = 0; + var magnitude: f64 = 0; + var max_abs: f64 = 0; + var worst: usize = 0; + var unequal: usize = 0; + var first: ?usize = null; + for (actual, expected, 0..) |a, r, i| { + const x = if (T == u16) math.f(a) else a; + const y = if (T == u16) math.f(r) else r; + if (!std.math.isFinite(x) or !std.math.isFinite(y)) return error.NonfiniteAttentionDiagnostic; + const delta = @as(f64, x) - y; + squared += delta * delta; + magnitude += @as(f64, y) * y; + if (@abs(delta) > max_abs) { + max_abs = @abs(delta); + worst = i; + } + const ab = if (T == u16) a else math.b(a); + const rb = if (T == u16) r else math.b(r); + if (ab != rb) { + unequal += 1; + if (first == null) first = i; + } + } + const line = try std.json.Stringify.valueAlloc(gpa, .{ + .name = name, + .comparison = comparison, + .elements = actual.len, + .output_dtype = if (T == u16) "bf16" else "f32", + .normalized_rms_error = if (magnitude == 0) @sqrt(squared) else @sqrt(squared / magnitude), + .max_absolute_error = max_abs, + .worst_index = worst, + .bf16_unequal_count = unequal, + .bf16_first_unequal_index = first, + .worst_native = if (T == u16) math.f(actual[worst]) else actual[worst], + .worst_reference = if (T == u16) math.f(expected[worst]) else expected[worst], + }, .{}); + defer gpa.free(line); + if (count.* != 0) try report.appendSlice(gpa, ",\n"); + try report.appendSlice(gpa, line); + count.* += 1; + std.debug.print("{s}\n", .{line}); +} + +pub fn attentionProjection(gpa: std.mem.Allocator, model: *Model, matrix: ex.Matrix, input: attention.View, name: []const u8, scratch: *projection.Scratch, expected: []f32, report: *std.ArrayList(u8), count: *usize) !void { + const input_row = @as([*]const u16, @ptrCast(@alignCast(input.buffer.contents() + input.offset)))[0..matrix.k]; + try projection.applyF32(.{ .trellis = matrix }, input_row, expected[0..matrix.n], scratch); + const output = model.workspace.view(.logits); + const cb = model.queue.commandBufferUnretained(); + const enc = cb.compute(.serial); + var encoding = true; + errdefer if (encoding) enc.end(); + ex.useMatrix(enc, matrix); + try ex.encode(&model.pipelines.projection, enc, matrix, .{ .buffer = input.buffer, .offset = input.offset }, .{ .buffer = output.buffer, .offset = output.offset }, model.workspace.projection(), .{ .count = 1, .x_stride = matrix.k, .y_stride = matrix.n, .output = .f32 }); + enc.end(); + encoding = false; + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.OperatorGpuFailure; + const actual = @as([*]const f32, @ptrCast(@alignCast(output.buffer.contents() + output.offset)))[0..matrix.n]; + try attentionMetric(f32, gpa, report, count, name, "same_native_input_fp64_dot", actual, expected[0..matrix.n]); +} + +pub fn diagnosticRow(comptime T: type, view: attention.View, len: usize) []T { + return @as([*]T, @ptrCast(@alignCast(view.buffer.contents() + view.offset)))[0..len]; +} +pub fn attentionWrite(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, name: []const u8, data: []const u8) !void { + const path = try std.fs.path.join(gpa, &.{ dir, name }); + defer gpa.free(path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = data }); +} diff --git a/zig/tests/deepseek_v41_reference/operator_hc.zig b/zig/tests/deepseek_v41_reference/operator_hc.zig new file mode 100644 index 0000000000..4aec23362a --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_hc.zig @@ -0,0 +1,98 @@ +//! Retained-input hyperconnection isolation, separate from qualification. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const math = @import("math.zig"); +const attention = tf.deepseek_v41.attention; +const Model = tf.deepseek_v41.Model; +const diagnostic = @import("operator_diagnostic.zig"); +const diagnosticRow = diagnostic.diagnosticRow; +const attentionMetric = diagnostic.attentionMetric; + +pub fn hyperconnections(gpa: std.mem.Allocator, model: *Model, stream_bytes: []const u8, branch: attention.View, report: *std.ArrayList(u8), count: *usize, layer: usize) !void { + const ws = &model.workspace; + const hc = model.weights.layers[layer].hc_ffn; + const cfg = model.cfg.text_config; + const stream = ws.view(.stream); + @memcpy(stream.buffer.contents()[stream.offset..][0..40960], stream_bytes[0..40960]); + var input: [20480]u16 = undefined; + @memcpy(&input, diagnosticRow(u16, stream, input.len)); + var projected: [24]f32 = undefined; + math.dense(hc.projection, &input, &projected); + const expected_mix = math.mixes(&input, hc, cfg); + var squares: f64 = 0; + for (input) |v| squares += @as(f64, math.f(v)) * math.f(v); + const mean = math.meanFromWide(squares, input.len); + const expected_inv = [_]f32{math.idealRsqrt(mean + cfg.rms_norm_eps)}; + const cb = model.queue.commandBufferUnretained(); + const enc = cb.compute(.serial); + for (model.checkpoint.shards.items) |shard| enc.useResource(shard.buffer, mtl.ResourceUsage.read); + tf.deepseek_v41.mhc.encodeRms(enc, &model.pipelines.mhc, stream, ws.view(.inv), .{ .rows = 1, .width = 5120, .stream_stride = 20480, .eps = cfg.rms_norm_eps }); + tf.deepseek_v41.ops.encodeDense(enc, &model.pipelines.ops, stream, .{ .buffer = hc.projection.buffer, .offset = hc.projection.offset }, ws.view(.mixes), .{ .rows = 1, .k = 20480, .n = 24, .x_stride = 20480, .w_stride = 20480, .y_stride = 24, .w_dtype = .f32, .y_dtype = .f32 }); + enc.barrier(); + tf.deepseek_v41.mhc.encodeSplit(enc, &model.pipelines.mhc, ws.view(.mixes), ws.view(.inv), .{ .buffer = hc.scale.buffer, .offset = hc.scale.offset }, .{ .buffer = hc.base.buffer, .offset = hc.base.offset }, ws.view(.attention_pre), ws.view(.post), ws.view(.comb), .{ .rows = 1, .eps = cfg.hc_eps }); + enc.barrier(); + tf.deepseek_v41.mhc.encodeCollapse(enc, &model.pipelines.mhc, stream, ws.view(.attention_pre), ws.view(.collapsed), .{ .rows = 1, .width = 5120, .stream_stride = 20480, .y_stride = 5120 }); + enc.barrier(); + tf.deepseek_v41.mhc.encodePost(enc, &model.pipelines.mhc, branch, stream, ws.view(.post), ws.view(.comb), stream, .{ .rows = 1, .width = 5120, .branch_stride = 5120, .residual_stride = 20480, .y_stride = 20480 }); + enc.end(); + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.OperatorGpuFailure; + try attentionMetric(f32, gpa, report, count, "hc_projection", "same_retained_stream_fp64_dot", diagnosticRow(f32, ws.view(.mixes), 24), &projected); + try attentionMetric(f32, gpa, report, count, "hc_inverse", "same_retained_stream_reference_statistic", diagnosticRow(f32, ws.view(.inv), 1), &expected_inv); + try attentionMetric(f32, gpa, report, count, "hc_pre", "same_retained_stream_chain", diagnosticRow(f32, ws.view(.attention_pre), 4), &expected_mix.pre); + try attentionMetric(f32, gpa, report, count, "hc_post", "same_retained_stream_chain", diagnosticRow(f32, ws.view(.post), 4), &expected_mix.post); + try attentionMetric(f32, gpa, report, count, "hc_comb", "same_retained_stream_chain", diagnosticRow(f32, ws.view(.comb), 16), &expected_mix.comb); + var native_mix: math.Mix = undefined; + @memcpy(&native_mix.pre, diagnosticRow(f32, ws.view(.attention_pre), 4)); + @memcpy(&native_mix.post, diagnosticRow(f32, ws.view(.post), 4)); + @memcpy(&native_mix.comb, diagnosticRow(f32, ws.view(.comb), 16)); + var collapsed: [5120]u16 = undefined; + math.collapse(&input, native_mix.pre, &collapsed); + try attentionMetric(u16, gpa, report, count, "hc_collapse", "same_retained_stream_and_native_coefficients", diagnosticRow(u16, ws.view(.collapsed), 5120), &collapsed); + math.expand(&input, diagnosticRow(u16, branch, 5120), native_mix); + try attentionMetric(u16, gpa, report, count, "hc_expand", "same_retained_stream_branch_and_native_coefficients", diagnosticRow(u16, stream, 20480), &input); + if (layer + 1 < model.weights.layers.len and stream_bytes.len >= 61488 + 40960) { + @memcpy(stream.buffer.contents()[stream.offset..][0..40960], stream_bytes[61488..][0..40960]); + @memcpy(&input, diagnosticRow(u16, stream, input.len)); + const norm = model.weights.layers[layer + 1].attn_norm; + const next_cb = model.queue.commandBufferUnretained(); + const next_enc = next_cb.compute(.serial); + next_enc.useResource(norm.buffer, mtl.ResourceUsage.read); + tf.deepseek_v41.mhc.encodeCollapse(next_enc, &model.pipelines.mhc, stream, ws.view(.attention_pre), ws.view(.collapsed), .{ .rows = 1, .width = 5120, .stream_stride = 20480, .y_stride = 5120 }); + next_enc.barrier(); + tf.deepseek_v41.ops.encodeRms(next_enc, &model.pipelines.ops, ws.view(.collapsed), .{ .buffer = norm.buffer, .offset = norm.offset }, ws.view(.branch), .{ .rows = 1, .width = 5120, .x_stride = 5120, .y_stride = 5120, .eps = cfg.rms_norm_eps }); + next_enc.end(); + next_cb.commit(); + next_cb.wait(); + if (next_cb.failure() != null) return error.OperatorGpuFailure; + math.collapse(&input, native_mix.pre, &collapsed); + try attentionMetric(u16, gpa, report, count, "next_collapse", "same_retained_ffn_stream_and_native_pre", diagnosticRow(u16, ws.view(.collapsed), 5120), &collapsed); + math.collapse(&input, expected_mix.pre, &collapsed); + try attentionMetric(u16, gpa, report, count, "next_collapse", "same_retained_ffn_stream_reference_pre", diagnosticRow(u16, ws.view(.collapsed), 5120), &collapsed); + math.rms(&collapsed, norm, cfg.rms_norm_eps, &collapsed); + try attentionMetric(u16, gpa, report, count, "next_normalized", "same_retained_ffn_stream_reference_pre_and_norm", diagnosticRow(u16, ws.view(.branch), 5120), &collapsed); + } +} + +pub fn runHyperconnections(gpa: std.mem.Allocator, io: std.Io, model: *Model, snapshot: []const u8, dir: []const u8, layer: usize) !void { + if (layer >= model.weights.layers.len) return error.InvalidHyperconnectionLayer; + const blob = try std.Io.Dir.cwd().readFileAlloc(io, snapshot, gpa, .limited(8 << 20)); + defer gpa.free(blob); + const offset = layer * 2 * 61488; + if (blob.len < offset + 2 * 61488) return error.InvalidHyperconnectionSnapshot; + try model.reset(); + const branch = model.workspace.view(.branch); + @memcpy(branch.buffer.contents()[branch.offset..][0..10240], blob[offset + 61488 + 51200 ..][0..10240]); + var report: std.ArrayList(u8) = .empty; + defer report.deinit(gpa); + var count: usize = 0; + var header: [256]u8 = undefined; + try report.appendSlice(gpa, try std.fmt.bufPrint(&header, "{{\"mode\":\"hc-sameinput\",\"token\":null,\"token_identity_source\":\"unknown_headerless_snapshot\",\"layer\":{d},\"qualification_gate_applied\":false,\"cases\":[\n", .{layer})); + try hyperconnections(gpa, model, blob[offset..], branch, &report, &count, layer); + try report.appendSlice(gpa, "\n]}\n"); + const path = try std.fs.path.join(gpa, &.{ dir, "hc-sameinput.json" }); + defer gpa.free(path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = report.items }); +} diff --git a/zig/tests/deepseek_v41_reference/operator_hc_amplify_fp64.zig b/zig/tests/deepseek_v41_reference/operator_hc_amplify_fp64.zig new file mode 100644 index 0000000000..78c54e00a7 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_hc_amplify_fp64.zig @@ -0,0 +1,284 @@ +//! mHC FP64 stage propagation and 1-ULP amplification test (DeepSeek-V4.1-Flash inference, MIT). +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const math = @import("math.zig"); +const h = @import("operator_hc_fp64.zig"); +const mhc = tf.deepseek_v41.mhc; +const ops = tf.deepseek_v41.ops; +const Hc = tf.deepseek_v41_weights.Hc; +const Config = tf.deepseek_v41_config.Config; + +// Ideal FP64 stage propagation: statistic, dense, split, collapse and post, pre-rounding outputs kept. +fn stageF64(stream: []const u16, hc: Hc, cfg: Config, branch: []const u16, y_pre: []f64, y_bf: *[h.stream_width]u16, coll_pre: []f64, inv_out: *f64, comb_out: *[16]f64) void { + var proj: [h.mix_count]f64 = undefined; + math.denseOf(f64, hc.projection, stream, &proj); + const inv = math.inverseOf(f64, stream, cfg.text_config.rms_norm_eps); + const mix = math.splitOf(f64, &proj, inv, hc, cfg.text_config); + inv_out.* = inv; + comb_out.* = mix.comb; + for (0..h.width) |d| { + var s: f64 = 0; + for (0..h.copies) |c| s += mix.pre[c] * @as(f64, math.f(stream[c * h.width + d])); + coll_pre[d] = s; + } + for (0..h.width) |d| { + for (0..h.copies) |k| { + var acc: f64 = mix.post[k] * @as(f64, math.f(branch[d])); + for (0..h.copies) |j| acc += mix.comb[j * 4 + k] * @as(f64, math.f(stream[j * h.width + d])); + y_pre[k * h.width + d] = acc; + } + } + var expanded: [h.stream_width]u16 = undefined; + @memcpy(&expanded, stream); + math.expandOf(f64, &expanded, branch, mix); + @memcpy(y_bf, &expanded); +} + +fn perturbOneUlp(x: u16) u16 { + return if ((x & 0x8000) == 0) x + 1 else x - 1; +} + +test "mHC 1-ULP input amplification through the hyperconnection mix" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try mhc.Pipelines.load(device); + defer p.deinit(); + var o = try ops.Pipelines.load(device); + defer o.deinit(); + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const gpa = arena.allocator(); + const cfg = try Config.parse(gpa, h.released_config); + + const fixture = try std.json.parseFromSlice(h.Fixture, gpa, @embedFile(h.fixture_path), .{ .ignore_unknown_fields = true }); + const eight_rows = try h.decodeU16(gpa, fixture.value.eight_rows_bf16); + const ffn_stream = try h.decodeU16(gpa, fixture.value.ffn_stream_bf16); + const ffn_branch = try h.decodeU16(gpa, fixture.value.ffn_branch_bf16); + const ffn_scale = try h.decodeF32(gpa, fixture.value.ffn_scale_f32); + const ffn_base = try h.decodeF32(gpa, fixture.value.ffn_base_f32); + const weight = try h.syntheticWeight(gpa); + + const scale = try device.buffer(12, mtl.ResourceOptions.shared); + defer scale.deinit(); + h.upload(f32, scale, ffn_scale); + const base = try device.buffer(h.mix_count * 4, mtl.ResourceOptions.shared); + defer base.deinit(); + h.upload(f32, base, ffn_base); + const weight_buffer = try device.buffer(h.mix_count * h.stream_width * 4, mtl.ResourceOptions.shared); + defer weight_buffer.deinit(); + h.upload(f32, weight_buffer, weight); + const hc = Hc{ + .projection = .{ .buffer = weight_buffer, .offset = 0, .bytes = h.mix_count * h.stream_width * 4, .dtype = .f32 }, + .scale = .{ .buffer = scale, .offset = 0, .bytes = 12, .dtype = .f32 }, + .base = .{ .buffer = base, .offset = 0, .bytes = h.mix_count * 4, .dtype = .f32 }, + }; + + const columns = 64; + const column_stride = h.width / columns; + const Row = struct { name: []const u8, stream: []u16 }; + var rows: [9]Row = undefined; + for (0..8) |r| { + rows[r] = .{ .name = "teacher", .stream = try h.fourCopies(gpa, eight_rows[r * h.width ..][0..h.width]) }; + } + rows[8] = .{ .name = "ffn_token1", .stream = try gpa.dupe(u16, ffn_stream) }; + + const y0 = try gpa.alloc(f64, h.stream_width); + const y1 = try gpa.alloc(f64, h.stream_width); + var y0_bf: [h.stream_width]u16 = undefined; + var y1_bf: [h.stream_width]u16 = undefined; + const coll0 = try gpa.alloc(f64, h.width); + const coll1 = try gpa.alloc(f64, h.width); + var inv0: f64 = undefined; + var inv1: f64 = undefined; + var comb0: [16]f64 = undefined; + var comb1: [16]f64 = undefined; + + for (rows, 0..) |row, row_index| { + stageF64(row.stream, hc, cfg, ffn_branch, y0, &y0_bf, coll0, &inv0, &comb0); + var sum_y2: f64 = 0; + for (y0) |v| sum_y2 += v * v; + var sum_x2: f64 = 0; + for (row.stream) |v| sum_x2 += blk: { + const f: f64 = @floatCast(math.f(v)); + break :blk f * f; + }; + var sum_c2: f64 = 0; + for (coll0) |v| sum_c2 += v * v; + var sum_b2: f64 = 0; + for (y0_bf) |v| { + const f: f64 = @floatCast(math.f(v)); + sum_b2 += f * f; + } + + var amp_sum: f64 = 0; + var amp_max: f64 = 0; + var amp_n: usize = 0; + var amp_c_sum: f64 = 0; + var amp_c_max: f64 = 0; + var amp_b_sum: f64 = 0; + var amp_b_max: f64 = 0; + var changed_any: usize = 0; + var changed_cells: usize = 0; + var inv_rel_sum: f64 = 0; + var comb_sq: f64 = 0; + var comb_n: usize = 0; + + for (0..h.copies) |j| { + for (0..columns) |k| { + const col = k * column_stride; + const idx = j * h.width + col; + const orig = row.stream[idx]; + if (math.f(orig) == 0) continue; + const perturbed = perturbOneUlp(orig); + const dx = @as(f64, @floatCast(math.f(perturbed))) - @as(f64, @floatCast(math.f(orig))); + row.stream[idx] = perturbed; + stageF64(row.stream, hc, cfg, ffn_branch, y1, &y1_bf, coll1, &inv1, &comb1); + row.stream[idx] = orig; + + var dy2: f64 = 0; + for (y0, y1) |a, b| { + const d = a - b; + dy2 += d * d; + } + const amp = @sqrt(dy2 / sum_y2) * @sqrt(sum_x2) / @abs(dx); + amp_sum += amp; + if (amp > amp_max) amp_max = amp; + amp_n += 1; + + var dc2: f64 = 0; + for (coll0, coll1) |a, b| { + const d = a - b; + dc2 += d * d; + } + const amp_c = @sqrt(dc2 / sum_c2) * @sqrt(sum_x2) / @abs(dx); + amp_c_sum += amp_c; + if (amp_c > amp_c_max) amp_c_max = amp_c; + + var changed: usize = 0; + var dyb2: f64 = 0; + for (y0_bf, y1_bf) |a, b| { + if (a != b) changed += 1; + const av: f64 = @floatCast(math.f(a)); + const bv: f64 = @floatCast(math.f(b)); + const d = av - bv; + dyb2 += d * d; + } + const amp_b = @sqrt(dyb2 / sum_b2) * @sqrt(sum_x2) / @abs(dx); + amp_b_sum += amp_b; + if (amp_b > amp_b_max) amp_b_max = amp_b; + if (changed > 0) { + changed_any += 1; + changed_cells += changed; + } + inv_rel_sum += @abs(inv1 - inv0) / inv0; + var comb_d2: f64 = 0; + var comb_m2: f64 = 0; + for (comb0, comb1) |a, b| { + const d = a - b; + comb_d2 += d * d; + comb_m2 += b * b; + } + comb_sq += @sqrt(comb_d2 / comb_m2); + comb_n += 1; + } + } + std.debug.print("HC64 {{\"amplification\":{{\"row\":{d},\"row_name\":\"{s}\",\"perturbations\":{d},\"amp_mean\":{d},\"amp_max\":{d},\"amp_collapse_mean\":{d},\"amp_collapse_max\":{d},\"amp_bf16_mean\":{d},\"amp_bf16_max\":{d},\"bf16_changed_fraction\":{d},\"mean_changed_cells\":{d},\"inv_rel_mean\":{d},\"comb_nrms_mean\":{d}}}}}\n", .{ row_index, row.name, amp_n, amp_sum / @as(f64, @floatFromInt(amp_n)), amp_max, amp_c_sum / @as(f64, @floatFromInt(amp_n)), amp_c_max, amp_b_sum / @as(f64, @floatFromInt(amp_n)), amp_b_max, changed_any, @as(f64, @floatFromInt(changed_cells)) / @as(f64, @floatFromInt(amp_n)), inv_rel_sum / @as(f64, @floatFromInt(amp_n)), comb_sq / @as(f64, @floatFromInt(comb_n)) }); + // The mix attenuates a single-ULP input in FP64 (gain < 1) and stays near one BF16-visible ULP. + try std.testing.expect(amp_sum / @as(f64, @floatFromInt(amp_n)) < 1.0); + try std.testing.expect(amp_max < 1.0); + try std.testing.expect(amp_c_max < 2.5); + try std.testing.expect(amp_b_sum / @as(f64, @floatFromInt(amp_n)) < 3.0); + try std.testing.expect(inv_rel_sum / @as(f64, @floatFromInt(amp_n)) < 1e-5); + } + + // Metal confirmation on the first teacher row: the base row plus 16 stream-cell perturbations through the kernels. + { + const metal_perturbations = 16; + const batch = metal_perturbations + 1; + var batch_stream = try gpa.alloc(u16, batch * h.stream_width); + var cells: [metal_perturbations]usize = undefined; + var dxs: [metal_perturbations]f64 = undefined; + var filled: usize = 0; + @memcpy(batch_stream[0..h.stream_width], rows[0].stream); + outer: for (0..h.copies) |j| { + for (0..columns) |k| { + const idx = j * h.width + k * column_stride; + const orig = rows[0].stream[idx]; + if (math.f(orig) == 0) continue; + cells[filled] = idx; + dxs[filled] = @as(f64, @floatCast(math.f(perturbOneUlp(orig)))) - @as(f64, @floatCast(math.f(orig))); + filled += 1; + if (filled == metal_perturbations) break :outer; + } + } + for (0..filled) |i| { + @memcpy(batch_stream[(i + 1) * h.stream_width ..][0..h.stream_width], rows[0].stream); + batch_stream[(i + 1) * h.stream_width + cells[i]] = perturbOneUlp(rows[0].stream[cells[i]]); + } + const stream_buf = try device.buffer(batch * h.stream_width * 2, mtl.ResourceOptions.shared); + defer stream_buf.deinit(); + h.upload(u16, stream_buf, batch_stream); + const inv_buf = try device.buffer(batch * 4, mtl.ResourceOptions.shared); + defer inv_buf.deinit(); + const mixes_buf = try device.buffer(batch * h.mix_count * 4, mtl.ResourceOptions.shared); + defer mixes_buf.deinit(); + const pre_buf = try device.buffer(batch * 4 * 4, mtl.ResourceOptions.shared); + defer pre_buf.deinit(); + const post_buf = try device.buffer(batch * 4 * 4, mtl.ResourceOptions.shared); + defer post_buf.deinit(); + const comb_buf = try device.buffer(batch * 16 * 4, mtl.ResourceOptions.shared); + defer comb_buf.deinit(); + const collapsed_buf = try device.buffer(batch * h.width * 2, mtl.ResourceOptions.shared); + defer collapsed_buf.deinit(); + const branch_buf = try device.buffer(batch * h.width * 2, mtl.ResourceOptions.shared); + defer branch_buf.deinit(); + for (0..batch) |r| @memcpy(branch_buf.slice(u16, batch * h.width)[r * h.width ..][0..h.width], ffn_branch); + const y_buf = try device.buffer(batch * h.stream_width * 2, mtl.ResourceOptions.shared); + defer y_buf.deinit(); + try h.runStage(queue, &p, &o, @intCast(batch), cfg, h.view(stream_buf), h.view(weight_buffer), h.view(inv_buf), h.view(mixes_buf), h.view(scale), h.view(base), h.view(pre_buf), h.view(post_buf), h.view(comb_buf), h.view(collapsed_buf), h.view(branch_buf), h.view(y_buf)); + + const y_metal = y_buf.slice(u16, batch * h.stream_width); + var nan_count: usize = 0; + var first_nan: usize = 0; + for (y_metal[0..h.stream_width], 0..) |v, i| { + const f = math.f(v); + if (std.math.isNan(f)) { + nan_count += 1; + if (nan_count == 1) first_nan = i; + } + } + var sum_x2_row0: f64 = 0; + for (rows[0].stream) |v| { + const f: f64 = @floatCast(math.f(v)); + sum_x2_row0 += f * f; + } + var sum_base2: f64 = 0; + for (0..h.stream_width) |i| { + const f: f64 = @floatCast(math.f(y_metal[i])); + sum_base2 += f * f; + } + var amp_sum: f64 = 0; + var changed_total: usize = 0; + for (0..filled) |i| { + var dy2: f64 = 0; + var changed: usize = 0; + for (0..h.stream_width) |c| { + const a: f64 = @floatCast(math.f(y_metal[(i + 1) * h.stream_width + c])); + const b: f64 = @floatCast(math.f(y_metal[c])); + const d = a - b; + if (d != 0) changed += 1; + dy2 += d * d; + } + changed_total += changed; + amp_sum += @sqrt(dy2 / sum_base2) * @sqrt(sum_x2_row0) / @abs(dxs[i]); + } + std.debug.print("HC64 {{\"amplification\":{{\"row\":0,\"metal\":true,\"perturbations\":{d},\"amp_mean\":{d},\"mean_changed_cells\":{d}}}}}\n", .{ filled, amp_sum / @as(f64, @floatFromInt(filled)), @as(f64, @floatFromInt(changed_total)) / @as(f64, @floatFromInt(filled)) }); + const metal_amp = amp_sum / @as(f64, @floatFromInt(filled)); + try std.testing.expect(metal_amp > 0.5 and metal_amp < 3.0); + } +} diff --git a/zig/tests/deepseek_v41_reference/operator_hc_fp64.zig b/zig/tests/deepseek_v41_reference/operator_hc_fp64.zig new file mode 100644 index 0000000000..952f497e61 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_hc_fp64.zig @@ -0,0 +1,515 @@ +//! Shared mHC FP64 ideal-reference harness; receipts print "HC64" (DeepSeek-V4.1-Flash inference, MIT). +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const math = @import("math.zig"); + +const mhc = tf.deepseek_v41.mhc; +const ops = tf.deepseek_v41.ops; +pub const View = tf.deepseek_v41.attention.View; +pub const Tensor = tf.checkpoint_metal.Tensor; +pub const Hc = tf.deepseek_v41_weights.Hc; +pub const Config = tf.deepseek_v41_config.Config; + +pub const width: usize = 5120; +pub const copies: usize = 4; +pub const stream_width: usize = width * copies; +pub const mix_count: usize = 24; + +pub const released_config = + \\{"model_type":"deepseek_v41","quantization_config":{"quant_method":"exl3","version":"1"}, + \\"text_config":{"vocab_size":129280,"hidden_size":5120,"moe_intermediate_size":2304, + \\"num_hidden_layers":40,"num_attention_heads":64,"num_key_value_heads":1,"head_dim":512, + \\"qk_rope_head_dim":64,"q_lora_rank":1280,"o_lora_rank":1024,"o_groups":8, + \\"swiglu_limit":10,"rms_norm_eps":0.000001,"max_position_embeddings":1048576, + \\"rope_theta":10000,"rope_scaling":{"rope_type":"yarn","factor":16,"beta_fast":32, + \\"beta_slow":1,"original_max_position_embeddings":65536},"n_routed_experts":384, + \\"n_shared_experts":1,"num_experts_per_tok":6,"scoring_func":"sqrtsoftplus", + \\"topk_method":"noaux_tc","norm_topk_prob":true,"routed_scaling_factor":1.5, + \\"sliding_window":128,"compress_ratios":[0,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, + \\1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0],"compress_rope_theta":10000, + \\"kv_source_layer_ids":[2,8,14,20],"index_source_layer_ids":[2,8,14,20,24,28,32,36], + \\"index_n_heads":32,"index_head_dim":128,"index_topk":512,"candidate_source_layer_id":20, + \\"candidate_topk_blocks":2048,"candidate_block_size":8,"hc_mult":4,"hc_sinkhorn_iters":20, + \\"hc_eps":0.000001,"engram_layer_ids":[1,14],"engram_num_embeddings":[384006168,384016682], + \\"engram_max_ngram_size":4,"engram_vocab_size":16000000,"engram_n_heads":8,"engram_head_dim":256, + \\"engram_pad_token_id":2,"engram_compressed_vocab_size":99092,"num_nextn_predict_layers":3}} +; + +/// The 16 retained HC unary arguments (hc-exp-variants-input.json), including 1.4393715858459473. +pub const retained_exp_argument_bits = [_]u32{ + 3217026610, 1081519143, 1049174916, 1078264130, 1073892416, 1078760804, 1083574034, 1069038932, + 1043563522, 1083490981, 1087210444, 1094911558, 3195364771, 1082252091, 1096635047, 1091866698, +}; + +/// The documented controlling exponent input: Metal FP32 0x4086fa39, ideal 0x4086fa38 (1 ULP). +pub const controlling_exponent_input: f32 = 1.4393715858459473; + +pub const fixture_path = "../../src/families/deepseek_v41/fixtures/hc_fp64_rows.json"; + +pub const Fixture = struct { + eight_rows_bf16: []const u8, + ffn_stream_bf16: []const u8, + ffn_branch_bf16: []const u8, + ffn_scale_f32: []const u8, + ffn_base_f32: []const u8, + provenance: []const u8 = "", +}; + +fn decodeBase64(gpa: std.mem.Allocator, text: []const u8) ![]u8 { + const size = try std.base64.standard.Decoder.calcSizeForSlice(text); + const out = try gpa.alloc(u8, size); + errdefer gpa.free(out); + try std.base64.standard.Decoder.decode(out, text); + return out; +} + +pub fn decodeU16(gpa: std.mem.Allocator, text: []const u8) ![]u16 { + const bytes = try decodeBase64(gpa, text); + defer gpa.free(bytes); + const out = try gpa.alloc(u16, bytes.len / 2); + for (out, 0..) |*v, i| v.* = std.mem.readInt(u16, bytes[i * 2 ..][0..2], .little); + return out; +} + +pub fn decodeF32(gpa: std.mem.Allocator, text: []const u8) ![]f32 { + const bytes = try decodeBase64(gpa, text); + defer gpa.free(bytes); + const out = try gpa.alloc(f32, bytes.len / 4); + for (out, 0..) |*v, i| v.* = @bitCast(std.mem.readInt(u32, bytes[i * 4 ..][0..4], .little)); + return out; +} + +pub const Lcg = struct { + state: u64 = 0x853c49e6748fea9b, + pub fn next(s: *Lcg) u64 { + s.state = s.state *% 6364136223846793005 +% 1442695040888963407; + return s.state >> 11; + } + /// Uniform [-1, 1) with 24-bit granularity, exactly representable in FP32. + pub fn uniform(s: *Lcg) f32 { + const v: f64 = @as(f64, @floatFromInt(s.next() & 0xffffff)) / 16777216.0; + return @floatCast(v * 2 - 1); + } +}; + +fn nextUpF32(x: f32) f32 { + if (std.math.isNan(x) or std.math.isInf(x)) return x; + const bits: u32 = @bitCast(x); + if (bits == 0 or bits == 0x80000000) return @bitCast(@as(u32, 1)); + return @bitCast(if (bits >> 31 == 0) bits + 1 else bits - 1); +} + +/// The local FP32 spacing at the FP64 reference value: the ULP unit for kernel deltas. +fn ulp32At(v: f64) f64 { + const f: f32 = @floatCast(v); + if (std.math.isNan(f) or std.math.isInf(f)) return @abs(v); + const a = @abs(f); + if (a == 0) return 0x1p-149; + const up = nextUpF32(a); + if (std.math.isInf(up)) return @as(f64, a) - @as(f64, -nextUpF32(-a)); + return @as(f64, up) - @as(f64, a); +} + +pub const Delta = struct { + nrms: f64 = 0, + max_abs: f64 = 0, + max_ulp: f64 = 0, + worst: usize = 0, + count: usize = 0, +}; + +pub fn deltaF32VsF64(actual: []const f32, ref: []const f64) Delta { + var d: Delta = .{ .count = actual.len }; + var squared: f64 = 0; + var magnitude: f64 = 0; + for (actual, ref, 0..) |a, r, i| { + const diff = @as(f64, a) - r; + squared += diff * diff; + magnitude += r * r; + const ad = @abs(diff); + if (ad > d.max_abs) { + d.max_abs = ad; + d.worst = i; + } + const u = ad / ulp32At(r); + if (u > d.max_ulp) d.max_ulp = u; + } + d.nrms = if (magnitude == 0) @sqrt(squared) else @sqrt(squared / magnitude); + return d; +} + +const BfDelta = struct { + nrms: f64 = 0, + unequal_ideal: usize = 0, + unequal_f32_ref: usize = 0, + count: usize = 0, + worst: usize = 0, + worst_gap: f64 = 0, +}; + +/// BF16 outputs: decoded NRMS against the FP64 ideal cells plus ideal/F32-reference cell counts. +fn deltaBf16(actual: []const u16, ideal: []const u16, f32_ref: []const u16) BfDelta { + var d: BfDelta = .{ .count = actual.len }; + var squared: f64 = 0; + var magnitude: f64 = 0; + for (actual, ideal, f32_ref, 0..) |a, id, fr, i| { + const av: f64 = @floatCast(math.f(a)); + const iv: f64 = @floatCast(math.f(id)); + const diff = av - iv; + squared += diff * diff; + magnitude += iv * iv; + if (a != id) { + d.unequal_ideal += 1; + if (@abs(diff) > d.worst_gap) { + d.worst_gap = @abs(diff); + d.worst = i; + } + } + if (a != fr) d.unequal_f32_ref += 1; + } + d.nrms = if (magnitude == 0) @sqrt(squared) else @sqrt(squared / magnitude); + return d; +} + +fn StageRefOf(comptime T: type) type { + return struct { + inv: T, + proj: [mix_count]T, + pre: [4]T, + post: [4]T, + comb: [16]T, + collapsed: [width]u16, + y: [stream_width]u16, + }; +} + +fn referenceRow(comptime T: type, stream: []const u16, hc: Hc, cfg: Config, branch: []const u16, collapsed: *[width]u16, y: *[stream_width]u16) StageRefOf(T) { + var out: StageRefOf(T) = undefined; + out.inv = math.inverseOf(T, stream, cfg.text_config.rms_norm_eps); + math.denseOf(T, hc.projection, stream, &out.proj); + const mix = math.splitOf(T, &out.proj, out.inv, hc, cfg.text_config); + out.pre = mix.pre; + out.post = mix.post; + out.comb = mix.comb; + math.collapseOf(T, stream, mix.pre, collapsed); + var h: [stream_width]u16 = undefined; + @memcpy(&h, stream); + math.expandOf(T, &h, branch, mix); + @memcpy(y, &h); + return out; +} + +pub const Buffers = struct { + device: mtl.Device, + items: [24]mtl.Buffer = undefined, + count: usize = 0, + + pub fn get(b: *Buffers, comptime T: type, n: usize) !mtl.Buffer { + const x = try b.device.buffer(try std.math.mul(usize, @sizeOf(T), n), mtl.ResourceOptions.shared); + b.items[b.count] = x; + b.count += 1; + @memset(x.slice(T, n), 0); + return x; + } + pub fn deinit(b: *Buffers) void { + for (b.items[0..b.count]) |x| x.deinit(); + } +}; + +pub fn upload(comptime T: type, buffer: mtl.Buffer, data: []const T) void { + @memcpy(buffer.slice(T, data.len), data); +} + +pub fn view(buffer: mtl.Buffer) View { + return .{ .buffer = buffer }; +} + +pub fn runStage( + queue: mtl.Queue, + p: *const mhc.Pipelines, + o: *const ops.Pipelines, + rows: u32, + cfg: Config, + stream: View, + weight: View, + inv: View, + mixes: View, + scale: View, + base: View, + pre: View, + post: View, + comb: View, + collapsed: View, + branch: View, + y: View, +) !void { + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + mhc.encodeRms(e, p, stream, inv, .{ .rows = rows, .width = width, .stream_stride = stream_width, .eps = cfg.text_config.rms_norm_eps }); + ops.encodeDense(e, o, stream, weight, mixes, .{ .rows = rows, .k = stream_width, .n = mix_count, .x_stride = stream_width, .w_stride = stream_width, .y_stride = mix_count, .w_dtype = .f32, .y_dtype = .f32 }); + e.barrier(); + mhc.encodeSplit(e, p, mixes, inv, scale, base, pre, post, comb, .{ .rows = rows, .eps = cfg.text_config.hc_eps }); + e.barrier(); + mhc.encodeCollapse(e, p, stream, pre, collapsed, .{ .rows = rows, .width = width, .stream_stride = stream_width, .y_stride = width }); + e.barrier(); + mhc.encodePost(e, p, branch, stream, post, comb, y, .{ .rows = rows, .width = width, .branch_stride = width, .residual_stride = stream_width, .y_stride = stream_width }); + e.end(); + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.MhcFp64GpuFailure; +} + +pub const Case = struct { + name: []const u8, + rows: usize, + stream: []u16, + weight: []f32, + branch: []u16, + scale: []f32, + base: []f32, +}; + +pub const StageMetric = struct { + name: []const u8, + metal: Delta = .{}, + f32ref: Delta = .{}, + bf16: ?BfDelta = null, +}; + +pub fn printMetric(case_name: []const u8, m: StageMetric) void { + if (m.bf16) |bf| { + std.debug.print("HC64 {{\"case\":\"{s}\",\"stage\":\"{s}\",\"elements\":{d},\"nrms_metal_vs_ideal\":{d},\"bf16_unequal_metal_vs_ideal\":{d},\"bf16_unequal_f32ref_vs_ideal\":{d},\"worst_gap\":{d},\"worst_index\":{d}}}\n", .{ case_name, m.name, bf.count, bf.nrms, m.bf16.?.unequal_ideal, m.bf16.?.unequal_f32_ref, m.bf16.?.worst_gap, m.bf16.?.worst }); + } else { + std.debug.print("HC64 {{\"case\":\"{s}\",\"stage\":\"{s}\",\"elements\":{d},\"nrms_metal_vs_f64\":{d},\"max_ulp_metal_vs_f64\":{d:.4},\"max_abs_metal_vs_f64\":{d},\"worst_metal\":{d},\"nrms_f32_vs_f64\":{d},\"max_ulp_f32_vs_f64\":{d:.4}}}\n", .{ case_name, m.name, m.metal.count, m.metal.nrms, m.metal.max_ulp, m.metal.max_abs, m.metal.worst, m.f32ref.nrms, m.f32ref.max_ulp }); + } +} + +/// One full HC stage over rows streams on identical inputs: Metal, the FP32 source contract and the FP64 ideal. +fn runCase( + gpa: std.mem.Allocator, + device: mtl.Device, + queue: mtl.Queue, + p: *const mhc.Pipelines, + o: *const ops.Pipelines, + cfg: Config, + case: Case, +) !void { + var b = Buffers{ .device = device }; + defer b.deinit(); + const rows: u32 = @intCast(case.rows); + const n = case.rows; + const stream = try b.get(u16, n * stream_width); + upload(u16, stream, case.stream); + const weight = try b.get(f32, mix_count * stream_width); + upload(f32, weight, case.weight); + const inv = try b.get(f32, n); + const mixes = try b.get(f32, n * mix_count); + const scale = try b.get(f32, 3); + upload(f32, scale, case.scale); + const base = try b.get(f32, mix_count); + upload(f32, base, case.base); + const pre = try b.get(f32, n * 4); + const post = try b.get(f32, n * 4); + const comb = try b.get(f32, n * 16); + const collapsed = try b.get(u16, n * width); + const branch = try b.get(u16, n * width); + upload(u16, branch, case.branch); + const y = try b.get(u16, n * stream_width); + try runStage(queue, p, o, rows, cfg, view(stream), view(weight), view(inv), view(mixes), view(scale), view(base), view(pre), view(post), view(comb), view(collapsed), view(branch), view(y)); + + const weight_tensor: Tensor = .{ .buffer = weight, .offset = 0, .bytes = mix_count * stream_width * 4, .dtype = .f32 }; + const scale_tensor: Tensor = .{ .buffer = scale, .offset = 0, .bytes = 12, .dtype = .f32 }; + const base_tensor: Tensor = .{ .buffer = base, .offset = 0, .bytes = mix_count * 4, .dtype = .f32 }; + const hc = Hc{ .projection = weight_tensor, .scale = scale_tensor, .base = base_tensor }; + + // Gathered arrays: Metal (m), FP32 source-contract reference (32), FP64 ideal (64). + const inv_m = try gpa.alloc(f32, n); + defer gpa.free(inv_m); + @memcpy(inv_m, inv.slice(f32, n)); + const proj_m = try gpa.alloc(f32, n * mix_count); + defer gpa.free(proj_m); + @memcpy(proj_m, mixes.slice(f32, n * mix_count)); + const pre_m = try gpa.alloc(f32, n * 4); + defer gpa.free(pre_m); + @memcpy(pre_m, pre.slice(f32, n * 4)); + const post_m = try gpa.alloc(f32, n * 4); + defer gpa.free(post_m); + @memcpy(post_m, post.slice(f32, n * 4)); + const comb_m = try gpa.alloc(f32, n * 16); + defer gpa.free(comb_m); + @memcpy(comb_m, comb.slice(f32, n * 16)); + const collapsed_m = try gpa.alloc(u16, n * width); + defer gpa.free(collapsed_m); + @memcpy(collapsed_m, collapsed.slice(u16, n * width)); + const y_m = try gpa.alloc(u16, n * stream_width); + defer gpa.free(y_m); + @memcpy(y_m, y.slice(u16, n * stream_width)); + + const inv_64 = try gpa.alloc(f64, n); + defer gpa.free(inv_64); + const proj_64 = try gpa.alloc(f64, n * mix_count); + defer gpa.free(proj_64); + const pre_64 = try gpa.alloc(f64, n * 4); + defer gpa.free(pre_64); + const post_64 = try gpa.alloc(f64, n * 4); + defer gpa.free(post_64); + const comb_64 = try gpa.alloc(f64, n * 16); + defer gpa.free(comb_64); + const collapsed_64 = try gpa.alloc(u16, n * width); + defer gpa.free(collapsed_64); + const y_64 = try gpa.alloc(u16, n * stream_width); + defer gpa.free(y_64); + + const inv_32 = try gpa.alloc(f32, n); + defer gpa.free(inv_32); + const proj_32 = try gpa.alloc(f32, n * mix_count); + defer gpa.free(proj_32); + const pre_32 = try gpa.alloc(f32, n * 4); + defer gpa.free(pre_32); + const post_32 = try gpa.alloc(f32, n * 4); + defer gpa.free(post_32); + const comb_32 = try gpa.alloc(f32, n * 16); + defer gpa.free(comb_32); + const collapsed_32 = try gpa.alloc(u16, n * width); + defer gpa.free(collapsed_32); + const y_32 = try gpa.alloc(u16, n * stream_width); + defer gpa.free(y_32); + + for (0..n) |r| { + const stream_row = case.stream[r * stream_width ..][0..stream_width]; + const branch_row = case.branch[r * width ..][0..width]; + var collapsed64: [width]u16 = undefined; + var y64: [stream_width]u16 = undefined; + const ref64 = referenceRow(f64, stream_row, hc, cfg, branch_row, &collapsed64, &y64); + var collapsed32: [width]u16 = undefined; + var y32: [stream_width]u16 = undefined; + const ref32 = referenceRow(f32, stream_row, hc, cfg, branch_row, &collapsed32, &y32); + inv_64[r] = ref64.inv; + inv_32[r] = ref32.inv; + @memcpy(proj_64[r * mix_count ..][0..mix_count], &ref64.proj); + @memcpy(proj_32[r * mix_count ..][0..mix_count], &ref32.proj); + @memcpy(pre_64[r * 4 ..][0..4], &ref64.pre); + @memcpy(pre_32[r * 4 ..][0..4], &ref32.pre); + @memcpy(post_64[r * 4 ..][0..4], &ref64.post); + @memcpy(post_32[r * 4 ..][0..4], &ref32.post); + @memcpy(comb_64[r * 16 ..][0..16], &ref64.comb); + @memcpy(comb_32[r * 16 ..][0..16], &ref32.comb); + @memcpy(collapsed_64[r * width ..][0..width], &collapsed64); + @memcpy(collapsed_32[r * width ..][0..width], &collapsed32); + @memcpy(y_64[r * stream_width ..][0..stream_width], &y64); + @memcpy(y_32[r * stream_width ..][0..stream_width], &y32); + } + + const metrics = [_]StageMetric{ + .{ .name = "inverse", .metal = deltaF32VsF64(inv_m, inv_64), .f32ref = deltaF32VsF64(inv_32, inv_64) }, + .{ .name = "projection", .metal = deltaF32VsF64(proj_m, proj_64), .f32ref = deltaF32VsF64(proj_32, proj_64) }, + .{ .name = "pre", .metal = deltaF32VsF64(pre_m, pre_64), .f32ref = deltaF32VsF64(pre_32, pre_64) }, + .{ .name = "post", .metal = deltaF32VsF64(post_m, post_64), .f32ref = deltaF32VsF64(post_32, post_64) }, + .{ .name = "comb", .metal = deltaF32VsF64(comb_m, comb_64), .f32ref = deltaF32VsF64(comb_32, comb_64) }, + .{ .name = "collapse", .bf16 = deltaBf16(collapsed_m, collapsed_64, collapsed_32) }, + .{ .name = "post_expand", .bf16 = deltaBf16(y_m, y_64, y_32) }, + }; + for (metrics) |m| printMetric(case.name, m); + + // Rounding-level gate: Metal must sit at the FP32 source contract's own distance from the FP64 ideal. + for (metrics, 0..) |m, i| { + if (i < 5) { + try std.testing.expect(m.metal.nrms < 2e-7); + try std.testing.expect(m.f32ref.nrms < 2e-7); + } + } + try std.testing.expect(metrics[0].metal.max_ulp < 1.0); // statistic tree matches the wide sum here + try std.testing.expect(metrics[1].metal.max_ulp < 1.0); // compensated dense dot + try std.testing.expect(metrics[2].metal.max_ulp < 4.0); // sigmoid affine + exp boundary + try std.testing.expect(metrics[3].metal.max_ulp < 4.0); + // comb ULPs inflate on near-cancelled Sinkhorn cells; the absolute error stays at FP32 level. + try std.testing.expect(metrics[4].metal.max_abs < 5e-7); + try std.testing.expect(metrics[4].f32ref.nrms < 2e-7); + try std.testing.expect(metrics[5].bf16.?.unequal_ideal * 4096 <= metrics[5].bf16.?.count + 4096); + try std.testing.expect(metrics[6].bf16.?.unequal_ideal * 4096 <= metrics[6].bf16.?.count + 4096); + try std.testing.expect(metrics[5].bf16.?.unequal_f32_ref * 4096 <= metrics[5].bf16.?.count + 4096); + try std.testing.expect(metrics[6].bf16.?.unequal_f32_ref * 4096 <= metrics[6].bf16.?.count + 4096); +} + +pub fn syntheticWeight(gpa: std.mem.Allocator) ![]f32 { + var lcg = Lcg{}; + const w = try gpa.alloc(f32, mix_count * stream_width); + for (w) |*v| v.* = lcg.uniform() * 0.03; + return w; +} + +pub fn fourCopies(gpa: std.mem.Allocator, row: []const u16) ![]u16 { + var stream = try gpa.alloc(u16, stream_width); + for (0..copies) |c| @memcpy(stream[c * width ..][0..width], row); + return stream; +} + +test "mHC fp64 rounding level and Metal deltas on retained streams" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try mhc.Pipelines.load(device); + defer p.deinit(); + var o = try ops.Pipelines.load(device); + defer o.deinit(); + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const gpa = arena.allocator(); + const cfg = try Config.parse(gpa, released_config); + + const fixture = try std.json.parseFromSlice(Fixture, gpa, @embedFile(fixture_path), .{ .ignore_unknown_fields = true }); + const eight_rows = try decodeU16(gpa, fixture.value.eight_rows_bf16); + const ffn_stream = try decodeU16(gpa, fixture.value.ffn_stream_bf16); + const ffn_branch = try decodeU16(gpa, fixture.value.ffn_branch_bf16); + const ffn_scale = try decodeF32(gpa, fixture.value.ffn_scale_f32); + const ffn_base = try decodeF32(gpa, fixture.value.ffn_base_f32); + const token671 = eight_rows[1 * width ..][0..width]; // hc_attention_rms.json provenance: token 671. + + const weight = try syntheticWeight(gpa); + + // Token 671 embedding in four copies against real FFN scale/base; the branch row feeds a branch-free case. + { + const stream = try fourCopies(gpa, token671); + const branch = try gpa.alloc(u16, width); + @memcpy(branch, ffn_branch); + try runCase(gpa, device, queue, &p, &o, cfg, .{ .name = "token671_attention", .rows = 1, .stream = stream, .weight = weight, .branch = branch, .scale = ffn_scale, .base = ffn_base }); + } + { + const branch = try gpa.alloc(u16, width); + @memcpy(branch, ffn_branch); + try runCase(gpa, device, queue, &p, &o, cfg, .{ .name = "token1_ffn", .rows = 1, .stream = ffn_stream, .weight = weight, .branch = branch, .scale = ffn_scale, .base = ffn_base }); + } + { + const rows = 8; + const stream = try gpa.alloc(u16, rows * stream_width); + for (0..rows) |r| { + const row = eight_rows[r * width ..][0..width]; + for (0..copies) |c| @memcpy(stream[r * stream_width + c * width ..][0..width], row); + } + const branch = try gpa.alloc(u16, rows * width); + for (0..rows) |r| @memcpy(branch[r * width ..][0..width], ffn_branch); + try runCase(gpa, device, queue, &p, &o, cfg, .{ .name = "eight_teacher_rows", .rows = rows, .stream = stream, .weight = weight, .branch = branch, .scale = ffn_scale, .base = ffn_base }); + } + // Wide dynamic range: copy magnitudes 2^24..2^-12 with rotating signs, branch 2^8..2^-4. + { + const stream = try gpa.alloc(u16, stream_width); + const exponents = [_]i32{ 24, 12, 0, -12 }; + for (0..copies) |c| { + for (0..width) |d| { + const sign: f32 = if ((d + c) % 2 == 0) 1 else -1; + stream[c * width + d] = math.b(sign * @as(f32, @floatCast(std.math.pow(f64, 2, @floatFromInt(exponents[c]))))); + } + } + const branch = try gpa.alloc(u16, width); + const branch_exponents = [_]i32{ 8, 4, 0, -4 }; + for (0..width) |d| { + const sign: f32 = if (d % 2 == 0) 1 else -1; + branch[d] = math.b(sign * @as(f32, @floatCast(std.math.pow(f64, 2, @floatFromInt(branch_exponents[d % 4]))))); + } + try runCase(gpa, device, queue, &p, &o, cfg, .{ .name = "wide_dynamic_range", .rows = 1, .stream = stream, .weight = weight, .branch = branch, .scale = ffn_scale, .base = ffn_base }); + } +} diff --git a/zig/tests/deepseek_v41_reference/operator_hc_split_fp64.zig b/zig/tests/deepseek_v41_reference/operator_hc_split_fp64.zig new file mode 100644 index 0000000000..6739259e8b --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_hc_split_fp64.zig @@ -0,0 +1,303 @@ +//! mHC split and collapse FP64 ideal-reference tests (DeepSeek-V4.1-Flash inference, MIT). +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const math = @import("math.zig"); +const h = @import("operator_hc_fp64.zig"); +const mhc = tf.deepseek_v41.mhc; +const Tensor = tf.checkpoint_metal.Tensor; +const Hc = tf.deepseek_v41_weights.Hc; +const Config = tf.deepseek_v41_config.Config; + +// The comptime parameterization must keep the FP32 source contract bitwise against the old algorithm. +test "mHC fp32 reference path stays bit-identical under the comptime parameterization" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const gpa = arena.allocator(); + const cfg = try Config.parse(gpa, h.released_config); + const fixture = try std.json.parseFromSlice(h.Fixture, gpa, @embedFile(h.fixture_path), .{ .ignore_unknown_fields = true }); + const eight_rows = try h.decodeU16(gpa, fixture.value.eight_rows_bf16); + const ffn_scale = try h.decodeF32(gpa, fixture.value.ffn_scale_f32); + const ffn_base = try h.decodeF32(gpa, fixture.value.ffn_base_f32); + const scale = try device.buffer(12, mtl.ResourceOptions.shared); + defer scale.deinit(); + h.upload(f32, scale, ffn_scale); + const base = try device.buffer(h.mix_count * 4, mtl.ResourceOptions.shared); + defer base.deinit(); + h.upload(f32, base, ffn_base); + var lcg = h.Lcg{}; + const weight = try device.buffer(h.mix_count * h.stream_width * 4, mtl.ResourceOptions.shared); + defer weight.deinit(); + const weight_host = try gpa.alloc(f32, h.mix_count * h.stream_width); + for (weight_host) |*v| v.* = lcg.uniform() * 0.03; + h.upload(f32, weight, weight_host); + const hc = Hc{ + .projection = .{ .buffer = weight, .offset = 0, .bytes = h.mix_count * h.stream_width * 4, .dtype = .f32 }, + .scale = .{ .buffer = scale, .offset = 0, .bytes = 12, .dtype = .f32 }, + .base = .{ .buffer = base, .offset = 0, .bytes = h.mix_count * 4, .dtype = .f32 }, + }; + for (0..8) |r| { + const row = eight_rows[r * h.width ..][0..h.width]; + var stream: [h.stream_width]u16 = undefined; + for (0..h.copies) |c| @memcpy(stream[c * h.width ..][0..h.width], row); + const parameterized = math.mixesOf(f32, &stream, hc, cfg.text_config); + const legacy = mixesLegacy(&stream, hc, cfg); + try std.testing.expect(std.meta.eql(parameterized, legacy)); + } +} + +fn mixesLegacy(x: []const u16, hc: Hc, c: Config) math.Mix { + @setFloatMode(.strict); + var projected: [24]f32 = undefined; + math.denseOf(f32, hc.projection, x, &projected); + var squares: f64 = 0; + for (x) |v| squares += @as(f64, math.f(v)) * math.f(v); + const mean = math.meanFromWide(squares, x.len); + const r = math.idealRsqrt(mean + c.text_config.rms_norm_eps); + var m: math.Mix = undefined; + for (0..4) |i| { + m.pre[i] = math.sigmoid(@mulAdd(f32, projected[i] * r, math.value(hc.scale, 0), math.value(hc.base, i))) + c.text_config.hc_eps; + m.post[i] = 2 * math.sigmoid(@mulAdd(f32, projected[i + 4] * r, math.value(hc.scale, 1), math.value(hc.base, i + 4))); + } + for (0..4) |i| { + var high: f32 = -std.math.inf(f32); + for (0..4) |j| { + const at = i * 4 + j; + m.comb[at] = @mulAdd(f32, projected[at + 8] * r, math.value(hc.scale, 2), math.value(hc.base, at + 8)); + high = @max(high, m.comb[at]); + } + for (0..4) |j| { + m.comb[i * 4 + j] = @exp(m.comb[i * 4 + j] - high); + } + const denominator = (m.comb[i * 4] + m.comb[i * 4 + 2]) + + (m.comb[i * 4 + 1] + m.comb[i * 4 + 3]); + for (0..4) |j| m.comb[i * 4 + j] = m.comb[i * 4 + j] / denominator + c.text_config.hc_eps; + } + for (0..c.text_config.hc_sinkhorn_iters) |iteration| { + if (iteration != 0) for (0..4) |i| { + const denominator = ((m.comb[i * 4] + m.comb[i * 4 + 2]) + + (m.comb[i * 4 + 1] + m.comb[i * 4 + 3])) + c.text_config.hc_eps; + for (0..4) |j| m.comb[i * 4 + j] /= denominator; + }; + for (0..4) |j| { + const denominator = ((m.comb[j] + m.comb[8 + j]) + + (m.comb[4 + j] + m.comb[12 + j])) + c.text_config.hc_eps; + for (0..4) |i| m.comb[i * 4 + j] /= denominator; + } + } + return m; +} + +// Split-stage isolation: identical projected mixes and inverse, so only unary or reduction boundaries differ. +test "mHC split unary boundaries against the FP64 ideal on retained arguments" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try mhc.Pipelines.load(device); + defer p.deinit(); + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const gpa = arena.allocator(); + const cfg = try Config.parse(gpa, h.released_config); + + const rows = 7 + 65; // retained arguments plus every integer exponent input -32..32 + var b = h.Buffers{ .device = device }; + defer b.deinit(); + const mixes_host = try gpa.alloc(f32, rows * h.mix_count); + @memset(mixes_host, 0); + // Rows 0..3: the 16 retained unary arguments rotated through pre, post and comb positions. + for (0..4) |i| { + for (0..4) |j| { + mixes_host[i * h.mix_count + j] = @bitCast(h.retained_exp_argument_bits[(4 * i + j) % 16]); + mixes_host[i * h.mix_count + 4 + j] = @bitCast(h.retained_exp_argument_bits[(4 * i + j) % 16]); + } + for (0..16) |q| mixes_host[i * h.mix_count + 8 + q] = @bitCast(h.retained_exp_argument_bits[(i + q) % 16]); + } + // Row 4: the documented controlling exponent input as a post argument (exp(+1.439...) path). + mixes_host[4 * h.mix_count + 5] = -h.controlling_exponent_input; + // Row 5: comb affine arguments spanning the exponent range. + mixes_host[5 * h.mix_count + 8] = -32; + mixes_host[5 * h.mix_count + 9] = -16; + mixes_host[5 * h.mix_count + 10] = 16; + mixes_host[5 * h.mix_count + 11] = 32; + // Row 6: the same span through the sigmoid pre/post arguments. + const span = [_]f32{ -32, -16, 16, 32 }; + for (span, 0..) |v, j| { + mixes_host[6 * h.mix_count + j] = v; + mixes_host[6 * h.mix_count + 4 + j] = v; + } + + // Rows 7..69: one integer exponent input per row at a pre, post and comb position each. + for (0..65) |i| { + const r = 7 + i; + const v: f32 = @floatFromInt(@as(i32, @intCast(i)) - 32); + mixes_host[r * h.mix_count] = v; + mixes_host[r * h.mix_count + 4] = v; + mixes_host[r * h.mix_count + 8] = v; + } + + const mixes = try b.get(f32, rows * h.mix_count); + h.upload(f32, mixes, mixes_host); + const inv = try b.get(f32, rows); + @memset(inv.slice(f32, rows), 1.0); + const scale = try b.get(f32, 3); + @memset(scale.slice(f32, 3), 1.0); + const base = try b.get(f32, h.mix_count); + const pre = try b.get(f32, rows * 4); + const post = try b.get(f32, rows * 4); + const comb = try b.get(f32, rows * 16); + + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + mhc.encodeSplit(e, &p, h.view(mixes), h.view(inv), h.view(scale), h.view(base), h.view(pre), h.view(post), h.view(comb), .{ .rows = rows, .eps = cfg.text_config.hc_eps }); + e.end(); + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.MhcFp64GpuFailure; + + const scale_tensor: Tensor = .{ .buffer = scale, .offset = 0, .bytes = 12, .dtype = .f32 }; + const base_tensor: Tensor = .{ .buffer = base, .offset = 0, .bytes = h.mix_count * 4, .dtype = .f32 }; + const hc = Hc{ .projection = base_tensor, .scale = scale_tensor, .base = base_tensor }; + + var max_ulp_metal: f64 = 0; + var max_ulp_f32: f64 = 0; + var worst_name: []const u8 = ""; + const stages = [_]struct { name: []const u8, offset: usize, len: usize }{ + .{ .name = "pre", .offset = 0, .len = 4 }, + .{ .name = "post", .offset = 4, .len = 4 }, + .{ .name = "comb", .offset = 8, .len = 16 }, + }; + for (stages) |stage| { + for (0..rows) |r| { + const m: []const f32 = mixes_host[r * h.mix_count ..][0..h.mix_count]; + var m64: [h.mix_count]f64 = undefined; + for (m, 0..) |v, i| m64[i] = v; + const mix64 = math.splitOf(f64, &m64, 1.0, hc, cfg.text_config); + const mix32 = math.splitOf(f32, m, 1.0, hc, cfg.text_config); + const src64: []const f64 = if (std.mem.eql(u8, stage.name, "pre")) &mix64.pre else if (std.mem.eql(u8, stage.name, "post")) &mix64.post else &mix64.comb; + const src32: []const f32 = if (std.mem.eql(u8, stage.name, "pre")) &mix32.pre else if (std.mem.eql(u8, stage.name, "post")) &mix32.post else &mix32.comb; + const metal: []const f32 = if (std.mem.eql(u8, stage.name, "pre")) pre.slice(f32, rows * 4)[r * 4 ..][0..4] else if (std.mem.eql(u8, stage.name, "post")) post.slice(f32, rows * 4)[r * 4 ..][0..4] else comb.slice(f32, rows * 16)[r * 16 ..][0..16]; + const d = h.deltaF32VsF64(metal[0..stage.len], src64[0..stage.len]); + const d32 = h.deltaF32VsF64(src32[0..stage.len], src64[0..stage.len]); + if (d.max_ulp > max_ulp_metal) { + max_ulp_metal = d.max_ulp; + worst_name = stage.name; + } + if (d32.max_ulp > max_ulp_f32) max_ulp_f32 = d32.max_ulp; + std.debug.print("HC64 {{\"case\":\"split_args\",\"stage\":\"{s}\",\"row\":{d},\"nrms_metal_vs_f64\":{d},\"max_ulp_metal_vs_f64\":{d:.4},\"max_ulp_f32_vs_f64\":{d:.4}}}\n", .{ stage.name, r, d.nrms, d.max_ulp, d32.max_ulp }); + } + } + std.debug.print("HC64 {{\"case\":\"split_args\",\"summary\":true,\"max_ulp_metal_vs_f64\":{d:.4},\"worst_stage\":\"{s}\",\"max_ulp_f32_vs_f64\":{d:.4}}}\n", .{ max_ulp_metal, worst_name, max_ulp_f32 }); + // Metal may exceed the FP32 reference by one exp ULP; cancelled Sinkhorn cells inflate ULPs, NRMS stays below 1e-7. + try std.testing.expect(max_ulp_metal <= max_ulp_f32 + 2.0); + try std.testing.expect(max_ulp_metal < 48.0); +} + +// Retained cancellation fixtures: the kernel keeps the FP32 accumulation contract; the ideal shows the half-ULP gap. +test "mHC collapse and post keep source FP32 accumulation under cancellation" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + defer queue.deinit(); + var p = try mhc.Pipelines.load(device); + defer p.deinit(); + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const gpa = arena.allocator(); + const cfg = try Config.parse(gpa, h.released_config); + + var b = h.Buffers{ .device = device }; + defer b.deinit(); + // Collapse fixture: stream {3,3,0,0} (width 1), previous_pre {-1, 1+2^-23, 0, 0}. + const stream_a = try b.get(u16, 4); + const values_a = [_]u16{ math.b(3), math.b(3), math.b(0), math.b(0) }; + h.upload(u16, stream_a, &values_a); + const previous_a = try b.get(f32, 4); + const coeff = 1 + 0x1p-23; + const previous_values = [_]f32{ -1, coeff, 0, 0 }; + h.upload(f32, previous_a, &previous_values); + const collapsed = try b.get(u16, 1); + + // Post fixture: stream {1,1,1,1}, branch {1}, post {-1,0,0,0}, comb column 0 = {1, 2^-24, 2^-24, 0}, two half-ULPs. + const stream_b = try b.get(u16, 4); + const values_b = [_]u16{ math.b(1), math.b(1), math.b(1), math.b(1) }; + h.upload(u16, stream_b, &values_b); + const branch_b = try b.get(u16, 1); + const branch_values = [_]u16{math.b(1)}; + h.upload(u16, branch_b, &branch_values); + const post_b = try b.get(f32, 4); + const post_values = [_]f32{ -1, 0, 0, 0 }; + h.upload(f32, post_b, &post_values); + const comb_b = try b.get(f32, 16); + comb_b.slice(f32, 16)[0] = 1; + comb_b.slice(f32, 16)[4] = 0x1p-24; + comb_b.slice(f32, 16)[8] = 0x1p-24; + const y_b = try b.get(u16, 4); + + // Branch-product fixture: stream {3,0,0,0}, post {1+2^-23,...}, comb[0] = -1, branch {3}. + const stream_c = try b.get(u16, 4); + const values_c = [_]u16{ math.b(3), math.b(0), math.b(0), math.b(0) }; + h.upload(u16, stream_c, &values_c); + const branch_c = try b.get(u16, 1); + const branch_values_c = [_]u16{math.b(3)}; + h.upload(u16, branch_c, &branch_values_c); + const post_c = try b.get(f32, 4); + const post_values_c = [_]f32{ coeff, 0, 0, 0 }; + h.upload(f32, post_c, &post_values_c); + const comb_c = try b.get(f32, 16); + comb_c.slice(f32, 16)[0] = -1; + const y_c = try b.get(u16, 4); + + const cb = queue.commandBuffer(); + const e = cb.compute(.serial); + mhc.encodeCollapse(e, &p, h.view(stream_a), h.view(previous_a), h.view(collapsed), .{ .rows = 1, .width = 1, .stream_stride = 4, .y_stride = 1 }); + mhc.encodePost(e, &p, h.view(branch_b), h.view(stream_b), h.view(post_b), h.view(comb_b), h.view(y_b), .{ .rows = 1, .width = 1, .branch_stride = 1, .residual_stride = 4, .y_stride = 4 }); + mhc.encodePost(e, &p, h.view(branch_c), h.view(stream_c), h.view(post_c), h.view(comb_c), h.view(y_c), .{ .rows = 1, .width = 1, .branch_stride = 1, .residual_stride = 4, .y_stride = 4 }); + e.end(); + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.MhcFp64GpuFailure; + + // Source-contract expectations (math.zig retained boundaries). + try std.testing.expectEqual(@as(u16, 0x3500), collapsed.slice(u16, 1)[0]); + try std.testing.expectEqual(@as(u16, 0), y_b.slice(u16, 4)[0]); + try std.testing.expectEqual(@as(u16, 0x3500), y_c.slice(u16, 4)[0]); + + // Reference columns for the same fixtures. + var collapsed32: [1]u16 = undefined; + var collapsed64: [1]u16 = undefined; + math.collapseOf(f32, &values_a, .{ -1, coeff, 0, 0 }, &collapsed32); + math.collapseOf(f64, &values_a, .{ -1, @as(f64, coeff), 0, 0 }, &collapsed64); + std.debug.print("HC64 {{\"case\":\"cancellation\",\"fixture\":\"collapse_half_ulps\",\"metal\":\"0x{x}\",\"f32_ref\":\"0x{x}\",\"f64_ideal\":\"0x{x}\"}}\n", .{ collapsed.slice(u16, 1)[0], collapsed32[0], collapsed64[0] }); + try std.testing.expectEqual(collapsed32[0], collapsed.slice(u16, 1)[0]); + + const mix_b = math.MixOf(f32){ .pre = .{ 1, 0, 0, 0 }, .post = .{ -1, 0, 0, 0 }, .comb = .{ 1, 0, 0, 0, 0x1p-24, 0, 0, 0, 0x1p-24, 0, 0, 0, 0, 0, 0, 0 } }; + var y32: [4]u16 = undefined; + var h32 = values_b; + math.expandOf(f32, &h32, &branch_values, mix_b); + @memcpy(&y32, &h32); + const mix_b64 = math.MixOf(f64){ .pre = .{ 1, 0, 0, 0 }, .post = .{ -1, 0, 0, 0 }, .comb = .{ 1, 0, 0, 0, 0x1p-24, 0, 0, 0, 0x1p-24, 0, 0, 0, 0, 0, 0, 0 } }; + var h64 = values_b; + math.expandOf(f64, &h64, &branch_values, mix_b64); + std.debug.print("HC64 {{\"case\":\"cancellation\",\"fixture\":\"post_half_ulps\",\"metal\":\"0x{x}\",\"f32_ref\":\"0x{x}\",\"f64_ideal\":\"0x{x}\"}}\n", .{ y_b.slice(u16, 4)[0], y32[0], h64[0] }); + try std.testing.expectEqual(y32[0], y_b.slice(u16, 4)[0]); + + const mix_c = math.MixOf(f32){ .pre = .{ 1, 0, 0, 0 }, .post = .{ coeff, 0, 0, 0 }, .comb = .{ -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; + var hc32 = values_c; + math.expandOf(f32, &hc32, &branch_values_c, mix_c); + const mix_c64 = math.MixOf(f64){ .pre = .{ 1, 0, 0, 0 }, .post = .{ @as(f64, coeff), 0, 0, 0 }, .comb = .{ -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; + var hc64 = values_c; + math.expandOf(f64, &hc64, &branch_values_c, mix_c64); + std.debug.print("HC64 {{\"case\":\"cancellation\",\"fixture\":\"branch_product\",\"metal\":\"0x{x}\",\"f32_ref\":\"0x{x}\",\"f64_ideal\":\"0x{x}\"}}\n", .{ y_c.slice(u16, 4)[0], hc32[0], hc64[0] }); + try std.testing.expectEqual(hc32[0], y_c.slice(u16, 4)[0]); + _ = cfg; +} diff --git a/zig/tests/deepseek_v41_reference/operator_history.zig b/zig/tests/deepseek_v41_reference/operator_history.zig new file mode 100644 index 0000000000..499a1cd463 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_history.zig @@ -0,0 +1,298 @@ +//! Retained-input attention history and source-layout receipts. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const projection = @import("projection.zig"); +const math = @import("math.zig"); +const attention = tf.deepseek_v41.attention; +const Model = tf.deepseek_v41.Model; +const diagnostic = @import("operator_diagnostic.zig"); +const diagnosticRow = diagnostic.diagnosticRow; +const attentionMetric = diagnostic.attentionMetric; +const attention_math = @import("attention_math.zig"); +const AttentionStage = diagnostic.AttentionStage; +const AttentionCapture = diagnostic.AttentionCapture; +const attention_widths = diagnostic.attention_widths; +const attention_elements = diagnostic.attention_elements; +const attentionOffset = diagnostic.attentionOffset; +const attentionProjection = diagnostic.attentionProjection; +const attentionWrite = diagnostic.attentionWrite; + +// Source indices address physical ring slots then compressed rows; reference reserves 128 window rows even at zero. +fn attentionHistoryIndices(position: usize, selected: []const i32, compressed_offset: usize, out: *[640]i32) []const i32 { + const window_columns: usize = if (position == 0) 1 else 128; + if (position == 0) { + out[0] = 0; + } else { + for (0..128) |slot| { + const at = (position % 128 + 1 + slot) % 128; + out[slot] = if (at > position) -1 else @intCast(at); + } + } + for (selected, 0..) |row, slot| + out[window_columns + slot] = if (row < 0) -1 else @intCast(compressed_offset + @as(usize, @intCast(row))); + return out[0 .. window_columns + selected.len]; +} + +/// Replay prerequisite layers at each position; native/reference cursors advance without borrowed intermediates. +pub fn runAttentionHistory(gpa: std.mem.Allocator, io: std.Io, model: *Model, trace_dir: []const u8, dir: []const u8, token: usize, layer: usize) !void { + if (layer >= model.weights.layers.len) return error.InvalidAttentionLayer; + if (token >= model.cache.plan.capacity) return error.InvalidAttentionHistory; + if (token >= attention_math.max_capacity) return error.DiagnosticHistoryTooLarge; + const prefix_elements = try std.math.mul(usize, token + 1, (layer + 1) * 5120); + if (prefix_elements > (256 << 20) / @sizeOf(u16)) return error.DiagnosticHistoryTooLarge; + const cfg = model.cfg.text_config; + const history_count = token + 1; + const ratio = cfg.compress_ratios[layer]; + const compressed_rows = if (ratio == 0) 0 else history_count / ratio; + const topk = @min(compressed_rows, cfg.index_topk); + const window_rows: usize = if (token == 0) 1 else 128; + const snapshot_bytes = 61488; + const snapshot_offset = layer * 2 * snapshot_bytes; + const weights = &model.weights.layers[layer].attn; + const input = model.workspace.view(.normalized); + const native = try model.device.buffer(attention_elements * 2, mtl.ResourceOptions.shared); + defer native.deinit(); + const decoded = try model.device.buffer((128 * 512 + compressed_rows * 640) * 2, mtl.ResourceOptions.shared); + defer decoded.deinit(); + const reference_rows = try gpa.alloc(u16, attention_elements); + defer gpa.free(reference_rows); + const sparse_snapshot = try gpa.alloc(u16, 32768); + defer gpa.free(sparse_snapshot); + const normalized_prefix = try gpa.alloc(u16, prefix_elements); + defer gpa.free(normalized_prefix); + var retained_branch: [5120]u16 = undefined; + var capture = AttentionCapture{ .model = model, .native = native, .reference = reference_rows }; + for ([_]mtl.Buffer{ native, decoded }) |buffer| model.residency.add(buffer); + model.residency.commit(); + model.residency.requestResidency(); + defer { + for ([_]mtl.Buffer{ native, decoded }) |buffer| model.residency.remove(buffer); + model.residency.commit(); + } + var scratch = try projection.Scratch.init(gpa, 8192, 32768); + defer scratch.deinit(); + var state = try attention_math.State.init(gpa, history_count); + defer state.deinit(); + var output: [5120]u16 = undefined; + try model.reset(); + for (0..history_count) |position| { + var name: [64]u8 = undefined; + const trace_name = try std.fmt.bufPrint(&name, "trace-{d}-gpu.bin", .{position}); + const path = try std.fs.path.join(gpa, &.{ trace_dir, trace_name }); + defer gpa.free(path); + const blob = try std.Io.Dir.cwd().readFileAlloc(io, path, gpa, .limited(8 << 20)); + defer gpa.free(blob); + if (blob.len != model.weights.layers.len * 2 * snapshot_bytes) return error.InvalidAttentionSnapshot; + if (model.cache.position != position) return error.InvalidAttentionHistoryCursor; + for (0..layer + 1) |prior| { + const offset = prior * 2 * snapshot_bytes + 40960; + const row = normalized_prefix[(position * (layer + 1) + prior) * 5120 ..][0..5120]; + @memcpy(std.mem.sliceAsBytes(row), blob[offset..][0..10240]); + @memcpy(input.buffer.contents()[input.offset..][0..10240], std.mem.sliceAsBytes(row)); + const target = position == token and prior == layer; + const cb = model.queue.commandBufferUnretained(); + const enc = cb.compute(.serial); + var encoding = true; + errdefer if (encoding) enc.end(); + for (model.checkpoint.shards.items) |shard| enc.useResource(shard.buffer, mtl.ResourceUsage.read); + if (target) { + try attention.encodeDiagnostic(&model.pipelines.attention, enc, &model.weights.layers[prior].attn, &model.cache, &model.attention_workspace, input, model.workspace.view(.branch), &model.pipelines.projection, model.workspace.projection(), @intCast(prior), 1, cfg, true, .{ .ctx = &capture, .capture = AttentionCapture.gpu }); + @memcpy(std.mem.sliceAsBytes(&retained_branch), blob[snapshot_offset + 51200 ..][0..10240]); + } else { + try attention.encode(&model.pipelines.attention, enc, &model.weights.layers[prior].attn, &model.cache, &model.attention_workspace, input, model.workspace.view(.branch), &model.pipelines.projection, model.workspace.projection(), @intCast(prior), 1, cfg); + } + enc.end(); + encoding = false; + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.OperatorGpuFailure; + // Wait before the next retained row replaces the native workspace input. + if (target) { + state.observer = .{ .ctx = &capture, .capture = AttentionCapture.cpu }; + state.sparse_snapshot = sparse_snapshot; + } + try state.apply(model.weights.layers[prior].attn, prior, row, position, cfg, &output, &scratch); + state.observer = null; + state.sparse_snapshot = null; + } + try model.cache.append(1); + } + + // Export post-insertion physical ring slots, masking never-written slots in source indices. + const decoded_view = attention.View{ .buffer = decoded }; + const native_window = decoded.slice(u16, 128 * 512 + compressed_rows * 640)[0 .. 128 * 512]; + const native_compressed = decoded.slice(u16, 128 * 512 + compressed_rows * 640)[128 * 512 ..][0 .. compressed_rows * 512]; + const native_index_keys = decoded.slice(u16, 128 * 512 + compressed_rows * 640)[128 * 512 + compressed_rows * 512 ..][0 .. compressed_rows * 128]; + const unpack_cb = model.queue.commandBufferUnretained(); + const unpack_enc = unpack_cb.compute(.serial); + var unpack_encoding = true; + errdefer if (unpack_encoding) unpack_enc.end(); + const ring = try model.cache.ring(@intCast(layer)); + try model.pipelines.attention.cache.encodeUnpack(unpack_enc, ring, decoded_view, .{ .rows = 128, .width = 512, .stride = 512, .packed_stride = 528, .block = 32, .kind = .window_fp8 }); + if (compressed_rows != 0) { + const source = try model.cache.source(@intCast(layer), ratio); + try model.pipelines.attention.cache.encodeUnpack(unpack_enc, source.main, decoded_view.shifted(128 * 512 * 2), .{ .rows = @intCast(compressed_rows), .width = 512, .stride = 512, .packed_stride = 288, .block = 16, .kind = .main_fp4 }); + try model.pipelines.attention.cache.encodeUnpack(unpack_enc, source.index, decoded_view.shifted((128 * 512 + compressed_rows * 512) * 2), .{ .rows = @intCast(compressed_rows), .width = 128, .stride = 128, .packed_stride = 68, .block = 32, .kind = .index_fp4 }); + } + unpack_enc.end(); + unpack_encoding = false; + unpack_cb.commit(); + unpack_cb.wait(); + if (unpack_cb.failure() != null) return error.OperatorGpuFailure; + const reference_window = state.window[layer * 128 * 512 ..][0 .. 128 * 512]; + const reference_compressed = try gpa.alloc(u16, compressed_rows * 640); + defer gpa.free(reference_compressed); + const reference_main = reference_compressed[0 .. compressed_rows * 512]; + const reference_index_keys = reference_compressed[compressed_rows * 512 ..]; + for (0..compressed_rows) |row| { + const source_row = state.sources[state.active_owner * state.capacity * 640 + row * 640 ..][0..640]; + @memcpy(reference_main[row * 512 ..][0..512], source_row[0..512]); + @memcpy(reference_index_keys[row * 128 ..][0..128], source_row[512..640]); + } + const native_selected = diagnosticRow(i32, model.attention_workspace.selection.positions(), 512)[0..topk]; + var reference_selected: [512]i32 = undefined; + for (state.selected[0..topk], 0..) |row, i| reference_selected[i] = @intCast(row); + var native_indices_storage: [640]i32 = undefined; + var reference_indices_storage: [640]i32 = undefined; + const native_indices = attentionHistoryIndices(token, native_selected, window_rows, &native_indices_storage); + const reference_indices = attentionHistoryIndices(token, reference_selected[0..topk], window_rows, &reference_indices_storage); + var sinks: [64]f32 = undefined; + for (&sinks, 0..) |*value, head| value.* = math.value(weights.sink, head); + + var first_stage: ?[]const u8 = null; + inline for (std.enums.values(AttentionStage)) |stage| + if (first_stage == null and !std.mem.eql(u16, capture.row(stage), capture.cpuRow(stage))) { + first_stage = @tagName(stage); + }; + const header = try std.json.Stringify.valueAlloc(gpa, .{ + .mode = "attention-history-sameinput", + .token = token, + .layer = layer, + .history_count = history_count, + .token_identity_source = "caller_supplied_TOKEN_and_trace_POSITION_gpu_filenames_raw_snapshots_have_no_token_header", + .normalized_prefix_replay = .{ .file_pattern = "trace-{position}-gpu.bin", .origin = "retained_native_GPU_normalized_attention_inputs", .first_position = 0, .last_position = token, .first_layer = 0, .last_layer = layer, .order = "position_major_then_layer_ascending_inclusive", .snapshot_count_per_file = model.weights.layers.len * 2, .snapshot_length_bytes = snapshot_bytes, .attention_snapshot_index = "2*layer", .normalized_input_offset_within_snapshot_bytes = 40960, .normalized_input_length_bytes = 10240, .native_cache_reset = true, .native_cache_cursor_matches_each_position = true, .native_cache_append_rows_per_position = 1, .reference_position_passed_explicitly = true, .reference_capacity = state.capacity }, + .identical_retained_normalized_inputs = true, + .independent_reference_history = true, + .native_history_replay_branch_matches_retained_trace = std.mem.eql(u16, capture.row(.wo_b), &retained_branch), + .first_differing_same_history_input_stage = first_stage, + .window_history_equal = std.mem.eql(u16, native_window, reference_window), + .compressed_main_equal = std.mem.eql(u16, native_compressed, reference_main), + .compressed_index_keys_equal = std.mem.eql(u16, native_index_keys, reference_index_keys), + .selection_equal = std.mem.eql(i32, native_selected, reference_selected[0..topk]), + .compression_ratio = ratio, + .compressed_rows = compressed_rows, + .selection_count = topk, + .reference_dot = "independent_EXL3_FP64", + .reference_sparse_dot = "independent_FP64", + .qualification_gate_applied = false, + .qualification_oracle_modified_to_reuse_native_values = false, + .same_native_query_cache_diagnostic = "separate_reference_State_with_captured_native_window_main_selection_and_q_not_used_in_independent_chain", + .official_revision = "2cba9e42aa026125f3ed06c6d98c1db82f7ca027", + .receipt_dtype_endianness = "little_endian", + .receipt_strides = "contiguous_element_strides", + .files = .{ + .stage_chain = .{ .native = "attention-native.bf16", .reference = "attention-reference.bf16", .dtype = "bf16", .shape = .{attention_elements}, .strides = .{1}, .layout = "stages_table_below" }, + .normalized_prefix = .{ .file = "attention-normalized-prefix.bf16", .dtype = "bf16", .shape = .{ history_count, layer + 1, 5120 }, .strides = .{ (layer + 1) * 5120, 5120, 1 } }, + .target_input = .{ .file = "attention-input.bf16", .dtype = "bf16", .shape = .{5120}, .strides = .{1} }, + .retained_branch = .{ .file = "attention-retained-branch.bf16", .dtype = "bf16", .shape = .{5120}, .strides = .{1} }, + .query = .{ .native = "attention-query-native.bf16", .reference = "attention-query-reference.bf16", .dtype = "bf16", .shape = .{ 1, 1, 64, 512 }, .strides = .{ 32768, 32768, 512, 1 } }, + .window = .{ .native = "attention-window-native.bf16", .reference = "attention-window-reference.bf16", .dtype = "bf16", .shape = .{ 1, 128, 512 }, .strides = .{ 65536, 512, 1 }, .layout = "physical_ring_slot_position_modulo_128_includes_current_quantized_KV" }, + .compressed_main = .{ .native = "attention-compressed-native.bf16", .reference = "attention-compressed-reference.bf16", .dtype = "bf16", .shape = .{ 1, compressed_rows, 512 }, .strides = .{ compressed_rows * 512, 512, 1 } }, + .compressed_index_keys = .{ .native = "attention-index-keys-native.bf16", .reference = "attention-index-keys-reference.bf16", .dtype = "bf16", .shape = .{ 1, compressed_rows, 128 }, .strides = .{ compressed_rows * 128, 128, 1 } }, + .selection = .{ .native = "attention-selection-native.i32", .reference = "attention-selection-reference.i32", .dtype = "i32", .shape = .{ 1, topk }, .strides = .{ topk, 1 }, .layout = "actual_compressed_row_ids_compact_export_native_workspace_row_stride_is_512" }, + .source_kv = .{ .native = "attention-source-kv-native.bf16", .reference = "attention-source-kv-reference.bf16", .dtype = "bf16", .shape = .{ 1, window_rows + compressed_rows, 512 }, .strides = .{ (window_rows + compressed_rows) * 512, 512, 1 }, .layout = "source_sparse_attn_KV_window_then_compressed_rows_no_requantization" }, + .source_indices = .{ .native = "attention-source-indices-native.i32", .reference = "attention-source-indices-reference.i32", .dtype = "i32", .shape = .{ 1, 1, native_indices.len }, .strides = .{ native_indices.len, native_indices.len, 1 }, .layout = "source_get_window_topk_idxs_then_actual_selection_plus_window_rows_minus_one_remains_masked" }, + .sinks = .{ .file = "attention-sinks.f32", .dtype = "f32", .shape = .{64}, .strides = .{1} }, + .sparse_output = .{ .native = "attention-sparse-native.bf16", .reference = "attention-sparse-reference.bf16", .same_native_input_reference = "attention-sparse-same-native-input-reference.bf16", .dtype = "bf16", .shape = .{ 1, 1, 64, 512 }, .strides = .{ 32768, 32768, 512, 1 } }, + }, + .source_sparse_call = "sparse_attn(query,source_kv,sinks,source_indices,1/sqrt(512)); output_before_inverse_RoPE", + .source_sparse_replay_is_operator_evidence_only = true, + }, .{}); + defer gpa.free(header); + var report: std.ArrayList(u8) = .empty; + defer report.deinit(gpa); + try report.appendSlice(gpa, header[0 .. header.len - 1]); + try report.appendSlice(gpa, ",\"cases\":[\n"); + var count: usize = 0; + inline for (std.enums.values(AttentionStage)) |stage| + try attentionMetric(u16, gpa, &report, &count, @tagName(stage), "same_retained_normalized_history_input_chain", capture.row(stage), capture.cpuRow(stage)); + try attentionMetric(u16, gpa, &report, &count, "retained_branch", "native_history_replay_vs_retained_native_trace", capture.row(.wo_b), &retained_branch); + try attentionMetric(u16, gpa, &report, &count, "window_history", "same_retained_normalized_history_input_chain", native_window, reference_window); + if (compressed_rows != 0) { + try attentionMetric(u16, gpa, &report, &count, "compressed_kv", "same_retained_normalized_history_input_chain", native_compressed, reference_main); + try attentionMetric(u16, gpa, &report, &count, "compressed_index_keys", "same_retained_normalized_history_input_chain", native_index_keys, reference_index_keys); + } + + // This separate diagnostic State never overwrites independent retained-history reference data. + var native_input_state = try attention_math.State.init(gpa, history_count); + defer native_input_state.deinit(); + for (0..compressed_rows) |row| + @memcpy(native_input_state.sources[row * 640 ..][0..512], native_compressed[row * 512 ..][0..512]); + var helper_indices_storage: [640]i32 = undefined; + const helper_indices = attentionHistoryIndices(token, native_selected, 128, &helper_indices_storage); + var same_input: [32768]u16 = undefined; + for (0..64) |head| + native_input_state.sparseHead(capture.row(.q)[head * 512 ..][0..512], native_window, helper_indices, sinks[head], same_input[head * 512 ..][0..512]); + try attentionMetric(u16, gpa, &report, &count, "sparse_o", "same_native_query_window_compressed_cache_selection_reference_sparse_diagnostic", capture.row(.sparse_o), &same_input); + try attentionWrite(gpa, io, dir, "attention-sparse-same-native-input-reference.bf16", std.mem.sliceAsBytes(&same_input)); + math.rms(capture.row(.qr_projection), weights.q_norm, cfg.rms_norm_eps, same_input[0..1280]); + try attentionMetric(u16, gpa, &report, &count, "qr_norm", "same_native_input_reference_rms_diagnostic", capture.row(.qr_norm), same_input[0..1280]); + math.rms(capture.row(.kv_projection), weights.kv_norm, cfg.rms_norm_eps, same_input[0..512]); + try attentionMetric(u16, gpa, &report, &count, "kv_norm", "same_native_input_reference_rms_diagnostic", capture.row(.kv_norm), same_input[0..512]); + @memcpy(&same_input, capture.row(.q_projection)); + for (0..64) |head| math.rope(same_input[head * 512 ..][0..512], token, ratio != 0, false, cfg); + try attentionMetric(u16, gpa, &report, &count, "q", "same_native_input_reference_rope_diagnostic", capture.row(.q), &same_input); + @memcpy(same_input[0..512], capture.row(.kv_norm)); + math.rope(same_input[0..512], token, ratio != 0, false, cfg); + try attentionMetric(u16, gpa, &report, &count, "kv_rope", "same_native_input_reference_rope_diagnostic", capture.row(.kv_rope), same_input[0..512]); + @memcpy(same_input[0..512], capture.row(.kv_rope)); + math.quantize(same_input[0..512], .window); + try attentionMetric(u16, gpa, &report, &count, "kv_quantized", "same_native_input_reference_quantize_diagnostic", capture.row(.kv_quantized), same_input[0..512]); + const expected = try gpa.alloc(f32, 32768); + defer gpa.free(expected); + try attentionProjection(gpa, model, weights.wq_a, input, "wq_a", &scratch, expected, &report, &count); + try attentionProjection(gpa, model, weights.wq_b, capture.view(.qr_norm), "wq_b", &scratch, expected, &report, &count); + try attentionProjection(gpa, model, weights.wkv, input, "wkv", &scratch, expected, &report, &count); + for (weights.wo_a, 0..) |matrix, group| { + var name: [32]u8 = undefined; + try attentionProjection(gpa, model, matrix, capture.view(.o).shifted(group * 4096 * 2), try std.fmt.bufPrint(&name, "wo_a_group{d}", .{group}), &scratch, expected, &report, &count); + } + try attentionProjection(gpa, model, weights.wo_b, capture.view(.grouped), "wo_b", &scratch, expected, &report, &count); + try report.appendSlice(gpa, "\n],\"stages\":["); + inline for (std.enums.values(AttentionStage), 0..) |stage, i| { + const descriptor = try std.json.Stringify.valueAlloc(gpa, .{ .name = @tagName(stage), .offset_bytes = attentionOffset(stage) * 2, .elements = attention_widths[i] }, .{}); + defer gpa.free(descriptor); + if (i != 0) try report.appendSlice(gpa, ","); + try report.appendSlice(gpa, descriptor); + } + try report.appendSlice(gpa, "]}\n"); + const source_kv = try gpa.alloc(u16, 2 * (window_rows + compressed_rows) * 512); + defer gpa.free(source_kv); + const source_native = source_kv[0 .. (window_rows + compressed_rows) * 512]; + const source_reference = source_kv[(window_rows + compressed_rows) * 512 ..]; + @memcpy(source_native[0 .. window_rows * 512], native_window[0 .. window_rows * 512]); + @memcpy(source_native[window_rows * 512 ..], native_compressed); + @memcpy(source_reference[0 .. window_rows * 512], reference_window[0 .. window_rows * 512]); + @memcpy(source_reference[window_rows * 512 ..], reference_main); + const input_row = normalized_prefix[(history_count * (layer + 1) - 1) * 5120 ..][0..5120]; + inline for (.{ + "attention-history-sameinput.json", "attention-native.bf16", "attention-reference.bf16", + "attention-normalized-prefix.bf16", "attention-input.bf16", "attention-retained-branch.bf16", + "attention-query-native.bf16", "attention-query-reference.bf16", "attention-window-native.bf16", + "attention-window-reference.bf16", "attention-compressed-native.bf16", "attention-compressed-reference.bf16", + "attention-index-keys-native.bf16", "attention-index-keys-reference.bf16", "attention-selection-native.i32", + "attention-selection-reference.i32", "attention-source-kv-native.bf16", "attention-source-kv-reference.bf16", + "attention-source-indices-native.i32", "attention-source-indices-reference.i32", "attention-sinks.f32", + "attention-sparse-native.bf16", "attention-sparse-reference.bf16", + }, .{ + report.items, std.mem.sliceAsBytes(native.slice(u16, attention_elements)), std.mem.sliceAsBytes(reference_rows), + std.mem.sliceAsBytes(normalized_prefix), std.mem.sliceAsBytes(input_row), std.mem.sliceAsBytes(&retained_branch), + std.mem.sliceAsBytes(capture.row(.q)), std.mem.sliceAsBytes(capture.cpuRow(.q)), std.mem.sliceAsBytes(native_window), + std.mem.sliceAsBytes(reference_window), std.mem.sliceAsBytes(native_compressed), std.mem.sliceAsBytes(reference_main), + std.mem.sliceAsBytes(native_index_keys), std.mem.sliceAsBytes(reference_index_keys), std.mem.sliceAsBytes(native_selected), + std.mem.sliceAsBytes(reference_selected[0..topk]), std.mem.sliceAsBytes(source_native), std.mem.sliceAsBytes(source_reference), + std.mem.sliceAsBytes(native_indices), std.mem.sliceAsBytes(reference_indices), std.mem.sliceAsBytes(&sinks), + std.mem.sliceAsBytes(capture.row(.sparse_o)), std.mem.sliceAsBytes(capture.cpuRow(.sparse_o)), + }) |name, data| try attentionWrite(gpa, io, dir, name, data); +} diff --git a/zig/tests/deepseek_v41_reference/operator_moe.zig b/zig/tests/deepseek_v41_reference/operator_moe.zig new file mode 100644 index 0000000000..558234569f --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operator_moe.zig @@ -0,0 +1,409 @@ +//! Retained-input MoE routing and expert isolation. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const projection = @import("projection.zig"); +const math = @import("math.zig"); +const attention = tf.deepseek_v41.attention; +const Model = tf.deepseek_v41.Model; +const ex = tf.exl3_projection; +const diagnosticRow = @import("operator_diagnostic.zig").diagnosticRow; + +const moe_reference = @import("reference.zig"); +const MoeRows = struct { + gate: [7][2304]u16 = undefined, + up: [7][2304]u16 = undefined, + activation: [7][2304]u16 = undefined, + down: [7][5120]u16 = undefined, + branch: [5120]u16 = undefined, +}; +const MoeFirst = struct { + source_chain: ?[]const u8 = null, + source_expert_stage: ?[]const u8 = null, + native_input_diagnostic: ?[]const u8 = null, + native_weights_reference_chain_diagnostic: ?[]const u8 = null, + retained_weights_reference_chain_diagnostic: ?[]const u8 = null, + native_down_reference_combine_diagnostic_differs: bool = false, +}; + +fn moeMetric(comptime T: type, gpa: std.mem.Allocator, report: *std.ArrayList(u8), count: *usize, name: []const u8, comparison: []const u8, diagnostic_reuse: bool, actual: []const T, expected: []const T) !bool { + if (actual.len == 0 or actual.len != expected.len) return error.InvalidMoeMetricShape; + var squared: f64 = 0; + var magnitude: f64 = 0; + var max_abs: f64 = 0; + var worst: usize = 0; + var unequal: usize = 0; + var first: ?usize = null; + for (actual, expected, 0..) |a, r, i| { + const x = if (T == u16) math.f(a) else a; + const y = if (T == u16) math.f(r) else r; + if (!std.math.isFinite(x) or !std.math.isFinite(y)) return error.NonfiniteMoeDiagnostic; + const delta = @as(f64, x) - y; + squared += delta * delta; + magnitude += @as(f64, y) * y; + if (@abs(delta) > max_abs) { + max_abs = @abs(delta); + worst = i; + } + const ab = if (T == u16) a else @as(u32, @bitCast(a)); + const rb = if (T == u16) r else @as(u32, @bitCast(r)); + if (ab != rb) { + unequal += 1; + if (first == null) first = i; + } + } + const line = try std.json.Stringify.valueAlloc(gpa, .{ + .name = name, + .comparison = comparison, + .diagnostic_native_value_reuse = diagnostic_reuse, + .qualification_gate_applied = false, + .elements = actual.len, + .output_dtype = if (T == u16) "bf16" else "f32", + .normalized_rms_error = if (magnitude == 0) @sqrt(squared) else @sqrt(squared / magnitude), + .max_absolute_error = max_abs, + .worst_index = worst, + .bit_unequal_count = unequal, + .first_bit_unequal_index = first, + .worst_native = if (T == u16) math.f(actual[worst]) else actual[worst], + .worst_reference = if (T == u16) math.f(expected[worst]) else expected[worst], + }, .{}); + defer gpa.free(line); + if (count.* != 0) try report.appendSlice(gpa, ",\n"); + try report.appendSlice(gpa, line); + count.* += 1; + std.debug.print("{s}\n", .{line}); + return unequal != 0; +} + +fn moeWrite(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, name: []const u8, bytes: []const u8) !void { + const path = try std.fs.path.join(gpa, &.{ dir, name }); + defer gpa.free(path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = bytes }); +} + +fn moeWriteRows(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, prefix: []const u8, rows: *const MoeRows) !void { + inline for (.{ "gate", "up", "activation", "down", "branch" }) |stage| { + const name = try std.fmt.allocPrint(gpa, "{s}-{s}.bf16", .{ prefix, stage }); + defer gpa.free(name); + try moeWrite(gpa, io, dir, name, std.mem.asBytes(&@field(rows.*, stage))); + } +} + +fn moeMatrices(model: *Model, layer: usize, expert: ?u32) ![3]ex.Matrix { + if (expert == null) return model.weights.layers[layer].moe.shared; + var matrices: [3]ex.Matrix = undefined; + var name: [128]u8 = undefined; + inline for (.{ "w1", "w3", "w2" }, 0..) |part, i| { + const key = try std.fmt.bufPrint(&name, "layers.{d}.ffn.experts.{d}.{s}", .{ layer, expert.?, part }); + matrices[i] = try tf.deepseek_v41_weights.lookup(&model.checkpoint, key, if (i == 2) 2304 else 5120, if (i == 2) 5120 else 2304); + } + return matrices; +} + +fn moeCombineDown(ids: *const [6]u32, down: *const [7][5120]u16, branch: *[5120]u16) void { + @setFloatMode(.strict); + var sum: [5120]f32 = @splat(0); + // Keep slots in top-six order in receipts, but match the official ascending expert traversal. + for (0..384) |expert| { + for (ids, 0..) |id, slot| { + if (id == expert) + for (&sum, down[slot]) |*s, value| { + s.* += math.f(value); + }; + } + } + for (branch, sum, down[6]) |*v, routed, shared| v.* = math.b(routed + math.f(shared)); +} + +fn moeCombine(ids: *const [6]u32, rows: *MoeRows) void { + moeCombineDown(ids, &rows.down, &rows.branch); +} + +fn moeReferenceRows(model: *Model, layer: usize, input: []const u16, ids: *const [6]u32, routing: *const [6]f32, rows: *MoeRows, scratch: *projection.Scratch, previous: ?*const MoeRows, previous_ids: ?*const [6]u32) !void { + for (0..7) |slot| { + const matrices = try moeMatrices(model, layer, if (slot == 6) null else ids[slot]); + var reuse: ?usize = if (slot == 6 and previous != null) 6 else null; + if (slot != 6 and previous_ids != null) + for (previous_ids.?, 0..) |id, prior| { + if (id == ids[slot]) { + reuse = prior; + break; + } + }; + if (reuse) |prior| { + rows.gate[slot] = previous.?.gate[prior]; + rows.up[slot] = previous.?.up[prior]; + } else { + try projection.apply(.{ .trellis = matrices[0] }, input, &rows.gate[slot], scratch); + try projection.apply(.{ .trellis = matrices[1] }, input, &rows.up[slot], scratch); + } + moe_reference.moeActivation(&rows.gate[slot], &rows.up[slot], if (slot == 6) 1 else routing[slot], model.cfg.text_config, &rows.activation[slot]); + // A shared or identical-weight expert result can be reused only when the BF16 input is identical. + if (reuse != null and std.mem.eql(u16, &rows.activation[slot], &previous.?.activation[reuse.?])) { + rows.down[slot] = previous.?.down[reuse.?]; + } else { + try projection.apply(.{ .trellis = matrices[2] }, &rows.activation[slot], &rows.down[slot], scratch); + } + } + moeCombine(ids, rows); +} + +fn moeEncodeNative(model: *Model, layer: usize, select_routes: bool) !void { + const ws = &model.workspace; + const p = &model.pipelines; + const w = &model.weights.layers[layer].moe; + const cb = model.queue.commandBufferUnretained(); + const enc = cb.compute(.serial); + var encoding = true; + errdefer if (encoding) enc.end(); + for (model.checkpoint.shards.items) |shard| enc.useResource(shard.buffer, mtl.ResourceUsage.read); + for (w.descriptors) |buffer| enc.useResource(buffer, mtl.ResourceUsage.read); + if (select_routes) { + tf.deepseek_v41.ops.encodeDense(enc, &p.ops, ws.view(.normalized), .{ .buffer = w.gate.buffer, .offset = w.gate.offset }, ws.view(.router_logits), .{ .rows = 1, .k = 5120, .n = 384, .x_stride = 5120, .w_stride = 5120, .y_stride = 384, .w_dtype = .f16, .y_dtype = .f32 }); + enc.barrier(); + // Bind the existing production router; no new native interface or diagnostic math kernel. + enc.setPipeline(p.route.select); + inline for (.{ ws.view(.router_logits), attention.View{ .buffer = w.bias.buffer, .offset = w.bias.offset }, ws.view(.route_ids), ws.view(.route_weights) }, 0..) |view, i| + enc.setBuffer(view.buffer, view.offset, i); + const Shape = extern struct { rows: u32, logit_stride: u32, output_stride: u32, temperature: f32, bias_dtype: u32 }; + enc.setValue(Shape{ .rows = 1, .logit_stride = 384, .output_stride = 6, .temperature = 1, .bias_dtype = 2 }, 4); + enc.dispatchGroups(mtl.Size.of(1, 1, 1), mtl.Size.of(128, 1, 1)); + enc.barrier(); + } + var routed_rows = ex.Rows{ .count = 6, .x_stride = 5120, .y_stride = 2304, .routes = .{ .ids = .{ .buffer = ws.view(.route_ids).buffer, .offset = ws.view(.route_ids).offset }, .repeat = 6 } }; + for (0..2) |i| { + const target = ws.view(if (i == 0) .gate else .up); + const input = ws.view(.normalized); + try ex.encodeIndexed(&p.projection, enc, .{ .descriptors = .{ .buffer = w.descriptors[i] }, .count = 384, .k = 5120, .n = 2304 }, .{ .buffer = input.buffer, .offset = input.offset }, .{ .buffer = target.buffer, .offset = target.offset }, ws.projection(), routed_rows); + enc.barrier(); + } + tf.deepseek_v41.ops.encodeSwiGLU(enc, &p.ops, ws.view(.gate), ws.view(.up), ws.view(.route_weights), ws.view(.activation), .{ .rows = 6, .width = 2304, .input_stride = 2304, .output_stride = 2304, .weighted = true }); + enc.barrier(); + routed_rows.x_stride = 2304; + routed_rows.y_stride = 5120; + routed_rows.routes.?.repeat = 1; + const activation = ws.view(.activation); + const down = ws.view(.routed); + try ex.encodeIndexed(&p.projection, enc, .{ .descriptors = .{ .buffer = w.descriptors[2] }, .count = 384, .k = 2304, .n = 5120 }, .{ .buffer = activation.buffer, .offset = activation.offset }, .{ .buffer = down.buffer, .offset = down.offset }, ws.projection(), routed_rows); + enc.barrier(); + for (0..2) |i| { + const target = ws.view(if (i == 0) .shared_gate else .shared_up); + const input = ws.view(.normalized); + ex.useMatrix(enc, w.shared[i]); + try ex.encode(&p.projection, enc, w.shared[i], .{ .buffer = input.buffer, .offset = input.offset }, .{ .buffer = target.buffer, .offset = target.offset }, ws.projection(), .{ .count = 1, .x_stride = 5120, .y_stride = 2304 }); + enc.barrier(); + } + tf.deepseek_v41.ops.encodeSwiGLU(enc, &p.ops, ws.view(.shared_gate), ws.view(.shared_up), ws.view(.route_weights), ws.view(.shared_activation), .{ .rows = 1, .width = 2304, .input_stride = 2304, .output_stride = 2304 }); + enc.barrier(); + const shared_activation = ws.view(.shared_activation); + const shared_down = ws.view(.shared_down); + ex.useMatrix(enc, w.shared[2]); + try ex.encode(&p.projection, enc, w.shared[2], .{ .buffer = shared_activation.buffer, .offset = shared_activation.offset }, .{ .buffer = shared_down.buffer, .offset = shared_down.offset }, ws.projection(), .{ .count = 1, .x_stride = 2304, .y_stride = 5120 }); + enc.barrier(); + tf.deepseek_v41.ops.encodeCombine(enc, &p.ops, ws.view(.routed), ws.view(.shared_down), ws.view(.route_ids), ws.view(.branch), .{ .rows = 1, .width = 5120, .routed_row_stride = 6 * 5120, .slot_stride = 5120, .shared_stride = 5120, .y_stride = 5120 }); + enc.end(); + encoding = false; + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.OperatorGpuFailure; +} + +fn moeCaptureNative(model: *Model, rows: *MoeRows) void { + const ws = &model.workspace; + inline for (.{ "gate", "up", "activation", "down" }, .{ .gate, .up, .activation, .routed }, .{ .shared_gate, .shared_up, .shared_activation, .shared_down }) |stage, routed, shared| { + const width = if (comptime std.mem.eql(u8, stage, "down")) 5120 else 2304; + const source = diagnosticRow(u16, ws.view(routed), 6 * width); + for (0..6) |slot| @memcpy(&@field(rows.*, stage)[slot], source[slot * width ..][0..width]); + @memcpy(&@field(rows.*, stage)[6], diagnosticRow(u16, ws.view(shared), width)); + } + @memcpy(&rows.branch, diagnosticRow(u16, ws.view(.branch), 5120)); +} + +fn moeCompareRows(gpa: std.mem.Allocator, report: *std.ArrayList(u8), count: *usize, comparison: []const u8, reuse: bool, actual: *const MoeRows, actual_ids: *const [6]u32, expected: *const MoeRows, expected_ids: *const [6]u32, first: *?[]const u8) !void { + // Compare like expert IDs rather than assuming independent top-six orders are identical. + inline for (.{ "gate", "up", "activation", "down" }) |stage| { + var differs = false; + for (0..7) |slot| { + var reference_slot: ?usize = if (slot == 6) 6 else null; + if (slot != 6) + for (expected_ids, 0..) |id, i| { + if (id == actual_ids[slot]) { + reference_slot = i; + break; + } + }; + if (reference_slot == null) continue; + var name: [64]u8 = undefined; + const label = if (slot == 6) try std.fmt.bufPrint(&name, "shared_{s}", .{stage}) else try std.fmt.bufPrint(&name, "expert_{d}_{s}", .{ actual_ids[slot], stage }); + differs = (try moeMetric(u16, gpa, report, count, label, comparison, reuse, &@field(actual.*, stage)[slot], &@field(expected.*, stage)[reference_slot.?])) or differs; + } + if (differs and first.* == null) first.* = stage; + } + if (try moeMetric(u16, gpa, report, count, "combined_branch", comparison, reuse, &actual.branch, &expected.branch)) + if (first.* == null) { + first.* = "combined_branch"; + }; +} + +/// Retained local snapshot isolation. MoE has no history; qualification computes every value independently. +pub fn runMoe(gpa: std.mem.Allocator, io: std.Io, model: *Model, snapshot: []const u8, dir: []const u8, layer: usize) !void { + if (layer >= model.weights.layers.len) return error.InvalidMoeLayer; + const blob = try std.Io.Dir.cwd().readFileAlloc(io, snapshot, gpa, .limited(8 << 20)); + defer gpa.free(blob); + const snapshot_offset = (layer * 2 + 1) * 61488; + if (blob.len != 80 * 61488 or blob.len < snapshot_offset + 61488) return error.InvalidMoeSnapshot; + try model.reset(); + const input = model.workspace.view(.normalized); + @memcpy(input.buffer.contents()[input.offset..][0..10240], blob[snapshot_offset + 40960 ..][0..10240]); + const input_row = diagnosticRow(u16, input, 5120); + var retained_ids: [6]u32 = undefined; + var retained_weights: [6]f32 = undefined; + var retained_branch: [5120]u16 = undefined; + for (&retained_ids, 0..) |*id, i| id.* = std.mem.readInt(u32, blob[snapshot_offset + 61440 + 4 * i ..][0..4], .little); + for (&retained_weights, 0..) |*weight, i| + weight.* = @bitCast(std.mem.readInt(u32, blob[snapshot_offset + 61464 + 4 * i ..][0..4], .little)); + @memcpy(std.mem.asBytes(&retained_branch), blob[snapshot_offset + 51200 ..][0..10240]); + for (retained_ids, retained_weights, 0..) |id, weight, i| { + if (id >= 384 or !std.math.isFinite(weight)) return error.InvalidMoeRetainedRouting; + for (retained_ids[0..i]) |prior| if (id == prior) return error.InvalidMoeRetainedRouting; + } + const native = try gpa.create(MoeRows); + defer gpa.destroy(native); + const source = try gpa.create(MoeRows); + defer gpa.destroy(source); + const native_weights_reference = try gpa.create(MoeRows); + defer gpa.destroy(native_weights_reference); + const retained_reference = try gpa.create(MoeRows); + defer gpa.destroy(retained_reference); + const native_inputs_reference = try gpa.create(MoeRows); + defer gpa.destroy(native_inputs_reference); + const retained_native = try gpa.create(MoeRows); + defer gpa.destroy(retained_native); + var scratch = try projection.Scratch.init(gpa, 5120, 5120); + defer scratch.deinit(); + + try moeEncodeNative(model, layer, true); + moeCaptureNative(model, native); + var native_ids: [6]u32 = undefined; + var native_weights: [6]f32 = undefined; + var native_logits_routing: moe_reference.MoeRouting = .{}; + @memcpy(&native_ids, diagnosticRow(u32, model.workspace.view(.route_ids), 6)); + @memcpy(&native_weights, diagnosticRow(f32, model.workspace.view(.route_weights), 6)); + @memcpy(&native_logits_routing.logits, diagnosticRow(f32, model.workspace.view(.router_logits), 384)); + moe_reference.moeSelect(model.weights.layers[layer].moe, model.cfg.text_config, &native_logits_routing.logits, &native_logits_routing.scores, &native_logits_routing.ids, &native_logits_routing.weights); + var source_routing: moe_reference.MoeRouting = .{}; + math.dense(model.weights.layers[layer].moe.gate, input_row, &source_routing.logits); + moe_reference.moeSelect(model.weights.layers[layer].moe, model.cfg.text_config, &source_routing.logits, &source_routing.scores, &source_routing.ids, &source_routing.weights); + try moeReferenceRows(model, layer, input_row, &source_routing.ids, &source_routing.weights, source, &scratch, null, null); + try moeReferenceRows(model, layer, input_row, &native_ids, &native_weights, native_weights_reference, &scratch, source, &source_routing.ids); + try moeReferenceRows(model, layer, input_row, &retained_ids, &retained_weights, retained_reference, &scratch, native_weights_reference, &native_ids); + for (0..7) |slot| { + native_inputs_reference.gate[slot] = native.gate[slot]; + native_inputs_reference.up[slot] = native.up[slot]; + moe_reference.moeActivation(&native.gate[slot], &native.up[slot], if (slot == 6) 1 else native_weights[slot], model.cfg.text_config, &native_inputs_reference.activation[slot]); + // Deliberately use the captured native activation, not the preceding reference activation. + if (std.mem.eql(u16, &native.activation[slot], &native_weights_reference.activation[slot])) { + native_inputs_reference.down[slot] = native_weights_reference.down[slot]; + } else { + const matrices = try moeMatrices(model, layer, if (slot == 6) null else native_ids[slot]); + try projection.apply(.{ .trellis = matrices[2] }, &native.activation[slot], &native_inputs_reference.down[slot], &scratch); + } + } + moeCombine(&native_ids, native_inputs_reference); + @memcpy(diagnosticRow(u32, model.workspace.view(.route_ids), 6), &retained_ids); + @memcpy(diagnosticRow(f32, model.workspace.view(.route_weights), 6), &retained_weights); + try moeEncodeNative(model, layer, false); + moeCaptureNative(model, retained_native); + + var report: std.ArrayList(u8) = .empty; + defer report.deinit(gpa); + const header = try std.json.Stringify.valueAlloc(gpa, .{ + .mode = "moe-sameinput", + .token = @as(?usize, null), + .layer = layer, + .token_identity_source = "caller_supplied_snapshot_argument_raw_snapshot_has_no_token_header", + .retained_route_origin = "native_GPU_snapshot_command_contract", + .snapshot_count = 80, + .snapshot_length_bytes = 61488, + .snapshot_index = layer * 2 + 1, + .snapshot_stage = "moe", + .input_offset_bytes = snapshot_offset + 40960, + .input_length_bytes = 10240, + .retained_ids_offset_bytes = snapshot_offset + 61440, + .retained_weights_offset_bytes = snapshot_offset + 61464, + .identical_normalized_input = true, + .reference_dot = "independent_EXL3_FP64", + .qualification_gate_applied = false, + .qualification_oracle_modified_to_reuse_native_values = false, + .official_revision = "2cba9e42aa026125f3ed06c6d98c1db82f7ca027", + .source_ids = source_routing.ids, + .source_weights = source_routing.weights, + .native_ids = native_ids, + .native_weights = native_weights, + .retained_ids = retained_ids, + .retained_weights = retained_weights, + .source_ids_equal_native_order = std.mem.eql(u32, &source_routing.ids, &native_ids), + .native_ids_equal_retained_order = std.mem.eql(u32, &native_ids, &retained_ids), + .source_select_on_native_logits_ids = native_logits_routing.ids, + .source_select_on_native_logits_weights = native_logits_routing.weights, + .score_receipt_origin = "source_sqrt_softplus_applied_to_captured_native_logits_not_GPU_score_capture", + .expert_receipt_layout = "slot_major_top6_ids_order_then_shared_slot6", + .expert_sum_order = "ascending_expert_id_FP32_shared_last_final_BF16", + .receipt_prefixes = .{ "moe-native", "moe-source", "moe-native-weights-reference-diagnostic", "moe-native-inputs-reference-diagnostic", "moe-retained-weights-reference-diagnostic", "moe-retained-route-native-diagnostic" }, + .receipt_shapes = .{ .gate = .{ 7, 2304 }, .up = .{ 7, 2304 }, .activation = .{ 7, 2304 }, .down = .{ 7, 5120 }, .branch = .{5120} }, + .native_inputs_reference_contract = "activation:reference_on_native_gate_up_and_weight;down:FP64_on_native_activation;branch:reference_combine_of_these_down_outputs", + }, .{}); + defer gpa.free(header); + try report.appendSlice(gpa, header[0 .. header.len - 1]); + try report.appendSlice(gpa, ",\"cases\":[\n"); + var count: usize = 0; + var first: MoeFirst = .{}; + if (try moeMetric(f32, gpa, &report, &count, "route_logits", "source_chain", false, &native_logits_routing.logits, &source_routing.logits)) first.source_chain = "route_logits"; + if (try moeMetric(f32, gpa, &report, &count, "route_scores_source_transform", "source_score_transform_on_captured_native_logits_diagnostic", true, &native_logits_routing.scores, &source_routing.scores)) + if (first.source_chain == null) { + first.source_chain = "route_scores_source_transform"; + }; + if (!std.mem.eql(u32, &source_routing.ids, &native_ids) and first.source_chain == null) first.source_chain = "route_ids"; + if (std.mem.eql(u32, &source_routing.ids, &native_ids)) { + if (try moeMetric(f32, gpa, &report, &count, "route_weights", "source_chain", false, &native_weights, &source_routing.weights)) + if (first.source_chain == null) { + first.source_chain = "route_weights"; + }; + } + if (std.mem.eql(u32, &native_logits_routing.ids, &native_ids)) + _ = try moeMetric(f32, gpa, &report, &count, "route_weights", "source_select_on_same_native_logits_diagnostic", true, &native_weights, &native_logits_routing.weights); + if (std.mem.eql(u32, &retained_ids, &native_ids)) + _ = try moeMetric(f32, gpa, &report, &count, "route_weights", "retained_native_weights_vs_reexecuted_native", true, &native_weights, &retained_weights); + try moeCompareRows(gpa, &report, &count, "source_chain", false, native, &native_ids, source, &source_routing.ids, &first.source_expert_stage); + if (first.source_chain == null) first.source_chain = first.source_expert_stage; + try moeCompareRows(gpa, &report, &count, "same_native_ids_weights_reference_chain_diagnostic", true, native, &native_ids, native_weights_reference, &native_ids, &first.native_weights_reference_chain_diagnostic); + inline for (.{ "activation", "down" }) |stage| { + var differs = false; + for (0..7) |slot| { + var name: [64]u8 = undefined; + const label = if (slot == 6) try std.fmt.bufPrint(&name, "shared_{s}", .{stage}) else try std.fmt.bufPrint(&name, "expert_{d}_{s}", .{ native_ids[slot], stage }); + differs = (try moeMetric(u16, gpa, &report, &count, label, if (comptime std.mem.eql(u8, stage, "activation")) "same_native_gate_up_weight_reference_activation_diagnostic" else "same_native_activation_FP64_down_diagnostic", true, &@field(native.*, stage)[slot], &@field(native_inputs_reference.*, stage)[slot])) or differs; + } + if (differs and first.native_input_diagnostic == null) first.native_input_diagnostic = stage; + } + _ = try moeMetric(u16, gpa, &report, &count, "combined_branch", "same_native_activation_FP64_down_then_reference_combine_diagnostic", true, &native.branch, &native_inputs_reference.branch); + // Isolate combination arithmetic independently of every projection and route-weight difference. + var native_down_reference_branch: [5120]u16 = undefined; + moeCombineDown(&native_ids, &native.down, &native_down_reference_branch); + first.native_down_reference_combine_diagnostic_differs = + try moeMetric(u16, gpa, &report, &count, "combined_branch", "same_native_down_reference_combine_diagnostic", true, &native.branch, &native_down_reference_branch); + try moeWrite(gpa, io, dir, "moe-native-down-reference-combine-diagnostic.bf16", std.mem.asBytes(&native_down_reference_branch)); + try moeCompareRows(gpa, &report, &count, "same_retained_native_ids_weights_reference_chain_diagnostic", true, retained_native, &retained_ids, retained_reference, &retained_ids, &first.retained_weights_reference_chain_diagnostic); + _ = try moeMetric(u16, gpa, &report, &count, "combined_branch", "retained_native_branch_vs_reexecuted_native_routing", true, &native.branch, &retained_branch); + _ = try moeMetric(u16, gpa, &report, &count, "combined_branch", "retained_native_branch_vs_reexecuted_retained_routes_diagnostic", true, &retained_native.branch, &retained_branch); + const finding = try std.json.Stringify.valueAlloc(gpa, first, .{}); + defer gpa.free(finding); + try report.appendSlice(gpa, "\n],\"first_differing_intermediate\":"); + try report.appendSlice(gpa, finding); + try report.appendSlice(gpa, "}\n"); + try moeWrite(gpa, io, dir, "moe-sameinput.json", report.items); + inline for (.{ "moe-native", "moe-source", "moe-native-weights-reference-diagnostic", "moe-native-inputs-reference-diagnostic", "moe-retained-weights-reference-diagnostic", "moe-retained-route-native-diagnostic" }, .{ native, source, native_weights_reference, native_inputs_reference, retained_reference, retained_native }) |prefix, rows| + try moeWriteRows(gpa, io, dir, prefix, rows); + inline for (.{ "moe-input.bf16", "moe-retained-branch.bf16", "moe-native-logits.f32", "moe-source-logits.f32", "moe-native-logits-source-scores.f32", "moe-source-scores.f32", "moe-native-ids.u32", "moe-source-ids.u32", "moe-retained-ids.u32", "moe-native-weights.f32", "moe-source-weights.f32", "moe-retained-weights.f32", "moe-native-logits-source-select-ids.u32", "moe-native-logits-source-select-weights.f32" }, .{ std.mem.sliceAsBytes(input_row), std.mem.asBytes(&retained_branch), std.mem.asBytes(&native_logits_routing.logits), std.mem.asBytes(&source_routing.logits), std.mem.asBytes(&native_logits_routing.scores), std.mem.asBytes(&source_routing.scores), std.mem.asBytes(&native_ids), std.mem.asBytes(&source_routing.ids), std.mem.asBytes(&retained_ids), std.mem.asBytes(&native_weights), std.mem.asBytes(&source_routing.weights), std.mem.asBytes(&retained_weights), std.mem.asBytes(&native_logits_routing.ids), std.mem.asBytes(&native_logits_routing.weights) }) |name, bytes| + try moeWrite(gpa, io, dir, name, bytes); +} diff --git a/zig/tests/deepseek_v41_reference/operators.zig b/zig/tests/deepseek_v41_reference/operators.zig new file mode 100644 index 0000000000..d7c8cb65de --- /dev/null +++ b/zig/tests/deepseek_v41_reference/operators.zig @@ -0,0 +1,92 @@ +//! Actual checkpoint projections on identical deterministic BF16 rows, against independent FP64 dot products. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const projection = @import("projection.zig"); +const math = @import("math.zig"); +const Model = tf.deepseek_v41.Model; +const ex = tf.exl3_projection; +const Case = struct { name: []const u8, matrix: ex.Matrix }; + +pub fn run(gpa: std.mem.Allocator, io: std.Io, model: *Model, dir: []const u8) !void { + const first = &model.weights.layers[0]; + const cases = [_]Case{ + .{ .name = "wq_a", .matrix = first.attn.wq_a }, + .{ .name = "wq_b", .matrix = first.attn.wq_b }, + .{ .name = "wkv", .matrix = first.attn.wkv }, + .{ .name = "wo_a_group0", .matrix = first.attn.wo_a[0] }, + .{ .name = "wo_b", .matrix = first.attn.wo_b }, + .{ .name = "engram_wkv", .matrix = model.weights.layers[1].engram.?.wkv }, + .{ .name = "routed_gate", .matrix = try tf.deepseek_v41_weights.lookup(&model.checkpoint, "layers.0.ffn.experts.0.w1", 5120, 2304) }, + .{ .name = "routed_up", .matrix = try tf.deepseek_v41_weights.lookup(&model.checkpoint, "layers.0.ffn.experts.0.w3", 5120, 2304) }, + .{ .name = "routed_down", .matrix = try tf.deepseek_v41_weights.lookup(&model.checkpoint, "layers.0.ffn.experts.0.w2", 2304, 5120) }, + .{ .name = "head", .matrix = model.weights.head }, + }; + const x = try model.device.buffer(8192 * 2, mtl.ResourceOptions.shared); + defer x.deinit(); + const y = try model.device.buffer(2 * 129280 * 4, mtl.ResourceOptions.shared); + defer y.deinit(); + const scratch = try model.device.buffer(try ex.scratchBytes(8192, 129280, 1), mtl.ResourceOptions.shared); + defer scratch.deinit(); + for (x.slice(u16, 8192), 0..) |*value, i| + value.* = math.b(@as(f32, @floatFromInt(@as(i32, @intCast((i * 17) % 97)) - 48)) / 32); + for ([_]mtl.Buffer{ x, y, scratch }) |buffer| model.residency.add(buffer); + model.residency.commit(); + model.residency.requestResidency(); + defer { + for ([_]mtl.Buffer{ x, y, scratch }) |buffer| model.residency.remove(buffer); + model.residency.commit(); + } + var reference = try projection.Scratch.init(gpa, 8192, 129280); + defer reference.deinit(); + const expected = try gpa.alloc(f32, 129280); + defer gpa.free(expected); + var report: std.ArrayList(u8) = .empty; + defer report.deinit(gpa); + try report.appendSlice(gpa, "{\"input\":\"bf16((17*i%97-48)/32)\",\"reference_dot\":\"FP64\",\"cases\":["); + var all_pass = true; + for (cases, 0..) |case, index| { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const matrix = case.matrix; + const cb = model.queue.commandBufferUnretained(); + const enc = cb.compute(.serial); + var encoding = true; + errdefer if (encoding) enc.end(); + ex.useMatrix(enc, matrix); + try ex.encode(&model.pipelines.projection, enc, matrix, .{ .buffer = x }, .{ .buffer = y }, .{ .buffer = scratch }, .{ .count = 1, .x_stride = matrix.k, .y_stride = matrix.n, .output = .f32 }); + enc.end(); + encoding = false; + cb.commit(); + cb.wait(); + if (cb.failure() != null) return error.OperatorGpuFailure; + const actual = y.slice(f32, 129280); + try projection.applyF32(.{ .trellis = matrix }, x.slice(u16, matrix.k), expected[0..matrix.n], &reference); + var max_abs: f64 = 0; + var worst: usize = 0; + var squared: f64 = 0; + var magnitude: f64 = 0; + for (actual[0..matrix.n], expected[0..matrix.n], 0..) |a, b, column| { + if (!std.math.isFinite(a) or !std.math.isFinite(b)) return error.NonfiniteOperatorOutput; + const delta: f64 = @as(f64, a) - b; + if (@abs(delta) > max_abs) { + max_abs = @abs(delta); + worst = column; + } + squared += delta * delta; + magnitude += @as(f64, b) * b; + } + const nrms = if (magnitude == 0) @sqrt(squared) else @sqrt(squared / magnitude); + all_pass = all_pass and nrms <= 1e-4; + const line = try std.fmt.allocPrint(gpa, "{{\"name\":\"{s}\",\"k\":{d},\"n\":{d},\"normalized_rms_error\":{d},\"max_absolute_error\":{d},\"worst_column\":{d},\"worst_gpu\":{d},\"worst_reference\":{d}}}", .{ case.name, matrix.k, matrix.n, nrms, max_abs, worst, actual[worst], expected[worst] }); + defer gpa.free(line); + if (index > 0) try report.appendSlice(gpa, ","); + try report.appendSlice(gpa, line); + std.debug.print("{s}\n", .{line}); + } + try report.appendSlice(gpa, "]}\n"); + const path = try std.fs.path.join(gpa, &.{ dir, "operators.json" }); + defer gpa.free(path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = report.items }); + if (!all_pass) return error.OperatorReferenceMismatch; +} diff --git a/zig/tests/deepseek_v41_reference/profile.zig b/zig/tests/deepseek_v41_reference/profile.zig new file mode 100644 index 0000000000..0fb15a8b90 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/profile.zig @@ -0,0 +1,52 @@ +//! Real-checkpoint calibrated GPU stage timings, with profile-on/off logits checked before reporting. +const std = @import("std"); +const tf = @import("tensorfold"); +const counters = tf.deepseek_v41_profiler; +const Model = tf.deepseek_v41.Model; + +pub fn run(gpa: std.mem.Allocator, io: std.Io, model: *Model, ids: []const u32, dir: []const u8) !void { + if (ids.len == 0 or ids.len > model.workspace.plan.rows) return error.InvalidProfileWindow; + const previous_grouping = model.grouped_prefill; + defer model.grouped_prefill = previous_grouping; + model.grouped_prefill = false; + try model.reset(); + const baseline = try model.forward(ids, .all); + const baseline_gpu_seconds = model.last_gpu_seconds; + const expected = try gpa.dupe(f32, baseline); + defer gpa.free(expected); + model.grouped_prefill = true; + try model.reset(); + const candidate = try model.forward(ids, .all); + const candidate_gpu_seconds = model.last_gpu_seconds; + try check(expected, candidate); + var recorder = try counters.Recorder.init(gpa, model.device, .{ .enabled = true, .max_spans = 512 }); + defer recorder.deinit() catch unreachable; + const previous_profile = model.profile; + model.profile = &recorder; + defer model.profile = previous_profile; + try model.reset(); + const actual = try model.forward(ids, .last); + try check(expected[expected.len - 129280 ..], actual); + const totals = recorder.aggregates(); + var sampled_gpu_seconds: f64 = 0; + for (totals) |value| sampled_gpu_seconds += value.gpu_seconds; + var text: std.ArrayList(u8) = .empty; + defer text.deinit(gpa); + try text.print(gpa, "{{\"tokens\":{d},\"comparison_logits\":\"all\",\"candidate_grouping\":true,\"same_logits\":true,\"baseline_command_gpu_seconds\":{d},\"candidate_command_gpu_seconds\":{d},\"profiled_logits\":\"last\",\"sampled_span_gpu_seconds\":{d},\"stages\":[", .{ ids.len, baseline_gpu_seconds, candidate_gpu_seconds, sampled_gpu_seconds }); + for (std.enums.values(counters.Stage), totals, 0..) |stage, value, i| { + try text.print(gpa, "{s}{{\"stage\":\"{s}\",\"calls\":{d},\"gpu_seconds\":{d}}}", .{ if (i == 0) "" else ",", @tagName(stage), value.calls, value.gpu_seconds }); + } + try text.appendSlice(gpa, "]}\n"); + const path = try std.fs.path.join(gpa, &.{ dir, "gpu-profile.json" }); + defer gpa.free(path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = text.items }); + std.debug.print("{s}", .{text.items}); +} + +fn check(expected: []const f32, actual: []const f32) !void { + if (expected.len != actual.len) return error.ProfileLogitShapeMismatch; + for (actual, expected) |a, b| { + if (!std.math.isFinite(a) or !std.math.isFinite(b)) return error.NonfiniteProfileLogits; + if (@as(u32, @bitCast(a)) != @as(u32, @bitCast(b))) return error.ProfileChangedLogits; + } +} diff --git a/zig/tests/deepseek_v41_reference/projection.zig b/zig/tests/deepseek_v41_reference/projection.zig new file mode 100644 index 0000000000..03174c9459 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/projection.zig @@ -0,0 +1,311 @@ +//! Independent scalar EXL3 oracle over resident bytes; FP64 dots and FP32 rotation/output boundaries, no GPU work. + +// EXL3 attribution: docs/recipes/exl3.md; pinned checkpoint provenance in THIRD_PARTY_NOTICES.md. + +const std = @import("std"); +const tf = @import("tensorfold"); +const exl3 = tf.exl3_projection; +const weights = tf.deepseek_v41_weights; +const format = exl3.format; +const block_size = format.hadamard_dim; +const inverse: f32 = 0.08838834764831845; +const states_count = 65536; + +/// Reuse bounded vector/codebook storage for heterogeneous matrices; never allocate decoded W[K,N]. +pub const Scratch = struct { + gpa: std.mem.Allocator, + rotated: []f64, + sums: []f64, + codebooks: []f16, + + pub fn init(gpa: std.mem.Allocator, max_k: usize, max_n: usize) !Scratch { + if (max_k == 0 or max_n == 0) return error.InvalidReferenceProjectionCapacity; + const rotated = try gpa.alloc(f64, max_k); + errdefer gpa.free(rotated); + const sums = try gpa.alloc(f64, max_n); + errdefer gpa.free(sums); + const codebooks = try gpa.alloc(f16, 3 * states_count); + errdefer gpa.free(codebooks); + for ([_]format.Codebook{ .inst3, .mcg, .mul1 }, 0..) |book, bi| { + for (0..states_count) |state| { + codebooks[bi * states_count + state] = format.codebookValue(book, @intCast(state)); + } + } + return .{ .gpa = gpa, .rotated = rotated, .sums = sums, .codebooks = codebooks }; + } + + pub fn deinit(s: *Scratch) void { + s.gpa.free(s.rotated); + s.gpa.free(s.sums); + s.gpa.free(s.codebooks); + s.* = undefined; + } +}; + +/// One BF16 input row and BF16 output row; caller must finish GPU writes before borrowing shared CPU bytes. +pub fn apply(matrix: weights.AnyMatrix, x: []const u16, y: []u16, scratch: *Scratch) !void { + switch (matrix) { + .trellis => |m| return applyTrellis(m, x, y, scratch), + .dense => |d| return applyDense(u16, d, x, y, scratch), + } +} + +pub fn applyF32(matrix: weights.AnyMatrix, x: []const u16, y: []f32, scratch: *Scratch) !void { + switch (matrix) { + .trellis => |m| return applyF32Trellis(m, x, y, scratch), + .dense => |d| return applyDense(f32, d, x, y, scratch), + } +} + +/// Kept-dense: quantized rotation and output stages; FP64 dot over the f16-rounded row replaces the tile decode. +fn applyDense(comptime T: type, d: weights.Dense, x: []const u16, y: []T, scratch: *Scratch) !void { + @setFloatMode(.strict); + if (x.len != d.k or y.len != d.n) return error.InvalidReferenceProjectionLength; + if (d.k > scratch.rotated.len or d.n > scratch.sums.len) + return error.ReferenceProjectionScratchTooSmall; + const su = try borrowed(.{ .buffer = d.suh.buffer, .offset = d.suh.offset }, d.k * 2); + const sv = try borrowed(.{ .buffer = d.svh.buffer, .offset = d.svh.offset }, d.n * 2); + const w = try borrowed(.{ .buffer = d.weight.buffer, .offset = d.weight.offset }, d.n * d.k * 2); + const rotated = scratch.rotated[0..d.k]; + const sums = scratch.sums[0..d.n]; + prepare(x, su, .f16, rotated); + for (sums) |*slot| slot.* = 0; + for (0..d.n) |j| { + var acc: f64 = 0; + const wrow = 2 * j * d.k; + for (0..d.k) |k| { + const word = std.mem.readInt(u16, w[wrow + 2 * k ..][0..2], .little); + acc += @as(f64, rotated[k]) * @as(f64, @as(f32, @bitCast(@as(u32, word) << 16))); + } + sums[j] = acc; + } + var base: usize = 0; + while (base < d.n) : (base += block_size) { + var values: [block_size]f32 = undefined; + for (&values, 0..) |*value, i| value.* = @floatCast(sums[base + i]); + hadamard(&values); + for (values, 0..) |value, i| { + const out: f32 = (value * inverse) * half(sv, base + i); + y[base + i] = if (T == u16) bf16(out) else out; + } + } +} + +fn applyTrellis(matrix: exl3.Matrix, x: []const u16, y: []u16, scratch: *Scratch) !void { + try project(matrix, x, y.len, scratch); + try finish(u16, matrix, y, scratch.sums[0..matrix.n]); +} + +/// The same arithmetic with an FP32 output boundary, for compressor projections and full-vocabulary logits. +pub fn applyF32Trellis(matrix: exl3.Matrix, x: []const u16, y: []f32, scratch: *Scratch) !void { + try project(matrix, x, y.len, scratch); + try finish(f32, matrix, y, scratch.sums[0..matrix.n]); +} + +fn borrowed(view: exl3.View, bytes: usize) ![]const u8 { + const len = view.buffer.length(); + if (view.offset % 2 != 0 or view.offset > len or bytes > len - view.offset) + return error.InvalidReferenceProjectionView; + const address = view.buffer.contents(); + if (@intFromPtr(address) == 0) return error.NonCpuAccessibleExl3Buffer; + return address[view.offset..][0..bytes]; +} + +fn scaleBytes(count: usize, kind: exl3.ScaleFormat) !usize { + return std.math.mul(usize, if (kind == .packed_signs) count / 16 else count, 2); +} + +fn half(bytes: []const u8, i: usize) f32 { + const value: f16 = @bitCast(std.mem.readInt(u16, bytes[2 * i ..][0..2], .little)); + return @floatCast(value); +} + +fn scale(bytes: []const u8, i: usize, kind: exl3.ScaleFormat) f32 { + if (kind == .f16) return half(bytes, i); + const word = std.mem.readInt(u16, bytes[2 * (i / 16) ..][0..2], .little); + return if (((word >> @as(u4, @intCast(i % 16))) & 1) != 0) -1.0 else 1.0; +} + +fn bf16(value: f32) u16 { + const bits: u32 = @bitCast(value); + // Preserve a quiet NaN instead of letting rounding turn a tiny NaN payload into infinity. + if ((bits & 0x7fffffff) > 0x7f800000) return @as(u16, @truncate(bits >> 16)) | 0x0040; + return @truncate((bits +% (0x7fff + ((bits >> 16) & 1))) >> 16); +} + +/// Seven ordinary Sylvester butterflies, with explicit FP32 stage boundaries and normalization afterward. +fn hadamard(values: *[block_size]f32) void { + @setFloatMode(.strict); + var stride: usize = 1; + while (stride < block_size) : (stride *= 2) { + var base: usize = 0; + while (base < block_size) : (base += 2 * stride) { + for (0..stride) |j| { + const a = values[base + j]; + const b = values[base + stride + j]; + values[base + j] = a + b; + values[base + stride + j] = a - b; + } + } + } +} + +fn prepare(x: []const u16, su: []const u8, kind: exl3.ScaleFormat, rotated: []f64) void { + @setFloatMode(.strict); + var base: usize = 0; + while (base < x.len) : (base += block_size) { + var values: [block_size]f32 = undefined; + for (&values, 0..) |*value, i| { + const input: f32 = @bitCast(@as(u32, x[base + i]) << 16); + value.* = input * scale(su, base + i, kind); + } + hadamard(&values); + for (values, 0..) |value, i| { + const rounded: f16 = @floatCast(value * inverse); + rotated[base + i] = @floatCast(rounded); + } + } +} + +/// CPU stream reader consumes MSB-first bits from little-endian int16 pairs, not the GPU indexed decoder. +const Stream = struct { + bytes: []const u8, + cursor: usize = 0, + pending: u32 = 0, + available: u6 = 0, + + fn take(s: *Stream, count: u5) u16 { + if (s.available == 0) { + s.pending = std.mem.readInt(u32, s.bytes[s.cursor..][0..4], .little); + s.cursor += 4; + s.available = 32; + } + const first: u5 = @intCast(@min(s.available, count)); + const head: u32 = s.pending >> @as(u5, @intCast(32 - @as(u6, first))); + s.pending <<= first; + s.available -= first; + if (first == count) return @intCast(head); + const rest = count - first; + s.pending = std.mem.readInt(u32, s.bytes[s.cursor..][0..4], .little); + s.cursor += 4; + const tail = s.pending >> @as(u5, @intCast(32 - @as(u6, rest))); + s.pending <<= rest; + s.available = 32 - @as(u6, rest); + return @intCast((head << rest) | tail); + } +}; + +/// Circular initial state uses final 16 stream bits; decode only a sixteen-column accumulator and live stream. +fn accumulateTile(comptime halves: u16, tile: []const u8, table: []const f16, x: []const f64, acc: *[16]f64) void { + @setFloatMode(.strict); + var stream: Stream = .{ .bytes = tile }; + var state: u16 = @truncate(std.mem.readInt(u32, tile[tile.len - 4 ..][0..4], .little)); + var p: usize = 0; + while (p < 256) : (p += 1) { + // Odd half-widths alternate floor(bits),ceil(bits), exactly as format.streamEnds specifies. + const advance: u4 = @intCast(halves / 2 + (if (halves % 2 == 1) p % 2 else 0)); + state = (state << advance) | stream.take(advance); + const lane = p / 8; + const j = p % 8; + const row = 2 * (lane % 4) + (j & 1) + 8 * ((j >> 1) & 1); + const col = lane / 4 + 8 * (j >> 2); + const weight: f64 = @floatCast(table[state]); + acc[col] += x[row] * weight; + } +} + +fn product(comptime halves: u16, matrix: exl3.Matrix, trellis: []const u8, table: []const f16, rotated: []const f64, sums: []f64) !void { + const ks = try matrix.kStride(); + const ns = matrix.nStride(); + const tile_bytes = @as(usize, halves) * 16; + @memset(sums, 0); + for (0..matrix.k / 16) |tk| { + const input = rotated[tk * 16 ..][0..16]; + for (0..matrix.n / 16) |tn| { + const at = tk * ks + tn * ns; + accumulateTile(halves, trellis[at..][0..tile_bytes], table, input, sums[tn * 16 ..][0..16]); + } + } +} + +fn project(matrix: exl3.Matrix, x: []const u16, output_len: usize, scratch: *Scratch) !void { + try matrix.validate(); + if (x.len != matrix.k or output_len != matrix.n) return error.InvalidReferenceProjectionLength; + if (matrix.k > scratch.rotated.len or matrix.n > scratch.sums.len) + return error.ReferenceProjectionScratchTooSmall; + const trellis = try borrowed(matrix.trellis, try matrix.storageBytes()); + const su = try borrowed(matrix.suh, try scaleBytes(matrix.k, matrix.suh_format)); + // Check every borrowed view before doing arithmetic, including the output-only scale and optional bias. + _ = try borrowed(matrix.svh, try scaleBytes(matrix.n, matrix.svh_format)); + if (matrix.bias) |bias| _ = try borrowed(bias, try std.math.mul(usize, matrix.n, 2)); + const rotated = scratch.rotated[0..matrix.k]; + const sums = scratch.sums[0..matrix.n]; + prepare(x, su, matrix.suh_format, rotated); + const book_index: usize = @intCast(@backingInt(matrix.codebook)); + const table = scratch.codebooks[book_index * states_count ..][0..states_count]; + switch (matrix.width.halves) { + inline 2...16 => |halves| try product(halves, matrix, trellis, table, rotated, sums), + else => unreachable, + } +} + +fn finish(comptime T: type, matrix: exl3.Matrix, y: []T, sums: []const f64) !void { + @setFloatMode(.strict); + const sv = try borrowed(matrix.svh, try scaleBytes(matrix.n, matrix.svh_format)); + const bias = if (matrix.bias) |view| try borrowed(view, try std.math.mul(usize, matrix.n, 2)) else null; + var base: usize = 0; + while (base < matrix.n) : (base += block_size) { + var values: [block_size]f32 = undefined; + // One FP64->FP32 boundary replaces GPU FP32 dot-product accumulation, not the output rotation. + for (&values, 0..) |*value, i| value.* = @floatCast(sums[base + i]); + hadamard(&values); + for (values, 0..) |value, i| { + var out: f32 = (value * inverse) * scale(sv, base + i, matrix.svh_format); + if (bias) |bytes| out += half(bytes, base + i); + y[base + i] = if (T == u16) bf16(out) else out; + } + } +} + +test "CPU circular streaming matches scalar format for every supported width and codebook" { + var scratch = try Scratch.init(std.testing.allocator, 128, 128); + defer scratch.deinit(); + var words: [128]i16 = undefined; + for (&words, 0..) |*word, i| word.* = @bitCast(@as(u16, @truncate(i * 40503 + 17329))); + var x: [16]f64 = undefined; + for (&x, 0..) |*value, i| value.* = @as(f64, @floatFromInt(i + 1)) * 0.0625; + var placement: [256]format.Placement = undefined; + format.tilePlacement(&placement); + inline for (2..17) |halves| { + const width: format.Width = .{ .halves = halves }; + const tile = words[0..width.tileWords()]; + var states: [256]u32 = undefined; + format.tileStates(width, tile, &states); + for ([_]format.Codebook{ .inst3, .mcg, .mul1 }, 0..) |book, bi| { + if (width.isHalf() and (book != .mul1 or halves > 7)) continue; + const table = scratch.codebooks[bi * states_count ..][0..states_count]; + var expected: [16]f64 = @splat(0); + for (states, placement) |state, where| { + const weight: f64 = @floatCast(format.codebookValue(book, @intCast(state))); + expected[where.col] += x[where.row] * weight; + } + var actual: [16]f64 = @splat(0); + accumulateTile(halves, std.mem.sliceAsBytes(tile), table, &x, &actual); + try std.testing.expectEqualSlices(f64, &expected, &actual); + } + } +} + +test "FP32 Hadamard then FP16 and BF16 boundaries are deterministic" { + var x: [128]u16 = @splat(0); + x[0] = 0x3f80; + var signs: [16]u8 = @splat(0); + signs[0] = 1; + var rotated: [128]f64 = undefined; + prepare(&x, &signs, .packed_signs, &rotated); + const rounded: f16 = @floatCast(-inverse); + for (rotated) |value| try std.testing.expectEqual(@as(f64, @floatCast(rounded)), value); + try std.testing.expectEqual(@as(u16, 0x3f80), bf16(@bitCast(@as(u32, 0x3f808000)))); + try std.testing.expectEqual(@as(u16, 0x3f82), bf16(@bitCast(@as(u32, 0x3f818000)))); + try std.testing.expectEqual(@as(u16, 0x7fc0), bf16(@bitCast(@as(u32, 0x7f800001)))); +} diff --git a/zig/tests/deepseek_v41_reference/qualify.zig b/zig/tests/deepseek_v41_reference/qualify.zig new file mode 100644 index 0000000000..a8e5bbbedd --- /dev/null +++ b/zig/tests/deepseek_v41_reference/qualify.zig @@ -0,0 +1,160 @@ +//! Actual checkpoint teacher comparison: independent CPU arithmetic versus the resident GPU decode path. +const std = @import("std"); +const mtl = @import("metal"); +const Model = @import("tensorfold").deepseek_v41.Model; +const Reference = @import("reference.zig").Reference; +const nrms_limit: f64 = 0.01; + +pub const Result = struct { + tokens: usize, + max_absolute_error: f64, + rms_error: f64, + normalized_rms_error: f64, + greedy_disagreements: usize, + gpu_seconds: f64, + cpu_seconds: f64, + normalized_rms_limit: f64 = nrms_limit, + numeric_gate_passed: bool = false, + greedy_gate_passed: bool = false, +}; + +/// Indexer-selection boundary per window: contract margin (BF16-rounded FP32 scores), FP64-ideal margin, support flips. +const MarginLog = struct { + records: [1024]MarginRecord = undefined, + len: usize = 0, + + const MarginRecord = struct { + layer: u32, + position: u32, + count: u32, + selected: u32, + margin: f32, + margin_ideal: f32, + flips: u32, + lowest_selected: f32, + highest_unselected: f32, + }; + + fn capture(ctx: *anyopaque, layer: usize, position: usize, count: usize, selected_len: usize, margin: f32, margin_ideal: f32, flips: usize, lowest: f32, highest_unselected: f32) void { + const self: *MarginLog = @ptrCast(@alignCast(ctx)); + if (self.len == self.records.len) return; + self.records[self.len] = .{ + .layer = @intCast(layer), + .position = @intCast(position), + .count = @intCast(count), + .selected = @intCast(selected_len), + .margin = margin, + .margin_ideal = margin_ideal, + .flips = @intCast(flips), + .lowest_selected = lowest, + .highest_unselected = highest_unselected, + }; + self.len += 1; + } + + fn write(self: *MarginLog, gpa: std.mem.Allocator, io: std.Io, dir: []const u8) !void { + var text: std.ArrayList(u8) = .empty; + defer text.deinit(gpa); + var flips: usize = 0; + var tight: usize = 0; + var total: usize = 0; + for (self.records[0..self.len]) |r| { + total += 1; + flips += r.flips; + if (r.margin <= 0) tight += 1; + var buf: [320]u8 = undefined; + const line = try std.fmt.bufPrint(&buf, "{{\"layer\":{d},\"position\":{d},\"count\":{d},\"selected\":{d},\"margin\":{d},\"margin_ideal\":{d},\"flips\":{d},\"lowest_selected\":{d},\"highest_unselected\":{d}}}\n", .{ r.layer, r.position, r.count, r.selected, r.margin, r.margin_ideal, r.flips, r.lowest_selected, r.highest_unselected }); + try text.appendSlice(gpa, line); + } + var head_buf: [192]u8 = undefined; + const head = try std.fmt.bufPrint(&head_buf, "{{\"records\":{d},\"total_flips\":{d},\"zero_or_negative_margin\":{d}}}\n", .{ total, flips, tight }); + const path = try std.fs.path.join(gpa, &.{ dir, "indexer-margins.jsonl" }); + defer gpa.free(path); + var file = try std.Io.Dir.cwd().createFile(io, path, .{}); + defer file.close(io); + try file.writeStreamingAll(io, head); + try file.writeStreamingAll(io, text.items); + std.debug.print("indexer margins: {d} windows, {d} flips contract-vs-ideal, {d} zero-or-negative margins\n", .{ total, flips, tight }); + } +}; + +fn maximum(row: []const f32) !u32 { + var id: u32 = 0; + var best = -std.math.inf(f32); + for (row, 0..) |v, i| { + if (!std.math.isFinite(v)) return error.NonfiniteTeacherLogits; + if (v > best) { + best = v; + id = @intCast(i); + } + } + return id; +} + +pub fn run(gpa: std.mem.Allocator, io: std.Io, model: *Model, ids: []const u32, token_map: []const u8, dir: []const u8) !Result { + if (ids.len == 0 or ids.len > 16) return error.InvalidOracleWindow; + var reference = try Reference.init(gpa, model.cfg.text_config, &model.weights, &model.checkpoint, token_map, @intCast(ids.len)); + defer reference.deinit(); + var margins = MarginLog{}; + reference.attention_state.score_probe = .{ .ctx = &margins, .record = &MarginLog.capture }; + const cpu = try gpa.alloc(f32, ids.len * 129280); + defer gpa.free(cpu); + const gpu = try gpa.alloc(f32, cpu.len); + defer gpa.free(gpu); + try model.reset(); + var result = Result{ .tokens = ids.len, .max_absolute_error = 0, .rms_error = 0, .normalized_rms_error = 0, .greedy_disagreements = 0, .gpu_seconds = 0, .cpu_seconds = 0 }; + var squared_error: f64 = 0; + var squared_reference: f64 = 0; + for (ids, 0..) |id, i| { + const actual = try model.forward(&.{id}, .all); + result.gpu_seconds += model.last_gpu_seconds; + @memcpy(gpu[i * 129280 ..][0..129280], actual); + const start = mtl.clock.seconds(); + const expected = try reference.step(id); + result.cpu_seconds += mtl.clock.seconds() - start; + @memcpy(cpu[i * 129280 ..][0..129280], expected); + if (try maximum(actual) != try maximum(expected)) result.greedy_disagreements += 1; + for (actual, expected) |a, b| { + if (!std.math.isFinite(a) or !std.math.isFinite(b)) return error.NonfiniteTeacherLogits; + const error_: f64 = @as(f64, a) - b; + result.max_absolute_error = @max(result.max_absolute_error, @abs(error_)); + squared_error += error_ * error_; + squared_reference += @as(f64, b) * b; + } + std.debug.print("oracle teacher {d}/{d}: GPU token={d}, CPU token={d}\n", .{ i + 1, ids.len, try maximum(actual), try maximum(expected) }); + } + result.rms_error = @sqrt(squared_error / @as(f64, @floatFromInt(cpu.len))); + result.normalized_rms_error = if (squared_reference == 0) result.rms_error else @sqrt(squared_error / squared_reference); + result.numeric_gate_passed = result.normalized_rms_error <= result.normalized_rms_limit; + result.greedy_gate_passed = result.greedy_disagreements == 0; + const cp = try std.fs.path.join(gpa, &.{ dir, "oracle-logits.f32" }); + defer gpa.free(cp); + const gp = try std.fs.path.join(gpa, &.{ dir, "gpu-logits.f32" }); + defer gpa.free(gp); + const report = try std.fs.path.join(gpa, &.{ dir, "oracle.json" }); + defer gpa.free(report); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = cp, .data = std.mem.sliceAsBytes(cpu) }); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = gp, .data = std.mem.sliceAsBytes(gpu) }); + const json = try std.json.Stringify.valueAlloc(gpa, result, .{}); + defer gpa.free(json); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = report, .data = json }); + try margins.write(gpa, io, dir); + std.debug.print("{s}\n", .{json}); + try enforceGate(result.normalized_rms_error, result.greedy_disagreements); + return result; +} + +fn enforceGate(normalized_rms_error: f64, greedy_disagreements: usize) !void { + if (!std.math.isFinite(normalized_rms_error)) return error.NonfiniteTeacherLogits; + if (normalized_rms_error > nrms_limit) return error.TeacherReferenceMismatch; + if (greedy_disagreements != 0) return error.TeacherGreedyMismatch; +} + +test "teacher numeric gate remains independent of greedy agreement" { + try std.testing.expectError(error.TeacherReferenceMismatch, enforceGate(0.04, 0)); + try std.testing.expectError(error.TeacherReferenceMismatch, enforceGate(0.010000001, 1)); + try std.testing.expectError(error.TeacherGreedyMismatch, enforceGate(0.01, 1)); + try enforceGate(0.01, 0); + try std.testing.expectError(error.NonfiniteTeacherLogits, enforceGate(std.math.nan(f64), 0)); + try std.testing.expectError(error.NonfiniteTeacherLogits, enforceGate(std.math.inf(f64), 0)); +} diff --git a/zig/tests/deepseek_v41_reference/reference.zig b/zig/tests/deepseek_v41_reference/reference.zig new file mode 100644 index 0000000000..98069068f7 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/reference.zig @@ -0,0 +1,176 @@ +//! Independent wide-sum qualification with explicit FP32/BF16 boundaries; not GPU self-reference or CUDA emulation. + +// Equations: pinned DeepSeek AI inference/{model,engram,kernel}.py; provenance in THIRD_PARTY_NOTICES.md. +const std = @import("std"); +const tf = @import("tensorfold"); +const weights = tf.deepseek_v41_weights; +const config = tf.deepseek_v41_config; +const cp = tf.checkpoint_metal; +const exl3 = tf.exl3_projection; +const projection = @import("projection.zig"); +const math = @import("math.zig"); +const attention = @import("attention_math.zig"); +const engram = @import("engram_math.zig"); +const Stage = tf.deepseek_v41_profiler.Stage; + +/// Local routing isolation also uses these source equations; no native values enter qualification. +pub const MoeRouting = struct { + logits: [384]f32 = undefined, + scores: [384]f32 = undefined, + ids: [6]u32 = undefined, + weights: [6]f32 = undefined, +}; + +pub fn moeSelect(moe_weights: weights.Moe, c: config.Text, logits: *const [384]f32, scores: *[384]f32, ids: *[6]u32, routing_weights: *[6]f32) void { + @setFloatMode(.strict); + for (logits, scores) |z, *score| { + const softplus = if (z > 20) z else std.math.log1p(@exp(z)); + score.* = @sqrt(softplus); + } + var used: [384]bool = @splat(false); + for (0..6) |pick| { + var best: usize = 0; + var high: f32 = -std.math.inf(f32); + for (0..384) |expert| { + const biased = scores[expert] + math.value(moe_weights.bias, expert); + if (!used[expert] and biased > high) { + best = expert; + high = biased; + } + } + ids[pick] = @intCast(best); + routing_weights[pick] = scores[best]; + used[best] = true; + } + var normalization: f64 = 0; + for (routing_weights) |v| normalization += v; + const denominator = @as(f32, @floatCast(normalization)) + 1e-20; + for (routing_weights) |*v| v.* = (v.* / denominator) * c.routed_scaling_factor; +} + +pub fn moeActivation(gate: []const u16, up: []const u16, route: f32, c: config.Text, activation: []u16) void { + @setFloatMode(.strict); + for (activation, gate, up) |*v, g_value, u_value| { + const g = @min(math.f(g_value), c.swiglu_limit); + const u = @max(-c.swiglu_limit, @min(math.f(u_value), c.swiglu_limit)); + v.* = math.b(route * ((g / (1 + @exp(-g))) * u)); + } +} + +pub const Reference = struct { + gpa: std.mem.Allocator, + c: config.Text, + w: *const weights.Model, + checkpoint: *const cp.Checkpoint, + capacity: u32, + position: u32 = 0, + projection_scratch: projection.Scratch, + attention_state: attention.State, + engram_state: engram.State, + logits: []f32, + h: [20480]u16 = undefined, + input: [5120]u16 = undefined, + output: [5120]u16 = undefined, + gate: [2304]u16 = undefined, + up: [2304]u16 = undefined, + activation: [2304]u16 = undefined, + expert_output: [5120]u16 = undefined, + moe_sum: [5120]f32 = undefined, + route_logits: [384]f32 = undefined, + route_scores: [384]f32 = undefined, + last_routes: [6]u32 = undefined, + last_routing: [6]f32 = undefined, + observer: ?Observer = null, + pub const Observer = struct { + ctx: *anyopaque, + read: *const fn (*anyopaque, usize, Stage, *const Reference) void, + }; + + /// Borrowed model/checkpoint/config/map must outlive this instance; capacity is the teacher sequence length. + pub fn init(gpa: std.mem.Allocator, c: config.Text, w: *const weights.Model, checkpoint: *const cp.Checkpoint, token_map: []const u8, capacity: u32) !Reference { + if (capacity == 0 or capacity > c.max_position_embeddings or capacity > attention.max_capacity) + return error.InvalidReferenceCapacity; + if (c.hidden_size != 5120 or c.num_hidden_layers != 40 or c.vocab_size != 129280 or + c.num_experts_per_tok != 6 or c.n_routed_experts != 384 or c.hc_mult != 4) + return error.UnsupportedReferenceArchitecture; + const engram_state = try engram.State.init(token_map); + var projection_scratch = try projection.Scratch.init(gpa, 8192, 129280); + errdefer projection_scratch.deinit(); + var attention_state = try attention.State.init(gpa, capacity); + errdefer attention_state.deinit(); + const logits = try gpa.alloc(f32, 129280); + return .{ .gpa = gpa, .c = c, .w = w, .checkpoint = checkpoint, .capacity = capacity, .projection_scratch = projection_scratch, .attention_state = attention_state, .engram_state = engram_state, .logits = logits }; + } + pub fn deinit(self: *Reference) void { + self.projection_scratch.deinit(); + self.attention_state.deinit(); + self.gpa.free(self.logits); + self.* = undefined; + } + /// Consume exactly one teacher token and return all actual checkpoint logits, borrowed until the next step. + pub fn step(self: *Reference, id: u32) ![]const f32 { + if (self.position >= self.capacity) return error.ContextFull; + const compressed = try self.engram_state.compressed(id); + for (0..5120) |d| { + const v = math.b(math.value(self.w.embed, @as(usize, id) * 5120 + d)); + for (0..4) |copy| self.h[copy * 5120 + d] = v; + } + var pre = [4]f32{ 1, 0, 0, 0 }; + for (&self.w.layers, 0..) |*layer, layer_id| { + if (layer.engram) |e| try self.engram_state.apply(e, if (layer_id == 1) 0 else 1, compressed, &self.h, self.c, &self.projection_scratch); + const attn_mix = math.mixes(&self.h, layer.hc_attn, self.c); + math.collapse(&self.h, pre, &self.input); + math.rms(&self.input, layer.attn_norm, self.c.rms_norm_eps, &self.input); + try self.attention_state.apply(layer.attn, layer_id, &self.input, self.position, self.c, &self.output, &self.projection_scratch); + math.expand(&self.h, &self.output, attn_mix); + if (self.observer) |observer| observer.read(observer.ctx, layer_id, .attention, self); + const ffn_mix = math.mixes(&self.h, layer.hc_ffn, self.c); + math.collapse(&self.h, attn_mix.pre, &self.input); + math.rms(&self.input, layer.ffn_norm, self.c.rms_norm_eps, &self.input); + try self.moe(layer.moe, layer_id); + math.expand(&self.h, &self.output, ffn_mix); + if (self.observer) |observer| observer.read(observer.ctx, layer_id, .moe, self); + pre = ffn_mix.pre; + } + math.collapse(&self.h, pre, &self.input); + math.rms(&self.input, self.w.norm, self.c.rms_norm_eps, &self.input); + try projection.applyF32(.{ .trellis = self.w.head }, &self.input, self.logits, &self.projection_scratch); + for (self.logits) |v| if (!std.math.isFinite(v)) return error.NonFiniteReferenceLogits; + self.engram_state.advance(compressed); + self.position += 1; + return self.logits; + } + fn moe(self: *Reference, moe_weights: weights.Moe, layer: usize) !void { + @setFloatMode(.strict); + math.dense(moe_weights.gate, &self.input, &self.route_logits); + moeSelect(moe_weights, self.c, &self.route_logits, &self.route_scores, &self.last_routes, &self.last_routing); + @memset(&self.moe_sum, 0); + // Official MoE visits expert ids in ascending order, accumulates FP32, then adds shared expert last. + for (0..384) |expert| { + var route: ?f32 = null; + for (self.last_routes, self.last_routing) |selected, weight| + if (selected == expert) { + route = weight; + }; + const expert_route = route orelse continue; + var matrices: [3]exl3.Matrix = undefined; + var name: [128]u8 = undefined; + inline for (.{ "w1", "w3", "w2" }, 0..) |part, i| { + const prefix = try std.fmt.bufPrint(&name, "layers.{d}.ffn.experts.{d}.{s}", .{ layer, expert, part }); + matrices[i] = try weights.lookup(self.checkpoint, prefix, if (i == 2) 2304 else 5120, if (i == 2) 5120 else 2304); + } + try self.applyExpert(matrices, expert_route); + for (&self.moe_sum, self.expert_output) |*sum, v| sum.* += math.f(v); + } + try self.applyExpert(moe_weights.shared, 1); + for (&self.output, self.moe_sum, self.expert_output) |*v, routed, shared| + v.* = math.b(routed + math.f(shared)); + } + fn applyExpert(self: *Reference, matrices: [3]exl3.Matrix, route: f32) !void { + @setFloatMode(.strict); + try projection.apply(.{ .trellis = matrices[0] }, &self.input, &self.gate, &self.projection_scratch); + try projection.apply(.{ .trellis = matrices[1] }, &self.input, &self.up, &self.projection_scratch); + moeActivation(&self.gate, &self.up, route, self.c, &self.activation); + try projection.apply(.{ .trellis = matrices[2] }, &self.activation, &self.expert_output, &self.projection_scratch); + } +}; diff --git a/zig/tests/deepseek_v41_reference/rope_frequencies.json b/zig/tests/deepseek_v41_reference/rope_frequencies.json new file mode 100644 index 0000000000..2ffe1d1556 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/rope_frequencies.json @@ -0,0 +1,14 @@ +{ + "reference": "deepseek-ai/DeepSeek-V4.1-Flash", + "revision": "2cba9e42aa026125f3ed06c6d98c1db82f7ca027", + "source_sha256": "4e9ae23620edc8028ccc5d5fef552ab7fdc7dcd6f79608754fe9f67644056f65", + "torch_version": "2.14.1", + "extraction": "precompute_freqs_cis FP32 frequency vector before outer/polar; CPU", + "factor": 16, + "beta_fast": 32, + "beta_slow": 1, + "vectors": [ + {"theta": 10000, "original": 0, "bits": [1065353216,1061157137,1058010522,1054337179,1050798235,1047712803,1043732615,1040747895,1036831949,1033475085,1030116802,1026341397,1023510243,1019364175,1016180025,1012562930,1008981770,1005959600,1001931932,998911607,995049016,991652109,988353832,984533390,981668463,977573004,974350870,970790693,967166816,964208661,960132947,957076594]}, + {"theta": 160000, "original": 65536, "bits": [1065353216,1060112954,1056054303,1051098369,1046804782,1042117749,1037601972,1033169251,1028443341,1024251131,1019326489,1015361742,1010249151,1006366088,1001209182,997081655,991197921,985076960,979287478,973418702,966505391,959469944,952012877,943627103,933484300,917783504,913828774,908753896,904558591,899759068,895336240,890797133]} + ] +} diff --git a/zig/tests/deepseek_v41_reference/trace.zig b/zig/tests/deepseek_v41_reference/trace.zig new file mode 100644 index 0000000000..79cdef0fa4 --- /dev/null +++ b/zig/tests/deepseek_v41_reference/trace.zig @@ -0,0 +1,123 @@ +//! Bounded investigation snapshots; file I/O and independent CPU work are not performance measurements. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const Model = tf.deepseek_v41.Model; +const Reference = @import("reference.zig").Reference; +const Stage = tf.deepseek_v41_profiler.Stage; +const widths = [_]usize{ 20480 * 2, 5120 * 2, 5120 * 2, 6 * 4, 6 * 4 }; +const stride = 61488; +const bytes = stride * 80; + +const Capture = struct { + buffer: mtl.Buffer, + pipeline: mtl.Pipeline, + cpu: []u8, + + fn gpu(ctx: *anyopaque, enc: mtl.ComputeEncoder, layer: u32, stage: Stage, model: *const Model) void { + const self: *Capture = @ptrCast(@alignCast(ctx)); + var destination: usize = (@as(usize, layer) * 2 + @intFromBool(stage == .moe)) * stride; + const ws = &model.workspace; + inline for (.{ ws.view(.stream), ws.view(.normalized), ws.view(.branch), ws.view(.route_ids), ws.view(.route_weights) }, widths, 0..) |source, count, field| { + if (stage == .moe or field < 3) { + enc.setPipeline(self.pipeline); + enc.setBuffer(source.buffer, source.offset, 0); + enc.setBuffer(self.buffer, destination, 1); + enc.setValue(@as(u32, @intCast(count / 4)), 2); + enc.dispatchThreads(.{ .width = count / 4 }, .{ .width = 256 }); + destination += count; + } + } + enc.barrier(); + } + fn reference(ctx: *anyopaque, layer: usize, stage: Stage, ref: *const Reference) void { + const self: *Capture = @ptrCast(@alignCast(ctx)); + const start = (layer * 2 + @intFromBool(stage == .moe)) * stride; + var destination: []u8 = self.cpu[start..][0..stride]; + inline for (.{ std.mem.sliceAsBytes(&ref.h), std.mem.sliceAsBytes(&ref.input), std.mem.sliceAsBytes(&ref.output), std.mem.sliceAsBytes(&ref.last_routes), std.mem.sliceAsBytes(&ref.last_routing) }, 0..) |source, field| { + if (stage == .moe or field < 3) { + @memcpy(destination[0..source.len], source); + destination = destination[source.len..]; + } + } + } +}; +fn normalizedError(a: []const u8, b: []const u8) !f64 { + var squared: f64 = 0; + var denominator: f64 = 0; + var i: usize = 0; + while (i < a.len) : (i += 2) { + const x: f32 = @bitCast(@as(u32, std.mem.readInt(u16, a[i..][0..2], .little)) << 16); + const y: f32 = @bitCast(@as(u32, std.mem.readInt(u16, b[i..][0..2], .little)) << 16); + if (!std.math.isFinite(x) or !std.math.isFinite(y)) return error.NonfiniteTraceValues; + squared += (@as(f64, x) - y) * (@as(f64, x) - y); + denominator += @as(f64, y) * y; + } + return if (denominator == 0) @sqrt(squared) else @sqrt(squared / denominator); +} + +pub fn run(gpa: std.mem.Allocator, io: std.Io, model: *Model, ids: []const u32, token_map: []const u8, dir: []const u8) !void { + if (ids.len == 0 or ids.len > 16) return error.InvalidOracleWindow; + const lib = try mtl.Library.fromSource(model.device, "#include \nusing namespace metal;\nkernel void trace_copy(const device uint* x [[buffer(0)]], device uint* y [[buffer(1)]], constant uint& n [[buffer(2)]], uint i [[thread_position_in_grid]]) { if(i 0.01 or !routes_match) std.debug.print("trace token={d} layer={d} stage={s} stream={d} input={d} branch={d} routes_match={}\n", .{ token, point / 2, if (point % 2 == 0) "attention" else "moe", stream_error, input_error, branch_error, routes_match }); + } + inline for (.{ "gpu", "cpu" }, .{ gpu, cpu }) |name, data| { + const path = try std.fmt.allocPrint(gpa, "{s}/trace-{d}-{s}.bin", .{ dir, token, name }); + defer gpa.free(path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = data }); + } + std.debug.print("trace captured token {d}/{d}, same_logits=true\n", .{ token + 1, ids.len }); + } + try report.appendSlice(gpa, "\n]\n"); + const path = try std.fs.path.join(gpa, &.{ dir, "layer-trace.json" }); + defer gpa.free(path); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = report.items }); +}