DeepSeek V4.1: the family — configuration, Engram tables, resident weights and the Metal operators

THIRD_PARTY_NOTICES.md

@@ -207,6 +207,13 @@ TensorFold ships no model weights. The `z-lab/Qwen3.8-27B-DFlash2` model card st
 The optional `incoai/GLM-5.3-Flash-DFlash2` model card states CC BY-NC-ND 4.0, for non-commercial use
 without derivatives; only the Python engine reads that drafter. Each checkpoint keeps its own license.
 
+The native `zig/src/families/deepseek_v41/` implementation and `zig/tests/deepseek_v41_reference/`
+follow [DeepSeek-V4.1-Flash's official inference](https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/tree/2cba9e42aa026125f3ed06c6d98c1db82f7ca027/inference),
+MIT License, Copyright (c) 2023 DeepSeek: the reference's equations, the Engram normalization,
+PCG64 multipliers and prime bucket layout, and the pinned chat template come from that revision.
+The EXL3 checkpoint reads through the ExLlamaV3 format covered above; no captured CUDA/Triton
+kernel assets, full model shards or binary traces are bundled.
+
 ## Structured output: libtfgrammar
 
 `lib/libtfgrammar` in the release archives is `zig/src/grammar/tf_grammar.cc` compiled with the C++ sources of

build.zig

@@ -19,6 +19,7 @@ const golden_build = @import("zig/build/golden.zig");
 const contract_build = @import("zig/build/kernel_contract.zig");
 const video_build = @import("zig/build/video.zig");
 const sys_build = @import("zig/build/sys.zig");
+const dsv41_build = @import("zig/build/deepseek_v41.zig");
 
 comptime {
     const required = std.mem.trim(u8, @embedFile(".zig-version"), "\r\n");
@@ -416,6 +417,10 @@ fn metalTargets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.b
     const load_admission_run = b.addRunArtifact(load_admission_cases);
     b.step("test-flashnext-load-admission", "CPU-only Flash Next runtime quantization admission").dependOn(&load_admission_run.step);
     test_step.dependOn(&load_admission_run.step);
+    test_step.dependOn(dsv41_build.weights(b, target, optimize, metal));
+    const dsv41 = dsv41_build.checks(b, target, optimize, metal, sources, engine);
+    test_step.dependOn(dsv41.device);
+    test_step.dependOn(dsv41.reference);
     const pack_cache_cases = b.addTest(.{ .root_module = b.createModule(.{
         .root_source_file = b.path("zig/flashnext_pack_cache_cases.zig"),
         .target = target,
@@ -425,8 +430,10 @@ fn metalTargets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.b
     const pack_cache_run = b.addRunArtifact(pack_cache_cases);
     b.step("test-flashnext-pack-cache", "CPU-only atomic Flash Next pack cache lifecycle").dependOn(&pack_cache_run.step);
     test_step.dependOn(&pack_cache_run.step);
-    const engine_programs = [_]struct { name: []const u8, path: []const u8, about: []const u8, c_source: ?[]const u8 = null }{
+    const engine_programs = [_]struct { name: []const u8, path: []const u8, about: []const u8, c_source: ?[]const u8 = null, host_tests: bool = false }{
         .{ .name = "tf-qwen35-check", .path = "zig/tests/qwen35_check.zig", .about = "Qwen3.5-2B checkpoint and native operations" },
+        .{ .name = "tf-dsv41-token-map", .path = "zig/dsv41_token_map.zig", .about = "Build the Engram token map from a tokenizer and inference config, byte-identical to the release's", .host_tests = true },
+        .{ .name = "tf-dsv41-run", .path = "zig/deepseek_v41_run.zig", .about = "Full-resident DeepSeek V4.1 admission, teacher-forced logits, chunk seams and decode timing" },
         .{ .name = "tf-qwen35-forward", .path = "zig/tests/qwen35_forward.zig", .about = "Qwen3.5-2B native teacher-forced logits" },
         .{ .name = "tf-qwen35-exact", .path = "zig/tests/qwen35_exact.zig", .about = "Qwen3.5-2B forward and committed-state exactness" },
         .{ .name = "tf-lfm2-forward", .path = "zig/tests/lfm2_forward.zig", .about = "LFM2-MoE native teacher-forced logits" },
@@ -479,6 +486,7 @@ fn metalTargets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.b
         const exe = b.addExecutable(.{ .name = p.name, .root_module = mod });
         b.installArtifact(exe);
         b.step(p.name, p.about).dependOn(&b.addInstallArtifact(exe, .{}).step);
+        if (p.host_tests) test_step.dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = mod })).step);
     }
 
     glm_checks.sampler(b, target, optimize, metal, lanes, sources);

tools/zig/lean_check.py

@@ -12,9 +12,9 @@
            ".sh": re.compile(r"^\s*#(?!!)")}
 BLOCK = re.compile(r"/\*")
 DOCSTRING = re.compile(r'^\s*(?:[rubf]{0,2})("""|\'\'\')')
-GENERATED = ("unicode_data.zig",)
+GENERATED = ("unicode_data.zig", "nfkd_data.zig")
 HEX_METAL = re.compile(r"_[0-9a-f]{16}\.metal$")
-# Generator outputs are skipped entirely. unicode_data.zig stays exempt from the length check only.
+# Generator outputs are skipped entirely. The Unicode table files stay exempt from the length check only.
 GENERATOR_OUTPUT = (
     "zig/src/families/flashnext/roles_gen.zig",
     "zig/kernels/metal/flashnext/sources_gen.zig",

tools/zig/nfkd_tables.py

@@ -0,0 +1,96 @@
+"""Emit the Engram token-map normalizer's NFKD, mark, lowercase and combining-class tables as Zig source."""
+import argparse
+import subprocess
+import sys
+import tempfile
+import unicodedata
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+OUT = ROOT / "zig" / "src" / "core" / "tokenizer" / "nfkd_data.zig"
+VERSION = "16.0.0"
+
+
+def rows(values, per_line, fmt):
+    """Comma-terminated lines of at most `per_line` values, four-space indented."""
+    for i in range(0, len(values), per_line):
+        yield "    " + " ".join(fmt(v) + "," for v in values[i:i + per_line])
+
+
+def render():
+    """The generated Zig source before the compiler's formatter aligns the columns."""
+    flat, decompositions = [], []
+    for cp in range(0x110000):
+        decomposed = unicodedata.normalize("NFKD", chr(cp))
+        if decomposed != chr(cp):
+            decompositions.append((cp, len(flat), len(decomposed)))
+            flat.extend(ord(c) for c in decomposed)
+    marks = [cp for cp in range(0x110000) if unicodedata.category(chr(cp)) in ("Mn", "Mc", "Me")]
+    lower_flat, lowercases = [], []
+    for cp in range(0x110000):
+        low = chr(cp).lower()
+        if low != chr(cp):
+            lowercases.append((cp, len(lower_flat), len(low)))
+            lower_flat.extend(ord(c) for c in low)
+    cccs = []
+    for cp in range(0x110000):
+        ccc = unicodedata.combining(chr(cp))
+        if not ccc:
+            continue
+        if cccs and cccs[-1][2] == ccc and cccs[-1][1] + 1 == cp:
+            cccs[-1][1] = cp
+        else:
+            cccs.append([cp, cp, ccc])
+    decomp = lambda e: ".{ .cp = 0x%04X, .start = %d, .len = %d }" % e
+    ccc = lambda t: ".{ 0x%04X, 0x%04X, 0x%04X }" % tuple(t)
+    return "\n".join([
+        "// Generated by tools/zig/nfkd_tables.py from the Unicode %s character database; do not edit." % VERSION,
+        'pub const version = "%s";' % VERSION,
+        "",
+        "pub const Decomp = struct { cp: u21, start: u32, len: u8 };",
+        "",
+        "// All NFKD sequences concatenated, indexed by Decomp.start/.len.",
+        "pub const sequences = [_]u21{", *rows(flat, 8, lambda v: "0x%04X" % v), "};",
+        "",
+        "pub const decompositions = [_]Decomp{", *rows(decompositions, 1, decomp), "};",
+        "",
+        "// General-category Mn/Mc/Me codepoints, sorted; the Rust tokenizers' StripAccents removes all of category M*.",
+        "pub const marks = [_]u21{", *rows(marks, 8, lambda v: "0x%04X" % v), "};",
+        "",
+        "pub const Lower = struct { cp: u21, start: u32, len: u8 };",
+        "",
+        "// Full lowercase mappings concatenated (Rust char::to_lowercase semantics).",
+        "pub const lower_sequences = [_]u21{", *rows(lower_flat, 8, lambda v: "0x%04X" % v), "};",
+        "",
+        "pub const lowercases = [_]Lower{", *rows(lowercases, 1, decomp), "};",
+        "",
+        "// Canonical combining class ranges [first, last, ccc], unicodedata %s." % VERSION,
+        "pub const cccs = [_][3]u21{", *rows(cccs, 4, ccc), "};",
+    ]) + "\n"
+
+
+def formatted(text, zig):
+    """`text` grown through the Zig compiler's formatter, which is how the committed file is shaped."""
+    with tempfile.TemporaryDirectory() as tmp:
+        path = Path(tmp) / "nfkd_data.zig"
+        path.write_text(text)
+        subprocess.run([zig, "fmt", str(path)], check=True, capture_output=True)
+        return path.read_text()
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--check", action="store_true", help="fail if the committed file differs")
+    parser.add_argument("--zig", default="zig", help="the Zig compiler whose formatter shapes the output")
+    args = parser.parse_args()
+    if unicodedata.unidata_version != VERSION:
+        sys.exit("Python has Unicode %s; the Engram tables need %s" % (unicodedata.unidata_version, VERSION))
+    text = formatted(render(), args.zig)
+    if args.check:
+        sys.exit(0 if OUT.read_text() == text else "%s is stale; rerun tools/zig/nfkd_tables.py" % OUT.relative_to(ROOT))
+    OUT.write_text(text)
+    print("Wrote %s (Unicode %s)" % (OUT.relative_to(ROOT), VERSION))
+
+
+if __name__ == "__main__":
+    main()

zig/build/deepseek_v41.zig

@@ -0,0 +1,60 @@
+//! Experimental DS4.1 weight-loading qualification targets.
+const std = @import("std");
+
+pub fn weights(
+    b: *std.Build,
+    target: std.Build.ResolvedTarget,
+    optimize: std.builtin.OptimizeMode,
+    metal: *std.Build.Module,
+) *std.Build.Step {
+    const host = b.addTest(.{ .root_module = b.createModule(.{
+        .root_source_file = b.path("zig/deepseek_v41_weights_checks.zig"),
+        .target = target,
+        .optimize = optimize,
+        .link_libc = true,
+        .imports = &.{
+            .{ .name = "metal", .module = metal },
+        },
+    }) });
+    const step = b.step("test-deepseek-v41-weights", "V4.1 configuration, Engram table plans and resident weight staging");
+    step.dependOn(&b.addRunArtifact(host).step);
+    return step;
+}
+
+pub fn checks(
+    b: *std.Build,
+    target: std.Build.ResolvedTarget,
+    optimize: std.builtin.OptimizeMode,
+    metal: *std.Build.Module,
+    sources: *std.Build.Module,
+    engine: *std.Build.Module,
+) struct { device: *std.Build.Step, reference: *std.Build.Step } {
+    const device = b.addTest(.{ .root_module = b.createModule(.{
+        .root_source_file = b.path("zig/deepseek_v41_checks.zig"),
+        .target = target,
+        .optimize = optimize,
+        .link_libc = true,
+        .imports = &.{
+            .{ .name = "metal", .module = metal },
+            .{ .name = "kernel_sources", .module = sources },
+        },
+    }) });
+    const device_step = b.step("test-deepseek-v41", "Experimental DS4.1 EXL3, Engram, mHC and sparse attention on Metal");
+    device_step.dependOn(&b.addRunArtifact(device).step);
+    const reference = b.addTest(.{
+        .root_module = b.createModule(.{
+            .root_source_file = b.path("zig/deepseek_v41_reference_checks.zig"),
+            .target = target,
+            .optimize = optimize,
+            .link_libc = true,
+            .imports = &.{
+                .{ .name = "tensorfold", .module = engine },
+                .{ .name = "metal", .module = metal },
+            },
+        }),
+        .filters = &.{"tests.deepseek_v41_reference."},
+    });
+    const reference_step = b.step("test-deepseek-v41-reference", "Independent CPU teacher-oracle arithmetic boundaries");
+    reference_step.dependOn(&b.addRunArtifact(reference).step);
+    return .{ .device = device_step, .reference = reference_step };
+}

zig/deepseek_v41_checks.zig

@@ -0,0 +1,13 @@
+//! Source-built V4.1 operator qualification; real Metal execution, no checkpoint needed.
+test {
+    _ = @import("src/core/exl3_projection_gpu_test.zig");
+    _ = @import("src/families/deepseek_v41/engram.zig");
+    _ = @import("src/families/deepseek_v41/csa2.zig");
+    _ = @import("src/families/deepseek_v41/ops_check.zig");
+    _ = @import("src/families/deepseek_v41/load.zig");
+    _ = @import("src/families/deepseek_v41/index_precision_test.zig");
+    _ = @import("src/families/deepseek_v41/config.zig");
+    _ = @import("src/families/deepseek_v41/profiler.zig");
+    _ = @import("src/core/gpu_counters.zig");
+    _ = @import("src/core/expert_groups.zig");
+}

zig/deepseek_v41_reference_checks.zig

@@ -0,0 +1,7 @@
+//! Independent CPU arithmetic boundaries used by the resident-checkpoint teacher oracle.
+test {
+    _ = @import("tests/deepseek_v41_reference/projection.zig");
+    _ = @import("tests/deepseek_v41_reference/math.zig");
+    _ = @import("tests/deepseek_v41_reference/attention_math.zig");
+    _ = @import("tests/deepseek_v41_reference/qualify.zig");
+}