server: graceful drain and pre-token keepalives (DeepSeek-V4.1 stack 7/14)

RUNBOOK.md

@@ -168,6 +168,12 @@ The server logs `heat_wait_s`, a cancel ends the wait, and the reply's tokens do
 On three GB10s serving GLM-5.3-Flash NVFP4 at TP=3, a 1,035,295-token prompt without the bands kept one host's hottest
 zone at 94 C or more for most of the prompt (97.7 C at the top); `TF_HEAT_HIGH=92` and `TF_HEAT_LOW=88` held all three
 at 93.0 C or under, and the first token came 10% later.
+`TF_GRAPH_CACHE=0` (or `off`, any case) runs every step eagerly in any family that installs the keyed CUDA graph cache,
+without a rebuild. Set it alike on every tensor-parallel rank: an off rank never joins the cache's per-miss vote, so a
+rank left on would wait at its first miss, and only the off rank logs the switch. Ranks that share a cache compare
+`Settings.digest` of their effective settings at start, before building it, and refuse to start when they differ.
+A tensor-parallel family passes its cache settings in the TP config; the bootstrap then compares them and refuses to
+start on every rank, naming TF_GRAPH_CACHE among the settings that must match.
 Archives whose names contain `host-only` are CPU verification artifacts and cannot serve CUDA inference.
 
 ## Flash Next and paired Metal serving

THIRD_PARTY_NOTICES.md

@@ -183,6 +183,12 @@ the one-shot all-gather protocol of [b12x](https://github.com/local-inference-la
 sequence parity and a timeout that fails the transport. Copyright 2026 Luke Alonso and the b12x contributors,
 Apache License 2.0 (see [the license text](LICENSES/Apache-2.0.txt)); b12x ships no NOTICE file.
 
+The Zig tensor-parallel fabric's RoCE path (`zig/src/fabric/tp/mailbox.cu`, `roce.zig`, `roce_verbs.zig`) adapts
+the one-shot all-gather protocol of [b12x](https://github.com/local-inference-lab/b12x)'s RoCEnante
+(`b12x/comm/roce/`, PRs #295 and #315): a device-resident epoch, sequence flags after each payload, two slots by
+sequence parity and a timeout that fails the transport. Copyright 2026 Luke Alonso and the b12x contributors,
+Apache License 2.0 (see [the license text](LICENSES/Apache-2.0.txt)); b12x ships no NOTICE file.
+
 ## Vendored code and weights
 
 In the Python engine, frozen at 0.6.6, `src/tensorfold/drafters/vendor/z_lab_dflash/model_mlx.py` is the unmodified

build.zig

@@ -19,6 +19,8 @@ const golden_build = @import("zig/build/golden.zig");
 const contract_build = @import("zig/build/kernel_contract.zig");
 const video_build = @import("zig/build/video.zig");
 const sys_build = @import("zig/build/sys.zig");
+const tp_build = @import("zig/build/tp.zig");
+const sessions_build = @import("zig/build/sessions.zig");
 
 comptime {
     const required = std.mem.trim(u8, @embedFile(".zig-version"), "\r\n");
@@ -202,6 +204,8 @@ pub fn build(b: *std.Build) void {
     b.step("test-learn-warning", "CPU production-log refusal, backoff and recovery test").dependOn(&warning_run.step);
     grammar_build.steps(b, target, xgrammar);
     contract_build.hostTests(b, gemma4_contract_host);
+    tp_build.add(b, target, optimize, test_step);
+    sessions_build.add(b, test_step);
     sys_build.wire(b);
     @import("zig/build/checkpoint_config.zig").wire(b);
 }

zig/build/sessions.zig

@@ -0,0 +1,20 @@
+//! Session memory: portable bookkeeping tests everywhere, NVMe tier tests on Linux.
+
+const std = @import("std");
+
+pub fn module(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode) *std.Build.Module {
+    return b.createModule(.{ .root_source_file = b.path("zig/src/sessions/sessions.zig"), .target = target, .optimize = optimize, .link_libc = true });
+}
+
+pub fn add(b: *std.Build, test_step: *std.Build.Step) void {
+    const host = b.graph.host;
+    const tests = if (host.result.os.tag == .linux) module(b, host, .debug) else b.createModule(.{
+        .root_source_file = b.path("zig/src/sessions/portable_test.zig"),
+        .target = host,
+        .optimize = .debug,
+        .link_libc = true,
+    });
+    const run = b.addRunArtifact(b.addTest(.{ .root_module = tests }));
+    b.step("test-sessions", "Session memory's host tests: pool, prefix trie, RAM tier, NVMe tier on Linux (no GPU)").dependOn(&run.step);
+    test_step.dependOn(&run.step);
+}

zig/build/tp.zig

@@ -0,0 +1,88 @@
+//! Tensor parallelism's build: Linux host tests, and on Linux the `tf-tp-test` GPU runner with the mailbox kernel.
+
+const std = @import("std");
+
+/// The CUDA runtime without the engine's kernels (tp only needs the driver, memory, streams and NCCL).
+fn runtime(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode) *std.Build.Module {
+    const options = b.addOptions();
+    options.addOption(bool, "with_kernels", false);
+    const cuda = b.createModule(.{ .root_source_file = b.path("zig/src/cuda/root.zig"), .target = target, .optimize = optimize, .link_libc = true });
+    cuda.addOptions("kernel_options", options);
+    cuda.addImport("stagger", b.createModule(.{ .root_source_file = b.path("zig/src/core/stagger.zig"), .target = target, .optimize = optimize }));
+    return cuda;
+}
+
+/// The `tp` module over `cuda` (an engine passes its own runtime module); verbs (zig/src/fabric/verbs.zig) for RoCE.
+pub fn module(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, cuda: *std.Build.Module) *std.Build.Module {
+    const m = b.createModule(.{ .root_source_file = b.path("zig/src/fabric/tp/tp.zig"), .target = target, .optimize = optimize, .link_libc = true });
+    m.addImport("cuda", cuda);
+    m.addImport("verbs", b.createModule(.{ .root_source_file = b.path("zig/src/fabric/verbs.zig"), .target = target, .optimize = optimize, .link_libc = true }));
+    return m;
+}
+
+/// mailbox.cu as a fatbin: sm_120 (RTX PRO 6000), sm_121 (GB10), and compute_90 PTX for anything else.
+fn mailboxFatbin(b: *std.Build, nvcc: []const u8) std.Build.LazyPath {
+    const run = b.addSystemCommand(&.{ nvcc, "-fatbin", "-O3", "-std=c++17", "-gencode=arch=compute_120,code=sm_120", "-gencode=arch=compute_121,code=sm_121", "-gencode=arch=compute_90,code=compute_90", "-o" });
+    const out = run.addOutputFileArg("tp_mailbox.fatbin");
+    run.addFileArg(b.path("zig/src/fabric/tp/mailbox.cu"));
+    return out;
+}
+
+/// `pub const mailbox: []const u8`, the fatbin's bytes or empty.
+fn kernelModule(b: *std.Build, image: ?std.Build.LazyPath) *std.Build.Module {
+    const wf = b.addWriteFiles();
+    if (image) |file| {
+        _ = wf.addCopyFile(file, "tp_mailbox.fatbin");
+        // the driver takes 8-byte aligned images (Module.load refuses others); @embedFile alone promises nothing
+        return b.createModule(.{ .root_source_file = wf.add("tp_kernel.zig", "const bytes align(16) = @embedFile(\"tp_mailbox.fatbin\").*;\npub const mailbox: []const u8 = &bytes;\n") });
+    }
+    return b.createModule(.{ .root_source_file = wf.add("tp_kernel.zig", "pub const mailbox: []const u8 = \"\";\n") });
+}
+
+pub fn add(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, test_step: *std.Build.Step) void {
+    const host = b.graph.host;
+    const cli_tests = b.addRunArtifact(b.addTest(.{ .root_module = b.createModule(.{ .root_source_file = b.path("zig/src/fabric/tp/gpu_cli.zig"), .target = host, .optimize = optimize }) }));
+    const cli_step = b.step("test-tp-cli", "TP runner command validation (no GPU)");
+    cli_step.dependOn(&cli_tests.step);
+    test_step.dependOn(&cli_tests.step);
+    if (host.result.os.tag == .linux) {
+        const tests = b.addRunArtifact(b.addTest(.{ .root_module = module(b, host, .debug, runtime(b, host, .debug)) }));
+        const tp_step = b.step("test-tp", "Tensor parallelism's host tests: bootstrap, fate channel, in-process ranks (no GPU)");
+        tp_step.dependOn(&tests.step);
+        tp_step.dependOn(cli_step);
+        test_step.dependOn(&tests.step);
+    }
+    if (target.result.os.tag != .linux) return;
+    const nvcc = b.option([]const u8, "tp-nvcc", "nvcc that builds the TP mailbox kernel");
+    const prebuilt = b.option([]const u8, "tp-fatbin", "absolute path of a prebuilt tp_mailbox.fatbin to embed");
+    const image: ?std.Build.LazyPath = if (prebuilt) |p| b.graph.cwdRelativePath(p) else if (nvcc) |tool| mailboxFatbin(b, tool) else null;
+    const cuda = runtime(b, target, optimize);
+    const runner = b.createModule(.{ .root_source_file = b.path("zig/src/fabric/tp/gpu_main.zig"), .target = target, .optimize = optimize, .link_libc = true });
+    runner.addImport("cuda", cuda);
+    runner.addImport("tp", module(b, target, optimize, cuda));
+    runner.addImport("tp_kernel", kernelModule(b, image));
+    const exe = b.addExecutable(.{ .name = "tf-tp-test", .root_module = runner });
+    if (host.result.os.tag == .linux and target.query.isNative()) {
+        const cli = @import("../src/fabric/tp/gpu_cli.zig");
+        for (cli.invalid) |args| {
+            const check = b.addRunArtifact(exe);
+            check.addArgs(args);
+            check.setEnvironmentVariable("TF_TP_WORLD", "invalid");
+            check.expectExitCode(2);
+            check.expectStdErrEqual(cli.usage);
+            cli_step.dependOn(&check.step);
+            test_step.dependOn(&check.step);
+        }
+        if (image == null) {
+            const kernel = b.addRunArtifact(exe);
+            kernel.addArg("kernel");
+            kernel.setEnvironmentVariable("TF_TP_WORLD", "invalid");
+            kernel.expectExitCode(1);
+            kernel.expectStdErrEqual("mailbox kernel 0 B\n");
+            cli_step.dependOn(&kernel.step);
+            test_step.dependOn(&kernel.step);
+        }
+    }
+    b.installArtifact(exe);
+    b.step("tf-tp-test", "TP=2 on two GPUs: bootstrap, every collective bit-exact, graph capture, fail-fast, latency").dependOn(&b.addInstallArtifact(exe, .{}).step);
+}

zig/src/cuda/abi.zig

@@ -39,6 +39,8 @@ pub const DeviceAttribute = enum(c_int) {
     compute_capability_minor = 76,
     max_shared_memory_per_multiprocessor = 81,
     max_shared_memory_per_block_optin = 97,
+    virtual_memory_management_supported = 102,
+    gpu_direct_rdma_with_cuda_vmm_supported = 110,
 };
 
 pub const FunctionAttribute = enum(c_int) {
@@ -242,3 +244,10 @@ comptime {
     std.debug.assert(@sizeOf(KernelNodeParams) == 72 and @offsetOf(KernelNodeParams, "params") == 40);
     std.debug.assert(@sizeOf(ExecUpdateResultInfo) == 24);
 }
+
+test "CUDA device attribute matches the RDMA VMM capability ABI" {
+    // CUDA's CUdevice_attribute assigns 110 to RDMA VMM support; 126 is the memory sync domain count.
+    try @import("std").testing.expectEqual(@as(c_int, 110), @backingInt(
+        DeviceAttribute.gpu_direct_rdma_with_cuda_vmm_supported,
+    ));
+}