cuda: the start-up budget counts the free pages parked on the kernel's per-CPU lists

zig/src/native/cuda.zig

@@ -283,6 +283,7 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O
         fn read(p: *anyopaque) ?u64 {
             const self: *@This() = @ptrCast(@alignCast(p));
             const now = (pool(self.a, self.io, self.ctx, self.problem) catch return null) orelse return null;
+            family_memory.noteParked(now, "the room this rank reports");
             return now.room(cuda.usage(false).device - self.held);
         }
     };
@@ -335,6 +336,7 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O
     defer if (!opened) served.close(gpa);
     const model = cuda.usage(false).device - held0;
     const after = try pool(a, io, &g.ctx, problem) orelse return null;
+    family_memory.noteParked(after, "the CUDA memory budget");
     const L = @TypeOf(loaded); // a family's Loaded may leave out a learner and a lone driver
     const learn_bytes: usize = if (@hasField(L, "learn_bytes")) loaded.learn_bytes else 0;
     const host_cache = try kv_pages.placeHost(F, a, loaded, o, after.room(model), @intCast(window), served.stream_bytes, problem) orelse return null;

zig/src/native/cuda_family_memory.zig

@@ -21,10 +21,14 @@ pub fn poolFor(comptime windows: bool, comptime Host: type, a: Allocator, io: st
     const host_counts = budget.hostCounts(unified, windows);
     const text = if (host_counts) Host.procText(a, io, "/proc/meminfo") else null;
     defer if (text) |bytes| a.free(bytes);
-    const available = budget.counts(host_counts, .{ .total = card.total, .available = card.free }, text) catch {
+    // read after MemAvailable, which the lists drain into: a page moving in between is missed, not counted twice
+    const zones = if (host_counts) Host.procText(a, io, "/proc/zoneinfo") else null;
+    defer if (zones) |bytes| a.free(bytes);
+    const measured = budget.counts(host_counts, .{ .total = card.total, .available = card.free }, text) catch {
         problem.* = "cannot read or parse /proc/meminfo's MemTotal and MemAvailable; refusing CUDA unified-memory admission";
         return null;
     };
+    const available = budget.counting(measured, zones);
     const free = available.available;
     const total = available.total;
     const reserve = budget.reserveBytes(Host.env("TENSORFOLD_MEMORY_RESERVE_GIB"), total, unified) catch {
@@ -35,7 +39,7 @@ pub fn poolFor(comptime windows: bool, comptime Host: type, a: Allocator, io: st
         problem.* = try std.fmt.allocPrint(a, "TENSORFOLD_CUDA_MEMORY_LIMIT_GB={s}: a positive number of GiB whose byte count fits in a 64-bit size", .{Host.env("TENSORFOLD_CUDA_MEMORY_LIMIT_GB").?});
         return null;
     };
-    return .{ .free = free, .total = total, .reserve = reserve, .limit = limit, .unified = unified };
+    return .{ .free = free, .total = total, .reserve = reserve, .limit = limit, .unified = unified, .parked = free - measured.available };
 }
 
 /// `F`'s weight bytes on this rank: tensors or files, a split's share, a tower beside; null if tensors are unreadable.
@@ -51,6 +55,12 @@ pub fn weights(comptime F: type, a: Allocator, io: std.Io, o: api.Open, problem:
     return text + try @import("cuda_vision_offer.zig").weightBytes(F, a, io, o);
 }
 
+/// Logs, from 0.25 GiB, the free pages on the per-CPU lists that a pool's reading counted.
+pub fn noteParked(p: budget.Pool, what: []const u8) void {
+    const line = "{s} counts the {d:.2} GiB free on the per-CPU page lists beyond their resting level (MemAvailable does not)";
+    if (p.parked >= 1 << 28) std.log.info(line, .{ what, toGib(p.parked) });
+}
+
 pub fn room(comptime F: type, ptr: *anyopaque, local: u64) !u64 {
     if (@hasDecl(F, "rankRoom")) return F.rankRoom(ptr, local);
     return local;

zig/src/native/cuda_family_memory_test.zig

@@ -6,7 +6,9 @@ const Fake = struct {
     var integrated: c_int = 1;
     var missing = false;
     var text: ?[]const u8 = null;
+    var zones: ?[]const u8 = null;
     var probes: usize = 0;
+    var zone_probes: usize = 0;
     fn attribute(out: *c_int, _: cuda.abi.DeviceAttribute, _: c_int) callconv(.c) cuda.abi.Result {
         out.* = integrated;
         return 0;
@@ -17,7 +19,11 @@ const Fake = struct {
         free.* = 40 << 30;
         return 0;
     }
-    pub fn procText(a: std.mem.Allocator, _: std.Io, _: []const u8) ?[]u8 {
+    pub fn procText(a: std.mem.Allocator, _: std.Io, path: []const u8) ?[]u8 {
+        if (std.mem.eql(u8, path, "/proc/zoneinfo")) {
+            zone_probes += 1;
+            return a.dupe(u8, zones orelse return null) catch null;
+        }
         probes += 1;
         return a.dupe(u8, text orelse return null) catch null;
     }
@@ -65,3 +71,66 @@ test "Windows pool: real CUDA counts, OS selection, unified reserves and missing
     try t.expectError(error.NotReady, poolFor(true, Fake, t.allocator, t.io, &ctx, &problem));
     try t.expectError(error.NotReady, poolFor(false, Fake, t.allocator, t.io, &ctx, &problem));
 }
+
+/// One zone whose CPU list holds `parked` bytes beyond its high_min, as kernel 6.7+ prints /proc/zoneinfo.
+fn zoneinfo(a: std.mem.Allocator, parked: u64) ![]u8 {
+    const hm = 2146;
+    const fmt = "Node 0, zone   Normal\n  pages free     10\n  pagesets\n    cpu: 0\n              count:    {d}\n" ++
+        "              high:     5000\n              batch:    63\n              high_min: {d}\n  vm stats threshold: 110\n" ++
+        "  start_pfn:           4096\n";
+    return std.fmt.allocPrint(a, fmt, .{ hm + parked / std.heap.pageSize(), hm });
+}
+
+test "a Linux unified pool counts the free pages parked on the per-CPU lists, at most MemTotal" {
+    var driver: cuda.Driver = undefined;
+    driver.api.cuDeviceGetAttribute = Fake.attribute;
+    driver.api.cuMemGetInfo_v2 = Fake.memInfo;
+    const ctx: cuda.Context = .{ .d = &driver, .device = 0, .handle = null };
+    var problem: []const u8 = "";
+    const g: u64 = 1 << 30;
+    Fake.missing = false;
+    Fake.integrated = 1;
+    Fake.text = "MemTotal: 134217728 kB\nMemAvailable: 100663296 kB\n";
+    defer Fake.zones = null;
+    const six = try zoneinfo(t.allocator, 6 * g);
+    defer t.allocator.free(six);
+    Fake.zones = six;
+    try t.expectEqual(102 * g, (try poolFor(false, Fake, t.allocator, t.io, &ctx, &problem)).?.free);
+    const forty = try zoneinfo(t.allocator, 40 * g);
+    defer t.allocator.free(forty);
+    Fake.zones = forty;
+    try t.expectEqual(128 * g, (try poolFor(false, Fake, t.allocator, t.io, &ctx, &problem)).?.free); // MemTotal
+    for ([_]?[]const u8{ null, "", "Node 0, zone Normal\n  pagesets\n    cpu: 0\n      count: x\n" }) |zones| {
+        Fake.zones = zones;
+        try t.expectEqual(96 * g, (try poolFor(false, Fake, t.allocator, t.io, &ctx, &problem)).?.free);
+    }
+    Fake.zones = six;
+    Fake.zone_probes = 0;
+    const windows = (try poolFor(true, Fake, t.allocator, t.io, &ctx, &problem)).?;
+    try t.expectEqual(@as(u64, 40 << 30), windows.free);
+    Fake.integrated = 0;
+    const card = (try poolFor(false, Fake, t.allocator, t.io, &ctx, &problem)).?;
+    try t.expectEqual(@as(u64, 40 << 30), card.free);
+    try t.expectEqual(@as(usize, 0), Fake.zone_probes);
+}
+
+test "the pool names the parked share it counted, for the start-up log line" {
+    var driver: cuda.Driver = undefined;
+    driver.api.cuDeviceGetAttribute = Fake.attribute;
+    driver.api.cuMemGetInfo_v2 = Fake.memInfo;
+    const ctx: cuda.Context = .{ .d = &driver, .device = 0, .handle = null };
+    var problem: []const u8 = "";
+    const g: u64 = 1 << 30;
+    Fake.missing = false;
+    Fake.integrated = 1;
+    Fake.text = "MemTotal: 134217728 kB\nMemAvailable: 130023424 kB\n";
+    defer Fake.zones = null;
+    const six = try zoneinfo(t.allocator, 6 * g);
+    defer t.allocator.free(six);
+    Fake.zones = six;
+    try t.expectEqual(4 * g, (try poolFor(false, Fake, t.allocator, t.io, &ctx, &problem)).?.parked); // to MemTotal
+    Fake.text = "MemTotal: 134217728 kB\nMemAvailable: 100663296 kB\n";
+    try t.expectEqual(6 * g, (try poolFor(false, Fake, t.allocator, t.io, &ctx, &problem)).?.parked);
+    Fake.zones = null;
+    try t.expectEqual(@as(u64, 0), (try poolFor(false, Fake, t.allocator, t.io, &ctx, &problem)).?.parked);
+}

zig/src/native/cuda_memory.zig

@@ -1,6 +1,7 @@
 //! CUDA memory admission arithmetic, and the host pool read from /proc/meminfo and validated, without a device.
 const cuda = @import("cuda");
 const std = @import("std");
+const parked_pages = @import("parked_pages.zig");
 const gib: f64 = 1 << 30;
 
 /// Bytes of the checkpoint's safetensors (the weight map's files, else the folder's own), near enough to refuse early.
@@ -80,11 +81,18 @@ pub fn meminfo(text: []const u8) ?MemInfo {
     return if (t > 0 and a <= t) .{ .total = t, .available = a } else null;
 }
 
-/// --prompt-cache-gib in pinned host memory (8 GiB unasked), at most half of what /proc/meminfo has available.
+/// MemAvailable plus the free pages parked on the per-CPU lists (`zones`, read after it), at most MemTotal.
+pub fn counting(m: MemInfo, zones: ?[]const u8) MemInfo {
+    const parked = parked_pages.bytes(zones orelse return m, std.heap.pageSize());
+    return .{ .total = m.total, .available = @min(m.total, m.available +| parked) };
+}
+
+/// --prompt-cache-gib in pinned host memory (8 GiB unasked), at most half of what is available (`counting`).
 pub fn hostCacheBytes(a: std.mem.Allocator, io: std.Io, asked: ?f64) u64 {
     const want: u64 = if (asked) |g| (if (g > 0) std.math.lossyCast(u64, g * (1 << 30)) else 0) else 8 << 30;
     const text = procText(a, io, "/proc/meminfo") orelse return 0;
-    const m = meminfo(text) orelse return 0;
+    const zones = procText(a, io, "/proc/zoneinfo");
+    const m = counting(meminfo(text) orelse return 0, zones);
     return @min(want, m.available / 2);
 }
 
@@ -104,6 +112,7 @@ pub const Pool = struct {
     reserve: u64,
     limit: ?u64,
     unified: bool,
+    parked: u64 = 0, // the part of `free` parked on the per-CPU page lists
 
     /// Bytes the engine may still allocate on top of `held`.
     pub fn room(p: Pool, held: u64) u64 {
@@ -260,6 +269,21 @@ test "unified admission refuses missing or malformed host memory" {
     try std.testing.expectEqual(MemInfo{ .total = 131072 * 1024, .available = 65536 * 1024 }, try counts(true, card, "MemTotal: 131072 kB\nMemAvailable: 65536 kB\n"));
 }
 
+test "counting: the per-CPU lists' parked pages raise MemAvailable, at most MemTotal; nothing without them" {
+    const page = std.heap.pageSize();
+    const m: MemInfo = .{ .total = 128 << 30, .available = 96 << 30 };
+    const zones = try std.fmt.allocPrint(std.testing.allocator, "  pagesets\n    cpu: 0\n      count: {d}\n      high_min: 10\n", .{10 + (6 << 30) / page});
+    defer std.testing.allocator.free(zones);
+    try std.testing.expectEqual(MemInfo{ .total = 128 << 30, .available = 102 << 30 }, counting(m, zones));
+    try std.testing.expectEqual(MemInfo{ .total = 128 << 30, .available = 128 << 30 }, counting(.{ .total = 128 << 30, .available = 125 << 30 }, zones));
+    try std.testing.expectEqual(m, counting(m, null));
+    try std.testing.expectEqual(m, counting(m, "  pagesets\n    cpu: 0\n      count: x\n"));
+}
+
+test {
+    _ = parked_pages;
+}
+
 test "memory limit refuses an oversized finite value and the rounded u64 boundary" {
     try std.testing.expectError(error.Invalid, limitBytes("1e300"));
     try std.testing.expectError(error.Invalid, limitBytes("17179869184"));
@@ -273,10 +297,13 @@ pub fn readPool(a: std.mem.Allocator, io: std.Io, ctx: *const cuda.Context, prob
     const host_counts = hostCounts(unified, @import("sys").windows);
     const text = if (host_counts) procText(a, io, "/proc/meminfo") else null;
     defer if (text) |bytes| a.free(bytes);
-    const available = counts(host_counts, .{ .total = card.total, .available = card.free }, text) catch {
+    const zones = if (host_counts) procText(a, io, "/proc/zoneinfo") else null;
+    defer if (zones) |bytes| a.free(bytes);
+    const measured = counts(host_counts, .{ .total = card.total, .available = card.free }, text) catch {
         problem.* = "cannot read or parse /proc/meminfo's MemTotal and MemAvailable; refusing CUDA unified-memory admission";
         return null;
     };
+    const available = counting(measured, zones);
     const reserve_text = if (std.c.getenv("TENSORFOLD_MEMORY_RESERVE_GIB")) |p| std.mem.span(p) else null;
     const limit_text = if (std.c.getenv("TENSORFOLD_CUDA_MEMORY_LIMIT_GB")) |p| std.mem.span(p) else null;
     const reserve = reserveBytes(reserve_text, available.total, unified) catch {
@@ -287,7 +314,7 @@ pub fn readPool(a: std.mem.Allocator, io: std.Io, ctx: *const cuda.Context, prob
         problem.* = try std.fmt.allocPrint(a, "TENSORFOLD_CUDA_MEMORY_LIMIT_GB={s}: a positive number of GiB whose byte count fits in a 64-bit size", .{limit_text orelse ""});
         return null;
     };
-    return .{ .free = available.available, .total = available.total, .reserve = reserve, .limit = limit, .unified = unified };
+    return .{ .free = available.available, .total = available.total, .reserve = reserve, .limit = limit, .unified = unified, .parked = available.available - measured.available };
 }
 
 test "the weights' bytes: each file the weight map names once, under a folder too; without an index the folder's own" {
@@ -353,3 +380,11 @@ test "an index allocation failure is reported instead of counting the flat direc
     var fixed = std.heap.FixedBufferAllocator.init(&bytes);
     try std.testing.expectError(error.OutOfMemory, checkedWeightBytes(fixed.allocator(), io, dir));
 }
+
+test "a GB10's /proc files: its lists' parked pages raise MemAvailable and stay under MemTotal" {
+    const m = meminfo(@embedFile("fixtures/gb10_parked/meminfo_draining")) orelse return error.TestUnexpectedResult;
+    const parked = parked_pages.bytes(@embedFile("fixtures/gb10_parked/zoneinfo_draining"), 4096); // GB10's page size
+    try std.testing.expectEqual(@as(u64, 32813016 * 1024), m.available);
+    try std.testing.expect(parked > 0 and m.available + parked < m.total);
+    try std.testing.expect(meminfo(@embedFile("fixtures/gb10_parked/meminfo_at_budget")) != null);
+}

zig/src/native/fixtures/gb10_parked/meminfo_at_budget

@@ -0,0 +1,55 @@
+MemTotal:       127535276 kB
+MemFree:         1412252 kB
+MemAvailable:   27922896 kB
+Buffers:           27172 kB
+Cached:         28350024 kB
+SwapCached:        59960 kB
+Active:          3665280 kB
+Inactive:       26514296 kB
+Active(anon):    2842388 kB
+Inactive(anon):   110140 kB
+Active(file):     822892 kB
+Inactive(file): 26404156 kB
+Unevictable:       23004 kB
+Mlocked:           22988 kB
+SwapTotal:      16777212 kB
+SwapFree:       15990404 kB
+Zswap:                 0 kB
+Zswapped:              0 kB
+Dirty:            122292 kB
+Writeback:             0 kB
+AnonPages:       1821348 kB
+Mapped:          1657408 kB
+Shmem:           1145772 kB
+KReclaimable:     424748 kB
+Slab:            1412576 kB
+SReclaimable:     424748 kB
+SUnreclaim:       987828 kB
+KernelStack:       14336 kB
+ShadowCallStack:    3664 kB
+PageTables:        26432 kB
+SecPageTables:    209100 kB
+NFS_Unstable:          0 kB
+Bounce:                0 kB
+WritebackTmp:          0 kB
+CommitLimit:    80544848 kB
+Committed_AS:   10111800 kB
+VmallocTotal:   135288315904 kB
+VmallocUsed:      813760 kB
+VmallocChunk:          0 kB
+Percpu:            30320 kB
+HardwareCorrupted:     0 kB
+AnonHugePages:         0 kB
+ShmemHugePages:        0 kB
+ShmemPmdMapped:        0 kB
+FileHugePages:    135168 kB
+FilePmdMapped:         0 kB
+CmaTotal:         131072 kB
+CmaFree:           69320 kB
+Balloon:               0 kB
+HugePages_Total:       0
+HugePages_Free:        0
+HugePages_Rsvd:        0
+HugePages_Surp:        0
+Hugepagesize:       2048 kB
+Hugetlb:               0 kB

zig/src/native/fixtures/gb10_parked/meminfo_draining

@@ -0,0 +1,55 @@
+MemTotal:       127535276 kB
+MemFree:         1193024 kB
+MemAvailable:   32813016 kB
+Buffers:           27096 kB
+Cached:         32372516 kB
+SwapCached:        59960 kB
+Active:          2127420 kB
+Inactive:       31636316 kB
+Active(anon):    1312032 kB
+Inactive(anon):   110144 kB
+Active(file):     815388 kB
+Inactive(file): 31526172 kB
+Unevictable:       23004 kB
+Mlocked:           22988 kB
+SwapTotal:      16777212 kB
+SwapFree:       15990384 kB
+Zswap:                 0 kB
+Zswapped:              0 kB
+Dirty:              1348 kB
+Writeback:             0 kB
+AnonPages:       1382828 kB
+Mapped:           559176 kB
+Shmem:             53936 kB
+KReclaimable:     419584 kB
+Slab:            1403816 kB
+SReclaimable:     419584 kB
+SUnreclaim:       984232 kB
+KernelStack:       14336 kB
+ShadowCallStack:    3664 kB
+PageTables:        23392 kB
+SecPageTables:    198668 kB
+NFS_Unstable:          0 kB
+Bounce:                0 kB
+WritebackTmp:          0 kB
+CommitLimit:    80544848 kB
+Committed_AS:    8583804 kB
+VmallocTotal:   135288315904 kB
+VmallocUsed:      756344 kB
+VmallocChunk:          0 kB
+Percpu:            30320 kB
+HardwareCorrupted:     0 kB
+AnonHugePages:         0 kB
+ShmemHugePages:        0 kB
+ShmemPmdMapped:        0 kB
+FileHugePages:     63488 kB
+FilePmdMapped:         0 kB
+CmaTotal:         131072 kB
+CmaFree:           69320 kB
+Balloon:               0 kB
+HugePages_Total:       0
+HugePages_Free:        0
+HugePages_Rsvd:        0
+HugePages_Surp:        0
+Hugepagesize:       2048 kB
+Hugetlb:               0 kB