diff --git a/zig/src/native/cuda.zig b/zig/src/native/cuda.zig index ab56df311d..f5ca856e98 100644 --- a/zig/src/native/cuda.zig +++ b/zig/src/native/cuda.zig @@ -283,6 +283,7 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O fn read(p: *anyopaque) ?u64 { const self: *@This() = @ptrCast(@alignCast(p)); const now = (pool(self.a, self.io, self.ctx, self.problem) catch return null) orelse return null; + family_memory.noteParked(now, "the room this rank reports"); return now.room(cuda.usage(false).device - self.held); } }; @@ -335,6 +336,7 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O defer if (!opened) served.close(gpa); const model = cuda.usage(false).device - held0; const after = try pool(a, io, &g.ctx, problem) orelse return null; + family_memory.noteParked(after, "the CUDA memory budget"); const L = @TypeOf(loaded); // a family's Loaded may leave out a learner and a lone driver const learn_bytes: usize = if (@hasField(L, "learn_bytes")) loaded.learn_bytes else 0; const host_cache = try kv_pages.placeHost(F, a, loaded, o, after.room(model), @intCast(window), served.stream_bytes, problem) orelse return null; diff --git a/zig/src/native/cuda_family_memory.zig b/zig/src/native/cuda_family_memory.zig index f15208264d..6cf103622a 100644 --- a/zig/src/native/cuda_family_memory.zig +++ b/zig/src/native/cuda_family_memory.zig @@ -21,10 +21,14 @@ pub fn poolFor(comptime windows: bool, comptime Host: type, a: Allocator, io: st const host_counts = budget.hostCounts(unified, windows); const text = if (host_counts) Host.procText(a, io, "/proc/meminfo") else null; defer if (text) |bytes| a.free(bytes); - const available = budget.counts(host_counts, .{ .total = card.total, .available = card.free }, text) catch { + // read after MemAvailable, which the lists drain into: a page moving in between is missed, not counted twice + const zones = if (host_counts) Host.procText(a, io, "/proc/zoneinfo") else null; + defer if (zones) |bytes| a.free(bytes); + const measured = budget.counts(host_counts, .{ .total = card.total, .available = card.free }, text) catch { problem.* = "cannot read or parse /proc/meminfo's MemTotal and MemAvailable; refusing CUDA unified-memory admission"; return null; }; + const available = budget.counting(measured, zones); const free = available.available; const total = available.total; const reserve = budget.reserveBytes(Host.env("TENSORFOLD_MEMORY_RESERVE_GIB"), total, unified) catch { @@ -35,7 +39,7 @@ pub fn poolFor(comptime windows: bool, comptime Host: type, a: Allocator, io: st problem.* = try std.fmt.allocPrint(a, "TENSORFOLD_CUDA_MEMORY_LIMIT_GB={s}: a positive number of GiB whose byte count fits in a 64-bit size", .{Host.env("TENSORFOLD_CUDA_MEMORY_LIMIT_GB").?}); return null; }; - return .{ .free = free, .total = total, .reserve = reserve, .limit = limit, .unified = unified }; + return .{ .free = free, .total = total, .reserve = reserve, .limit = limit, .unified = unified, .parked = free - measured.available }; } /// `F`'s weight bytes on this rank: tensors or files, a split's share, a tower beside; null if tensors are unreadable. @@ -51,6 +55,12 @@ pub fn weights(comptime F: type, a: Allocator, io: std.Io, o: api.Open, problem: return text + try @import("cuda_vision_offer.zig").weightBytes(F, a, io, o); } +/// Logs, from 0.25 GiB, the free pages on the per-CPU lists that a pool's reading counted. +pub fn noteParked(p: budget.Pool, what: []const u8) void { + const line = "{s} counts the {d:.2} GiB free on the per-CPU page lists beyond their resting level (MemAvailable does not)"; + if (p.parked >= 1 << 28) std.log.info(line, .{ what, toGib(p.parked) }); +} + pub fn room(comptime F: type, ptr: *anyopaque, local: u64) !u64 { if (@hasDecl(F, "rankRoom")) return F.rankRoom(ptr, local); return local; diff --git a/zig/src/native/cuda_family_memory_test.zig b/zig/src/native/cuda_family_memory_test.zig index fea897f207..4e48552e23 100644 --- a/zig/src/native/cuda_family_memory_test.zig +++ b/zig/src/native/cuda_family_memory_test.zig @@ -6,7 +6,9 @@ const Fake = struct { var integrated: c_int = 1; var missing = false; var text: ?[]const u8 = null; + var zones: ?[]const u8 = null; var probes: usize = 0; + var zone_probes: usize = 0; fn attribute(out: *c_int, _: cuda.abi.DeviceAttribute, _: c_int) callconv(.c) cuda.abi.Result { out.* = integrated; return 0; @@ -17,7 +19,11 @@ const Fake = struct { free.* = 40 << 30; return 0; } - pub fn procText(a: std.mem.Allocator, _: std.Io, _: []const u8) ?[]u8 { + pub fn procText(a: std.mem.Allocator, _: std.Io, path: []const u8) ?[]u8 { + if (std.mem.eql(u8, path, "/proc/zoneinfo")) { + zone_probes += 1; + return a.dupe(u8, zones orelse return null) catch null; + } probes += 1; return a.dupe(u8, text orelse return null) catch null; } @@ -65,3 +71,66 @@ test "Windows pool: real CUDA counts, OS selection, unified reserves and missing try t.expectError(error.NotReady, poolFor(true, Fake, t.allocator, t.io, &ctx, &problem)); try t.expectError(error.NotReady, poolFor(false, Fake, t.allocator, t.io, &ctx, &problem)); } + +/// One zone whose CPU list holds `parked` bytes beyond its high_min, as kernel 6.7+ prints /proc/zoneinfo. +fn zoneinfo(a: std.mem.Allocator, parked: u64) ![]u8 { + const hm = 2146; + const fmt = "Node 0, zone Normal\n pages free 10\n pagesets\n cpu: 0\n count: {d}\n" ++ + " high: 5000\n batch: 63\n high_min: {d}\n vm stats threshold: 110\n" ++ + " start_pfn: 4096\n"; + return std.fmt.allocPrint(a, fmt, .{ hm + parked / std.heap.pageSize(), hm }); +} + +test "a Linux unified pool counts the free pages parked on the per-CPU lists, at most MemTotal" { + var driver: cuda.Driver = undefined; + driver.api.cuDeviceGetAttribute = Fake.attribute; + driver.api.cuMemGetInfo_v2 = Fake.memInfo; + const ctx: cuda.Context = .{ .d = &driver, .device = 0, .handle = null }; + var problem: []const u8 = ""; + const g: u64 = 1 << 30; + Fake.missing = false; + Fake.integrated = 1; + Fake.text = "MemTotal: 134217728 kB\nMemAvailable: 100663296 kB\n"; + defer Fake.zones = null; + const six = try zoneinfo(t.allocator, 6 * g); + defer t.allocator.free(six); + Fake.zones = six; + try t.expectEqual(102 * g, (try poolFor(false, Fake, t.allocator, t.io, &ctx, &problem)).?.free); + const forty = try zoneinfo(t.allocator, 40 * g); + defer t.allocator.free(forty); + Fake.zones = forty; + try t.expectEqual(128 * g, (try poolFor(false, Fake, t.allocator, t.io, &ctx, &problem)).?.free); // MemTotal + for ([_]?[]const u8{ null, "", "Node 0, zone Normal\n pagesets\n cpu: 0\n count: x\n" }) |zones| { + Fake.zones = zones; + try t.expectEqual(96 * g, (try poolFor(false, Fake, t.allocator, t.io, &ctx, &problem)).?.free); + } + Fake.zones = six; + Fake.zone_probes = 0; + const windows = (try poolFor(true, Fake, t.allocator, t.io, &ctx, &problem)).?; + try t.expectEqual(@as(u64, 40 << 30), windows.free); + Fake.integrated = 0; + const card = (try poolFor(false, Fake, t.allocator, t.io, &ctx, &problem)).?; + try t.expectEqual(@as(u64, 40 << 30), card.free); + try t.expectEqual(@as(usize, 0), Fake.zone_probes); +} + +test "the pool names the parked share it counted, for the start-up log line" { + var driver: cuda.Driver = undefined; + driver.api.cuDeviceGetAttribute = Fake.attribute; + driver.api.cuMemGetInfo_v2 = Fake.memInfo; + const ctx: cuda.Context = .{ .d = &driver, .device = 0, .handle = null }; + var problem: []const u8 = ""; + const g: u64 = 1 << 30; + Fake.missing = false; + Fake.integrated = 1; + Fake.text = "MemTotal: 134217728 kB\nMemAvailable: 130023424 kB\n"; + defer Fake.zones = null; + const six = try zoneinfo(t.allocator, 6 * g); + defer t.allocator.free(six); + Fake.zones = six; + try t.expectEqual(4 * g, (try poolFor(false, Fake, t.allocator, t.io, &ctx, &problem)).?.parked); // to MemTotal + Fake.text = "MemTotal: 134217728 kB\nMemAvailable: 100663296 kB\n"; + try t.expectEqual(6 * g, (try poolFor(false, Fake, t.allocator, t.io, &ctx, &problem)).?.parked); + Fake.zones = null; + try t.expectEqual(@as(u64, 0), (try poolFor(false, Fake, t.allocator, t.io, &ctx, &problem)).?.parked); +} diff --git a/zig/src/native/cuda_memory.zig b/zig/src/native/cuda_memory.zig index ee00e372d5..aebbf76ea7 100644 --- a/zig/src/native/cuda_memory.zig +++ b/zig/src/native/cuda_memory.zig @@ -1,6 +1,7 @@ //! CUDA memory admission arithmetic, and the host pool read from /proc/meminfo and validated, without a device. const cuda = @import("cuda"); const std = @import("std"); +const parked_pages = @import("parked_pages.zig"); const gib: f64 = 1 << 30; /// Bytes of the checkpoint's safetensors (the weight map's files, else the folder's own), near enough to refuse early. @@ -80,11 +81,18 @@ pub fn meminfo(text: []const u8) ?MemInfo { return if (t > 0 and a <= t) .{ .total = t, .available = a } else null; } -/// --prompt-cache-gib in pinned host memory (8 GiB unasked), at most half of what /proc/meminfo has available. +/// MemAvailable plus the free pages parked on the per-CPU lists (`zones`, read after it), at most MemTotal. +pub fn counting(m: MemInfo, zones: ?[]const u8) MemInfo { + const parked = parked_pages.bytes(zones orelse return m, std.heap.pageSize()); + return .{ .total = m.total, .available = @min(m.total, m.available +| parked) }; +} + +/// --prompt-cache-gib in pinned host memory (8 GiB unasked), at most half of what is available (`counting`). pub fn hostCacheBytes(a: std.mem.Allocator, io: std.Io, asked: ?f64) u64 { const want: u64 = if (asked) |g| (if (g > 0) std.math.lossyCast(u64, g * (1 << 30)) else 0) else 8 << 30; const text = procText(a, io, "/proc/meminfo") orelse return 0; - const m = meminfo(text) orelse return 0; + const zones = procText(a, io, "/proc/zoneinfo"); + const m = counting(meminfo(text) orelse return 0, zones); return @min(want, m.available / 2); } @@ -104,6 +112,7 @@ pub const Pool = struct { reserve: u64, limit: ?u64, unified: bool, + parked: u64 = 0, // the part of `free` parked on the per-CPU page lists /// Bytes the engine may still allocate on top of `held`. pub fn room(p: Pool, held: u64) u64 { @@ -260,6 +269,21 @@ test "unified admission refuses missing or malformed host memory" { try std.testing.expectEqual(MemInfo{ .total = 131072 * 1024, .available = 65536 * 1024 }, try counts(true, card, "MemTotal: 131072 kB\nMemAvailable: 65536 kB\n")); } +test "counting: the per-CPU lists' parked pages raise MemAvailable, at most MemTotal; nothing without them" { + const page = std.heap.pageSize(); + const m: MemInfo = .{ .total = 128 << 30, .available = 96 << 30 }; + const zones = try std.fmt.allocPrint(std.testing.allocator, " pagesets\n cpu: 0\n count: {d}\n high_min: 10\n", .{10 + (6 << 30) / page}); + defer std.testing.allocator.free(zones); + try std.testing.expectEqual(MemInfo{ .total = 128 << 30, .available = 102 << 30 }, counting(m, zones)); + try std.testing.expectEqual(MemInfo{ .total = 128 << 30, .available = 128 << 30 }, counting(.{ .total = 128 << 30, .available = 125 << 30 }, zones)); + try std.testing.expectEqual(m, counting(m, null)); + try std.testing.expectEqual(m, counting(m, " pagesets\n cpu: 0\n count: x\n")); +} + +test { + _ = parked_pages; +} + test "memory limit refuses an oversized finite value and the rounded u64 boundary" { try std.testing.expectError(error.Invalid, limitBytes("1e300")); try std.testing.expectError(error.Invalid, limitBytes("17179869184")); @@ -273,10 +297,13 @@ pub fn readPool(a: std.mem.Allocator, io: std.Io, ctx: *const cuda.Context, prob const host_counts = hostCounts(unified, @import("sys").windows); const text = if (host_counts) procText(a, io, "/proc/meminfo") else null; defer if (text) |bytes| a.free(bytes); - const available = counts(host_counts, .{ .total = card.total, .available = card.free }, text) catch { + const zones = if (host_counts) procText(a, io, "/proc/zoneinfo") else null; + defer if (zones) |bytes| a.free(bytes); + const measured = counts(host_counts, .{ .total = card.total, .available = card.free }, text) catch { problem.* = "cannot read or parse /proc/meminfo's MemTotal and MemAvailable; refusing CUDA unified-memory admission"; return null; }; + const available = counting(measured, zones); const reserve_text = if (std.c.getenv("TENSORFOLD_MEMORY_RESERVE_GIB")) |p| std.mem.span(p) else null; const limit_text = if (std.c.getenv("TENSORFOLD_CUDA_MEMORY_LIMIT_GB")) |p| std.mem.span(p) else null; const reserve = reserveBytes(reserve_text, available.total, unified) catch { @@ -287,7 +314,7 @@ pub fn readPool(a: std.mem.Allocator, io: std.Io, ctx: *const cuda.Context, prob problem.* = try std.fmt.allocPrint(a, "TENSORFOLD_CUDA_MEMORY_LIMIT_GB={s}: a positive number of GiB whose byte count fits in a 64-bit size", .{limit_text orelse ""}); return null; }; - return .{ .free = available.available, .total = available.total, .reserve = reserve, .limit = limit, .unified = unified }; + return .{ .free = available.available, .total = available.total, .reserve = reserve, .limit = limit, .unified = unified, .parked = available.available - measured.available }; } test "the weights' bytes: each file the weight map names once, under a folder too; without an index the folder's own" { @@ -353,3 +380,11 @@ test "an index allocation failure is reported instead of counting the flat direc var fixed = std.heap.FixedBufferAllocator.init(&bytes); try std.testing.expectError(error.OutOfMemory, checkedWeightBytes(fixed.allocator(), io, dir)); } + +test "a GB10's /proc files: its lists' parked pages raise MemAvailable and stay under MemTotal" { + const m = meminfo(@embedFile("fixtures/gb10_parked/meminfo_draining")) orelse return error.TestUnexpectedResult; + const parked = parked_pages.bytes(@embedFile("fixtures/gb10_parked/zoneinfo_draining"), 4096); // GB10's page size + try std.testing.expectEqual(@as(u64, 32813016 * 1024), m.available); + try std.testing.expect(parked > 0 and m.available + parked < m.total); + try std.testing.expect(meminfo(@embedFile("fixtures/gb10_parked/meminfo_at_budget")) != null); +} diff --git a/zig/src/native/fixtures/gb10_parked/meminfo_at_budget b/zig/src/native/fixtures/gb10_parked/meminfo_at_budget new file mode 100644 index 0000000000..f54b63141f --- /dev/null +++ b/zig/src/native/fixtures/gb10_parked/meminfo_at_budget @@ -0,0 +1,55 @@ +MemTotal: 127535276 kB +MemFree: 1412252 kB +MemAvailable: 27922896 kB +Buffers: 27172 kB +Cached: 28350024 kB +SwapCached: 59960 kB +Active: 3665280 kB +Inactive: 26514296 kB +Active(anon): 2842388 kB +Inactive(anon): 110140 kB +Active(file): 822892 kB +Inactive(file): 26404156 kB +Unevictable: 23004 kB +Mlocked: 22988 kB +SwapTotal: 16777212 kB +SwapFree: 15990404 kB +Zswap: 0 kB +Zswapped: 0 kB +Dirty: 122292 kB +Writeback: 0 kB +AnonPages: 1821348 kB +Mapped: 1657408 kB +Shmem: 1145772 kB +KReclaimable: 424748 kB +Slab: 1412576 kB +SReclaimable: 424748 kB +SUnreclaim: 987828 kB +KernelStack: 14336 kB +ShadowCallStack: 3664 kB +PageTables: 26432 kB +SecPageTables: 209100 kB +NFS_Unstable: 0 kB +Bounce: 0 kB +WritebackTmp: 0 kB +CommitLimit: 80544848 kB +Committed_AS: 10111800 kB +VmallocTotal: 135288315904 kB +VmallocUsed: 813760 kB +VmallocChunk: 0 kB +Percpu: 30320 kB +HardwareCorrupted: 0 kB +AnonHugePages: 0 kB +ShmemHugePages: 0 kB +ShmemPmdMapped: 0 kB +FileHugePages: 135168 kB +FilePmdMapped: 0 kB +CmaTotal: 131072 kB +CmaFree: 69320 kB +Balloon: 0 kB +HugePages_Total: 0 +HugePages_Free: 0 +HugePages_Rsvd: 0 +HugePages_Surp: 0 +Hugepagesize: 2048 kB +Hugetlb: 0 kB diff --git a/zig/src/native/fixtures/gb10_parked/meminfo_draining b/zig/src/native/fixtures/gb10_parked/meminfo_draining new file mode 100644 index 0000000000..5bb7f5b003 --- /dev/null +++ b/zig/src/native/fixtures/gb10_parked/meminfo_draining @@ -0,0 +1,55 @@ +MemTotal: 127535276 kB +MemFree: 1193024 kB +MemAvailable: 32813016 kB +Buffers: 27096 kB +Cached: 32372516 kB +SwapCached: 59960 kB +Active: 2127420 kB +Inactive: 31636316 kB +Active(anon): 1312032 kB +Inactive(anon): 110144 kB +Active(file): 815388 kB +Inactive(file): 31526172 kB +Unevictable: 23004 kB +Mlocked: 22988 kB +SwapTotal: 16777212 kB +SwapFree: 15990384 kB +Zswap: 0 kB +Zswapped: 0 kB +Dirty: 1348 kB +Writeback: 0 kB +AnonPages: 1382828 kB +Mapped: 559176 kB +Shmem: 53936 kB +KReclaimable: 419584 kB +Slab: 1403816 kB +SReclaimable: 419584 kB +SUnreclaim: 984232 kB +KernelStack: 14336 kB +ShadowCallStack: 3664 kB +PageTables: 23392 kB +SecPageTables: 198668 kB +NFS_Unstable: 0 kB +Bounce: 0 kB +WritebackTmp: 0 kB +CommitLimit: 80544848 kB +Committed_AS: 8583804 kB +VmallocTotal: 135288315904 kB +VmallocUsed: 756344 kB +VmallocChunk: 0 kB +Percpu: 30320 kB +HardwareCorrupted: 0 kB +AnonHugePages: 0 kB +ShmemHugePages: 0 kB +ShmemPmdMapped: 0 kB +FileHugePages: 63488 kB +FilePmdMapped: 0 kB +CmaTotal: 131072 kB +CmaFree: 69320 kB +Balloon: 0 kB +HugePages_Total: 0 +HugePages_Free: 0 +HugePages_Rsvd: 0 +HugePages_Surp: 0 +Hugepagesize: 2048 kB +Hugetlb: 0 kB diff --git a/zig/src/native/fixtures/gb10_parked/zoneinfo_at_budget b/zig/src/native/fixtures/gb10_parked/zoneinfo_at_budget new file mode 100644 index 0000000000..dd328af2cd --- /dev/null +++ b/zig/src/native/fixtures/gb10_parked/zoneinfo_at_budget @@ -0,0 +1,431 @@ +Node 0, zone DMA + per-node stats + nr_inactive_anon 27535 + nr_active_anon 710597 + nr_inactive_file 6601039 + nr_active_file 205723 + nr_unevictable 5751 + nr_slab_reclaimable 106187 + nr_slab_unreclaimable 246957 + nr_isolated_anon 0 + nr_isolated_file 0 + workingset_nodes 452445 + workingset_refault_anon 2244775 + workingset_refault_file 494634477 + workingset_activate_anon 1488521 + workingset_activate_file 605354 + workingset_restore_anon 226966 + workingset_restore_file 146105 + workingset_nodereclaim 8653937 + nr_anon_pages 455337 + nr_mapped 414352 + nr_file_pages 7109289 + nr_dirty 30573 + nr_writeback 0 + nr_shmem 286443 + nr_shmem_hugepages 0 + nr_shmem_pmdmapped 0 + nr_file_hugepages 66 + nr_file_pmdmapped 0 + nr_anon_transparent_hugepages 0 + nr_vmscan_write 11418948 + nr_vmscan_immediate_reclaim 1699 + nr_dirtied 890679311 + nr_written 878843442 + nr_throttled_written 0 + nr_kernel_misc_reclaimable 0 + nr_foll_pin_acquired 7722644832 + nr_foll_pin_released 7722355338 + nr_kernel_stack 14336 + nr_shadow_call_stack 3664 + nr_page_table_pages 6608 + nr_sec_page_table_pages 52275 + nr_iommu_pages 52275 + nr_swapcached 14990 + pgpromote_success 0 + pgpromote_candidate 0 + pgdemote_kswapd 0 + pgdemote_direct 0 + pgdemote_khugepaged 0 + pgdemote_proactive 0 + nr_hugetlb 0 + nr_balloon_pages 0 + pages free 123858 + boost 0 + min 56 + low 214 + high 372 + promo 530 + spanned 524288 + present 426977 + managed 158398 + cma 32768 + protection: (0, 0, 123927, 123927, 123927) + nr_free_pages 123858 + nr_free_pages_blocks 122880 + nr_zone_inactive_anon 3 + nr_zone_active_anon 1 + nr_zone_inactive_file 19097 + nr_zone_active_file 1 + nr_zone_unevictable 0 + nr_zone_write_pending 0 + nr_mlock 0 + nr_zspages 0 + nr_free_cma 17330 + numa_hit 12291892 + numa_miss 0 + numa_foreign 0 + numa_interleave 0 + numa_local 12293807 + numa_other 0 + pagesets + cpu: 0 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 7 + cpu: 1 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 7 + cpu: 2 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 7 + cpu: 3 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 7 + cpu: 4 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 7 + cpu: 5 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 7 + cpu: 6 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 7 + cpu: 7 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 7 + cpu: 8 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 7 + cpu: 9 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 7 + cpu: 10 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 7 + cpu: 11 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 7 + cpu: 12 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 7 + cpu: 13 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 7 + cpu: 14 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 7 + cpu: 15 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 7 + cpu: 16 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 7 + cpu: 17 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 7 + cpu: 18 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 7 + cpu: 19 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 7 + node_unreclaimable: 0 + start_pfn: 524288 +Node 0, zone DMA32 + pages free 0 + boost 0 + min 0 + low 0 + high 0 + promo 0 + spanned 0 + present 0 + managed 0 + cma 0 + protection: (0, 0, 0, 0, 0) +Node 0, zone Normal + pages free 229091 + boost 0 + min 11231 + low 42932 + high 74633 + promo 106334 + spanned 33030144 + present 33030144 + managed 31725421 + cma 0 + protection: (0, 0, 0, 0, 0) + nr_free_pages 229091 + nr_free_pages_blocks 1536 + nr_zone_inactive_anon 27532 + nr_zone_active_anon 710596 + nr_zone_inactive_file 6581942 + nr_zone_active_file 205722 + nr_zone_unevictable 5751 + nr_zone_write_pending 30573 + nr_mlock 5747 + nr_zspages 0 + nr_free_cma 0 + numa_hit 46371395493 + numa_miss 0 + numa_foreign 0 + numa_interleave 3491 + numa_local 46374884324 + numa_other 0 + pagesets + cpu: 0 + count: 0 + high: 2146 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 1 + count: 0 + high: 2146 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 2 + count: 195 + high: 2398 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 3 + count: 0 + high: 2146 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 4 + count: 0 + high: 2146 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 5 + count: 24 + high: 2146 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 6 + count: 0 + high: 2146 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 7 + count: 0 + high: 2146 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 8 + count: 47 + high: 2146 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 9 + count: 0 + high: 2146 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 10 + count: 0 + high: 2146 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 11 + count: 0 + high: 2146 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 12 + count: 0 + high: 2209 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 13 + count: 0 + high: 2146 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 14 + count: 0 + high: 2146 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 15 + count: 0 + high: 10767 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 16 + count: 86 + high: 53180 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 17 + count: 67 + high: 2272 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 18 + count: 0 + high: 52483 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 19 + count: 1098 + high: 12289 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + node_unreclaimable: 0 + start_pfn: 1048576 +Node 0, zone Movable + pages free 0 + boost 0 + min 32 + low 32 + high 32 + promo 32 + spanned 0 + present 0 + managed 0 + cma 0 + protection: (0, 0, 0, 0, 0) +Node 0, zone Device + pages free 0 + boost 0 + min 0 + low 0 + high 0 + promo 0 + spanned 0 + present 0 + managed 0 + cma 0 + protection: (0, 0, 0, 0, 0) diff --git a/zig/src/native/fixtures/gb10_parked/zoneinfo_draining b/zig/src/native/fixtures/gb10_parked/zoneinfo_draining new file mode 100644 index 0000000000..6c92496491 --- /dev/null +++ b/zig/src/native/fixtures/gb10_parked/zoneinfo_draining @@ -0,0 +1,431 @@ +Node 0, zone DMA + per-node stats + nr_inactive_anon 27536 + nr_active_anon 328008 + nr_inactive_file 7881543 + nr_active_file 203847 + nr_unevictable 5751 + nr_slab_reclaimable 104896 + nr_slab_unreclaimable 246058 + nr_isolated_anon 0 + nr_isolated_file 0 + workingset_nodes 431730 + workingset_refault_anon 2244774 + workingset_refault_file 494634473 + workingset_activate_anon 1488521 + workingset_activate_file 605354 + workingset_restore_anon 226966 + workingset_restore_file 146105 + workingset_nodereclaim 8653937 + nr_anon_pages 345707 + nr_mapped 139794 + nr_file_pages 8114893 + nr_dirty 337 + nr_writeback 0 + nr_shmem 13484 + nr_shmem_hugepages 0 + nr_shmem_pmdmapped 0 + nr_file_hugepages 31 + nr_file_pmdmapped 0 + nr_anon_transparent_hugepages 0 + nr_vmscan_write 11418948 + nr_vmscan_immediate_reclaim 1699 + nr_dirtied 890648887 + nr_written 878843254 + nr_throttled_written 0 + nr_kernel_misc_reclaimable 0 + nr_foll_pin_acquired 7722371873 + nr_foll_pin_released 7722355338 + nr_kernel_stack 14336 + nr_shadow_call_stack 3664 + nr_page_table_pages 5848 + nr_sec_page_table_pages 49667 + nr_iommu_pages 49667 + nr_swapcached 14990 + pgpromote_success 0 + pgpromote_candidate 0 + pgdemote_kswapd 0 + pgdemote_direct 0 + pgdemote_khugepaged 0 + pgdemote_proactive 0 + nr_hugetlb 0 + nr_balloon_pages 0 + pages free 123858 + boost 0 + min 56 + low 214 + high 372 + promo 530 + spanned 524288 + present 426977 + managed 158398 + cma 32768 + protection: (0, 0, 123927, 123927, 123927) + nr_free_pages 123858 + nr_free_pages_blocks 122880 + nr_zone_inactive_anon 3 + nr_zone_active_anon 1 + nr_zone_inactive_file 19097 + nr_zone_active_file 1 + nr_zone_unevictable 0 + nr_zone_write_pending 0 + nr_mlock 0 + nr_zspages 0 + nr_free_cma 17330 + numa_hit 12291892 + numa_miss 0 + numa_foreign 0 + numa_interleave 0 + numa_local 12293807 + numa_other 0 + pagesets + cpu: 0 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 40 + cpu: 1 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 40 + cpu: 2 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 40 + cpu: 3 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 40 + cpu: 4 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 40 + cpu: 5 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 40 + cpu: 6 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 40 + cpu: 7 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 40 + cpu: 8 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 40 + cpu: 9 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 40 + cpu: 10 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 40 + cpu: 11 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 40 + cpu: 12 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 40 + cpu: 13 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 40 + cpu: 14 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 40 + cpu: 15 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 40 + cpu: 16 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 40 + cpu: 17 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 40 + cpu: 18 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 40 + cpu: 19 + count: 0 + high: 124 + batch: 31 + high_min: 124 + high_max: 989 + vm stats threshold: 40 + node_unreclaimable: 0 + start_pfn: 524288 +Node 0, zone DMA32 + pages free 0 + boost 0 + min 0 + low 0 + high 0 + promo 0 + spanned 0 + present 0 + managed 0 + cma 0 + protection: (0, 0, 0, 0, 0) +Node 0, zone Normal + pages free 174398 + boost 0 + min 11231 + low 42932 + high 74633 + promo 106334 + spanned 33030144 + present 33030144 + managed 31725421 + cma 0 + protection: (0, 0, 0, 0, 0) + nr_free_pages 174398 + nr_free_pages_blocks 0 + nr_zone_inactive_anon 27533 + nr_zone_active_anon 328007 + nr_zone_inactive_file 7862446 + nr_zone_active_file 203846 + nr_zone_unevictable 5751 + nr_zone_write_pending 337 + nr_mlock 5747 + nr_zspages 0 + nr_free_cma 0 + numa_hit 46370054001 + numa_miss 0 + numa_foreign 0 + numa_interleave 3491 + numa_local 46373542801 + numa_other 0 + pagesets + cpu: 0 + count: 4857 + high: 4937 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 1 + count: 1682 + high: 11500 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 2 + count: 3657 + high: 5165 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 3 + count: 1393 + high: 13735 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 4 + count: 426 + high: 2650 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 5 + count: 6928 + high: 7393 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 6 + count: 11447 + high: 24315 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 7 + count: 5491 + high: 9949 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 8 + count: 1716 + high: 2448 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 9 + count: 2207 + high: 13746 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 10 + count: 1688 + high: 2146 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 11 + count: 329 + high: 2146 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 12 + count: 3259 + high: 3282 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 13 + count: 656 + high: 2146 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 14 + count: 237 + high: 2154 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 15 + count: 47727 + high: 88344 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 16 + count: 153589 + high: 153589 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 17 + count: 13847 + high: 34570 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 18 + count: 34500 + high: 44303 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + cpu: 19 + count: 553 + high: 3569 + batch: 63 + high_min: 2146 + high_max: 198132 + vm stats threshold: 110 + node_unreclaimable: 0 + start_pfn: 1048576 +Node 0, zone Movable + pages free 0 + boost 0 + min 32 + low 32 + high 32 + promo 32 + spanned 0 + present 0 + managed 0 + cma 0 + protection: (0, 0, 0, 0, 0) +Node 0, zone Device + pages free 0 + boost 0 + min 0 + low 0 + high 0 + promo 0 + spanned 0 + present 0 + managed 0 + cma 0 + protection: (0, 0, 0, 0, 0) diff --git a/zig/src/native/parked_pages.zig b/zig/src/native/parked_pages.zig new file mode 100644 index 0000000000..d300c51c63 --- /dev/null +++ b/zig/src/native/parked_pages.zig @@ -0,0 +1,144 @@ +//! Free pages parked on the kernel's per-CPU lists, read from /proc/zoneinfo: free, but not in MemFree or MemAvailable. +const std = @import("std"); + +/// What the per-CPU lists hold at rest when /proc/zoneinfo prints no high_min (kernels before 6.7). +pub const resting: u64 = 1 << 29; + +/// Bytes of the `count:` lines under every `pagesets` less their `high_min:` (else `resting`); 0 on a bad field. +pub fn bytes(text: []const u8, page: u64) u64 { + var count: u64 = 0; + var rest: u64 = 0; + var listed = false; + var rows = std.mem.splitScalar(u8, text, '\n'); + while (rows.next()) |row| { + var words = std.mem.tokenizeAny(u8, row, " \t\r"); + const first = words.next() orelse ""; + if (std.mem.eql(u8, std.mem.trim(u8, row, " \t\r"), "pagesets")) { + listed = true; + } else if (listed and !std.mem.startsWith(u8, row, " ") and !std.mem.startsWith(u8, row, " vm stats")) { + listed = false; // the next zone, or this zone's next field + } else if (listed and (std.mem.eql(u8, first, "count:") or std.mem.eql(u8, first, "high_min:"))) { + const n = std.fmt.parseInt(u64, words.next() orelse return 0, 10) catch return 0; + const sum = if (first[0] == 'c') &count else &rest; + sum.* = std.math.add(u64, sum.*, n) catch return 0; + } + } + if (count == 0) return 0; + const held = std.math.mul(u64, count, page) catch return 0; + const rests = if (rest > 0) std.math.mul(u64, rest, page) catch return 0 else resting; + return held -| rests; +} + +/// One zone's /proc/zoneinfo block as kernel 6.7+ prints it, with page lists when `counts` is given. +fn zone(a: std.mem.Allocator, out: *std.ArrayList(u8), node: u32, name: []const u8, counts: ?[]const u64, high_min: ?u64) !void { + try out.print(a, "Node {d}, zone {s:>8}\n pages free 10\n managed 1000\n nr_free_cma 0\n", .{ node, name }); + if (counts) |cs| { + try out.appendSlice(a, " pagesets\n"); + for (cs, 0..) |c, n| { + try out.print(a, " cpu: {d}\n count: {d}\n high: 5000\n batch: 63\n", .{ n, c }); + if (high_min) |h| try out.print(a, " high_min: {d}\n", .{h}); + try out.appendSlice(a, " high_max: 198283\n vm stats threshold: 110\n"); + } + } + try out.appendSlice(a, " node_unreclaimable: 0\n start_pfn: 4096\n"); +} + +test "parked pages, kernel 6.7+: every CPU, zone and node, less each list's high_min" { + const a = std.testing.allocator; + const page: u64 = 4096; + const hm: u64 = 2146; + var t: std.ArrayList(u8) = .empty; + defer t.deinit(a); + const ca = [_]u64{ 200000, 3143, 2332, 17 }; + const cb = [_]u64{ 100000, 100000, 100000 }; + const cc = [_]u64{400000}; + try zone(a, &t, 0, "DMA", &ca, hm); + try zone(a, &t, 0, "DMA32", null, hm); + try zone(a, &t, 0, "Normal", &cb, hm); + try zone(a, &t, 1, "Normal", &cc, hm); + try zone(a, &t, 1, "Movable", null, hm); + var sum: u64 = 0; + for (ca ++ cb ++ cc) |c| sum += c; + try std.testing.expectEqual((sum - 8 * hm) * page, bytes(t.items, page)); + t.clearRetainingCapacity(); + try zone(a, &t, 0, "Normal", &@as([20]u64, @splat(100)), hm); + try std.testing.expectEqual(@as(u64, 0), bytes(t.items, page)); // under the resting level: nothing +} + +test "parked pages, before 6.7: no high_min rests the first 512 MiB" { + const a = std.testing.allocator; + const page: u64 = 4096; + var t: std.ArrayList(u8) = .empty; + defer t.deinit(a); + const half = ((1 << 30) + resting) / page / 2; + try zone(a, &t, 0, "Normal", &.{ half, half }, null); + try std.testing.expectEqual(@as(u64, 1 << 30), bytes(t.items, page)); + t.clearRetainingCapacity(); + try zone(a, &t, 0, "Normal", &.{ 1000, 1000, 1000, 1000 }, null); + try std.testing.expectEqual(@as(u64, 0), bytes(t.items, page)); +} + +test "parked pages: only the count: and high_min: lines under pagesets" { + const a = std.testing.allocator; + const page: u64 = 4096; + var t: std.ArrayList(u8) = .empty; + defer t.deinit(a); + try t.appendSlice(a, " count: 5\n count: 6\n"); + try zone(a, &t, 0, "Normal", &.{2146 + 10}, 2146); + try t.appendSlice(a, " count: 99999\n high_min: 7\n"); // after start_pfn: not a list + try t.appendSlice(a, "Node 0, zone Other\n count: 77777\n pagesets\n cpu: 0\n high: 1\n"); + try std.testing.expectEqual(10 * page, bytes(t.items, page)); + const next_field = "Node 0, zone Normal\n pagesets\n cpu: 0\n count: 2200\n high: 9\n" ++ + " high_min: 2000\n vm stats threshold: 8\n cpu: 1\n count: 2100\n" ++ + " high_min: 2000\n all_unreclaimable: 0\n count: 9999\n"; + try std.testing.expectEqual(300 * page, bytes(next_field, page)); +} + +test "parked pages: no lists, empty, garbled, binary or overflowing text counts nothing" { + const a = std.testing.allocator; + const page: u64 = 4096; + var t: std.ArrayList(u8) = .empty; + defer t.deinit(a); + try zone(a, &t, 0, "DMA", null, 2146); + try zone(a, &t, 0, "Normal", null, 2146); + try std.testing.expectEqual(@as(u64, 0), bytes(t.items, page)); + try std.testing.expectEqual(@as(u64, 0), bytes("", page)); + var good: std.ArrayList(u8) = .empty; + defer good.deinit(a); + try zone(a, &good, 0, "Normal", &.{1000000}, 2146); + try std.testing.expect(bytes(good.items, page) > 0); + const swaps = [_][2][]const u8{ + .{ "count: 1000000", "count: x" }, + .{ "count: 1000000", "count:" }, + .{ "count: 1000000", "count: -5" }, + .{ "high_min: 2146", "high_min: x" }, + .{ "count: 1000000", "count: \xff\xfe" }, + }; + for (swaps) |swap| { + const garbled = try std.mem.replaceOwned(u8, a, good.items, swap[0], swap[1]); + defer a.free(garbled); + try std.testing.expectEqual(@as(u64, 0), bytes(garbled, page)); + } + try std.testing.expectEqual(@as(u64, 0), bytes("\x00\xff\xfe\x00\xff\xfe\n pagesets\x00", page)); + const wraps = "Node 0, zone Normal\n pagesets\n cpu: 0\n count: 18446744073709551615\n count: 1\n"; + try std.testing.expectEqual(@as(u64, 0), bytes(wraps, page)); + try std.testing.expectEqual(@as(u64, 0), bytes(good.items, std.math.maxInt(u64))); // pages times size overflows +} + +test "this host's /proc/zoneinfo parses to less than its memory (Linux)" { + if (@import("builtin").os.tag != .linux) return error.SkipZigTest; + const a = std.testing.allocator; + const procText = @import("cuda_memory.zig").procText; + const zones = procText(a, std.testing.io, "/proc/zoneinfo") orelse return error.Unreadable; + defer a.free(zones); + const mem = procText(a, std.testing.io, "/proc/meminfo") orelse return error.Unreadable; + defer a.free(mem); + try std.testing.expect(std.mem.indexOf(u8, zones, "pagesets") != null); + try std.testing.expect(bytes(zones, std.heap.pageSize()) < @import("cuda_memory.zig").meminfo(mem).?.total); +} + +test "a GB10's /proc/zoneinfo: lists still draining after a server exit count, lists at rest count nothing" { + const page = 4096; // GB10's page size + try std.testing.expectEqual(@as(u64, 1027231744), bytes(@embedFile("fixtures/gb10_parked/zoneinfo_draining"), page)); + try std.testing.expectEqual(@as(u64, 0), bytes(@embedFile("fixtures/gb10_parked/zoneinfo_at_budget"), page)); +}