From 8e20fd0c87d4d99e891410adb498c47c6b98248a Mon Sep 17 00:00:00 2001 From: tensorfold-expert Date: Fri, 23 Oct 2026 22:07:46 +0300 Subject: [PATCH 1/2] flashnext: admit a shard in a folder inside the model folder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit nvidia/Qwen3.8-Flash-Next-FP4-Local maps every tensor to tensors/model-000NN-of-00021.safetensors. Core's index check accepts that since #1179, but the family's own admission refused any '/' with UnsafeFlashShard, so serve stopped before CUDA init. - families/flashnext/index.zig: shardName/plainPart take the same rule as core/checkpoint_index.zig, kept local so this file still imports no core; escapes (../, absolute, a drive, a hidden or reserved part) stay refused. Tests: the 30-shard fixture admitted flat, under tensors/, and refused under ../. - core/checkpoint.zig: a checkpoint whose index names shards in a folder opens and reads them (shardFiles + openModel + get). - cli/pull_test.zig: the fake hub serves tensors/model-…safetensors; the main pull test asserts the snapshot's subfolder link, its blob and the second pull's "cached", and a security test pins the nested path. - cli/hub.zig: sizeOf counts shards in a subfolder (the info memory floor); the registry test no longer hardcodes a Metal-only family. - build: test-pull-host runs pull's tests with the engine registry; the older test-pull-security step lacks native_engines and does not compile. Evidence: zig build test-pull-host (21 tests) and zig build test-cuda-host (176/176 steps) pass on Linux x86_64 with Zig 0.17.0; zig fmt --check clean. --- zig/build/cuda.zig | 11 +++++ zig/src/cli/hub.zig | 17 +++++++- zig/src/cli/pull_test.zig | 45 ++++++++++++++++++- zig/src/core/checkpoint.zig | 23 ++++++++++ zig/src/families/flashnext/index.zig | 65 +++++++++++++++++++++++++++- 5 files changed, 157 insertions(+), 4 deletions(-) diff --git a/zig/build/cuda.zig b/zig/build/cuda.zig index 71e260a3ff..f2e4d3cbeb 100644 --- a/zig/build/cuda.zig +++ b/zig/build/cuda.zig @@ -522,6 +522,17 @@ pub fn hostTests(b: *std.Build, draft_ids: *std.Build.Module, build_options: *st memory_cli.root_source_file = b.path("zig/src/cli/tests.zig"); b.step("test-cli-memory-host", "Actual checkpoint info memory tests without device or shader producers").dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = memory_cli, .filters = &.{ "info prints family", "issue 934:" } })).step); b.step("test-info-config", "Info shares the checkpoint config reader and diagnostics").dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = cli, .filters = &.{"issue 934:"} })).step); + // `pull`'s tests need the engine registry (`hub.family`) and EXLR8's import; nothing else pulls them in whole. + const pull_tests = b.createModule(.{ + .root_source_file = b.path("zig/src/cli/pull_test.zig"), + .target = host, + .optimize = .debug, + .link_libc = true, + .imports = &.{ .{ .name = "native_engines", .module = native_modules.engines }, exlr8_build.cliImport(b, host, .debug, mods.core) }, + }); + const pull_run = b.addRunArtifact(b.addTest(.{ .root_module = pull_tests })); + b.step("test-pull-host", "Checkpoint download paths: a nested shard folder, retries, resume and the whole-file fallback").dependOn(&pull_run.step); + step.dependOn(&pull_run.step); step.dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = cli })).step); } diff --git a/zig/src/cli/hub.zig b/zig/src/cli/hub.zig index 65395cde50..4896d3b345 100644 --- a/zig/src/cli/hub.zig +++ b/zig/src/cli/hub.zig @@ -197,7 +197,8 @@ test "security: negative quantization metadata is refused without a cast trap" { test "family detection matches the registry and refuses unknown types" { try std.testing.expectEqualStrings("Nemotron 3.5 Lightning", family("nemotron_h").?.title); try std.testing.expectEqualStrings("Qwen3.8 Flash Next", family("qwen4_exp").?.title); - try std.testing.expectEqualStrings("Gemma 4 12B", family("gemma4_unified").?.title); + // Every family the linked registry serves resolves; the titles are display names, so only their presence is fixed. + for (engines.families) |f| try std.testing.expect(family(f.model_type) != null); try std.testing.expect(family("gemma3n") == null); try std.testing.expect(family("unknown") == null); } @@ -236,3 +237,17 @@ test "cache layout helpers read a synthetic cache" { const missing = try cachedSnapshot(a, io, hub, "Org/Other"); try std.testing.expect(missing == null); } + +test "a checkpoint's size counts shards in a folder inside it" { + const io = std.testing.io; + const gpa = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + // The publisher's export keeps every shard in `tensors/`, so a walk that stopped at the top level undercounted it. + try tmp.dir.createDirPath(io, "tensors"); + try tmp.dir.writeFile(io, .{ .sub_path = "config.json", .data = "{}" }); + try tmp.dir.writeFile(io, .{ .sub_path = "tensors/model-00001-of-00001.safetensors", .data = "xxxxxxx" }); + var buf: [64]u8 = undefined; + const dir = try std.fmt.bufPrint(&buf, ".zig-cache/tmp/{s}", .{tmp.sub_path}); + try std.testing.expectEqual(@as(u64, 9), try sizeOf(gpa, io, dir)); +} diff --git a/zig/src/cli/pull_test.zig b/zig/src/cli/pull_test.zig index a877e896b1..73ed435602 100644 --- a/zig/src/cli/pull_test.zig +++ b/zig/src/cli/pull_test.zig @@ -16,16 +16,22 @@ const FakeHub = struct { range_starts: std.atomic.Value(u64) = .init(0), weights: []const u8, config: []const u8, + nested: []const u8, weights_sha_hex: []const u8, + nested_sha_hex: []const u8, config_git_hex: [40]u8, flaky_git_hex: [40]u8, port: u16, const weights_path = "/Org/Flash/resolve/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1/weights.safetensors"; const config_path = "/Org/Flash/resolve/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1/config.json"; + const nested_path = "/Org/Flash/resolve/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1/tensors/model-00001-of-00001.safetensors"; const flaky_body = "{\"chat_template\": \"{{ messages }}\"}"; const drafter_config = "{\"model_type\": \"qwen3\", \"architectures\": [\"DFlash2DraftModel\"], \"dflash_config\": {\"block_size\": 8}}"; + var nested_body_storage: [1234]u8 = @splat(0x5A); + const nested_body: []const u8 = &nested_body_storage; + fn open() !FakeHub { const posix = std.posix; const rc = posix.system.socket(posix.AF.INET, posix.SOCK.STREAM, 0); @@ -45,11 +51,17 @@ const FakeHub = struct { hash.update(weights_body); var digest: [32]u8 = undefined; hash.final(&digest); + var nested_hash = std.crypto.hash.sha2.Sha256.init(.{}); + nested_hash.update(nested_body); + var nested_digest: [32]u8 = undefined; + nested_hash.final(&nested_digest); return .{ .server_fd = fd, .weights = weights_body, .config = config_body, + .nested = nested_body, .weights_sha_hex = try std.fmt.allocPrint(std.testing.allocator, "{s}", .{std.fmt.bytesToHex(digest, .lower)}), + .nested_sha_hex = try std.fmt.allocPrint(std.testing.allocator, "{s}", .{std.fmt.bytesToHex(nested_digest, .lower)}), .config_git_hex = gitHex(config_body), .flaky_git_hex = gitHex(flaky_body), .port = port, @@ -162,7 +174,7 @@ const FakeHub = struct { respond(fd, drafter_config); } else if (std.mem.eql(u8, path, "/api/models/Org/Flash/tree/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1") or std.mem.eql(u8, path, "/api/models/Org/Whole/tree/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb2")) { // The hub's tree sends LFS oids as bare hex. - const tree = std.fmt.allocPrint(std.testing.allocator, "[{{\"type\": \"file\", \"oid\": \"{s}\", \"path\": \"config.json\", \"size\": {d}}}, {{\"type\": \"file\", \"oid\": \"1111111111111111111111111111111111111111\", \"path\": \"weights.safetensors\", \"size\": {d}, \"lfs\": {{\"oid\": \"{s}\", \"size\": {d}}}}}, {{\"type\": \"file\", \"oid\": \"{s}\", \"path\": \"flaky.json\", \"size\": {d}}}]", .{ &fake.config_git_hex, fake.config.len, fake.weights.len, fake.weights_sha_hex, fake.weights.len, &fake.flaky_git_hex, flaky_body.len }) catch return false; + const tree = std.fmt.allocPrint(std.testing.allocator, "[{{\"type\": \"file\", \"oid\": \"{s}\", \"path\": \"config.json\", \"size\": {d}}}, {{\"type\": \"file\", \"oid\": \"1111111111111111111111111111111111111111\", \"path\": \"weights.safetensors\", \"size\": {d}, \"lfs\": {{\"oid\": \"{s}\", \"size\": {d}}}}}, {{\"type\": \"file\", \"oid\": \"{s}\", \"path\": \"flaky.json\", \"size\": {d}}}, {{\"type\": \"file\", \"oid\": \"3333333333333333333333333333333333333333\", \"path\": \"tensors/model-00001-of-00001.safetensors\", \"size\": {d}, \"lfs\": {{\"oid\": \"{s}\", \"size\": {d}}}}}]", .{ &fake.config_git_hex, fake.config.len, fake.weights.len, fake.weights_sha_hex, fake.weights.len, &fake.flaky_git_hex, flaky_body.len, fake.nested.len, fake.nested_sha_hex, fake.nested.len }) catch return false; defer std.testing.allocator.free(tree); respond(fd, tree); } else if (std.mem.eql(u8, path, config_path) or std.mem.eql(u8, path, "/Org/Whole/resolve/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb2/config.json")) { @@ -179,6 +191,8 @@ const FakeHub = struct { } else if (std.mem.eql(u8, path, "/Org/Whole/resolve/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb2/weights.safetensors")) { _ = fake.weight_requests.fetchAdd(1, .monotonic); whole(fd, fake.weights); + } else if (std.mem.eql(u8, path, nested_path) or std.mem.eql(u8, path, "/Org/Whole/resolve/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb2/tensors/model-00001-of-00001.safetensors")) { + whole(fd, fake.nested); } else if (std.mem.eql(u8, path, weights_path)) { _ = fake.weight_requests.fetchAdd(1, .monotonic); const from = range_start orelse { @@ -238,6 +252,7 @@ const FakeHub = struct { if (fake.thread) |t| t.join(); _ = std.posix.system.close(fake.server_fd); std.testing.allocator.free(fake.weights_sha_hex); + std.testing.allocator.free(fake.nested_sha_hex); } }; @@ -271,6 +286,11 @@ test "pull downloads, verifies, retries a cut body, resumes and refuses a family try std.testing.expectEqualStrings(FakeHub.config_body, config_read); const weights_read = try std.Io.Dir.cwd().readFileAlloc(io, weights_link, a, .limited(1 << 20)); try std.testing.expectEqualSlices(u8, FakeHub.weights_body, weights_read); + // A shard in a folder inside the model folder lands as a real subfolder in the snapshot, still a blob link. + const nested_link = try std.fs.path.join(a, &.{ snapshot, "tensors/model-00001-of-00001.safetensors" }); + const nested_read = try std.Io.Dir.cwd().readFileAlloc(io, nested_link, a, .limited(1 << 20)); + try std.testing.expectEqualSlices(u8, FakeHub.nested_body, nested_read); + try std.Io.Dir.cwd().access(io, try std.fs.path.join(a, &.{ root, "models--Org--Flash/blobs", fake_hub.nested_sha_hex }), .{}); const ref = try std.Io.Dir.cwd().readFileAlloc(io, try std.fs.path.join(a, &.{ root, "models--Org--Flash/refs/main" }), a, .limited(64)); try std.testing.expectEqualStrings("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", std.mem.trim(u8, ref, " \r\n")); try std.testing.expect(std.mem.indexOf(u8, out.written(), "weights.safetensors") != null); @@ -288,6 +308,7 @@ test "pull downloads, verifies, retries a cut body, resumes and refuses a family try std.testing.expectEqual(@as(u8, 0), try pull.run(a, io, &out2.writer, &err_out.writer, &env, root, "Org/Flash")); try std.testing.expectEqual(@as(u32, 1), fake_hub.weight_requests.load(.monotonic)); try std.testing.expect(std.mem.indexOf(u8, out2.written(), "weights.safetensors: cached") != null); + try std.testing.expect(std.mem.indexOf(u8, out2.written(), "tensors/model-00001-of-00001.safetensors: cached") != null); try std.testing.expect(std.mem.indexOf(u8, out2.written(), "flaky.json: cached") != null); try std.testing.expectEqual(@as(u32, 2), fake_hub.flaky_requests.load(.monotonic)); @@ -448,3 +469,25 @@ test "security: a hub response cannot write beyond the manifest file size" { }; try std.testing.expect(written <= 8); } + +test "security: a shard in a folder inside the model folder lands inside the snapshot" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const a = arena_state.allocator(); + const io = std.testing.io; + var fake_hub = FakeHub.open() catch return error.SkipZigTest; + fake_hub.start(); + defer fake_hub.stopServer(); + const root = try std.fmt.allocPrint(a, ".tf-pull-nested-{d}", .{std.Io.Clock.awake.now(io).toNanoseconds()}); + defer std.Io.Dir.cwd().deleteTree(io, root) catch {}; + const env = try testEnv(a, try fake_hub.endpoint(a)); + var out: std.Io.Writer.Allocating = .init(a); + var err_out: std.Io.Writer.Allocating = .init(a); + try std.testing.expectEqual(@as(u8, 0), try pull.run(a, io, &out.writer, &err_out.writer, &env, root, "Org/Flash")); + const snapshot = try std.fs.path.join(a, &.{ root, "models--Org--Flash/snapshots/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1" }); + // NVIDIA's own export puts every shard in `tensors/`; the snapshot keeps that folder and still links the blob. + const link = try std.fs.path.join(a, &.{ snapshot, "tensors/model-00001-of-00001.safetensors" }); + const body = try std.Io.Dir.cwd().readFileAlloc(io, link, a, .limited(1 << 20)); + try std.testing.expectEqualSlices(u8, FakeHub.nested_body, body); + try std.Io.Dir.cwd().access(io, try std.fs.path.join(a, &.{ root, "models--Org--Flash/blobs", fake_hub.nested_sha_hex }), .{}); +} diff --git a/zig/src/core/checkpoint.zig b/zig/src/core/checkpoint.zig index 6bb36f9ba1..9c4c7ee7da 100644 --- a/zig/src/core/checkpoint.zig +++ b/zig/src/core/checkpoint.zig @@ -345,3 +345,26 @@ test "Save recovery: an unresolved partial save refuses actual checkpoint reopen return error.UncertainCheckpointReopened; } else |err| try std.testing.expectEqual(error.CheckpointSaveUncertain, err); } + +test "a checkpoint whose index names shards in a folder opens and reads them" { + const io = std.testing.io; + const gpa = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + // NVIDIA's own export writes `tensors/model-00001-of-00001.safetensors`. + try tmp.dir.createDirPath(io, "tensors"); + const image = try testImage(gpa, "a", 4); + defer gpa.free(image); + try tmp.dir.writeFile(io, .{ .sub_path = "tensors/model-00001-of-00001.safetensors", .data = image }); + try tmp.dir.writeFile(io, .{ .sub_path = "model.safetensors.index.json", .data = "{\"weight_map\":{\"a\":\"tensors/model-00001-of-00001.safetensors\"}}" }); + var buf: [64]u8 = undefined; + const dir = try std.fmt.bufPrint(&buf, ".zig-cache/tmp/{s}", .{tmp.sub_path}); + const listed = try shardFiles(gpa, io, dir); + defer freeShardFiles(gpa, listed); + try std.testing.expectEqual(@as(usize, 1), listed.len); + try std.testing.expectEqualStrings("tensors/model-00001-of-00001.safetensors", listed[0][dir.len + 1 ..]); + var ck = try Checkpoint.openModel(gpa, io, dir); + defer ck.close(); + _ = try ck.get("a"); + try std.testing.expectEqual(@as(usize, 0), ck.unused()); +} diff --git a/zig/src/families/flashnext/index.zig b/zig/src/families/flashnext/index.zig index 2950971acd..a4984aa2ce 100644 --- a/zig/src/families/flashnext/index.zig +++ b/zig/src/families/flashnext/index.zig @@ -1,7 +1,29 @@ -//! Public index admission proves required tensor names and safe shard basenames before payload headers are available. +//! Public index admission proves required tensor names and safe shard names before payload headers are available. +//! A shard may sit in a folder inside the model folder (`tensors/model-…safetensors`); names that leave it are refused. const std = @import("std"); const Config = @import("config.zig").Config; +/// A weight map's shard, as core/checkpoint_index.zig's `shardName` takes it (kept here so this file imports no core): +/// plain names joined by `/`, no part hidden or reserved, and the file must be a safetensors one. +pub fn shardName(name: []const u8) bool { + if (!std.mem.endsWith(u8, name, ".safetensors")) return false; + var parts = std.mem.splitScalar(u8, name, '/'); + while (parts.next()) |part| if (!plainPart(part)) return false; + return true; +} + +fn plainPart(part: []const u8) bool { + if (part.len == 0 or part[0] == '.' or part[part.len - 1] == '.' or part[part.len - 1] == ' ') return false; + for (part) |c| if (c < 0x20 or c == 0x7f) return false; + if (std.mem.indexOfAny(u8, part, "\\:<>\"|?*") != null) return false; + const stem = part[0 .. std.mem.indexOfScalar(u8, part, '.') orelse part.len]; + for ([_][]const u8{ "CON", "PRN", "AUX", "NUL" }) |reserved| + if (std.ascii.eqlIgnoreCase(stem, reserved)) return false; + if (stem.len == 4 and stem[3] >= '1' and stem[3] <= '9' and + (std.ascii.eqlIgnoreCase(stem[0..3], "COM") or std.ascii.eqlIgnoreCase(stem[0..3], "LPT"))) return false; + return true; +} + /// A declared width, the fields of core/quant's mlx.Width, so this file imports no core. pub const Width = struct { bits: u8, group: u16 }; @@ -142,7 +164,7 @@ fn admitWith(comptime Runtime: type, gpa: std.mem.Allocator, bytes: []const u8, var it = map.object.iterator(); while (it.next()) |item| { const file = item.value_ptr.*; - if (file != .string or file.string.len == 0 or file.string[0] == '.' or std.mem.indexOfAny(u8, file.string, "/\\:\x00") != null or !std.mem.endsWith(u8, file.string, ".safetensors")) return error.UnsafeFlashShard; + if (file != .string or !shardName(file.string)) return error.UnsafeFlashShard; try files.put(gpa, file.string, {}); } var c = Check(Runtime){ .map = map.object, .runtime = runtime, .refusal = refusal }; @@ -169,3 +191,42 @@ fn admitWith(comptime Runtime: type, gpa: std.mem.Allocator, bytes: []const u8, } return .{ .tensor_names = map.object.count(), .required_names = c.names, .shards = files.count() }; } + +test "admission takes NVIDIA's tensors/ shard folder and keeps the escape refusal" { + const a = std.testing.allocator; + var c = try @import("config.zig").parse(a, @embedFile("fixtures/config.json")); + defer c.deinit(); + const bytes = @embedFile("fixtures/index.json"); + const flat = try admit(a, bytes, &c); + try std.testing.expect(flat.shards > 1); + // The publisher's own export maps every tensor to `tensors/model-000NN-of-00021.safetensors`. + const nested = try std.mem.replaceOwned(u8, a, bytes, "\":\"model-", "\":\"tensors/model-"); + defer a.free(nested); + try std.testing.expectEqual(flat, try admit(a, nested, &c)); + const escaped = try std.mem.replaceOwned(u8, a, bytes, "\":\"model-", "\":\"../model-"); + defer a.free(escaped); + try std.testing.expectError(error.UnsafeFlashShard, admit(a, escaped, &c)); +} + +test "a shard may sit in a folder inside the model folder; escapes keep their refusal" { + // NVIDIA's own export writes `tensors/model-000NN-of-00021.safetensors`. + for ([_][]const u8{ + "model-00001-of-00021.safetensors", + "tensors/model-00001-of-00021.safetensors", + "a/b/c.safetensors", + }) |name| try std.testing.expect(shardName(name)); + for ([_][]const u8{ + "", // nothing to open + "model.bin", // not a safetensors file + "/model.safetensors", // absolute + "../model.safetensors", // leaves the folder + "tensors/../model.safetensors", // leaves it through a subfolder + "tensors//model.safetensors", // an empty part + ".cache/model.safetensors", // hidden folder + "tensors/.hidden.safetensors", // hidden file + "C:/model.safetensors", // a drive name + "tensors\\model.safetensors", // a Windows separator + "tensors/nul.safetensors", // a reserved device stem + "tensors/model.safetensors ", // trailing space + }) |name| try std.testing.expect(!shardName(name)); +} From 9ec3f523831bb12c8711d5abb91e668e80394171 Mon Sep 17 00:00:00 2001 From: tensorfold-expert Date: Fri, 23 Oct 2026 22:42:18 +0300 Subject: [PATCH 2/2] build: the pull tests behind the engine registry, and not on Windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `pull_test.zig` ran in no passing no-GPU step: `test-pull-security` had not compiled since `cli/hub.zig` began reading `native_engines` for `family()`, and `cli/tests.zig` is built with two name filters only. - new `zig/build/pull_tests.zig` owns the pull test module — the engine registry, EXLR8's import, the `test-pull-host` step and its dependency on `test-cuda-host`; it returns early on Windows, where the fake hub's POSIX socket (`sockaddr.in`) cannot compile. - `golden.step` takes the platform's engine registry, so `test-pull-security` compiles again: the CUDA host passes `native_modules.engines`, the Metal host its own `engines`. - `cuda.zig` stays within the 600-line lean limit; the call replaces a blank line beside the other Windows-guarded host test. Verified: `zig build test-pull-host test-pull-security` passes; lean check reports 0 new problems against 8e49cf8; `zig fmt --check` clean. --- zig/build/cuda.zig | 15 ++------------- zig/build/golden.zig | 5 +++-- zig/build/metal_targets.zig | 2 +- zig/build/pull_tests.zig | 16 ++++++++++++++++ zig/src/families/flashnext/index.zig | 6 ++---- 5 files changed, 24 insertions(+), 20 deletions(-) create mode 100644 zig/build/pull_tests.zig diff --git a/zig/build/cuda.zig b/zig/build/cuda.zig index f2e4d3cbeb..1f038f0ec8 100644 --- a/zig/build/cuda.zig +++ b/zig/build/cuda.zig @@ -418,7 +418,7 @@ pub fn hostTests(b: *std.Build, draft_ids: *std.Build.Module, build_options: *st const h3_checkpoint = b.createModule(.{ .root_source_file = b.path("zig/src/families/h3/checkpoint.zig"), .target = host, .optimize = .debug, .link_libc = true }); // the checkpoint maps its file with mmap: not Windows if (host.result.os.tag != .windows) step.dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = h3_checkpoint })).step); - + @import("pull_tests.zig").add(b, host, .debug, native_modules.engines, mods.core, step); b.step("test-save-reopen-host", "Unresolved save through actual checkpoint admission").dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = mods.core, .filters = &.{"Save recovery:"} })).step); b.step("test-trainer-close-host", "Actual native close and real Flash Next trainer ownership sentinels").dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = native, .filters = &.{"Save recovery:"} })).step); // the HTTP server's unit tests (zig/src/server/root.zig), which the macOS build runs, on Linux hosts too @@ -479,7 +479,7 @@ pub fn hostTests(b: *std.Build, draft_ids: *std.Build.Module, build_options: *st } // the server goldens and Cortheon's fuzz step here when the target is not macOS, whose Metal build defines them if (server_steps) { - golden_build.step(b, host, server_tests, native_modules.api, mods.tokenizer, template); + golden_build.step(b, host, server_tests, native_modules.api, mods.tokenizer, template, native_modules.engines); @import("cortheon_fuzz.zig").add(b, host, native_modules.api, mods.tokenizer, template); } // the tokenizer's own tests (memo, steps, regex, unicode) run here: no other test binary reaches them @@ -522,17 +522,6 @@ pub fn hostTests(b: *std.Build, draft_ids: *std.Build.Module, build_options: *st memory_cli.root_source_file = b.path("zig/src/cli/tests.zig"); b.step("test-cli-memory-host", "Actual checkpoint info memory tests without device or shader producers").dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = memory_cli, .filters = &.{ "info prints family", "issue 934:" } })).step); b.step("test-info-config", "Info shares the checkpoint config reader and diagnostics").dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = cli, .filters = &.{"issue 934:"} })).step); - // `pull`'s tests need the engine registry (`hub.family`) and EXLR8's import; nothing else pulls them in whole. - const pull_tests = b.createModule(.{ - .root_source_file = b.path("zig/src/cli/pull_test.zig"), - .target = host, - .optimize = .debug, - .link_libc = true, - .imports = &.{ .{ .name = "native_engines", .module = native_modules.engines }, exlr8_build.cliImport(b, host, .debug, mods.core) }, - }); - const pull_run = b.addRunArtifact(b.addTest(.{ .root_module = pull_tests })); - b.step("test-pull-host", "Checkpoint download paths: a nested shard folder, retries, resume and the whole-file fallback").dependOn(&pull_run.step); - step.dependOn(&pull_run.step); step.dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = cli })).step); } diff --git a/zig/build/golden.zig b/zig/build/golden.zig index 9955b9b2a9..e1b8c82b8c 100644 --- a/zig/build/golden.zig +++ b/zig/build/golden.zig @@ -2,8 +2,8 @@ const std = @import("std"); -/// `zig build test-golden` over the server tests' module and the imports it was built with. -pub fn step(b: *std.Build, target: std.Build.ResolvedTarget, server: *std.Build.Module, api: *std.Build.Module, tokenizer: *std.Build.Module, template: *std.Build.Module) void { +/// `zig build test-golden` over the server tests' module; `engines` is the platform registry `cli/hub.zig` reads. +pub fn step(b: *std.Build, target: std.Build.ResolvedTarget, server: *std.Build.Module, api: *std.Build.Module, tokenizer: *std.Build.Module, template: *std.Build.Module, engines: *std.Build.Module) void { const fake_serve = b.addExecutable(.{ .name = "fake_serve", .root_module = b.createModule(.{ .root_source_file = b.path("zig/tests/server/fake_serve.zig"), .target = target, @@ -45,6 +45,7 @@ pub fn step(b: *std.Build, target: std.Build.ResolvedTarget, server: *std.Build. .target = target, .optimize = .Debug, .link_libc = true, + .imports = &.{.{ .name = "native_engines", .module = engines }}, }), .filters = &.{"security:"} }); const run = b.addRunArtifact(checks); run.setCwd(b.path(".")); diff --git a/zig/build/metal_targets.zig b/zig/build/metal_targets.zig index 531e8b10b8..8b723a1816 100644 --- a/zig/build/metal_targets.zig +++ b/zig/build/metal_targets.zig @@ -113,7 +113,7 @@ pub fn nativeServer(b: *std.Build, target: std.Build.ResolvedTarget, optimize: s // the engine seam's own tests (lane_host.zig), as zig/tests/server/zig_test.sh runs them test_step.?.dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = api })).step); // the frozen server parity goldens, checked by Zig driving fake_serve - golden_build.step(b, target, server_tests, api, tokenizer, template); + golden_build.step(b, target, server_tests, api, tokenizer, template, engines); b.step("test-native", "The Metal engines module's host-side tests").dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = engines })).step); const reuse = b.addExecutable(.{ .name = "tf-flashnext-reuse", .root_module = b.createModule(.{ .root_source_file = b.path("zig/tests/flashnext_reuse.zig"), diff --git a/zig/build/pull_tests.zig b/zig/build/pull_tests.zig new file mode 100644 index 0000000000..a0ce780430 --- /dev/null +++ b/zig/build/pull_tests.zig @@ -0,0 +1,16 @@ +//! `pull`'s own tests: `cli/hub.zig` reads the engine registry, so they need `native_engines` and EXLR8's import. +const std = @import("std"); + +pub fn add(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, engines: *std.Build.Module, core: *std.Build.Module, test_step: *std.Build.Step) void { + if (target.result.os.tag == .windows) return; // the fake hub listens on a POSIX socket + const mod = b.createModule(.{ + .root_source_file = b.path("zig/src/cli/pull_test.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + .imports = &.{ .{ .name = "native_engines", .module = engines }, @import("exlr8.zig").cliImport(b, target, optimize, core) }, + }); + const run = b.addRunArtifact(b.addTest(.{ .root_module = mod })); + b.step("test-pull-host", "Checkpoint download paths: a nested shard folder, retries, resume and the whole-file fallback").dependOn(&run.step); + test_step.dependOn(&run.step); +} diff --git a/zig/src/families/flashnext/index.zig b/zig/src/families/flashnext/index.zig index a4984aa2ce..3b94b1fb6d 100644 --- a/zig/src/families/flashnext/index.zig +++ b/zig/src/families/flashnext/index.zig @@ -1,10 +1,8 @@ -//! Public index admission proves required tensor names and safe shard names before payload headers are available. -//! A shard may sit in a folder inside the model folder (`tensors/model-…safetensors`); names that leave it are refused. +//! Public index admission proves required tensor names and safe shard names before any payload header is read. const std = @import("std"); const Config = @import("config.zig").Config; -/// A weight map's shard, as core/checkpoint_index.zig's `shardName` takes it (kept here so this file imports no core): -/// plain names joined by `/`, no part hidden or reserved, and the file must be a safetensors one. +/// A weight map's shard, as core's `shardName` takes it: plain names joined by `/`, none hidden, `.safetensors`. pub fn shardName(name: []const u8) bool { if (!std.mem.endsWith(u8, name, ".safetensors")) return false; var parts = std.mem.splitScalar(u8, name, '/');