Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 7 additions & 10 deletions zig/src/native/cuda.zig
Original file line number Diff line number Diff line change
Expand Up @@ -330,14 +330,11 @@ const Host = struct {
}
};

/// What the backend refuses before any device work, whatever the family; null when `o` may open.
fn refusal(o: api.Open) ?[]const u8 {
return if (o.learn != null) "--learn learns prompt prefixes on Macs only: serve without --learn, --learn-dir or --learn-gib on CUDA" else null;
}
const refusals = @import("cuda_refusal.zig");

/// The engine for `o.dir`, or null with `problem` set when `refusal` names a flag or no CUDA family reads it.
pub fn open(a: Allocator, gpa: Allocator, io: std.Io, o: api.Open, problem: *[]const u8) !?api.Opened {
if (refusal(o)) |why| {
if (refusals.backend(o)) |why| {
problem.* = why;
return null;
}
Expand All @@ -349,6 +346,10 @@ pub fn open(a: Allocator, gpa: Allocator, io: std.Io, o: api.Open, problem: *[]c
}

fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.Open, problem: *[]const u8) !?api.Opened {
if (refusals.family(F, o)) |why| {
problem.* = why;
return null;
}
const native = modelContext(a, io, o.dir);
const window = contextWindow(o.context, native, F.default_context) catch |e| {
problem.* = switch (e) {
Expand Down Expand Up @@ -485,11 +486,7 @@ fn toGib(bytes: u64) f64 {

test {
_ = budget; // cuda_memory.zig's own tests, which no other test binary reaches
}

test "--learn is refused whatever the family, and --slide or a learned-disk size alone is not" {
try std.testing.expectStringStartsWith(refusal(.{ .dir = "d", .model_type = "nemotron_h", .learn = "/x" }) orelse "", "--learn learns prompt prefixes on Macs only");
try std.testing.expect(refusal(.{ .dir = "d", .model_type = "nemotron_h", .slide = true, .learn_gib = 8, .learn_min_free_gib = 2 }) == null);
_ = refusals;
}

test "chip classes name the compute capability as gate entries do" {
Expand Down
38 changes: 38 additions & 0 deletions zig/src/native/cuda_refusal.zig
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
//! What the native CUDA server refuses before any device work: flags no CUDA family serves, and --drafter per family.
const std = @import("std");
const api = @import("engine_api");

/// Flags refused whatever the family; null when `o` may open.
pub fn backend(o: api.Open) ?[]const u8 {
if (o.learn != null) return "--learn learns prompt prefixes on Macs only: serve without --learn, --learn-dir or --learn-gib on CUDA";
if (o.speed_up != null) return "--speed-up links two Macs: serve without --speed-up on CUDA";
if (o.prompt_cache_over_cap) return "--prompt-cache-over-cap lifts a Mac's 70% cap: on CUDA the prompt cache takes at most the memory the weights and streams leave, so serve without it";
return null;
}

/// A family takes --drafter only if it says so (`takes_drafter`); null when `o` may open `F`.
pub fn family(comptime F: type, o: api.Open) ?[]const u8 {
if (o.drafter != null and !@hasDecl(F, "takes_drafter")) return "--drafter loads a separate drafter, and this CUDA family drafts with its own MTP head: serve without --drafter";
return null;
}

test "--learn is refused whatever the family, and --slide or a learned-disk size alone is not" {
try std.testing.expectStringStartsWith(backend(.{ .dir = "d", .model_type = "nemotron_h", .learn = "/x" }) orelse "", "--learn learns prompt prefixes on Macs only");
try std.testing.expect(backend(.{ .dir = "d", .model_type = "nemotron_h", .slide = true, .learn_gib = 8, .learn_min_free_gib = 2 }) == null);
}

test "flags with nothing to do on CUDA are refused, not ignored" {
try std.testing.expectStringStartsWith(backend(.{ .dir = "d", .model_type = "nemotron_h", .speed_up = "rank0.json" }) orelse "", "--speed-up links two Macs");
try std.testing.expectStringStartsWith(backend(.{ .dir = "d", .model_type = "nemotron_h", .prompt_cache_over_cap = true }) orelse "", "--prompt-cache-over-cap");
try std.testing.expect(backend(.{ .dir = "d", .model_type = "nemotron_h", .prompt_cache_gib = 4 }) == null);
}

test "--drafter opens only a family that takes one" {
const Mtp = struct {};
const Drafted = struct {
pub const takes_drafter = true;
};
try std.testing.expectStringStartsWith(family(Mtp, .{ .dir = "d", .model_type = "m", .drafter = "/x" }) orelse "", "--drafter loads a separate drafter");
try std.testing.expect(family(Drafted, .{ .dir = "d", .model_type = "m", .drafter = "/x" }) == null);
try std.testing.expect(family(Mtp, .{ .dir = "d", .model_type = "m" }) == null);
}
Loading