Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ Each release's page on GitHub has its notes and measurements. See [the 1.0.0 rel
- The Qwen3.5-2B recipe says which of the family's listed formats load at its geometry: only `mlx-q4g64` (#2139).

- Flash Next on a Mac without tensor units, with `--parallel` above one: a long prompt beside a running reply no longer slows both. Replies take 0.8 of that time by default and the prompt's steps grow from 32 to 512 rows after eight. On an M3 Ultra, a 1,024-token reply beside a 64K prompt went from 48 s to 8 s and the prompt's first token from 90 s to 54 s, with the same tokens. `--decode-share 0` keeps the old order (#2167, #1893). Thanks to @dinadur.
- `--profile stable`'s registry covers GLM's host KV tier: `TF_GLM_KV_HOST` and `TF_GLM_KV_HOST_SHARED` are now rows in the table, with [#1795](https://github.com/ashhart/TensorFold/pull/1795)'s identical tokens at TP=2, and their readers resolve through the registry. A switch looked up by name now goes through `core.profile.lookup`, which serves registered names from the table and every other name from the environment as before. Stable supplies no value, so nothing is enabled or changed by default (#1660).

- GLM-5.3-Flash on a Mac without tensor units learns what a shared round's row costs on that chip instead of pricing it from an M5 Ultra's table, so an M3 Ultra stops over-drafting with several requests at once: about 14% more at four streams when drafts land often, with the same tokens (measured on 40 of the 45 layers).

Expand Down
10 changes: 8 additions & 2 deletions docs/profiles.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,15 +38,21 @@ profile supplies no value. Registry metadata comes directly from the same table
Initial coverage is:

- Qwen3.5 dense/MoE on CUDA: `TF_QWEN_KV_PAGES`, `TF_QWEN_KV_PAGES_ALL`, `TF_QWEN_KV_SHARE`.
- GLM on CUDA: `TF_GLM_KV_SHARE`, `TF_KV_PAGES`.
- GLM on CUDA: `TF_GLM_KV_SHARE`, `TF_KV_PAGES`, `TF_GLM_KV_HOST`, `TF_GLM_KV_HOST_SHARED`.
- Nemotron and Nemotron Omni on CUDA: `TF_KV_PAGES`, with their existing default-on parser.
- Gemma 4, Kolibri-1, and Spark2.5 on the core CUDA page adapter: `TF_KV_SHARE`, where the adapter is served.
- Flash Next on CUDA: `TF_FLASHNEXT_KV_PAGES`, `TF_FLASHNEXT_KV_PAGES_ALL`.

`identical` records the existing switch's token-preserving design and linked evidence; it is not a claim that the
switch is faster for every shape, device, or workload. The `*_ALL` checks stay `debug`; Flash Next paging remains
`experimental`. No class turns a switch on. GLM's existing automatic paging default is preserved; other default-on
paging outside this registry is unchanged.
paging outside this registry is unchanged. The two GLM host-memory switches carry [#1795](https://github.com/ashhart/TensorFold/pull/1795)'s
identical tokens at TP=2 on two GB10s. `TF_GLM_KV_HOST_GIB` stays outside the registry: it sizes those pages rather
than choosing a policy, and its reader keeps its own default.

A reader that resolves its switch by name uses `core.profile.lookup`: a registered name comes from this registry, and
every other name keeps the libc lookup and default it had. GLM's rank settings read that way, and their fingerprint
still carries each resolved value, so ranks that disagree still stop before weights load.

## Startup reporting

Expand Down
26 changes: 25 additions & 1 deletion zig/src/core/profile.zig
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ pub const Switch = enum {
TF_FLASHNEXT_KV_PAGES,
TF_FLASHNEXT_KV_PAGES_ALL,
TF_KV_PAGES,
TF_GLM_KV_HOST,
TF_GLM_KV_HOST_SHARED,
};
pub const Syntax = enum { count, one, auto };
pub const Entry = struct {
Expand Down Expand Up @@ -62,6 +64,8 @@ pub const entries = [_]Entry{
.{ .id = .TF_FLASHNEXT_KV_PAGES_ALL, .families = flash, .class = .debug, .syntax = .one, .effect = "Put every stream on pool pages for checks", .evidence = "#1660" },
.{ .id = .TF_KV_PAGES, .families = &.{"glm5_next"}, .class = .identical, .syntax = .auto, .effect = "GLM KV pool pages; auto from two ranks with BF16, windows with FP8", .evidence = "#1322" },
.{ .id = .TF_KV_PAGES, .families = &.{ "nemotron_h", "nemotron_h_omni" }, .class = .identical, .report_values = &.{ "0", "all" }, .effect = "Nemotron KV pool pages by default; exact 0 disables, exact all pages every stream", .evidence = "#781" },
.{ .id = .TF_GLM_KV_HOST, .families = &.{"glm5_next"}, .class = .identical, .syntax = .one, .effect = "Keep the DSA layers' latent rows on pinned host pages", .evidence = "#1795" },
.{ .id = .TF_GLM_KV_HOST_SHARED, .families = &.{"glm5_next"}, .class = .identical, .syntax = .one, .effect = "One copy of those host pages across the ranks on one host", .evidence = "#1795" },
};

comptime {
Expand Down Expand Up @@ -98,8 +102,28 @@ pub fn resolve(profile: Profile, id: Switch, environment: ?[:0]const u8) ?[:0]co

/// The only served profile is stable; no global setting or environment mutation is needed to select it.
pub inline fn get(comptime id: Switch) ?[:0]const u8 {
return value(.stable, id);
}

/// One switch's value for a profile chosen at runtime: the same environment-first rule as `get`.
pub fn value(profile: Profile, id: Switch) ?[:0]const u8 {
const raw = std.c.getenv(entry(id).name());
return resolve(.stable, id, if (raw) |v| std.mem.span(v) else null);
return resolve(profile, id, if (raw) |v| std.mem.span(v) else null);
}

/// The registry row a switch's environment name belongs to; null for a name the table does not cover.
pub fn registered(name: []const u8) ?Switch {
inline for (@typeInfo(Switch).@"enum".field_names, @typeInfo(Switch).@"enum".field_values) |n, v| {
if (std.mem.eql(u8, name, n)) return @fromBackingInt(@intCast(v));
}
return null;
}

/// A runtime name lookup: a registered switch resolves through the profile, any other name through libc as before.
pub fn lookup(name: [:0]const u8) ?[:0]const u8 {
if (registered(name)) |id| return value(.stable, id);
const raw = std.c.getenv(name) orelse return null;
return std.mem.span(raw);
}

test {
Expand Down
25 changes: 25 additions & 0 deletions zig/src/core/profile_test.zig
Original file line number Diff line number Diff line change
Expand Up @@ -89,3 +89,28 @@ test "stable profile: GLM paging registers automatic policy without supplying or
for ([_][]const u8{ "off", "0", "on", "1", "all", "", "AUTO", "false", " 0 " }) |raw|
try t.expect(e.differs(raw));
}

test "stable profile: the GLM host KV tier registers as an exact-one switch a family, off unless set" {
inline for (.{ .TF_GLM_KV_HOST, .TF_GLM_KV_HOST_SHARED }) |id| {
const e = p.entry(id);
try t.expectEqual(p.Class.identical, e.class);
try t.expectEqual(p.Syntax.one, e.syntax);
try t.expect(e.applies("glm5_next", "cuda"));
try t.expect(!e.applies("qwen3_5", "cuda"));
try t.expect(!e.applies("glm5_next", "mlx"));
try t.expectEqualStrings("#1795", e.evidence);
try t.expect(p.resolve(.stable, id, null) == null);
try t.expect(e.differs("1"));
for ([_][]const u8{ "0", "", "01", "true", " 1 " }) |raw| try t.expect(!e.differs(raw));
}
}

test "stable profile: a runtime name lookup covers the registry and leaves every other name to libc" {
try t.expectEqual(p.Switch.TF_GLM_KV_HOST, p.registered("TF_GLM_KV_HOST").?);
try t.expectEqual(p.Switch.TF_QWEN_KV_PAGES, p.registered("TF_QWEN_KV_PAGES").?);
for ([_][]const u8{ "TF_GLM_KV_HOST_GIB", "TF_UNREGISTERED_FIXTURE", "", "tf_glm_kv_host" }) |name|
try t.expect(p.registered(name) == null);
// An unset variable is null through both paths, so an unregistered name keeps the reader's old default.
try t.expect(p.lookup("TF_PROFILE_FIXTURE_UNSET") == null);
inline for (p.entries) |e| try t.expectEqual(p.get(e.id), p.lookup(e.name()));
}
11 changes: 6 additions & 5 deletions zig/src/families/glm/cuda_kv_host.zig
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,18 @@ const cuda = @import("cuda");
const posix = std.posix;

pub fn onHost() bool {
return set("TF_GLM_KV_HOST");
return enabled(.TF_GLM_KV_HOST);
}

/// TF_GLM_KV_HOST_SHARED=1: the ranks on this host share one copy of the pages (their latent rows are identical).
pub fn shared() bool {
return set("TF_GLM_KV_HOST_SHARED");
return enabled(.TF_GLM_KV_HOST_SHARED);
}

fn set(name: [*:0]const u8) bool {
const v = std.c.getenv(name) orelse return false;
return std.mem.eql(u8, std.mem.span(v), "1");
/// An exact-one switch through the profile registry, which is the one reader of a registered name.
fn enabled(comptime id: @import("profile").Switch) bool {
const v = @import("profile").get(id) orelse return false;
return std.mem.eql(u8, v, "1");
}

/// Host pages of `page` bytes for a `capacity`-row window: `gib` GiB of them, else one full window and 16.
Expand Down
4 changes: 2 additions & 2 deletions zig/src/families/glm/cuda_ranks.zig
Original file line number Diff line number Diff line change
Expand Up @@ -151,9 +151,9 @@ pub const Settings = struct {
/// The most ranks a world holds: rank 0's greedy pick gathers every rank's top-1 into one host array of this size.
pub const max_world = 16;

/// The process environment's value of `name` (null when unset).
/// A setting by name (null when unset): a registered switch through the profile registry, the rest as before.
pub fn environment(name: [:0]const u8) ?[]const u8 {
return std.mem.span(std.c.getenv(name) orelse return null);
return @import("profile").lookup(name);
}

/// What every rank must agree on, as bytes the communicator carries.
Expand Down
17 changes: 17 additions & 0 deletions zig/src/server/profile_line.zig
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,17 @@ const TestEnv = struct {
all: ?[]const u8 = null,
share: ?[]const u8 = null,
glm_pages: ?[]const u8 = null,
glm_host: ?[]const u8 = null,
glm_shared: ?[]const u8 = null,
pub fn get(e: @This(), name: []const u8) ?[]const u8 {
if (std.mem.eql(u8, name, "TF_QWEN_KV_PAGES")) return e.qwen;
if (std.mem.eql(u8, name, "TF_QWEN_KV_PAGES_ALL")) return e.all;
if (std.mem.eql(u8, name, "TF_QWEN_KV_SHARE")) return e.share;
if (std.mem.eql(u8, name, "TF_KV_PAGES")) return e.glm_pages;
if (std.mem.eql(u8, name, "TF_GLM_KV_HOST")) return e.glm_host;
if (std.mem.eql(u8, name, "TF_GLM_KV_HOST_SHARED")) return e.glm_shared;
if (std.mem.eql(u8, name, "TF_GLM_KV_SHARE")) return null;
if (std.mem.eql(u8, name, "TF_KV_SHARE")) return null;
return "fixture-secret-must-not-be-read";
}
};
Expand Down Expand Up @@ -72,6 +77,18 @@ test "profile line: GLM reports explicit page policies including off without tre
try std.testing.expectEqualStrings("profile: stable; registered overrides: none", try format(a, .stable, scope[0], scope[1], TestEnv{ .glm_pages = "off" }));
}

test "profile line: the GLM host KV tier is reported for its own family and backend only" {
var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);
defer arena.deinit();
const a = arena.allocator();
try std.testing.expectEqualStrings("profile: stable; registered overrides: TF_GLM_KV_HOST=1 (identical, env)", try format(a, .stable, "glm5_next", "cuda", TestEnv{ .glm_host = "1" }));
try std.testing.expectEqualStrings("profile: stable; registered overrides: TF_GLM_KV_HOST=1 (identical, env); TF_GLM_KV_HOST_SHARED=1 (identical, env)", try format(a, .stable, "glm5_next", "cuda", TestEnv{ .glm_host = "1", .glm_shared = "1" }));
for ([_][]const u8{ "0", "", "01", "true", " 1 " }) |raw|
try std.testing.expectEqualStrings("profile: stable; registered overrides: none", try format(a, .stable, "glm5_next", "cuda", TestEnv{ .glm_host = raw, .glm_shared = raw }));
for ([_][2][]const u8{ .{ "qwen3_5", "cuda" }, .{ "gemma4", "cuda" }, .{ "glm5_next", "mlx" }, .{ "unknown", "cuda" } }) |scope|
try std.testing.expectEqualStrings("profile: stable; registered overrides: none", try format(a, .stable, scope[0], scope[1], TestEnv{ .glm_host = "1", .glm_shared = "1" }));
}

test "profile line: control bytes and overlong values stay bounded on a single line" {
var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);
defer arena.deinit();
Expand Down
Loading