From e9a410b72772b00080ec61dba1aa6ce82411a314 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= Date: Sat, 17 Oct 2026 16:30:21 +0200 Subject: [PATCH 1/2] test(tp): retain verbs load failures in HCA diagnostics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 569577afd3c74c70525a4160e236b7654ac257e7) Reported-by: Christian Schröder (cherry picked from commit 4df28ced9871f0d32ecdb18a399efbc30c939a57) --- zig/src/fabric/tp/roce_pick.zig | 59 ++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 5 deletions(-) diff --git a/zig/src/fabric/tp/roce_pick.zig b/zig/src/fabric/tp/roce_pick.zig index b9355c401a..66589ba1ce 100644 --- a/zig/src/fabric/tp/roce_pick.zig +++ b/zig/src/fabric/tp/roce_pick.zig @@ -6,6 +6,7 @@ const bootstrap = @import("bootstrap.zig"); const sock = @import("sock.zig"); const Config = @import("config.zig").Config; const Verbs = @import("verbs").Verbs; +const OpenFn = *const fn (?[:0]const u8) @import("verbs").Error!Verbs; /// Ports a rank lists to its peers (a DGX Spark's ConnectX-7 shows four RDMA devices). pub const max_ports = 8; @@ -90,9 +91,13 @@ pub fn resolve(cfg: Config, b: *bootstrap.Bootstrap, out: *[rv.max_hcas]rv.HcaSp /// `resolve` with how the devices were chosen, over this host's ports (a collective, as `resolve`). pub fn select(cfg: Config, b: *bootstrap.Bootstrap, out: *[rv.max_hcas]rv.HcaSpec) !Pick { + return selectWithOpen(cfg, b, out, Verbs.open); +} + +fn selectWithOpen(cfg: Config, b: *bootstrap.Bootstrap, out: *[rv.max_hcas]rv.HcaSpec, open: OpenFn) !Pick { var ports: [max_ports]rv.Port = undefined; var n_ports: usize = 0; - if (Verbs.open(null)) |opened| { + if (open(null)) |opened| { var v = opened; defer v.close(); n_ports = rv.scanPorts(&v, &ports); @@ -338,14 +343,14 @@ test "the port scan reads a host's devices through verbs: state, Ethernet, the I } /// One rank's `selectFrom` over a real bootstrap (rank 0 on `l`), its hca line or error into `out`. -fn hcaRank(cfg: Config, l: ?sock.Fd, ports: []const rv.Port, out: *[list_bytes + 16]u8, line: *HcaError![]const u8, err: *?anyerror) void { +fn hcaRank(cfg: Config, l: ?sock.Fd, ports: []const rv.Port, open: ?OpenFn, out: *[list_bytes + 16]u8, line: *HcaError![]const u8, err: *?anyerror) void { var b = (if (l) |fd| bootstrap.Bootstrap.acceptOn(fd, 2, 5 * std.time.ns_per_s) else bootstrap.Bootstrap.open(cfg)) catch |e| { err.* = e; return; }; defer b.close(); var specs: [rv.max_hcas]rv.HcaSpec = undefined; - const pick = selectFrom(cfg, &b, ports, &specs) catch |e| { + const pick = (if (open) |f| selectWithOpen(cfg, &b, &specs, f) else selectFrom(cfg, &b, ports, &specs)) catch |e| { err.* = e; return; }; @@ -353,18 +358,62 @@ fn hcaRank(cfg: Config, l: ?sock.Fd, ports: []const rv.Port, out: *[list_bytes + } fn hcaPair(cfgs: [2]Config, ports: [2][]const rv.Port, lines: *[2]HcaError![]const u8, bufs: *[2][list_bytes + 16]u8) !void { + return hcaPairWithOpen(cfgs, ports, .{ null, null }, lines, bufs); +} + +fn hcaPairWithOpen(cfgs: [2]Config, ports: [2][]const rv.Port, open: [2]?OpenFn, lines: *[2]HcaError![]const u8, bufs: *[2][list_bytes + 16]u8) !void { const l = try bootstrap.testListener(); defer sock.close(l.fd); var errs: [2]?anyerror = @splat(null); var c1 = cfgs[1]; c1.link_port = l.port; c1.connect_timeout_ns = 5 * std.time.ns_per_s; - const t = try std.Thread.spawn(.{}, hcaRank, .{ c1, null, ports[1], &bufs[1], &lines[1], &errs[1] }); - hcaRank(cfgs[0], l.fd, ports[0], &bufs[0], &lines[0], &errs[0]); + const t = try std.Thread.spawn(.{}, hcaRank, .{ c1, null, ports[1], open[1], &bufs[1], &lines[1], &errs[1] }); + hcaRank(cfgs[0], l.fd, ports[0], open[0], &bufs[0], &lines[0], &errs[0]); t.join(); for (errs) |e| if (e) |x| return x; } +fn missingLibrary(_: ?[:0]const u8) @import("verbs").Error!Verbs { + return error.LibraryUnavailable; +} + +fn missingSymbol(_: ?[:0]const u8) @import("verbs").Error!Verbs { + return error.MissingSymbol; +} + +test "hca keeps a verbs load failure while completing the peer's port exchange" { + var lines: [2]HcaError![]const u8 = undefined; + var bufs: [2][list_bytes + 16]u8 = undefined; + var scanned: [max_ports]rv.Port = undefined; + const peer = ScanFixture.scan(&hosts[1], &scanned); + const cfgs: [2]Config = .{ .{ .rank = 0 }, .{ .rank = 1 } }; + try hcaPairWithOpen(cfgs, .{ &.{}, peer }, .{ missingLibrary, null }, &lines, &bufs); + try std.testing.expectError(error.LibraryUnavailable, lines[0]); + try std.testing.expectError(error.NoSharedSubnet, lines[1]); +} + +test "hca distinguishes missing verbs symbols from an empty successful scan" { + const cfg: Config = .{ .world = 1 }; + var b = try bootstrap.Bootstrap.open(cfg); + defer b.close(); + var specs: [rv.max_hcas]rv.HcaSpec = undefined; + var buf: [list_bytes + 16]u8 = undefined; + const pick = try selectWithOpen(cfg, &b, &specs, missingSymbol); + try std.testing.expectError(error.MissingSymbol, hcaLine(pick, &specs, &buf)); + try std.testing.expectError(error.NoRoceDevice, hcaLine(try selectFrom(cfg, &b, &.{}, &specs), &specs, &buf)); +} + +test "hca keeps an explicit list when the verbs scan cannot open its library" { + const cfg: Config = .{ .world = 1, .roce_hca = "fx_rocep1s0f0:3" }; + var b = try bootstrap.Bootstrap.open(cfg); + defer b.close(); + var specs: [rv.max_hcas]rv.HcaSpec = undefined; + var buf: [list_bytes + 16]u8 = undefined; + const pick = try selectWithOpen(cfg, &b, &specs, missingLibrary); + try std.testing.expectEqualStrings("NCCL_IB_HCA=fx_rocep1s0f0\n", try hcaLine(pick, &specs, &buf)); +} + test "hca over a real bootstrap: both ranks print their peer-linked ports; an explicit rank keeps its list; no ports fail both" { var lines: [2]HcaError![]const u8 = undefined; var bufs: [2][list_bytes + 16]u8 = undefined; From 06f87d528df6401562cd391a8d7f27312933f3b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= Date: Sat, 17 Oct 2026 16:33:30 +0200 Subject: [PATCH 2/2] fix(tp): distinguish verbs load errors from absent HCA ports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 8f75a6aa22c9c71d1342bbf0d914a67ae3a5318e) Reported-by: Christian Schröder (cherry picked from commit 0a80da001f05f5225cceeaa16190bb25422c786a) --- zig/src/fabric/tp/gpu_main.zig | 9 ++++++--- zig/src/fabric/tp/roce_pick.zig | 23 ++++++++++++++++++----- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/zig/src/fabric/tp/gpu_main.zig b/zig/src/fabric/tp/gpu_main.zig index 82fc1edd5b..3bb0e30f39 100644 --- a/zig/src/fabric/tp/gpu_main.zig +++ b/zig/src/fabric/tp/gpu_main.zig @@ -73,10 +73,13 @@ fn hca(cfg: tp.Config, io: std.Io) !u8 { var buf: [tp.roce_pick.list_bytes + 16]u8 = undefined; const line = tp.roce_pick.hcaLine(pick, &specs, &buf) catch |e| { const why = switch (e) { - error.NoRoceDevice => "no active Ethernet RDMA port with an IPv4 RoCE v2 GID on this host", - error.NoSharedSubnet => "no RoCE port here shares an IPv4 /24 with the peer's", + error.LibraryUnavailable => "could not load libibverbs.so.1 (LibraryUnavailable); check the RDMA userspace installation", + error.MissingSymbol => "libibverbs.so.1 lacks required symbols (MissingSymbol); check the RDMA userspace installation", + error.NoRoceDevice => "no active Ethernet RDMA port with an IPv4 RoCE v2 GID on this host: name the peer's ports in TF_TP_ROCE_HCA or NCCL_IB_HCA", + error.NoSharedSubnet => "no RoCE port here shares an IPv4 /24 with the peer's: name the peer's ports in TF_TP_ROCE_HCA or NCCL_IB_HCA", + else => @errorName(e), }; - std.debug.print("FAIL hca rank {d}: {s}: name the peer's ports in TF_TP_ROCE_HCA or NCCL_IB_HCA\n", .{ cfg.rank, why }); + std.debug.print("FAIL hca rank {d}: {s}\n", .{ cfg.rank, why }); return 1; }; var names: [tp.roce_pick.list_bytes]u8 = undefined; diff --git a/zig/src/fabric/tp/roce_pick.zig b/zig/src/fabric/tp/roce_pick.zig index 66589ba1ce..d1ed7a6529 100644 --- a/zig/src/fabric/tp/roce_pick.zig +++ b/zig/src/fabric/tp/roce_pick.zig @@ -13,7 +13,12 @@ pub const max_ports = 8; const blob_bytes = 1 + 4 * max_ports; /// How many devices, and how: on a /24 the peer shares (`shared`), named by TF_TP_ROCE_HCA (`explicit`), or neither. -pub const Pick = struct { n: usize, shared: bool, explicit: bool = false }; +pub const Pick = struct { + n: usize, + shared: bool, + explicit: bool = false, + verbs_open_error: ?@import("verbs").Error = null, +}; fn usable(p: rv.Port) bool { return p.active and p.ethernet and p.ipv4 != null; @@ -97,12 +102,17 @@ pub fn select(cfg: Config, b: *bootstrap.Bootstrap, out: *[rv.max_hcas]rv.HcaSpe fn selectWithOpen(cfg: Config, b: *bootstrap.Bootstrap, out: *[rv.max_hcas]rv.HcaSpec, open: OpenFn) !Pick { var ports: [max_ports]rv.Port = undefined; var n_ports: usize = 0; + var open_error: ?@import("verbs").Error = null; if (open(null)) |opened| { var v = opened; defer v.close(); n_ports = rv.scanPorts(&v, &ports); - } else |_| {} - return selectFrom(cfg, b, ports[0..n_ports], out); + } else |e| { + open_error = e; + } + var pick = try selectFrom(cfg, b, ports[0..n_ports], out); + pick.verbs_open_error = open_error; + return pick; } /// `select` over given ports (the scan's, or a test's fixture ports). @@ -140,11 +150,14 @@ pub const HcaError = error{ NoRoceDevice, /// this host's ports share no IPv4 /24 with the peer's: NCCL would not reach the peer on them either NoSharedSubnet, -}; +} || @import("verbs").Error; /// `tf-tp-test hca`'s line for run_hosts.sh: `NCCL_IB_HCA=` the picked devices; never the first local ports alone. pub fn hcaLine(pick: Pick, specs: []const rv.HcaSpec, buf: *[list_bytes + 16]u8) HcaError![]const u8 { - if (pick.n == 0) return error.NoRoceDevice; + if (pick.n == 0) { + if (pick.verbs_open_error) |e| return e; + return error.NoRoceDevice; + } if (!pick.shared and !pick.explicit) return error.NoSharedSubnet; const prefix = "NCCL_IB_HCA="; @memcpy(buf[0..prefix.len], prefix);