Nemotron Parse on HIP: NVIDIA's checkpoint, token-exact against its HF example

LICENSES/NVIDIA-OpenMDW-1.1.txt

@@ -0,0 +1,57 @@
+Unless separately indicated, the model weights and NVIDIA-provided source code
+and documentation are licensed under the OpenMDW-1.1 License.
+The tokenizer included with this model is separately licensed under the
+MIT license. See TOKENIZER_ATTRIBUTION.md for the license text,
+original copyright notice, source, and NVIDIA modification history.
+
+------------
+Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+
+OpenMDW License Agreement, version 1.1 (OpenMDW-1.1)
+
+By exercising rights granted to you under this agreement, you accept and agree
+to its terms.
+
+As used in this agreement, "Model Materials" means the materials provided to
+you under this agreement, consisting of: (1) one or more machine learning
+models (including architecture and parameters); and (2) all related artifacts
+(including associated data, documentation and software) that are provided to
+you hereunder.
+
+Subject to your compliance with this agreement, permission is hereby granted,
+free of charge, to deal in the Model Materials without restriction, including
+under all copyright, patent, database, and trade secret rights included or
+embodied therein.
+
+If you distribute any portion of the Model Materials, you shall retain in your
+distribution (1) a copy of this agreement, and (2) all copyright notices and
+other notices of origin included in the Model Materials that are applicable to
+your distribution.
+If you file, maintain, or voluntarily participate in a lawsuit against any
+person or entity asserting that the Model Materials directly or indirectly
+infringe any patent or copyright, then all rights and grants made to you
+hereunder are terminated, unless that lawsuit was in response to a
+corresponding lawsuit first brought against you.
+
+This agreement does not impose any restrictions or obligations with respect to
+any use, modification, or sharing of any outputs generated by using the Model
+Materials.
+
+THE MODEL MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE, NONINFRINGEMENT, ACCURACY, OR THE
+ABSENCE OF LATENT OR OTHER DEFECTS OR ERRORS, WHETHER OR NOT DISCOVERABLE, ALL
+TO THE GREATEST EXTENT PERMISSIBLE UNDER APPLICABLE LAW.
+
+YOU ARE SOLELY RESPONSIBLE FOR (1) CLEARING RIGHTS OF OTHER PERSONS THAT MAY
+APPLY TO THE MODEL MATERIALS OR ANY USE THEREOF, INCLUDING WITHOUT LIMITATION
+ANY PERSON'S COPYRIGHTS OR OTHER RIGHTS INCLUDED OR EMBODIED IN THE MODEL
+MATERIALS; (2) OBTAINING ANY NECESSARY CONSENTS, PERMISSIONS OR OTHER RIGHTS
+REQUIRED FOR ANY USE OF THE MODEL MATERIALS; OR (3) PERFORMING ANY DUE
+DILIGENCE OR UNDERTAKING ANY OTHER INVESTIGATIONS INTO THE MODEL MATERIALS OR
+ANYTHING INCORPORATED OR EMBODIED THEREIN.
+IN NO EVENT SHALL THE PROVIDERS OF THE MODEL MATERIALS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE MODEL MATERIALS, THE
+USE THEREOF OR OTHER DEALINGS THEREIN.

build.zig

@@ -191,6 +191,7 @@ pub fn build(b: *std.Build) void {
     const with_hip = b.option(bool, "hip", "Enable the opt-in HIP backend and its host/device targets") orelse false;
     const hip_options = hip_build.options(b); // declared on every target (#1165), used only with -Dhip=true
     hip_build.loaderTests(b, test_step);
+    @import("zig/build/nemotron_parse.zig").hostTests(b, test_step);
     if (with_hip) hip_build.steps(b, target, optimize, build_options, test_step, hip_options);
     const with_vulkan = b.option(bool, "vulkan", "Enable the opt-in Vulkan backend and its host/device targets") orelse false;
     const vulkan_options = vulkan_build.options(b); // declared on every target, used only with -Dvulkan=true

docs/recipes/README.md

@@ -8,6 +8,7 @@ For agent-assisted contributions, read the [TensorFold contributor skill](SKILL.
 | Family | Recipe |
 | --- | --- |
 | Nemotron 3.5 Lightning | [MLX](nemotron-3.5.md), [Intel Arc, experimental](xpu-arc.md) |
+| Nemotron 3.5 Parse | [Native HIP OCR, validation and qualification limits](nemotron-parse.md) |
 | Nemotron 3.5 Super VL | [Native CUDA on one or two GB10s, images](nemotron-3.5-super-vl.md) |
 | Qwen3.8-27B | [MLX, quantization and CUDA](qwen3.8-27b.md) |
 | Qwen3.5-2B | [Native Metal preview, tied affine 4-bit](qwen3.5-2b-native.md) |

docs/recipes/nemotron-parse.md

@@ -0,0 +1,141 @@
+# Nemotron 3.5 Parse on native HIP
+
+This port runs `nvidia/Nemotron-3.5-Parse` on one HIP rank. The
+validated checkpoint revision is
+`4035ffb1d8f8bafcdb867dfaf6f7514f101464f9`. The family is
+`nemotron_parse`; it is separate from Nemotron-H and Super VL.
+
+The original FP32 safetensors stay unchanged. The loader validates all
+510 tensors, then owns BF16 device copies of the 509 floating tensors.
+Activations and KV use BF16 with FP32 sums. The image tower, horizontal
+neck, causal mBART decoder and trained one-step proposal run in native HIP
+kernels. Python is used only for independent development references.
+
+## Build and serve
+
+Use the repository Zig version. On ROCm installations with a different
+layout, give the actual compiler and device-library paths:
+
+```sh
+zig build native-hip -Dhip=true -Dhip-arch=gfx1201 \
+  -Dhipcc=/opt/rocm/core/bin/hipcc \
+  -Dhip-device-lib=/opt/rocm/core/lib/llvm/amdgcn/bitcode
+
+zig-out/native-hip/bin/tensorfold-native serve "$PARSE_MODEL" \
+  --backend rocm --device 0 --vision --name parse --parallel 2 --context 4096 \
+  --mtp-drafts 1 --prompt-cache-gib 0.25 --host 127.0.0.1 --port 8000
+```
+
+`PARSE_MODEL` names the already cached snapshot. Select the intended
+visible device and verify its PCI identity; management and HIP ordinals
+can differ. The engine checks driver free memory at load and admission.
+The developer checks require at least six GiB free before a bounded
+two-lane run.
+
+Send one user message with one PNG `image_url` data URL to
+`/v1/chat/completions`. For example, with an ordinary OpenAI client:
+
+```python
+import base64
+from pathlib import Path
+from openai import OpenAI
+
+image = base64.b64encode(Path("page.png").read_bytes()).decode()
+client = OpenAI(base_url="http://127.0.0.1:8000/v1", api_key="local")
+reply = client.chat.completions.create(
+    model="parse",
+    messages=[{"role": "user", "content": [
+        {"type": "image_url", "image_url": {"url": "data:image/png;base64," + image}}
+    ]}],
+    temperature=0,
+    max_tokens=1024,
+)
+print(reply.choices[0].message.content)
+```
+
+Use the served model name reported by `/health`, or set `--name` explicitly.
+The built-in media template puts the visual prefix before the example's
+six task tokens. Optional text parts follow those tokens. Generated OCR,
+including its location/class markers, is returned in `content`. A custom
+`--chat-template` still overrides the built-in bridge.
+
+The image processor shrinks into 2048-by-1664 without enlargement, pads
+bottom/right to 16/64, and preserves the visible content size for RADIO
+positions. A full page uses 3329 visual context rows. Visual and text rows
+share the 16384-row model context; `--context` may lower this limit. One
+image is required. Multi-turn chat, video, tensor parallelism, lossy KV,
+Sliding Weights and multiple proposal heads are not part of this port.
+
+Default generation is greedy with repetition penalty 1.1. The shared
+sampler widens raw logits to FP32 before applying penalties, and forces
+EOS at the configured final output position. `"draft": false` selects
+plain target decoding; normal drafted requests use the trained head and
+the existing target verifier. Shared rounds batch dense work across all
+admitted streams. Prompt snapshots use image content and frame identities.
+
+## Reproduce the checks
+
+Create offline BF16 references from the checkpoint's own example workflow:
+
+```sh
+python tools/zig/nemotron_parse_reference.py --model "$PARSE_MODEL" \
+  --image page.png --output "$REFERENCE"
+python tools/zig/nemotron_parse_reference.py positions --model "$PARSE_MODEL" \
+  --output "$POSITION_REFERENCE"
+
+zig build test-nemotron-parse-host -Dparse-model-dir="$PARSE_MODEL" \
+  -Dparse-reference-dir="$REFERENCE"
+
+TF_PARSE_DIR="$PARSE_MODEL" TF_PARSE_POSITION_DIR="$POSITION_REFERENCE" \
+  zig build hip-parse-ops-test -Dhip=true -Dhip-arch=gfx1201 \
+  -Dhipcc=/opt/rocm/core/bin/hipcc \
+  -Dhip-device-lib=/opt/rocm/core/lib/llvm/amdgcn/bitcode
+```
+
+The reference environment used CPU Torch 2.11.0, TorchVision 0.26.0,
+Transformers 5.14.1 and timm 1.0.22. It loads only local assets. The tool
+retains NVIDIA's OpenMDW notice; the license is
+[LICENSES/NVIDIA-OpenMDW-1.1.txt](../../LICENSES/NVIDIA-OpenMDW-1.1.txt).
+Reference directories must be new; large tensors are not checked into Git.
+`nemotron_parse_reference.py processor` prints the model-free
+`zig/src/families/nemotron_parse/fixtures/processor.json`.
+
+`tf-nemotron-parse-test` builds a developer probe. Give it the model,
+reference and output directories. It compares intermediate seams and
+teacher-forced logits, then checks full/chunked and solo/shared KV/hidden
+bytes and suffix rollback. `nemotron_parse_reference.py compare
+--reference "$REFERENCE" --native "$PROBE_OUTPUT"` applies the original HF
+logits processor to saved native rows.
+
+The model-free contract runner registers twelve new operator cases.
+`hip-vision-record-objects` builds the actual ops image for gfx1030,
+gfx1100, gfx1151 and gfx1201. `hip_vision_records.zig`, with the
+`core/contract/hsaco.zig` module imported as `hsaco`, emits their measured
+launch records. The runner's `vision` mode is diagnostic; its default
+mode runs the complete contract and planted controls.
+
+## Evidence and limits
+
+Native HTTP token arrays matched three PNG references (two 512-by-256
+pages, one 1664-by-2048 page), plain/drafted and solo/concurrent. The
+full page uses 3335 prompt rows including the six text tokens. Prompt KV
+and final-hidden bytes agree across full/chunked and solo/shared forwards
+and after suffix rollback. No speed target is claimed.
+
+All twelve new operator cases pass arithmetic gates 1–4, including five
+repeat runs, guards, non-finites, tile edges and accumulator stress. New
+launch records match actual compiles on all four recorded chips and the
+live gfx1201 device. The complete contract still reports 120 inherited
+launch-record mismatches for older kernels. Fresh compiles of the exact
+base have identical usage metadata for all 456 pre-existing symbols.
+Do not describe the complete contract as passing.
+
+BF16 intermediate tensors differ from HF's CPU GEMM rounding; saved
+seam errors are diagnostic, while actual output token arrays are checked
+separately. Runtime model checks cover gfx1201 only. CUDA, Metal, other
+AMD hardware have not been qualified here. Sampled plain/drafted equality
+passes at two small-page seeds, one with presence and frequency penalties;
+this does not compare TensorFold's keyed sampler with HF's random draws.
+PNG is tested. JPEG uses the repository's pinned libjpeg-turbo decoder
+and has no qualification receipt yet. Structured output needs the
+existing grammar library and has no real-model receipt yet.

tools/zig/hip_vision_records.zig

@@ -0,0 +1,26 @@
+//! Emit measured records from hip-vision-record-objects; import core/contract/hsaco.zig as the hsaco module.
+const std = @import("std");
+const hsaco = @import("hsaco");
+
+pub fn main(init: std.process.Init) !void {
+    const a = init.arena.allocator();
+    const args = try init.minimal.args.toSlice(a);
+    if (args.len != 2) return error.Usage;
+    const chips = [_][]const u8{ "gfx1030", "gfx1100", "gfx1151", "gfx1201" };
+    var objects: [chips.len][]hsaco.Kernel = undefined;
+    for (chips, 0..) |chip, i| {
+        const path = try std.fmt.allocPrint(a, "{s}/{s}.hsaco", .{ args[1], chip });
+        const bytes = try std.Io.Dir.cwd().readFileAlloc(init.io, path, a, .limited(1 << 24));
+        objects[i] = try hsaco.kernels(a, bytes);
+    }
+    std.debug.print("//! Measured with hip-vision-record-objects; hipcc flags and caps match the served ops image.\nconst Record = @import(\"hip_records.zig\").Record;\n\npub const records = [_]Record{{\n", .{});
+    for ([_][]const u8{ "tf_vl_embed", "tf_vl_linear", "tf_vl_layernorm", "tf_vl_pointwise", "tf_vl_layout", "tf_vl_resize", "tf_vl_position" }) |name| {
+        std.debug.print("    .{{\n        .launch = .{{ .name = \"vision {s}\", .symbol = \"{s}\", .image = \"ops\", .threads = 256 }},\n        .usage = .{{\n", .{ name, name });
+        for (objects) |object| {
+            const u = (hsaco.find(object, name) orelse return error.NoSymbol).usage;
+            std.debug.print("            .{{ .vgprs = {d}, .sgprs = {d}, .lds = {d}, .private = {d}, .bound = {d}, .wave = {d} }},\n", .{ u.vgprs, u.sgprs, u.lds, u.private, u.bound, u.wave });
+        }
+        std.debug.print("        }},\n    }},\n", .{});
+    }
+    std.debug.print("}};\n", .{});
+}