[MiniMax-H3][CUDA] Add deterministic FFN down projection candidate

.gitignore

@@ -208,6 +208,7 @@ __marimo__/
 
 # Cross-configuration alignment local run artifacts
 /runs/
+/artifacts/h3-down/
 
 # Local dev notes (not for upstream)
 _dev_notes/

benchmarks/operators/gemm/plot_h3_down_report.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2026 RL-Kernel Contributors
+"""Produce a readable report and static charts from measured H3 evidence only."""
+
+import argparse
+import csv
+import json
+import math
+from pathlib import Path
+
+
+def plot_reference_accuracy(axis, cases):
+    """Keep failed or unavailable accuracy checks visible without inventing errors."""
+    counts = [case["rows"] for case in cases]
+    for tensor in ("y", "dx", "dw"):
+        values, flagged = [], []
+        for case in cases:
+            checks = [
+                row
+                for row in case["checks"]
+                if row["tensor"] == tensor and row["transform"] == "fp32_reference"
+            ]
+            ratios = [row.get("max_error_over_tolerance") for row in checks]
+            available = bool(ratios) and all(
+                value is not None and math.isfinite(value) for value in ratios
+            )
+            values.append(max(ratios) if available else math.nan)
+            if not available or not all(row["passed"] for row in checks):
+                flagged.append(case["rows"])
+        (line,) = axis.plot(counts, values, marker="o", label=tensor)
+        if flagged:
+            # These markers use axes height, not the numerical error scale.
+            axis.scatter(
+                flagged,
+                [1.02] * len(flagged),
+                transform=axis.get_xaxis_transform(),
+                clip_on=False,
+                marker="x",
+                color=line.get_color(),
+                label=f"{tensor}: failed / unavailable",
+            )
+    axis.axhline(1, color="red", linestyle="--", label="shared tolerance boundary")
+    axis.set(
+        xlabel="Logical rows M",
+        ylabel="Max error / allowed error",
+        xscale="log",
+        title="Accuracy against independent FP32 reference",
+    )
+    axis.legend(fontsize=8)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("report")
+    parser.add_argument("--output-dir", required=True)
+    args = parser.parse_args()
+    report = json.loads(Path(args.report).read_text())
+    output = Path(args.output_dir)
+    output.mkdir(parents=True, exist_ok=True)
+    cases = report["cases"]
+    # Render earlier native-midtree reports alongside the versioned FP32 candidate.
+    for case in cases:
+        case["benchmarks"] = {
+            key.replace("native_", "candidate_", 1): value
+            for key, value in case["benchmarks"].items()
+        }
+    lines = [
+        "# H3 down projection GPU evidence",
+        "",
+        f"Run status: **{report['status']}**.",
+        "",
+        "Operator-only evidence; production promotion requires review of the matrix, "
+        "real checkpoint fixtures and the arithmetic contract.",
+        "",
+    ]
+    if report.get("error"):
+        lines += ["```text", report["error"], "```", ""]
+    if report.get("fixture"):
+        metadata = report["fixture"]["metadata"]
+        lines += [f"Checkpoint fixture activation kind: **{metadata['activation_kind']}**.", ""]
+    lines += [
+        "| Input | M | Checks passed | Total | Candidate fwd ms | torch BF16 fwd ms |",
+        "|---|---:|---:|---:|---:|---:|",
+    ]
+    for case in cases:
+        bench = case["benchmarks"]
+        lines.append(
+            f"| {case['input_family']} | {case['rows']} | "
+            f"{sum(row['passed'] for row in case['checks'])} | {len(case['checks'])} | "
+            f"{bench['candidate_forward']['median_ms']:.4f} | "
+            f"{bench['torch_bf16_forward']['median_ms']:.4f} |"
+        )
+    lines += ["", "Remaining acceptance:", ""]
+    lines += [f"- {value}" for value in report.get("remaining_acceptance", [])]
+    (output / "summary.md").write_text("\n".join(lines) + "\n")
+    with (output / "checks.csv").open("w", newline="") as stream:
+        writer = csv.writer(stream)
+        writer.writerow(
+            [
+                "input",
+                "M",
+                "judgment",
+                "tensor",
+                "transform",
+                "passed",
+                "max_abs_error",
+                "max_error_over_tolerance",
+                "mismatched_bytes",
+            ]
+        )
+        for case in cases:
+            for row in case["checks"]:
+                writer.writerow(
+                    [
+                        case["input_family"],
+                        case["rows"],
+                        row["judgment"],
+                        row["tensor"],
+                        row["transform"],
+                        row["passed"],
+                        row.get("max_abs_error"),
+                        row.get("max_error_over_tolerance"),
+                        row.get("mismatched_bytes"),
+                    ]
+                )
+    if not cases:
+        return
+    import matplotlib
+
+    matplotlib.use("Agg")
+    import matplotlib.pyplot as plt
+
+    for family in sorted({case["input_family"] for case in cases}):
+        subset = sorted(
+            (case for case in cases if case["input_family"] == family),
+            key=lambda case: case["rows"],
+        )
+        fig, axes = plt.subplots(1, 2, figsize=(11, 4), constrained_layout=True)
+        counts = [case["rows"] for case in subset]
+        for key, label in (
+            ("candidate_forward", "Candidate forward"),
+            ("torch_bf16_forward", "torch BF16 forward"),
+            ("candidate_forward_backward", "Candidate forward+backward"),
+            ("torch_bf16_forward_backward", "torch BF16 forward+backward"),
+        ):
+            axes[0].plot(
+                counts,
+                [case["benchmarks"][key]["median_ms"] for case in subset],
+                marker="o",
+                label=label,
+            )
+        axes[0].set(
+            xlabel="Logical rows M",
+            ylabel="Synchronized wall time (ms)",
+            xscale="log",
+            yscale="log",
+            title=family,
+        )
+        axes[0].legend(fontsize=8)
+        plot_reference_accuracy(axes[1], subset)
+        fig.savefig(output / f"{family}.png", dpi=180)
+        plt.close(fig)
+
+
+if __name__ == "__main__":
+    main()

ci/scripts/run_h3_down_h100.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2026 RL-Kernel Contributors
+set -euo pipefail
+
+repo_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$repo_dir"
+mode="${1:-smoke}"
+case "$mode" in smoke|full) ;; *) echo 'Usage: bash ci/scripts/run_h3_down_h100.sh [smoke|full]' >&2; exit 2 ;; esac
+result_dir="${H3_RESULT_DIR:-$repo_dir/artifacts/h3-down/$(date -u +%Y%m%dT%H%M%SZ)}"
+mkdir -p "$result_dir"
+export KERNEL_ALIGN_USE_FAST_MATH=0
+export KERNEL_ALIGN_DET_GEMM_SM90=1
+export KERNEL_ALIGN_FORCE_SM90=0
+export RL_KERNEL_REQUIRE_EXT=1
+export RL_KERNEL_DET_GEMM_BACKEND=sm90
+export MAX_JOBS="${MAX_JOBS:-8}"
+export PYTHONUNBUFFERED=1
+# Non-login development shells may omit the CUDA toolkit from PATH.
+if ! command -v nvcc >/dev/null 2>&1 && [[ -x "${CUDA_HOME:-/usr/local/cuda}/bin/nvcc" ]]; then
+    export CUDA_HOME="${CUDA_HOME:-/usr/local/cuda}"
+    export PATH="$CUDA_HOME/bin:$PATH"
+fi
+# BuildExtension's explicit 90a flags cover the TMA path; do not add plain compute_90 PTX.
+unset TORCH_CUDA_ARCH_LIST
+
+python - <<'PY'
+from pathlib import Path
+import os
+import sysconfig
+import torch
+if torch.version.hip is not None or not torch.cuda.is_available():
+    raise SystemExit('An NVIDIA H100 GPU and a CUDA PyTorch build are required')
+if torch.cuda.get_device_capability() != (9, 0):
+    raise SystemExit('This profile requires SM90')
+if torch.version.cuda != '12.4':
+    raise SystemExit('This initial runbook pins a CUDA 12.4 PyTorch build')
+print(torch.__version__, torch.cuda.get_device_name(), torch.cuda.get_device_capability())
+if os.environ.get('H3_BUILD_NATIVE', '0') == '1' and not (
+    Path(sysconfig.get_path('include')) / 'Python.h'
+).is_file():
+    raise SystemExit('Python development headers are required; install the matching pythonX.Y-dev')
+PY
+nvcc --version | tee "$result_dir/nvcc.txt"
+nvcc_version="$(< "$result_dir/nvcc.txt")"
+if [[ "$nvcc_version" != *"release 12.4"* ]]; then
+    echo 'This initial runbook requires nvcc 12.4' >&2
+    exit 2
+fi
+nvidia-smi -q > "$result_dir/nvidia-smi.txt"
+python -m pip freeze > "$result_dir/pip-freeze.txt"
+git status --short > "$result_dir/git-status.txt"
+git diff HEAD > "$result_dir/source.patch"
+python -m pytest -q tests/ops/gemm/test_h3_ffn_down.py -k 'not on_gpu' | tee "$result_dir/cpu-tests.txt"
+if [[ "${H3_BUILD_NATIVE:-0}" == 1 ]]; then
+    python -m pip install --no-deps --no-build-isolation -v -e . 2>&1 | tee "$result_dir/build.log"
+fi
+set +e
+python -m pytest -q tests/ops/gemm/test_h3_ffn_down.py | tee "$result_dir/gpu-tests.txt"
+gpu_test_status=${PIPESTATUS[0]}
+set -e
+
+args=(--output "$result_dir/report.json")
+if [[ -f "$result_dir/build.log" ]]; then args+=(--build-log "$result_dir/build.log"); fi
+if [[ -n "${H3_BACKEND:-}" ]]; then args+=(--backend "$H3_BACKEND"); fi
+if [[ "$mode" == smoke ]]; then
+    args+=(--rows 1,33,129 --families random --repeats 3)
+fi
+if [[ -n "${H3_FIXTURE:-}" ]]; then args+=(--fixture "$H3_FIXTURE"); fi
+if [[ "${H3_COMPARE_VLLM:-0}" == 1 ]]; then args+=(--compare-vllm); fi
+set +e
+python tools/validation/operators/validate_h3_ffn_down.py "${args[@]}" 2>&1 | tee "$result_dir/validation.log"
+validation_status=${PIPESTATUS[0]}
+set -e
+python benchmarks/operators/gemm/plot_h3_down_report.py "$result_dir/report.json" --output-dir "$result_dir"
+echo "Evidence directory: $result_dir"
+if [[ "$gpu_test_status" -ne 0 ]]; then exit "$gpu_test_status"; fi
+exit "$validation_status"